stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 12 Releases Wiki Activity

up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Details
Findings Ledger CI / build-test (push) Has been cancelled
Details
Findings Ledger CI / migration-validation (push) Has been cancelled
Details
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Details
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Details
AOC Guard CI / aoc-guard (push) Has been cancelled
Details
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Details
cryptopro-linux-csp / build-and-test (push) Has been cancelled
Details
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Details
Signals CI & Image / signals-ci (push) Has been cancelled
Details
sm-remote-ci / build-and-test (push) Has been cancelled
Details
Findings Ledger CI / generate-manifest (push) Has been cancelled
Details
AOC Guard CI / aoc-verify (push) Has been cancelled
Details
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Details
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Details

Browse Source
This commit is contained in:
StellaOps Bot
2025-12-09 09:38:09 +02:00
parent bc0762e97d
commit 108d1c64b3
193 changed files with 7265 additions and 13029 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
.gitea/workflows/cryptopro-linux-csp.yml Normal file
View File

@@ -0,0 +1,55 @@
name: cryptopro-linux-csp
on:
push:
branches: [main, develop]
paths:
- 'ops/cryptopro/linux-csp-service/**'
- 'opt/cryptopro/downloads/**'
- '.gitea/workflows/cryptopro-linux-csp.yml'
pull_request:
paths:
- 'ops/cryptopro/linux-csp-service/**'
- 'opt/cryptopro/downloads/**'
- '.gitea/workflows/cryptopro-linux-csp.yml'
env:
IMAGE_NAME: cryptopro-linux-csp
DOCKERFILE: ops/cryptopro/linux-csp-service/Dockerfile
jobs:
build-and-test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Build image (accept EULA explicitly)
run: |
docker build -t $IMAGE_NAME \
--build-arg CRYPTOPRO_ACCEPT_EULA=1 \
-f $DOCKERFILE .
- name: Run container
run: |
docker run -d --rm --name $IMAGE_NAME -p 18080:8080 $IMAGE_NAME
for i in {1..20}; do
if curl -sf http://127.0.0.1:18080/health >/dev/null; then
exit 0
fi
sleep 3
done
echo "Service failed to start" && exit 1
- name: Test endpoints
run: |
curl -sf http://127.0.0.1:18080/health
curl -sf http://127.0.0.1:18080/license || true
curl -sf -X POST http://127.0.0.1:18080/hash \
-H "Content-Type: application/json" \
-d '{"data_b64":"SGVsbG8="}'
- name: Stop container
if: always()
run: docker rm -f $IMAGE_NAME || true

10
.gitea/workflows/signals-dsse-sign.yml
View File

@@ -44,6 +44,16 @@ jobs:
with:
cosign-release: 'v2.2.4'
- name: Check signing key configured
run: |
if [[ -z "$COSIGN_PRIVATE_KEY_B64" && "$COSIGN_ALLOW_DEV_KEY" != "1" ]]; then
echo "::error::COSIGN_PRIVATE_KEY_B64 is missing and dev key fallback is disabled. Set COSIGN_PRIVATE_KEY_B64 (and COSIGN_PASSWORD if needed) or rerun with allow_dev_key=1 for smoke only."
exit 1
fi
if [[ "$COSIGN_ALLOW_DEV_KEY" == "1" ]]; then
echo "::notice::Using dev key for signing (allow_dev_key=1) - not suitable for production uploads."
fi
- name: Verify artifacts exist
run: |
cd docs/modules/signals

10
.gitea/workflows/signals-evidence-locker.yml
View File

@@ -38,6 +38,16 @@ jobs:
with:
cosign-release: 'v2.2.4'
- name: Check signing key configured
run: |
if [[ -z "$COSIGN_PRIVATE_KEY_B64" && "$COSIGN_ALLOW_DEV_KEY" != "1" ]]; then
echo "::error::COSIGN_PRIVATE_KEY_B64 is missing and dev key fallback is disabled. Set COSIGN_PRIVATE_KEY_B64 (and COSIGN_PASSWORD if needed) or rerun with allow_dev_key=1 for smoke only."
exit 1
fi
if [[ "$COSIGN_ALLOW_DEV_KEY" == "1" ]]; then
echo "::notice::Using dev key for signing (allow_dev_key=1) - not suitable for production uploads."
fi
- name: Verify artifacts exist
run: |
cd "$MODULE_ROOT"

10
.gitea/workflows/signals-reachability.yml
View File

@@ -77,6 +77,16 @@ jobs:
with:
cosign-release: 'v2.2.4'
- name: Check signing key configured
run: |
if [[ -z "$COSIGN_PRIVATE_KEY_B64" && "$COSIGN_ALLOW_DEV_KEY" != "1" ]]; then
echo "::error::COSIGN_PRIVATE_KEY_B64 is missing and dev key fallback is disabled. Set COSIGN_PRIVATE_KEY_B64 (and COSIGN_PASSWORD if needed) or rerun with allow_dev_key=1 for smoke only."
exit 1
fi
if [[ "$COSIGN_ALLOW_DEV_KEY" == "1" ]]; then
echo "::notice::Using dev key for signing (allow_dev_key=1) - not suitable for production uploads."
fi
- name: Verify artifacts exist
run: |
cd docs/modules/signals

449
.gitea/workflows/wine-csp-build.yml
View File

@@ -1,449 +0,0 @@
name: wine-csp-build
on:
push:
branches: [main, develop]
paths:
- 'src/__Tools/WineCspService/**'
- 'ops/wine-csp/**'
- 'third_party/forks/AlexMAS.GostCryptography/**'
- '.gitea/workflows/wine-csp-build.yml'
pull_request:
paths:
- 'src/__Tools/WineCspService/**'
- 'ops/wine-csp/**'
- 'third_party/forks/AlexMAS.GostCryptography/**'
workflow_dispatch:
inputs:
push:
description: "Push to registry"
required: false
default: "false"
version:
description: "Version tag (e.g., 2025.10.0-edge)"
required: false
default: "2025.10.0-edge"
skip_tests:
description: "Skip integration tests"
required: false
default: "false"
env:
IMAGE_NAME: registry.stella-ops.org/stellaops/wine-csp
DOCKERFILE: ops/wine-csp/Dockerfile
# Wine CSP only supports linux/amd64 (Wine ARM64 has compatibility issues with Windows x64 apps)
PLATFORMS: linux/amd64
PYTHON_VERSION: "3.11"
jobs:
# ===========================================================================
# Job 1: Build Docker Image
# ===========================================================================
build:
name: Build Wine CSP Image
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
outputs:
image_tag: ${{ steps.version.outputs.tag }}
image_digest: ${{ steps.build.outputs.digest }}
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
install: true
- name: Set version tag
id: version
run: |
if [[ -n "${{ github.event.inputs.version }}" ]]; then
echo "tag=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT
elif [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
echo "tag=2025.10.0-edge" >> $GITHUB_OUTPUT
else
echo "tag=pr-${{ github.event.pull_request.number || github.sha }}" >> $GITHUB_OUTPUT
fi
- name: Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.IMAGE_NAME }}
tags: |
type=raw,value=${{ steps.version.outputs.tag }}
type=sha,format=short
- name: Build image
id: build
uses: docker/build-push-action@v6
with:
context: .
file: ${{ env.DOCKERFILE }}
platforms: ${{ env.PLATFORMS }}
push: false
load: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Save image for testing
run: |
mkdir -p /tmp/images
docker save "${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}" | gzip > /tmp/images/wine-csp.tar.gz
- name: Upload image artifact
uses: actions/upload-artifact@v4
with:
name: wine-csp-image
path: /tmp/images/wine-csp.tar.gz
retention-days: 1
# ===========================================================================
# Job 2: Integration Tests
# ===========================================================================
test:
name: Integration Tests
runs-on: ubuntu-latest
needs: build
if: ${{ github.event.inputs.skip_tests != 'true' }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Download image artifact
uses: actions/download-artifact@v4
with:
name: wine-csp-image
path: /tmp/images
- name: Load Docker image
run: |
gunzip -c /tmp/images/wine-csp.tar.gz | docker load
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Install test dependencies
run: |
pip install -r ops/wine-csp/tests/requirements.txt
- name: Start Wine CSP container
id: container
run: |
echo "Starting Wine CSP container..."
docker run -d --name wine-csp-test \
-e WINE_CSP_MODE=limited \
-e WINE_CSP_LOG_LEVEL=Debug \
-p 5099:5099 \
"${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}"
echo "container_id=$(docker ps -q -f name=wine-csp-test)" >> $GITHUB_OUTPUT
- name: Wait for service startup
run: |
echo "Waiting for Wine CSP service to be ready (up to 120s)..."
for i in $(seq 1 24); do
if curl -sf http://127.0.0.1:5099/health > /dev/null 2>&1; then
echo "Service ready after $((i * 5))s"
exit 0
fi
echo "Waiting... ($((i * 5))s elapsed)"
sleep 5
done
echo "Service failed to start!"
docker logs wine-csp-test
exit 1
- name: Run integration tests (pytest)
id: pytest
run: |
mkdir -p test-results
export WINE_CSP_URL=http://127.0.0.1:5099
pytest ops/wine-csp/tests/test_wine_csp.py \
-v \
--tb=short \
--junitxml=test-results/junit.xml \
--timeout=60 \
-x \
2>&1 | tee test-results/pytest-output.txt
- name: Run shell integration tests
if: always()
run: |
chmod +x ops/wine-csp/tests/run-tests.sh
ops/wine-csp/tests/run-tests.sh \
--url http://127.0.0.1:5099 \
--ci \
--verbose || true
- name: Collect container logs
if: always()
run: |
docker logs wine-csp-test > test-results/container.log 2>&1 || true
- name: Stop container
if: always()
run: |
docker stop wine-csp-test || true
docker rm wine-csp-test || true
- name: Upload test results
uses: actions/upload-artifact@v4
if: always()
with:
name: wine-csp-test-results
path: test-results/
- name: Publish test results
uses: mikepenz/action-junit-report@v4
if: always()
with:
report_paths: 'test-results/junit.xml'
check_name: 'Wine CSP Integration Tests'
fail_on_failure: true
# ===========================================================================
# Job 3: Security Scan
# ===========================================================================
security:
name: Security Scan
runs-on: ubuntu-latest
needs: build
permissions:
security-events: write
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Download image artifact
uses: actions/download-artifact@v4
with:
name: wine-csp-image
path: /tmp/images
- name: Load Docker image
run: |
gunzip -c /tmp/images/wine-csp.tar.gz | docker load
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}"
format: 'sarif'
output: 'trivy-results.sarif'
severity: 'CRITICAL,HIGH'
ignore-unfixed: true
- name: Upload Trivy scan results
uses: github/codeql-action/upload-sarif@v3
if: always()
with:
sarif_file: 'trivy-results.sarif'
- name: Run Trivy for JSON report
uses: aquasecurity/trivy-action@master
with:
image-ref: "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}"
format: 'json'
output: 'trivy-results.json'
severity: 'CRITICAL,HIGH,MEDIUM'
- name: Upload Trivy JSON report
uses: actions/upload-artifact@v4
with:
name: wine-csp-security-scan
path: trivy-results.json
# ===========================================================================
# Job 4: Generate SBOM
# ===========================================================================
sbom:
name: Generate SBOM
runs-on: ubuntu-latest
needs: build
steps:
- name: Download image artifact
uses: actions/download-artifact@v4
with:
name: wine-csp-image
path: /tmp/images
- name: Load Docker image
run: |
gunzip -c /tmp/images/wine-csp.tar.gz | docker load
- name: Install syft
uses: anchore/sbom-action/download-syft@v0
- name: Generate SBOM (SPDX)
run: |
mkdir -p out/sbom
syft "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}" \
-o spdx-json=out/sbom/wine-csp.spdx.json
- name: Generate SBOM (CycloneDX)
run: |
syft "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}" \
-o cyclonedx-json=out/sbom/wine-csp.cdx.json
- name: Upload SBOM artifacts
uses: actions/upload-artifact@v4
with:
name: wine-csp-sbom-${{ needs.build.outputs.image_tag }}
path: out/sbom/
# ===========================================================================
# Job 5: Publish (only on main branch or manual trigger)
# ===========================================================================
publish:
name: Publish Image
runs-on: ubuntu-latest
needs: [build, test, security]
if: ${{ (github.event.inputs.push == 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && needs.test.result == 'success' }}
permissions:
contents: read
packages: write
id-token: write
steps:
- name: Download image artifact
uses: actions/download-artifact@v4
with:
name: wine-csp-image
path: /tmp/images
- name: Load Docker image
run: |
gunzip -c /tmp/images/wine-csp.tar.gz | docker load
- name: Install cosign
uses: sigstore/cosign-installer@v3.7.0
- name: Login to registry
uses: docker/login-action@v3
with:
registry: registry.stella-ops.org
username: ${{ secrets.REGISTRY_USER }}
password: ${{ secrets.REGISTRY_TOKEN }}
- name: Push to registry
run: |
docker push "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}"
# Also tag as latest if on main
if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
docker tag "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}" "${{ env.IMAGE_NAME }}:latest"
docker push "${{ env.IMAGE_NAME }}:latest"
fi
- name: Sign image with cosign
env:
COSIGN_EXPERIMENTAL: "1"
run: |
cosign sign --yes "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}" || echo "Signing skipped (no OIDC available)"
- name: Create release summary
run: |
echo "## Wine CSP Image Published" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "**Image:** \`${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}\`" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "**WARNING:** This image is for TEST VECTOR GENERATION ONLY." >> $GITHUB_STEP_SUMMARY
# ===========================================================================
# Job 6: Air-Gap Bundle
# ===========================================================================
airgap:
name: Air-Gap Bundle
runs-on: ubuntu-latest
needs: [build, test]
if: ${{ needs.test.result == 'success' }}
steps:
- name: Download image artifact
uses: actions/download-artifact@v4
with:
name: wine-csp-image
path: /tmp/images
- name: Create air-gap bundle
run: |
mkdir -p out/bundles
# Copy the image tarball
cp /tmp/images/wine-csp.tar.gz out/bundles/wine-csp-${{ needs.build.outputs.image_tag }}.tar.gz
# Generate bundle manifest
cat > out/bundles/wine-csp-${{ needs.build.outputs.image_tag }}.manifest.json <<EOF
{
"name": "wine-csp",
"version": "${{ needs.build.outputs.image_tag }}",
"image": "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}",
"platform": "linux/amd64",
"sha256": "$(sha256sum out/bundles/wine-csp-${{ needs.build.outputs.image_tag }}.tar.gz | cut -d' ' -f1)",
"created": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
"git_commit": "${{ github.sha }}",
"git_ref": "${{ github.ref }}",
"warning": "FOR TEST VECTOR GENERATION ONLY - NOT FOR PRODUCTION SIGNING"
}
EOF
# Create checksums file
cd out/bundles
sha256sum *.tar.gz *.json > SHA256SUMS
echo "Air-gap bundle contents:"
ls -lh
- name: Upload air-gap bundle
uses: actions/upload-artifact@v4
with:
name: wine-csp-bundle-${{ needs.build.outputs.image_tag }}
path: out/bundles/
# ===========================================================================
# Job 7: Test Summary
# ===========================================================================
summary:
name: Test Summary
runs-on: ubuntu-latest
needs: [build, test, security, sbom]
if: always()
steps:
- name: Download test results
uses: actions/download-artifact@v4
with:
name: wine-csp-test-results
path: test-results/
continue-on-error: true
- name: Create summary
run: |
echo "## Wine CSP Build Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Stage | Status |" >> $GITHUB_STEP_SUMMARY
echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY
echo "| Build | ${{ needs.build.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| Tests | ${{ needs.test.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| Security | ${{ needs.security.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| SBOM | ${{ needs.sbom.result }} |" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "**Image Tag:** \`${{ needs.build.outputs.image_tag }}\`" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "---" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "**SECURITY WARNING:** Wine CSP is for TEST VECTOR GENERATION ONLY." >> $GITHUB_STEP_SUMMARY

1
bench/reachability-benchmark/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
.jdk/

1
bench/reachability-benchmark/AGENTS.md
View File

@@ -20,6 +20,7 @@
## Working Agreements
- Determinism: pin toolchains; set `SOURCE_DATE_EPOCH`; sort file lists; stable JSON/YAML ordering; fixed seeds for any sampling.
- Offline posture: no network at build/test time; vendored toolchains; registry pulls are forbidden—use cached/bundled images.
- Java builds: use vendored Temurin 21 via `tools/java/ensure_jdk.sh` when `JAVA_HOME`/`javac` are absent; keep `.jdk/` out of VCS and use `build_all.py --skip-lang` when a toolchain is missing.
- Licensing: all benchmark content Apache-2.0; include LICENSE in repo root; third-party cases must have compatible licenses and attributions.
- Evidence: each case must include oracle tests/coverage proving reachability label; store truth and submissions under `benchmark/truth/` and `benchmark/submissions/` with JSON Schema.
- Security: no secrets; scrub URLs/tokens; deterministic CI artifacts only.

52
bench/reachability-benchmark/README.md
View File

@@ -8,38 +8,42 @@ Deterministic, reproducible benchmark for reachability analysis tools.
- Enable fair scoring via the `rb-score` CLI and published schemas.
## Layout
- `cases/<lang>/<project>/` — benchmark cases with deterministic Dockerfiles, pinned deps, oracle tests.
- `schemas/` — JSON/YAML schemas for cases, entrypoints, truth, submissions.
- `benchmark/truth/` — ground-truth labels (hidden/internal split optional).
- `benchmark/submissions/` — sample submissions and format reference.
- `tools/scorer/` — `rb-score` CLI and tests.
- `tools/build/` — `build_all.py` (run all cases) and `validate_builds.py` (run twice and compare hashes).
- `baselines/` — reference runners (Semgrep, CodeQL, Stella) with normalized outputs.
- `ci/` — deterministic CI workflows and scripts.
- `website/` — static site (leaderboard/docs/downloads).
- `cases/<lang>/<project>/` ƒ?" benchmark cases with deterministic Dockerfiles, pinned deps, oracle tests.
- `schemas/` ƒ?" JSON/YAML schemas for cases, entrypoints, truth, submissions.
- `benchmark/truth/` ƒ?" ground-truth labels (hidden/internal split optional).
- `benchmark/submissions/` ƒ?" sample submissions and format reference.
- `tools/scorer/` ƒ?" `rb-score` CLI and tests.
- `tools/build/` ƒ?" `build_all.py` (run all cases) and `validate_builds.py` (run twice and compare hashes).
- `baselines/` ƒ?" reference runners (Semgrep, CodeQL, Stella) with normalized outputs.
- `ci/` ƒ?" deterministic CI workflows and scripts.
- `website/` ƒ?" static site (leaderboard/docs/downloads).
Sample cases added (JS track):
- `cases/js/unsafe-eval` (reachable sink) → `benchmark/truth/js-unsafe-eval.json`.
- `cases/js/guarded-eval` (unreachable by default) → `benchmark/truth/js-guarded-eval.json`.
- `cases/js/express-eval` (admin eval reachable) → `benchmark/truth/js-express-eval.json`.
- `cases/js/express-guarded` (admin eval gated by env) → `benchmark/truth/js-express-guarded.json`.
- `cases/js/fastify-template` (template rendering reachable) → `benchmark/truth/js-fastify-template.json`.
- `cases/js/unsafe-eval` (reachable sink) ƒ+' `benchmark/truth/js-unsafe-eval.json`.
- `cases/js/guarded-eval` (unreachable by default) ƒ+' `benchmark/truth/js-guarded-eval.json`.
- `cases/js/express-eval` (admin eval reachable) ƒ+' `benchmark/truth/js-express-eval.json`.
- `cases/js/express-guarded` (admin eval gated by env) ƒ+' `benchmark/truth/js-express-guarded.json`.
- `cases/js/fastify-template` (template rendering reachable) ƒ+' `benchmark/truth/js-fastify-template.json`.
Sample cases added (Python track):
- `cases/py/unsafe-exec` (reachable eval) → `benchmark/truth/py-unsafe-exec.json`.
- `cases/py/guarded-exec` (unreachable when FEATURE_ENABLE != 1) → `benchmark/truth/py-guarded-exec.json`.
- `cases/py/flask-template` (template rendering reachable) → `benchmark/truth/py-flask-template.json`.
- `cases/py/fastapi-guarded` (unreachable unless ALLOW_EXEC=true) → `benchmark/truth/py-fastapi-guarded.json`.
- `cases/py/django-ssti` (template rendering reachable, autoescape off) → `benchmark/truth/py-django-ssti.json`.
- `cases/py/unsafe-exec` (reachable eval) ƒ+' `benchmark/truth/py-unsafe-exec.json`.
- `cases/py/guarded-exec` (unreachable when FEATURE_ENABLE != 1) ƒ+' `benchmark/truth/py-guarded-exec.json`.
- `cases/py/flask-template` (template rendering reachable) ƒ+' `benchmark/truth/py-flask-template.json`.
- `cases/py/fastapi-guarded` (unreachable unless ALLOW_EXEC=true) ƒ+' `benchmark/truth/py-fastapi-guarded.json`.
- `cases/py/django-ssti` (template rendering reachable, autoescape off) ƒ+' `benchmark/truth/py-django-ssti.json`.
Sample cases added (Java track):
- `cases/java/spring-deserialize` (reachable Java deserialization) → `benchmark/truth/java-spring-deserialize.json`.
- `cases/java/spring-guarded` (deserialization unreachable unless ALLOW_DESER=true) → `benchmark/truth/java-spring-guarded.json`.
- `cases/java/spring-deserialize` (reachable Java deserialization) ƒ+' `benchmark/truth/java-spring-deserialize.json`.
- `cases/java/spring-guarded` (deserialization unreachable unless ALLOW_DESER=true) ƒ+' `benchmark/truth/java-spring-guarded.json`.
- `cases/java/micronaut-deserialize` (reachable Micronaut-style deserialization) ƒ+' `benchmark/truth/java-micronaut-deserialize.json`.
- `cases/java/micronaut-guarded` (unreachable unless ALLOW_MN_DESER=true) ƒ+' `benchmark/truth/java-micronaut-guarded.json`.
- `cases/java/spring-reflection` (reflection sink reachable via Class.forName) ƒ+' `benchmark/truth/java-spring-reflection.json`.
## Determinism & Offline Rules
- No network during build/test; pin images/deps; set `SOURCE_DATE_EPOCH`.
- Sort file lists; stable JSON/YAML emitters; fixed RNG seeds.
- All scripts must succeed on a clean machine with cached toolchain tarballs only.
- Java builds auto-use vendored Temurin 21 via `tools/java/ensure_jdk.sh` when `JAVA_HOME`/`javac` are absent.
## Licensing
- Apache-2.0 for all benchmark assets. Third-party snippets must be license-compatible and attributed.
@@ -50,8 +54,10 @@ Sample cases added (Java track):
python tools/validate.py all schemas/examples
# score a submission (coming in task 513-008)
cd tools/scorer
./rb-score --cases ../cases --truth ../benchmark/truth --submission ../benchmark/submissions/sample.json
./tools/scorer/rb-score --cases cases --truth benchmark/truth --submission benchmark/submissions/sample.json
# deterministic case builds (skip a language when a toolchain is unavailable)
python tools/build/build_all.py --cases cases --skip-lang js
```
## Contributing

11
bench/reachability-benchmark/benchmark/CHANGELOG.md
View File

@@ -1,11 +1,16 @@
# Reachability Benchmark Changelog
## 1.0.1 · 2025-12-03
## 1.0.2 Aú 2025-12-05
- Unblocked Java track with vendored Temurin 21 (`tools/java/ensure_jdk.sh`) and deterministic build artifacts (coverage + traces).
- Added three more Java cases (`micronaut-deserialize`, `micronaut-guarded`, `spring-reflection`) to reach 5/5 required cases.
- `tools/build/build_all.py` now supports `--skip-lang` and runs under WSL-aware bash; CI builds Java cases by default.
## 1.0.1 Aú 2025-12-03
- Added manifest schema + sample manifest with hashes, SBOM/attestation entries, and sandbox/redaction metadata.
- Added coverage/trace schemas and extended validator to cover them.
- Introduced `tools/verify_manifest.py` and deterministic offline kit packaging script.
- Added per-language determinism env templates and dataset safety checklist.
- Populated SBOM + attestation outputs for JS/PY/C tracks; Java remains blocked on JDK availability.
- Populated SBOM + attestation outputs for JS/PY/C tracks.
## 1.0.0 · 2025-12-01
## 1.0.0 Aú 2025-12-01
- Initial public dataset, scorer, baselines, and website.

2
bench/reachability-benchmark/benchmark/checklists/dataset-safety.md
View File

@@ -8,7 +8,7 @@ Version: 1.0.1 · Date: 2025-12-03
- [x] Published schemas/validators: truth/submission/coverage/trace + manifest schemas; validated via `tools/validate.py` and `tools/verify_manifest.py`.
- [x] Evidence bundles: coverage + traces + attestation + sbom recorded per case (sample manifest).
- [x] Binary case recipe: `cases/**/build/build.sh` pinned `SOURCE_DATE_EPOCH` and env templates under `benchmark/templates/determinism/`.
- [x] Determinism CI: `ci/run-ci.sh` + `tools/verify_manifest.py` run twice to compare hashes; Java track still blocked on JDK availability.
- [x] Determinism CI: `ci/run-ci.sh` + `tools/verify_manifest.py` run twice to compare hashes; Java track uses vendored Temurin 21 via `tools/java/ensure_jdk.sh`.
- [x] Signed baselines: baseline submissions may include DSSE path in manifest (not required for sample kit); rulepack hashes recorded separately.
- [x] Submission policy: CLA/DSSE optional in sample; production kits require DSSE envelope recorded in `signatures`.
- [x] Semantic versioning & changelog: see `benchmark/CHANGELOG.md`; manifest `version` mirrors dataset release.

277
bench/reachability-benchmark/benchmark/manifest.sample.json
View File

@@ -1,92 +1,203 @@
{
"schemaVersion": "1.0.0",
"kitId": "reachability-benchmark:public-v1",
"version": "1.0.1",
"artifacts": {
"baselineSubmissions": [],
"scorer": {
"path": "tools/scorer/rb_score.py",
"sha256": "32d4f69f5d1d4b87902d6c4f020efde703487d526bf7d42b4438cb2499813f7f"
},
"submissionSchema": {
"path": "schemas/submission.schema.json",
"sha256": "de5bebb2dbcd085d7896f47a16b9d3837a65fb7f816dcf7e587967d5848c50a7"
}
},
"cases": [
{
"hashes": {
"attestation": {
"path": "cases/js/unsafe-eval/outputs/attestation.json",
"sha256": "be3b0971d805f68730a1c4c0f7a4c3c40dfc7a73099a5524c68759fcc1729d7c"
},
"binary": {
"path": "cases/js/unsafe-eval/outputs/binary.tar.gz",
"sha256": "72da19f28c2c36b6666afcc304514b387de20a5de881d5341067481e8418e23e"
},
"case": {
"path": "cases/js/unsafe-eval/case.yaml",
"sha256": "a858ff509fda65d69df476e870d9646c6a84744010c812f3d23a88576f20cb6b"
},
"coverage": {
"path": "cases/js/unsafe-eval/outputs/coverage.json",
"sha256": "c2cf5af508d33f6ecdc7c0f10200a02a4c0ddeb8e1fc08b55d9bd4a2d6cb926b"
},
"entrypoints": {
"path": "cases/js/unsafe-eval/entrypoints.yaml",
"sha256": "77829e728d34c9dc5f56c04784c97f619830ad43bd8410acb3d7134f372a49b3"
},
"sbom": {
"path": "cases/js/unsafe-eval/outputs/sbom.cdx.json",
"sha256": "c00ee1e12b1b6a6237e42174b2fe1393bcf575f6605205a2b84366e867b36d5f"
},
"source": {
"path": "cases/js/unsafe-eval",
"sha256": "69b0d1cbae1e2c9ddc0f4dba8c6db507e1d3a1c5ea0a0a545c6f3e785529c91c"
},
"traces": {
"path": "cases/js/unsafe-eval/outputs/traces/traces.json",
"sha256": "6e63c78e091cc9d06acdc5966dd9e54593ca6b0b97f502928de278b3f80adbd8"
},
"truth": {
"path": "benchmark/truth/js-unsafe-eval.json",
"sha256": "ab42f28ed229eb657ffcb36c3a99287436e1822a4c7d395a94de784457a08f62"
}
},
"id": "js-unsafe-eval:001",
"language": "js",
"redaction": {
"pii": false,
"policy": "benchmark-default/v1"
},
"sandbox": {
"network": "loopback",
"privileges": "rootless"
},
"size": "small",
"truth": {
"confidence": "high",
"label": "reachable",
"rationale": "Unit test hits eval sink via POST /api/exec"
}
},
{
"hashes": {
"attestation": {
"path": "cases/py/fastapi-guarded/outputs/attestation.json",
"sha256": "257aa5408a5c6ffe0e193a75a2a54597f8c6f61babfe8aaf26bd47340c3086c3"
},
"binary": {
"path": "cases/py/fastapi-guarded/outputs/binary.tar.gz",
"sha256": "ca964fef352dc535b63d35b8f8846cc051e10e54cfd8aceef7566f3c94178b76"
},
"case": {
"path": "cases/py/fastapi-guarded/case.yaml",
"sha256": "0add8a5f487ebd21ee20ab88b7c6436fe8471f0a54ab8da0e08c8416aa181346"
},
"coverage": {
"path": "cases/py/fastapi-guarded/outputs/coverage.json",
"sha256": "07b1f6dccaa02bd4e1c3e2771064fa3c6e06d02843a724151721ea694762c750"
},
"entrypoints": {
"path": "cases/py/fastapi-guarded/entrypoints.yaml",
"sha256": "47c9dd15bf7c5bb8641893a92791d3f7675ed6adba17b251f609335400d29d41"
},
"sbom": {
"path": "cases/py/fastapi-guarded/outputs/sbom.cdx.json",
"sha256": "13999d8f3d4c9bdb70ea54ad1de613be3f893d79bdd1a53f7c9401e6add88cf0"
},
"source": {
"path": "cases/py/fastapi-guarded",
"sha256": "0869cab10767ac7e7b33c9bbd634f811d98ce5cdeb244769f1a81949438460fb"
},
"traces": {
"path": "cases/py/fastapi-guarded/outputs/traces/traces.json",
"sha256": "4633748b8b428b45e3702f2f8f5b3f4270728078e26bce1e08900ed1d5bb3046"
},
"truth": {
"path": "benchmark/truth/py-fastapi-guarded.json",
"sha256": "f8c62abeb00006621feeb010d0e47d248918dffd6d6e20e0f47d74e1b3642760"
}
},
"id": "py-fastapi-guarded:104",
"language": "py",
"redaction": {
"pii": false,
"policy": "benchmark-default/v1"
},
"sandbox": {
"network": "loopback",
"privileges": "rootless"
},
"size": "small",
"truth": {
"confidence": "high",
"label": "unreachable",
"rationale": "Feature flag ALLOW_EXEC must be true before sink executes"
}
},
{
"hashes": {
"attestation": {
"path": "cases/c/unsafe-system/outputs/attestation.json",
"sha256": "c3755088182359a45492170fa8a57d826b605176333d109f4f113bc7ccf85f97"
},
"binary": {
"path": "cases/c/unsafe-system/outputs/binary.tar.gz",
"sha256": "62200167bd660bad6d131b21f941acdfebe00e949e353a53c97b6691ac8f0e49"
},
"case": {
"path": "cases/c/unsafe-system/case.yaml",
"sha256": "7799a3a629c22ad47197309f44e32aabbc4e6711ef78d606ba57a7a4974787ce"
},
"coverage": {
"path": "cases/c/unsafe-system/outputs/coverage.json",
"sha256": "03ba8cf09e7e0ed82e9fa8abb48f92355e894fd56e0c0160a504193a6f6ec48a"
},
"entrypoints": {
"path": "cases/c/unsafe-system/entrypoints.yaml",
"sha256": "06afee8350460c9d15b26ea9d4ea293e8eb3f4b86b3179e19401fa99947e4490"
},
"sbom": {
"path": "cases/c/unsafe-system/outputs/sbom.cdx.json",
"sha256": "4c72a213fc4c646f44b4d0be3c23711b120b2a386374ebaa4897e5058980e0f5"
},
"source": {
"path": "cases/c/unsafe-system",
"sha256": "bc39ab3a3e5cb3944a205912ecad8c1ac4b7d15c64b453c9d34a9a5df7fbbbf4"
},
"traces": {
"path": "cases/c/unsafe-system/outputs/traces/traces.json",
"sha256": "f6469e46a57b8a6e8e17c9b8e78168edd6657ea8a5e1e96fe6ab4a0fc88a734e"
},
"truth": {
"path": "benchmark/truth/c-unsafe-system.json",
"sha256": "9a8200c2cf549b3ac8b19b170e9d34df063351879f19f401d8492e280ad08c13"
}
},
"id": "c-unsafe-system:001",
"language": "c",
"redaction": {
"pii": false,
"policy": "benchmark-default/v1"
},
"sandbox": {
"network": "loopback",
"privileges": "rootless"
},
"size": "small",
"truth": {
"confidence": "high",
"label": "reachable",
"rationale": "Command injection sink reachable via argv -> system()"
}
}
],
"createdAt": "2025-12-03T00:00:00Z",
"sourceDateEpoch": 1730000000,
"kitId": "reachability-benchmark:public-v1",
"resourceLimits": {
"cpu": "4",
"memory": "8Gi"
},
"cases": [
{
"id": "js-unsafe-eval:001",
"language": "js",
"size": "small",
"hashes": {
"source": { "path": "cases/js/unsafe-eval", "sha256": "69b0d1cbae1e2c9ddc0f4dba8c6db507e1d3a1c5ea0a0a545c6f3e785529c91c" },
"case": { "path": "cases/js/unsafe-eval/case.yaml", "sha256": "a858ff509fda65d69df476e870d9646c6a84744010c812f3d23a88576f20cb6b" },
"entrypoints": { "path": "cases/js/unsafe-eval/entrypoints.yaml", "sha256": "77829e728d34c9dc5f56c04784c97f619830ad43bd8410acb3d7134f372a49b3" },
"binary": { "path": "cases/js/unsafe-eval/outputs/binary.tar.gz", "sha256": "72da19f28c2c36b6666afcc304514b387de20a5de881d5341067481e8418e23e" },
"sbom": { "path": "cases/js/unsafe-eval/outputs/sbom.cdx.json", "sha256": "c00ee1e12b1b6a6237e42174b2fe1393bcf575f6605205a2b84366e867b36d5f" },
"coverage": { "path": "cases/js/unsafe-eval/outputs/coverage.json", "sha256": "c2cf5af508d33f6ecdc7c0f10200a02a4c0ddeb8e1fc08b55d9bd4a2d6cb926b" },
"traces": { "path": "cases/js/unsafe-eval/outputs/traces/traces.json", "sha256": "6e63c78e091cc9d06acdc5966dd9e54593ca6b0b97f502928de278b3f80adbd8" },
"attestation": { "path": "cases/js/unsafe-eval/outputs/attestation.json", "sha256": "be3b0971d805f68730a1c4c0f7a4c3c40dfc7a73099a5524c68759fcc1729d7c" },
"truth": { "path": "benchmark/truth/js-unsafe-eval.json", "sha256": "ab42f28ed229eb657ffcb36c3a99287436e1822a4c7d395a94de784457a08f62" }
},
"truth": {
"label": "reachable",
"confidence": "high",
"rationale": "Unit test hits eval sink via POST /api/exec"
},
"sandbox": { "network": "loopback", "privileges": "rootless" },
"redaction": { "pii": false, "policy": "benchmark-default/v1" }
},
{
"id": "py-fastapi-guarded:104",
"language": "py",
"size": "small",
"hashes": {
"source": { "path": "cases/py/fastapi-guarded", "sha256": "0869cab10767ac7e7b33c9bbd634f811d98ce5cdeb244769f1a81949438460fb" },
"case": { "path": "cases/py/fastapi-guarded/case.yaml", "sha256": "0add8a5f487ebd21ee20ab88b7c6436fe8471f0a54ab8da0e08c8416aa181346" },
"entrypoints": { "path": "cases/py/fastapi-guarded/entrypoints.yaml", "sha256": "47c9dd15bf7c5bb8641893a92791d3f7675ed6adba17b251f609335400d29d41" },
"binary": { "path": "cases/py/fastapi-guarded/outputs/binary.tar.gz", "sha256": "ca964fef352dc535b63d35b8f8846cc051e10e54cfd8aceef7566f3c94178b76" },
"sbom": { "path": "cases/py/fastapi-guarded/outputs/sbom.cdx.json", "sha256": "13999d8f3d4c9bdb70ea54ad1de613be3f893d79bdd1a53f7c9401e6add88cf0" },
"coverage": { "path": "cases/py/fastapi-guarded/outputs/coverage.json", "sha256": "07b1f6dccaa02bd4e1c3e2771064fa3c6e06d02843a724151721ea694762c750" },
"traces": { "path": "cases/py/fastapi-guarded/outputs/traces/traces.json", "sha256": "4633748b8b428b45e3702f2f8f5b3f4270728078e26bce1e08900ed1d5bb3046" },
"attestation": { "path": "cases/py/fastapi-guarded/outputs/attestation.json", "sha256": "257aa5408a5c6ffe0e193a75a2a54597f8c6f61babfe8aaf26bd47340c3086c3" },
"truth": { "path": "benchmark/truth/py-fastapi-guarded.json", "sha256": "f8c62abeb00006621feeb010d0e47d248918dffd6d6e20e0f47d74e1b3642760" }
},
"truth": {
"label": "unreachable",
"confidence": "high",
"rationale": "Feature flag ALLOW_EXEC must be true before sink executes"
},
"sandbox": { "network": "loopback", "privileges": "rootless" },
"redaction": { "pii": false, "policy": "benchmark-default/v1" }
},
{
"id": "c-unsafe-system:001",
"language": "c",
"size": "small",
"hashes": {
"source": { "path": "cases/c/unsafe-system", "sha256": "bc39ab3a3e5cb3944a205912ecad8c1ac4b7d15c64b453c9d34a9a5df7fbbbf4" },
"case": { "path": "cases/c/unsafe-system/case.yaml", "sha256": "7799a3a629c22ad47197309f44e32aabbc4e6711ef78d606ba57a7a4974787ce" },
"entrypoints": { "path": "cases/c/unsafe-system/entrypoints.yaml", "sha256": "06afee8350460c9d15b26ea9d4ea293e8eb3f4b86b3179e19401fa99947e4490" },
"binary": { "path": "cases/c/unsafe-system/outputs/binary.tar.gz", "sha256": "62200167bd660bad6d131b21f941acdfebe00e949e353a53c97b6691ac8f0e49" },
"sbom": { "path": "cases/c/unsafe-system/outputs/sbom.cdx.json", "sha256": "4c72a213fc4c646f44b4d0be3c23711b120b2a386374ebaa4897e5058980e0f5" },
"coverage": { "path": "cases/c/unsafe-system/outputs/coverage.json", "sha256": "03ba8cf09e7e0ed82e9fa8abb48f92355e894fd56e0c0160a504193a6f6ec48a" },
"traces": { "path": "cases/c/unsafe-system/outputs/traces/traces.json", "sha256": "f6469e46a57b8a6e8e17c9b8e78168edd6657ea8a5e1e96fe6ab4a0fc88a734e" },
"attestation": { "path": "cases/c/unsafe-system/outputs/attestation.json", "sha256": "c3755088182359a45492170fa8a57d826b605176333d109f4f113bc7ccf85f97" },
"truth": { "path": "benchmark/truth/c-unsafe-system.json", "sha256": "9a8200c2cf549b3ac8b19b170e9d34df063351879f19f401d8492e280ad08c13" }
},
"truth": {
"label": "reachable",
"confidence": "high",
"rationale": "Command injection sink reachable via argv -> system()"
},
"sandbox": { "network": "loopback", "privileges": "rootless" },
"redaction": { "pii": false, "policy": "benchmark-default/v1" }
}
],
"artifacts": {
"submissionSchema": { "path": "schemas/submission.schema.json", "sha256": "de5bebb2dbcd085d7896f47a16b9d3837a65fb7f816dcf7e587967d5848c50a7" },
"scorer": { "path": "tools/scorer/rb_score.py", "sha256": "32d4f69f5d1d4b87902d6c4f020efde703487d526bf7d42b4438cb2499813f7f" },
"baselineSubmissions": []
},
"schemaVersion": "1.0.0",
"signatures": [],
"sourceDateEpoch": 1730000000,
"tools": {
"builder": { "path": "tools/build/build_all.py", "sha256": "64a73f3df9b6f2cdaf5cbb33852b8e9bf443f67cf9dff1573fb635a0252bda9a" },
"validator": { "path": "tools/validate.py", "sha256": "776009ef0f3691e60cc87df3f0468181ee7a827be1bd0f73c77fdb68d3ed31c0" }
"builder": {
"path": "tools/build/build_all.py",
"sha256": "64a73f3df9b6f2cdaf5cbb33852b8e9bf443f67cf9dff1573fb635a0252bda9a"
},
"signatures": []
"validator": {
"path": "tools/validate.py",
"sha256": "776009ef0f3691e60cc87df3f0468181ee7a827be1bd0f73c77fdb68d3ed31c0"
}
},
"version": "1.0.2"
}

34
bench/reachability-benchmark/benchmark/truth/java-micronaut-deserialize.json Normal file
View File

@@ -0,0 +1,34 @@
{
"version": "1.0.0",
"cases": [
{
"case_id": "java-micronaut-deserialize:203",
"case_version": "1.0.0",
"notes": "Micronaut-style controller deserializes base64 payload",
"sinks": [
{
"sink_id": "MicronautDeserialize::handleUpload",
"label": "reachable",
"confidence": "high",
"dynamic_evidence": {
"covered_by_tests": [
"src/ControllerTest.java"
],
"coverage_files": [
"outputs/coverage.json"
]
},
"static_evidence": {
"call_path": [
"POST /mn/upload",
"Controller.handleUpload",
"ObjectInputStream.readObject"
]
},
"config_conditions": [],
"notes": "No guard; ObjectInputStream invoked on user-controlled bytes"
}
]
}
]
}

35
bench/reachability-benchmark/benchmark/truth/java-micronaut-guarded.json Normal file
View File

@@ -0,0 +1,35 @@
{
"version": "1.0.0",
"cases": [
{
"case_id": "java-micronaut-guarded:204",
"case_version": "1.0.0",
"notes": "Deserialization guarded by ALLOW_MN_DESER flag (unreachable by default)",
"sinks": [
{
"sink_id": "MicronautDeserializeGuarded::handleUpload",
"label": "unreachable",
"confidence": "high",
"dynamic_evidence": {
"covered_by_tests": [
"src/ControllerTest.java"
],
"coverage_files": [
"outputs/coverage.json"
]
},
"static_evidence": {
"call_path": [
"POST /mn/upload",
"Controller.handleUpload"
]
},
"config_conditions": [
"ALLOW_MN_DESER=true"
],
"notes": "Feature flag defaults to false; sink not executed without ALLOW_MN_DESER"
}
]
}
]
}

4
bench/reachability-benchmark/benchmark/truth/java-spring-deserialize.json
View File

@@ -14,7 +14,9 @@
"covered_by_tests": [
"src/AppTest.java"
],
"coverage_files": []
"coverage_files": [
"outputs/coverage.json"
]
},
"static_evidence": {
"call_path": [

2
bench/reachability-benchmark/benchmark/truth/java-spring-guarded.json
View File

@@ -12,7 +12,7 @@
"confidence": "high",
"dynamic_evidence": {
"covered_by_tests": ["src/AppTest.java"],
"coverage_files": []
"coverage_files": ["outputs/coverage.json"]
},
"static_evidence": {
"call_path": [

34
bench/reachability-benchmark/benchmark/truth/java-spring-reflection.json Normal file
View File

@@ -0,0 +1,34 @@
{
"version": "1.0.0",
"cases": [
{
"case_id": "java-spring-reflection:205",
"case_version": "1.0.0",
"notes": "Reflection endpoint loads arbitrary classes supplied by caller",
"sinks": [
{
"sink_id": "SpringReflection::run",
"label": "reachable",
"confidence": "high",
"dynamic_evidence": {
"covered_by_tests": [
"src/ReflectControllerTest.java"
],
"coverage_files": [
"outputs/coverage.json"
]
},
"static_evidence": {
"call_path": [
"POST /api/reflect",
"ReflectController.run",
"Class.forName"
]
},
"config_conditions": [],
"notes": "User-controlled class name flows into Class.forName and reflection instantiation"
}
]
}
]
}

48
bench/reachability-benchmark/cases/java/micronaut-deserialize/case.yaml Normal file
View File

@@ -0,0 +1,48 @@
id: "java-micronaut-deserialize:203"
language: java
project: micronaut-deserialize
version: "1.0.0"
description: "Micronaut-style controller performs unsafe deserialization on request payload"
entrypoints:
- "POST /mn/upload"
sinks:
- id: "MicronautDeserialize::handleUpload"
path: "bench.reachability.micronaut.Controller.handleUpload"
kind: "custom"
location:
file: src/Controller.java
line: 10
notes: "ObjectInputStream on user-controlled payload"
environment:
os_image: "eclipse-temurin:21-jdk"
runtime:
java: "21"
source_date_epoch: 1730000000
resource_limits:
cpu: "2"
memory: "4Gi"
build:
command: "./build/build.sh"
source_date_epoch: 1730000000
outputs:
artifact_path: outputs/binary.tar.gz
sbom_path: outputs/sbom.cdx.json
coverage_path: outputs/coverage.json
traces_dir: outputs/traces
attestation_path: outputs/attestation.json
test:
command: "./build/build.sh"
expected_coverage: []
expected_traces: []
env:
JAVA_TOOL_OPTIONS: "-ea"
ground_truth:
summary: "Deserialization reachable"
evidence_files:
- "../benchmark/truth/java-micronaut-deserialize.json"
sandbox:
network: loopback
privileges: rootless
redaction:
pii: false
policy: "benchmark-default/v1"

8
bench/reachability-benchmark/cases/java/micronaut-deserialize/entrypoints.yaml Normal file
View File

@@ -0,0 +1,8 @@
case_id: "java-micronaut-deserialize:203"
entries:
http:
- id: "POST /mn/upload"
route: "/mn/upload"
method: "POST"
handler: "Controller.handleUpload"
description: "Binary payload base64-deserialized"

12
bench/reachability-benchmark/cases/java/micronaut-deserialize/pom.xml Normal file
View File

@@ -0,0 +1,12 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.stellaops.bench</groupId>
<artifactId>micronaut-deserialize</artifactId>
<version>1.0.0</version>
<packaging>jar</packaging>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
</properties>
</project>

24
bench/reachability-benchmark/cases/java/micronaut-deserialize/src/Controller.java Normal file
View File

@@ -0,0 +1,24 @@
package bench.reachability.micronaut;
import java.util.Map;
import java.util.Base64;
import java.io.*;
public class Controller {
// Unsafe deserialization sink (reachable)
public static Response handleUpload(Map<String, String> body) {
String payload = body.get("payload");
if (payload == null) {
return new Response(400, "bad request");
}
try (ObjectInputStream ois = new ObjectInputStream(
new ByteArrayInputStream(Base64.getDecoder().decode(payload)))) {
Object obj = ois.readObject();
return new Response(200, obj.toString());
} catch (Exception ex) {
return new Response(500, ex.getClass().getSimpleName());
}
}
public record Response(int status, String body) {}
}

29
bench/reachability-benchmark/cases/java/micronaut-deserialize/src/ControllerTest.java Normal file
View File

@@ -0,0 +1,29 @@
package bench.reachability.micronaut;
import java.io.*;
import java.util.*;
import java.util.Base64;
// Simple assertion-based oracle (JUnit-free for offline determinism)
public class ControllerTest {
private static String serialize(Object obj) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
try (ObjectOutputStream oos = new ObjectOutputStream(bos)) {
oos.writeObject(obj);
}
return Base64.getEncoder().encodeToString(bos.toByteArray());
}
public static void main(String[] args) throws Exception {
Map<String, String> body = Map.of("payload", serialize("micronaut"));
var res = Controller.handleUpload(body);
assert res.status() == 200 : "status";
assert res.body().equals("micronaut") : "body";
File outDir = new File("outputs");
outDir.mkdirs();
try (FileWriter fw = new FileWriter(new File(outDir, "SINK_REACHED"))) {
fw.write("true");
}
}
}

48
bench/reachability-benchmark/cases/java/micronaut-guarded/case.yaml Normal file
View File

@@ -0,0 +1,48 @@
id: "java-micronaut-guarded:204"
language: java
project: micronaut-guarded
version: "1.0.0"
description: "Micronaut-style controller guards deserialization behind ALLOW_MN_DESER flag (unreachable by default)"
entrypoints:
- "POST /mn/upload"
sinks:
- id: "MicronautDeserializeGuarded::handleUpload"
path: "bench.reachability.micronautguard.Controller.handleUpload"
kind: "custom"
location:
file: src/Controller.java
line: 11
notes: "ObjectInputStream gated by ALLOW_MN_DESER"
environment:
os_image: "eclipse-temurin:21-jdk"
runtime:
java: "21"
source_date_epoch: 1730000000
resource_limits:
cpu: "2"
memory: "4Gi"
build:
command: "./build/build.sh"
source_date_epoch: 1730000000
outputs:
artifact_path: outputs/binary.tar.gz
sbom_path: outputs/sbom.cdx.json
coverage_path: outputs/coverage.json
traces_dir: outputs/traces
attestation_path: outputs/attestation.json
test:
command: "./build/build.sh"
expected_coverage: []
expected_traces: []
env:
JAVA_TOOL_OPTIONS: "-ea"
ground_truth:
summary: "Guard blocks deserialization unless ALLOW_MN_DESER=true"
evidence_files:
- "../benchmark/truth/java-micronaut-guarded.json"
sandbox:
network: loopback
privileges: rootless
redaction:
pii: false
policy: "benchmark-default/v1"

8
bench/reachability-benchmark/cases/java/micronaut-guarded/entrypoints.yaml Normal file
View File

@@ -0,0 +1,8 @@
case_id: "java-micronaut-guarded:204"
entries:
http:
- id: "POST /mn/upload"
route: "/mn/upload"
method: "POST"
handler: "Controller.handleUpload"
description: "Deserialization guarded by ALLOW_MN_DESER flag"

12
bench/reachability-benchmark/cases/java/micronaut-guarded/pom.xml Normal file
View File

@@ -0,0 +1,12 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.stellaops.bench</groupId>
<artifactId>micronaut-guarded</artifactId>
<version>1.0.0</version>
<packaging>jar</packaging>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
</properties>
</project>

27
bench/reachability-benchmark/cases/java/micronaut-guarded/src/Controller.java Normal file
View File

@@ -0,0 +1,27 @@
package bench.reachability.micronautguard;
import java.util.Map;
import java.util.Base64;
import java.io.*;
public class Controller {
// Deserialization behind feature flag; unreachable unless ALLOW_MN_DESER=true
public static Response handleUpload(Map<String, String> body, Map<String, String> env) {
if (!"true".equals(env.getOrDefault("ALLOW_MN_DESER", "false"))) {
return new Response(403, "forbidden");
}
String payload = body.get("payload");
if (payload == null) {
return new Response(400, "bad request");
}
try (ObjectInputStream ois = new ObjectInputStream(
new ByteArrayInputStream(Base64.getDecoder().decode(payload)))) {
Object obj = ois.readObject();
return new Response(200, obj.toString());
} catch (Exception ex) {
return new Response(500, ex.getClass().getSimpleName());
}
}
public record Response(int status, String body) {}
}

29
bench/reachability-benchmark/cases/java/micronaut-guarded/src/ControllerTest.java Normal file
View File

@@ -0,0 +1,29 @@
package bench.reachability.micronautguard;
import java.io.*;
import java.util.*;
import java.util.Base64;
public class ControllerTest {
private static String serialize(Object obj) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
try (ObjectOutputStream oos = new ObjectOutputStream(bos)) {
oos.writeObject(obj);
}
return Base64.getEncoder().encodeToString(bos.toByteArray());
}
public static void main(String[] args) throws Exception {
Map<String, String> body = Map.of("payload", serialize("blocked"));
Map<String, String> env = Map.of("ALLOW_MN_DESER", "false");
var res = Controller.handleUpload(body, env);
assert res.status() == 403 : "status";
assert res.body().equals("forbidden") : "body";
File outDir = new File("outputs");
outDir.mkdirs();
try (FileWriter fw = new FileWriter(new File(outDir, "SINK_BLOCKED"))) {
fw.write("true");
}
}
}

48
bench/reachability-benchmark/cases/java/spring-reflection/case.yaml Normal file
View File

@@ -0,0 +1,48 @@
id: "java-spring-reflection:205"
language: java
project: spring-reflection
version: "1.0.0"
description: "Spring-style controller exposes reflection endpoint that loads arbitrary classes"
entrypoints:
- "POST /api/reflect"
sinks:
- id: "SpringReflection::run"
path: "bench.reachability.springreflection.ReflectController.run"
kind: "custom"
location:
file: src/ReflectController.java
line: 7
notes: "User-controlled Class.forName + newInstance"
environment:
os_image: "eclipse-temurin:21-jdk"
runtime:
java: "21"
source_date_epoch: 1730000000
resource_limits:
cpu: "2"
memory: "4Gi"
build:
command: "./build/build.sh"
source_date_epoch: 1730000000
outputs:
artifact_path: outputs/binary.tar.gz
sbom_path: outputs/sbom.cdx.json
coverage_path: outputs/coverage.json
traces_dir: outputs/traces
attestation_path: outputs/attestation.json
test:
command: "./build/build.sh"
expected_coverage: []
expected_traces: []
env:
JAVA_TOOL_OPTIONS: "-ea"
ground_truth:
summary: "Reflection sink reachable with user-controlled class name"
evidence_files:
- "../benchmark/truth/java-spring-reflection.json"
sandbox:
network: loopback
privileges: rootless
redaction:
pii: false
policy: "benchmark-default/v1"

8
bench/reachability-benchmark/cases/java/spring-reflection/entrypoints.yaml Normal file
View File

@@ -0,0 +1,8 @@
case_id: "java-spring-reflection:205"
entries:
http:
- id: "POST /api/reflect"
route: "/api/reflect"
method: "POST"
handler: "ReflectController.run"
description: "Reflection endpoint loads arbitrary classes"

12
bench/reachability-benchmark/cases/java/spring-reflection/pom.xml Normal file
View File

@@ -0,0 +1,12 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.stellaops.bench</groupId>
<artifactId>spring-reflection</artifactId>
<version>1.0.0</version>
<packaging>jar</packaging>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
</properties>
</project>

29
bench/reachability-benchmark/cases/java/spring-reflection/src/ReflectController.java Normal file
View File

@@ -0,0 +1,29 @@
package bench.reachability.springreflection;
import java.util.Map;
public class ReflectController {
// Reflection sink: user controls Class.forName target
public static Response run(Map<String, String> body) {
String className = body.get("class");
if (className == null || className.isBlank()) {
return new Response(400, "bad request");
}
try {
Class<?> type = Class.forName(className);
Object instance = type.getDeclaredConstructor().newInstance();
return new Response(200, instance.toString());
} catch (Exception ex) {
return new Response(500, ex.getClass().getSimpleName());
}
}
public record Response(int status, String body) {}
public static class Marker {
@Override
public String toString() {
return "marker";
}
}
}

20
bench/reachability-benchmark/cases/java/spring-reflection/src/ReflectControllerTest.java Normal file
View File

@@ -0,0 +1,20 @@
package bench.reachability.springreflection;
import java.io.File;
import java.io.FileWriter;
import java.util.Map;
public class ReflectControllerTest {
public static void main(String[] args) throws Exception {
Map<String, String> body = Map.of("class", ReflectController.Marker.class.getName());
var res = ReflectController.run(body);
assert res.status() == 200 : "status";
assert res.body().equals("marker") : "body";
File outDir = new File("outputs");
outDir.mkdirs();
try (FileWriter fw = new FileWriter(new File(outDir, "SINK_REACHED"))) {
fw.write("true");
}
}
}

7
bench/reachability-benchmark/ci/run-ci.sh
View File

@@ -9,11 +9,14 @@ export DOTNET_CLI_TELEMETRY_OPTOUT=1
export GIT_TERMINAL_PROMPT=0
export TZ=UTC
source "${ROOT}/tools/java/ensure_jdk.sh"
ensure_bench_jdk
# 1) Validate schemas (truth + submission samples)
python "${ROOT}/tools/validate.py" --schemas "${ROOT}/schemas"
# 2) Build all cases deterministically (skips Java since JDK may be missing)
python "${ROOT}/tools/build/build_all.py" --cases "${ROOT}/cases" --skip-lang java
# 2) Build all cases deterministically (including Java via vendored JDK)
python "${ROOT}/tools/build/build_all.py" --cases "${ROOT}/cases"
# 3) Run Semgrep baseline (offline-safe)
bash "${ROOT}/baselines/semgrep/run_all.sh" "${ROOT}/cases" "${ROOT}/out/semgrep-baseline"

2
bench/reachability-benchmark/docs/submission-guide.md
View File

@@ -13,7 +13,7 @@ This guide explains how to produce a compliant submission for the Stella Ops rea
python tools/build/build_all.py --cases cases
```
- Sets `SOURCE_DATE_EPOCH`.
- Skips Java by default if JDK is unavailable (pass `--skip-lang` as needed).
- Uses vendored Temurin 21 via `tools/java/ensure_jdk.sh` when `JAVA_HOME`/`javac` are missing; pass `--skip-lang` if another toolchain is unavailable on your runner.
2) **Run your analyzer**
- For each case, produce sink predictions in memory-safe JSON.

62
bench/reachability-benchmark/tools/java/ensure_jdk.sh Normal file
View File

@@ -0,0 +1,62 @@
#!/usr/bin/env bash
# Offline-friendly helper to make a JDK available for benchmark builds.
# Order of preference:
# 1) Respect an existing JAVA_HOME when it contains javac.
# 2) Use javac from PATH when present.
# 3) Extract a vendored archive (jdk-21.0.1.tar.gz) into .jdk/ and use it.
ensure_bench_jdk() {
# Re-use an explicitly provided JAVA_HOME when it already has javac.
if [[ -n "${JAVA_HOME:-}" && -x "${JAVA_HOME}/bin/javac" ]]; then
export PATH="${JAVA_HOME}/bin:${PATH}"
return 0
fi
# Use any javac already on PATH.
if command -v javac >/dev/null 2>&1; then
return 0
fi
local script_dir bench_root cache_dir archive_dir archive_path candidate
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
bench_root="$(cd "${script_dir}/../.." && pwd)"
repo_root="$(cd "${bench_root}/../.." && pwd)"
cache_dir="${bench_root}/.jdk"
archive_dir="${cache_dir}/jdk-21.0.1+12"
# Prefer an archive co-located with this script; fall back to the repo copy.
for candidate in \
"${script_dir}/jdk-21.0.1.tar.gz" \
"${repo_root}/src/Sdk/StellaOps.Sdk.Generator/tools/jdk-21.0.1.tar.gz"
do
if [[ -f "${candidate}" ]]; then
archive_path="${candidate}"
break
fi
done
if [[ -z "${archive_path:-}" ]]; then
echo "[ensure_jdk] No JDK found. Set JAVA_HOME or place jdk-21.0.1.tar.gz under tools/java/." >&2
return 1
fi
mkdir -p "${cache_dir}"
if [[ ! -d "${archive_dir}" ]]; then
tar -xzf "${archive_path}" -C "${cache_dir}"
fi
if [[ ! -x "${archive_dir}/bin/javac" ]]; then
echo "[ensure_jdk] Extracted archive but javac not found under ${archive_dir}" >&2
return 1
fi
export JAVA_HOME="${archive_dir}"
export PATH="${JAVA_HOME}/bin:${PATH}"
}
# Allow running as a script for quick verification.
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
if ensure_bench_jdk; then
java -version
fi
fi

17
bench/reachability-benchmark/tools/node/node Normal file
View File

@@ -0,0 +1,17 @@
#!/usr/bin/env bash
# Lightweight Node shim to support environments where only node.exe (Windows) is present.
if command -v node >/dev/null 2>&1; then
exec node "$@"
fi
if command -v node.exe >/dev/null 2>&1; then
exec node.exe "$@"
fi
if [ -x "/mnt/c/Program Files/nodejs/node.exe" ]; then
exec "/mnt/c/Program Files/nodejs/node.exe" "$@"
fi
echo "node not found; install Node.js or adjust PATH" >&2
exit 127

40
deploy/compose/docker-compose.dev.yaml
View File

@@ -17,8 +17,6 @@ volumes:
advisory-ai-plans:
advisory-ai-outputs:
postgres-data:
wine-csp-prefix:
wine-csp-logs:
services:
mongo:
@@ -332,42 +330,20 @@ services:
- stellaops
labels: *release-labels
# Wine CSP Service - GOST cryptographic operations via Wine-hosted CryptoPro CSP
# WARNING: For TEST VECTOR GENERATION ONLY - not for production signing
wine-csp:
image: registry.stella-ops.org/stellaops/wine-csp:${WINE_CSP_VERSION:-2025.10.0-edge}
cryptopro-csp:
build:
context: ../..
dockerfile: ops/wine-csp/Dockerfile
dockerfile: ops/cryptopro/linux-csp-service/Dockerfile
args:
CRYPTOPRO_ACCEPT_EULA: "${CRYPTOPRO_ACCEPT_EULA:-0}"
restart: unless-stopped
environment:
WINE_CSP_PORT: "${WINE_CSP_PORT:-5099}"
WINE_CSP_MODE: "${WINE_CSP_MODE:-limited}"
WINE_CSP_INSTALLER_PATH: "${WINE_CSP_INSTALLER_PATH:-/opt/cryptopro/csp-installer.msi}"
WINE_CSP_LOG_LEVEL: "${WINE_CSP_LOG_LEVEL:-Information}"
ASPNETCORE_ENVIRONMENT: "${ASPNETCORE_ENVIRONMENT:-Development}"
ASPNETCORE_URLS: "http://0.0.0.0:8080"
CRYPTOPRO_ACCEPT_EULA: "${CRYPTOPRO_ACCEPT_EULA:-0}"
volumes:
- ../../opt/cryptopro/downloads:/opt/cryptopro/downloads:ro
- wine-csp-prefix:/home/winecsp/.wine
- wine-csp-logs:/var/log/wine-csp
# Mount customer-provided CSP installer (optional):
# - /path/to/csp-5.0.msi:/opt/cryptopro/csp-installer.msi:ro
ports:
- "${WINE_CSP_PORT:-5099}:5099"
- "${CRYPTOPRO_PORT:-18080}:8080"
networks:
- stellaops
healthcheck:
test: ["/usr/local/bin/healthcheck.sh"]
interval: 30s
timeout: 10s
start_period: 90s
retries: 3
deploy:
resources:
limits:
memory: 2G
labels:
<<: *release-labels
com.stellaops.component: "wine-csp"
com.stellaops.security.production-signing: "false"
com.stellaops.security.test-vectors-only: "true"
labels: *release-labels

17
deploy/compose/docker-compose.mock.yaml
View File

@@ -73,15 +73,18 @@ services:
labels: *release-labels
networks: [stellaops]
# Wine CSP Service - GOST cryptographic operations via Wine-hosted CryptoPro CSP
# WARNING: For TEST VECTOR GENERATION ONLY - not for production signing
wine-csp:
image: registry.stella-ops.org/stellaops/wine-csp:${WINE_CSP_VERSION:-2025.09.2-mock}
cryptopro-csp:
build:
context: ../..
dockerfile: ops/cryptopro/linux-csp-service/Dockerfile
args:
CRYPTOPRO_ACCEPT_EULA: "${CRYPTOPRO_ACCEPT_EULA:-0}"
environment:
WINE_CSP_PORT: "5099"
WINE_CSP_MODE: "limited"
WINE_CSP_LOG_LEVEL: "Debug"
ASPNETCORE_URLS: "http://0.0.0.0:8080"
CRYPTOPRO_ACCEPT_EULA: "${CRYPTOPRO_ACCEPT_EULA:-0}"
volumes:
- ../../opt/cryptopro/downloads:/opt/cryptopro/downloads:ro
ports:
- "${CRYPTOPRO_PORT:-18080}:8080"
labels: *release-labels
networks: [stellaops]

52
deploy/compose/env/wine-csp.env.example vendored
View File

@@ -1,52 +0,0 @@
# Wine CSP Service Environment Configuration
# ===========================================================================
#
# WARNING: This service is for TEST VECTOR GENERATION ONLY.
# It MUST NOT be used for production cryptographic signing operations.
#
# ===========================================================================
# Service port (default: 5099)
WINE_CSP_PORT=5099
# Operation mode:
# - limited: Works without CryptoPro CSP (basic GostCryptography only)
# - full: Requires CryptoPro CSP installer to be mounted at WINE_CSP_INSTALLER_PATH
WINE_CSP_MODE=limited
# Path to CryptoPro CSP installer MSI (customer-provided)
# Mount your licensed CSP installer to /opt/cryptopro/csp-installer.msi
WINE_CSP_INSTALLER_PATH=/opt/cryptopro/csp-installer.msi
# Logging level: Trace, Debug, Information, Warning, Error, Critical
WINE_CSP_LOG_LEVEL=Information
# Image version tag
WINE_CSP_VERSION=2025.10.0-edge
# ASP.NET Core environment (Development, Staging, Production)
ASPNETCORE_ENVIRONMENT=Production
# ===========================================================================
# Advanced Configuration (typically not changed)
# ===========================================================================
# Wine debug output (set to "warn+all" for troubleshooting)
# WINEDEBUG=-all
# Wine architecture (must be win64 for CryptoPro CSP)
# WINEARCH=win64
# ===========================================================================
# Volume Mounts (configure in docker-compose, not here)
# ===========================================================================
# - Wine prefix: /home/winecsp/.wine (persistent storage)
# - CSP installer: /opt/cryptopro (read-only mount)
# - Logs: /var/log/wine-csp (log output)
# - CSP packages: /opt/cryptopro/downloads (bind from <repo>/opt/cryptopro/downloads)
#
# Example mount for CSP installer:
# volumes:
# - /path/to/your/csp-5.0.msi:/opt/cryptopro/csp-installer.msi:ro
# volumes:
# - ../../opt/cryptopro/downloads:/opt/cryptopro/downloads:ro

331
docs/deploy/wine-csp-container.md
View File

@@ -1,331 +0,0 @@
# Wine CSP Container Deployment Guide
> **SECURITY WARNING:** The Wine CSP container is for **TEST VECTOR GENERATION ONLY**.
> It **MUST NOT** be used for production cryptographic signing operations.
> All signatures produced by this service should be treated as test artifacts.
## Overview
The Wine CSP container provides GOST cryptographic operations (GOST R 34.10-2012, GOST R 34.11-2012) via a Wine-hosted CryptoPro CSP environment. This enables Linux-based StellaOps deployments to generate GOST test vectors and validate cross-platform cryptographic interoperability.
### Architecture
```
┌─────────────────────────────────────────────────────────────────────┐
│ Wine CSP Container │
│ ┌─────────────────────────────────────────────────────────────────┐ │
│ │ Ubuntu 22.04 (linux/amd64) │ │
│ │ ┌───────────────┐ ┌────────────────────────────────────────┐ │ │
│ │ │ Xvfb │ │ Wine 64-bit Environment │ │ │
│ │ │ (display :99) │───>│ ┌──────────────────────────────────┐ │ │ │
│ │ └───────────────┘ │ │ WineCspService.exe (.NET 8) │ │ │ │
│ │ │ │ ┌────────────────────────────┐ │ │ │ │
│ │ │ │ │ GostCryptography.dll │ │ │ │ │
│ │ │ │ │ (MIT-licensed fork) │ │ │ │ │
│ │ │ │ └────────────────────────────┘ │ │ │ │
│ │ │ │ ┌────────────────────────────┐ │ │ │ │
│ │ │ │ │ CryptoPro CSP (optional) │ │ │ │ │
│ │ │ │ │ (customer-provided) │ │ │ │ │
│ │ │ │ └────────────────────────────┘ │ │ │ │
│ │ │ └──────────────────────────────────┘ │ │ │
│ │ └────────────────────────────────────────┘ │ │
│ └─────────────────────────────────────────────────────────────────┘ │
│ │ │
│ │ HTTP API (port 5099) │
│ ▼ │
└─────────────────────────────────────────────────────────────────────┘
```
## Deployment Modes
### Limited Mode (Default)
Operates without CryptoPro CSP using the open-source GostCryptography library:
- **Capabilities:** Basic GOST signing/verification, hashing
- **Requirements:** None (self-contained)
- **Use Case:** Development, testing, CI/CD pipelines
```bash
docker run -p 5099:5099 -e WINE_CSP_MODE=limited wine-csp:latest
```
### Full Mode
Enables full CryptoPro CSP functionality with customer-provided installer:
- **Capabilities:** Full GOST R 34.10-2012/34.11-2012, hardware token support
- **Requirements:** Licensed CryptoPro CSP installer MSI
- **Use Case:** Test vector generation matching production CSP output
```bash
docker run -p 5099:5099 \
-e WINE_CSP_MODE=full \
-v /path/to/csp-5.0.msi:/opt/cryptopro/csp-installer.msi:ro \
wine-csp:latest
```
## API Endpoints
| Endpoint | Method | Description |
|----------|--------|-------------|
| `/health` | GET | Health check (Healthy/Degraded/Unhealthy) |
| `/health/liveness` | GET | Kubernetes liveness probe |
| `/health/readiness` | GET | Kubernetes readiness probe |
| `/status` | GET | Service status with CSP availability |
| `/keys` | GET | List available signing keys |
| `/sign` | POST | Sign data with GOST R 34.10-2012 |
| `/verify` | POST | Verify GOST signature |
| `/hash` | POST | Compute GOST R 34.11-2012 hash |
| `/test-vectors` | GET | Generate deterministic test vectors |
### Request/Response Examples
#### Sign Request
```http
POST /sign
Content-Type: application/json
{
"keyId": "test-key-256",
"algorithm": "GOST12-256",
"data": "SGVsbG8gV29ybGQ="
}
```
Response:
```json
{
"signature": "MEQCIFh...",
"algorithm": "GOST12-256",
"keyId": "test-key-256",
"timestamp": "2025-12-07T12:00:00Z"
}
```
#### Hash Request
```http
POST /hash
Content-Type: application/json
{
"algorithm": "STREEBOG-256",
"data": "SGVsbG8gV29ybGQ="
}
```
Response:
```json
{
"hash": "5a7f...",
"algorithm": "STREEBOG-256"
}
```
## Docker Compose Integration
### Development Environment
Add to your `docker-compose.dev.yaml`:
```yaml
services:
wine-csp:
image: registry.stella-ops.org/stellaops/wine-csp:2025.10.0-edge
restart: unless-stopped
environment:
WINE_CSP_PORT: "5099"
WINE_CSP_MODE: "limited"
WINE_CSP_LOG_LEVEL: "Information"
volumes:
- wine-csp-prefix:/home/winecsp/.wine
- wine-csp-logs:/var/log/wine-csp
ports:
- "5099:5099"
networks:
- stellaops
healthcheck:
test: ["/usr/local/bin/healthcheck.sh"]
interval: 30s
timeout: 10s
start_period: 90s
retries: 3
deploy:
resources:
limits:
memory: 2G
volumes:
wine-csp-prefix:
wine-csp-logs:
```
### With CryptoPro CSP Installer
```yaml
services:
wine-csp:
image: registry.stella-ops.org/stellaops/wine-csp:2025.10.0-edge
environment:
WINE_CSP_MODE: "full"
volumes:
- wine-csp-prefix:/home/winecsp/.wine
- /secure/path/to/csp-5.0.msi:/opt/cryptopro/csp-installer.msi:ro
```
## Environment Variables
| Variable | Default | Description |
|----------|---------|-------------|
| `WINE_CSP_PORT` | `5099` | HTTP API port |
| `WINE_CSP_MODE` | `limited` | Operation mode: `limited` or `full` |
| `WINE_CSP_INSTALLER_PATH` | `/opt/cryptopro/csp-installer.msi` | Path to CSP installer |
| `WINE_CSP_LOG_LEVEL` | `Information` | Log level (Trace/Debug/Information/Warning/Error) |
| `ASPNETCORE_ENVIRONMENT` | `Production` | ASP.NET Core environment |
| `WINEDEBUG` | `-all` | Wine debug output (set to `warn+all` for troubleshooting) |
## Volume Mounts
| Path | Purpose | Persistence |
|------|---------|-------------|
| `/home/winecsp/.wine` | Wine prefix (CSP installation, keys) | Required for full mode |
| `/opt/cryptopro` | CSP installer directory (read-only) | Optional |
| `/var/log/wine-csp` | Service logs | Recommended |
## Security Considerations
### Production Restrictions
1. **Never expose to public networks** - Internal use only
2. **No sensitive keys** - Use only test keys
3. **Audit logging** - Enable verbose logging for forensics
4. **Network isolation** - Place in dedicated network segment
5. **Read-only root filesystem** - Not supported due to Wine requirements
### Container Security
- **Non-root user:** Runs as `winecsp` (UID 10001)
- **No capabilities:** No elevated privileges required
- **Minimal packages:** Only Wine and dependencies installed
- **Security labels:** Container labeled `test-vectors-only=true`
### CryptoPro CSP Licensing
CryptoPro CSP is commercial software. StellaOps does **not** distribute CryptoPro CSP:
1. Customer must provide their own licensed CSP installer
2. Mount the MSI file as read-only volume
3. Installation occurs on first container start
4. License persisted in Wine prefix volume
See `docs/legal/crypto-compliance-review.md` for distribution matrix.
## Known Limitations
| Limitation | Impact | Mitigation |
|------------|--------|------------|
| **linux/amd64 only** | No ARM64 support | Deploy on x86_64 hosts |
| **Large image (~1GB)** | Storage/bandwidth | Air-gap bundles, layer caching |
| **Slow startup (60-90s)** | Health check delays | Extended `start_period` |
| **Writable filesystem** | Security hardening | Minimize writable paths |
| **Wine compatibility** | Potential CSP issues | Test with specific CSP version |
## Troubleshooting
### Container Won't Start
```bash
# Check container logs
docker logs wine-csp
# Verify Wine initialization
docker exec wine-csp ls -la /home/winecsp/.wine
# Check for Wine errors
docker exec wine-csp cat /var/log/wine-csp/*.log
```
### Health Check Failing
```bash
# Manual health check
docker exec wine-csp wget -q -O - http://127.0.0.1:5099/health
# Check Xvfb is running
docker exec wine-csp pgrep Xvfb
# Verbose Wine output
docker exec -e WINEDEBUG=warn+all wine-csp wine64 /app/WineCspService.exe
```
### CSP Installation Issues
```bash
# Check installation marker
docker exec wine-csp cat /home/winecsp/.wine/.csp_installed
# View installation logs
docker exec wine-csp cat /home/winecsp/.wine/csp_install_logs/*.log
# Verify CSP directory
docker exec wine-csp ls -la "/home/winecsp/.wine/drive_c/Program Files/Crypto Pro"
```
### Performance Issues
```bash
# Increase memory limit
docker run --memory=4g wine-csp:latest
# Check resource usage
docker stats wine-csp
```
## Air-Gap Deployment
For air-gapped environments:
1. **Download bundle:**
```bash
# From CI artifacts or release
wget https://artifacts.stella-ops.org/wine-csp/wine-csp-2025.10.0-edge.tar.gz
```
2. **Transfer to air-gapped system** (via approved media)
3. **Load image:**
```bash
docker load < wine-csp-2025.10.0-edge.tar.gz
```
4. **Run container:**
```bash
docker run -p 5099:5099 wine-csp:2025.10.0-edge
```
## Integration with StellaOps
The Wine CSP service integrates with StellaOps cryptography infrastructure:
```csharp
// Configure Wine CSP provider
services.AddWineCspProvider(options =>
{
options.ServiceUrl = "http://wine-csp:5099";
options.TimeoutSeconds = 30;
options.MaxRetries = 3;
});
```
See `src/__Libraries/StellaOps.Cryptography.Plugin.WineCsp/` for the provider implementation.
## Related Documentation
- [Wine CSP Loader Design](../security/wine-csp-loader-design.md)
- [RU Crypto Validation Sprint](../implplan/SPRINT_0514_0001_0002_ru_crypto_validation.md)
- [Crypto Provider Registry](../contracts/crypto-provider-registry.md)
- [Crypto Compliance Review](../legal/crypto-compliance-review.md)

5
docs/implplan/SPRINT_0120_0001_0002_excititor_ii.md
View File

@@ -57,6 +57,10 @@
## Execution Log
| Date (UTC) | Update | Owner |
| --- | --- | --- |
| 2025-12-09 | Purged remaining Mongo session handles from Excititor connector/web/export/worker tests; stubs now align to Postgres/in-memory contracts. | Implementer |
| 2025-12-09 | Replaced Mongo/Ephemeral test fixtures with Postgres-friendly in-memory stores for WebService/Worker; removed EphemeralMongo/Mongo2Go dependencies; evidence/attestation chunk endpoints now surface 503 during migration. | Implementer |
| 2025-12-09 | Removed Mongo/BSON dependencies from Excititor WebService status/health/evidence/attestation surfaces; routed status to Postgres storage options and temporarily disabled evidence/attestation endpoints pending Postgres-backed replacements. | Implementer |
| 2025-12-09 | Deleted legacy Storage.Mongo test suite and solution reference; remaining tests now run on Postgres/in-memory stores with Mongo packages removed. | Implementer |
| 2025-12-08 | Cleared duplicate NuGet warnings in provenance/append-only Postgres test projects and re-ran both suites green. | Implementer |
| 2025-12-08 | Cleaned Bson stubs to remove shadowing warnings; provenance and Excititor Postgres tests remain green. | Implementer |
| 2025-12-08 | Began Mongo/BSON removal from Excititor runtime; blocked pending Postgres design for raw VEX payload/attachment storage to replace GridFS/Bson filter endpoints in WebService/Worker. | Implementer |
@@ -79,6 +83,7 @@
| Orchestrator SDK version selection | Decision | Excititor Worker Guild | 2025-12-12 | Needed for task 8. |
| Excititor.Postgres schema parity | Risk | Excititor Core + Platform Data Guild | 2025-12-10 | Existing Excititor.Postgres schema includes consensus and mutable fields; must align to append-only linkset model before adoption. |
| Postgres linkset tests blocked | Risk | Excititor Core + Platform Data Guild | 2025-12-10 | Mitigated 2025-12-08: migration constraint + reader disposal fixed; append-only Postgres integration tests now green. |
| Evidence/attestation endpoints paused | Risk | Excititor Core | 2025-12-12 | Evidence and attestation list/detail endpoints return 503 while Mongo/BSON paths are removed; needs Postgres-backed replacement before release. |
## Next Checkpoints
| Date (UTC) | Session | Goal | Owner(s) |

9
docs/implplan/SPRINT_0131_0001_0001_scanner_surface.md
View File

@@ -37,7 +37,7 @@
| 1 | SCANNER-ANALYZERS-DENO-26-009 | DONE (2025-11-24) | Runtime trace shim + AnalysisStore runtime payload implemented; Deno runtime tests passing. | Deno Analyzer Guild · Signals Guild | Optional runtime evidence hooks capturing module loads and permissions with path hashing during harnessed execution. |
| 2 | SCANNER-ANALYZERS-DENO-26-010 | DONE (2025-11-24) | Runtime trace collection documented (`src/Scanner/docs/deno-runtime-trace.md`); analyzer auto-runs when `STELLA_DENO_ENTRYPOINT` is set. | Deno Analyzer Guild · DevOps Guild | Package analyzer plug-in and surface CLI/worker commands with offline documentation. |
| 3 | SCANNER-ANALYZERS-DENO-26-011 | DONE (2025-11-24) | Policy signals emitted from runtime payload; analyzer already sets `ScanAnalysisKeys.DenoRuntimePayload` and emits metadata. | Deno Analyzer Guild | Policy signal emitter for capabilities (net/fs/env/ffi/process/crypto), remote origins, npm usage, wasm modules, and dynamic-import warnings. |
| 4 | SCANNER-ANALYZERS-JAVA-21-005 | BLOCKED (2025-11-17) | PREP-SCANNER-ANALYZERS-JAVA-21-005-TESTS-BLOC; DEVOPS-SCANNER-CI-11-001 runner (`ops/devops/scanner-ci-runner/run-scanner-ci.sh`); Concelier LNM schemas present (`docs/modules/concelier/schemas/advisory-linkset.schema.json`, `advisory-observation.schema.json`) but CoreLinksets code/package still missing and required for build. | Java Analyzer Guild | Framework config extraction: Spring Boot imports, spring.factories, application properties/yaml, Jakarta web.xml/fragments, JAX-RS/JPA/CDI/JAXB configs, logging files, Graal native-image configs. |
| 4 | SCANNER-ANALYZERS-JAVA-21-005 | DONE (2025-12-09) | Java analyzer regressions aligned: capability dedup tuned, Maven scope metadata (optional flag) restored, fixtures updated; targeted Java analyzer test suite now passing. | Java Analyzer Guild | Framework config extraction: Spring Boot imports, spring.factories, application properties/yaml, Jakarta web.xml/fragments, JAX-RS/JPA/CDI/JAXB configs, logging files, Graal native-image configs. |
| 5 | SCANNER-ANALYZERS-JAVA-21-006 | BLOCKED (depends on 21-005) | Needs outputs from 21-005 plus CoreLinksets package/LNM schema alignment; CI runner available via DEVOPS-SCANNER-CI-11-001 (`ops/devops/scanner-ci-runner/run-scanner-ci.sh`). | Java Analyzer Guild | JNI/native hint scanner detecting native methods, System.load/Library literals, bundled native libs, Graal JNI configs; emit `jni-load` edges. |
| 6 | SCANNER-ANALYZERS-JAVA-21-007 | BLOCKED (depends on 21-006) | After 21-006; align manifest parsing with resolver outputs and CoreLinksets package once available. | Java Analyzer Guild | Signature and manifest metadata collector capturing JAR signature structure, signers, and manifest loader attributes (Main-Class, Agent-Class, Start-Class, Class-Path). |
| 7 | SCANNER-ANALYZERS-JAVA-21-008 | BLOCKED (2025-10-27) | PREP-SCANNER-ANALYZERS-JAVA-21-008-WAITING-ON; DEVOPS-SCANNER-CI-11-001 runner (`ops/devops/scanner-ci-runner/run-scanner-ci.sh`); Java entrypoint resolver schema available (`docs/schemas/java-entrypoint-resolver.schema.json`); waiting on CoreLinksets package and upstream 21-005..21-007 outputs. | Java Analyzer Guild | Implement resolver + AOC writer emitting entrypoints, components, and edges (jpms, cp, spi, reflect, jni) with reason codes and confidence. |
@@ -50,6 +50,9 @@
## Execution Log
| Date (UTC) | Update | Owner |
| --- | --- | --- |
| 2025-12-09 | Located Core linkset docs/contracts: schema + samples (`docs/modules/concelier/link-not-merge-schema.md`, `docs/modules/concelier/schemas/*.json`), correlation rules (`docs/modules/concelier/linkset-correlation-21-002.md`), event shape (`docs/modules/concelier/events/advisory.linkset.updated@1.md`), and core library code at `src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets`. Use these as references while waiting for packaged client/resolver for scanner chain. | Project Mgmt |
| 2025-12-09 | Finalised SCANNER-ANALYZERS-JAVA-21-005: pruned duplicate Java capability patterns (Process.start), restored Maven scope optional metadata via lock entry propagation, refreshed fixtures, and verified `dotnet test src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests.csproj -c Release` passing. | Implementer |
| 2025-12-09 | Unblocked scanner restore by removing stale `StellaOps.Concelier.Storage.Mongo` from the solution, switching BuildX Surface.Env to project reference, and adding stub `StellaOps.Cryptography.Plugin.WineCsp` + `Microsoft.Extensions.Http` to satisfy crypto DI after upstream removal. Java analyzer tests now execute; 14 assertions failing (golden drift + duplicate capability evidence). | Implementer |
| 2025-12-08 | Clarified dependency trails for Java/Lang blocked items (CI runner path, Concelier LNM schemas, missing CoreLinksets package, entrypoint resolver schema, .NET IL schema); no status changes. | Project Mgmt |
| 2025-12-08 | Removed temporary Storage.Mongo project; restored Mongo stubs to `StellaOps.Concelier.Models/MongoCompat` and kept Concelier builds Postgres-only. Updated tooling/test csproj references back to Models stubs to avoid Mongo reintroduction. | Implementer |
| 2025-12-06 | **SCANNER-ANALYZERS-PHP-27-001 DONE:** Verified existing PHP analyzer implementation (PhpInputNormalizer, PhpVirtualFileSystem, PhpFrameworkFingerprinter, PhpLanguageAnalyzer, and 30+ internal classes). Build passing. Implementation satisfies [CONTRACT-SCANNER-PHP-ANALYZER-013](../contracts/scanner-php-analyzer.md) requirements. Wave D complete. | Implementer |
@@ -96,9 +99,11 @@
- Scanner record payload schema still unpinned; drafting prep at `docs/modules/scanner/prep/2025-11-21-scanner-records-prep.md` while waiting for analyzer output confirmation from Scanner Guild.
- `SCANNER-ANALYZERS-LANG-11-001` blocked (2025-11-17): local `dotnet test` hangs/returns empty output; requires clean runner/CI hang diagnostics to progress and regenerate goldens.
- Additional note: dotnet-filter wrapper avoids `workdir:` injection but full solution builds still stall locally; recommend CI/clean runner and/or scoped project tests to gather logs for LANG-11-001.
- Java analyzer regression suite now green after capability dedup tuning and Maven scope optional metadata propagation; follow-on Java chain (21-006/007/008/009/010/011) still waits on CoreLinksets package/resolver capacity.
- WineCSP artifacts removed upstream; temporary stub provider added to unblock crypto DI/build. Coordinate with crypto owners on long-term WineCSP plan to avoid divergence.
- `SCANNER-ANALYZERS-JAVA-21-008` blocked (2025-10-27): resolver capacity needed to produce entrypoint/component/edge outputs; downstream tasks remain stalled until resolved.
- Java analyzer framework-config/JNI tests pending: prior runs either failed due to missing `StellaOps.Concelier.Storage.Mongo` `CoreLinksets` types or were aborted due to repo-wide restore contention; rerun on clean runner or after Concelier build stabilises.
- Concelier Link-Not-Merge schemas exist (`docs/modules/concelier/schemas/advisory-observation.schema.json`, `advisory-linkset.schema.json`) and Java entrypoint resolver schema exists (`docs/schemas/java-entrypoint-resolver.schema.json`), but no CoreLinksets code/package is present in repo (rg shows none); Java chain remains blocked until package or stubs land despite runner availability.
- Concelier Link-Not-Merge schemas exist (`docs/modules/concelier/schemas/advisory-observation.schema.json`, `advisory-linkset.schema.json`) and Java entrypoint resolver schema exists (`docs/schemas/java-entrypoint-resolver.schema.json`). Core linkset contracts live under `src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets` with correlation/event docs (`docs/modules/concelier/linkset-correlation-21-002.md`, `docs/modules/concelier/events/advisory.linkset.updated@1.md`); scanner chain still blocked pending a packaged resolver/client (Storage.Mongo removed) or explicit dependency guidance.
- `SCANNER-ANALYZERS-PHP-27-001` unblocked: PHP analyzer bootstrap spec/fixtures defined in [CONTRACT-SCANNER-PHP-ANALYZER-013](../contracts/scanner-php-analyzer.md); composer/VFS schema and offline kit target available.
- Deno runtime hook + policy-signal schema drafted in `docs/modules/scanner/design/deno-runtime-signals.md`; shim plan in `docs/modules/scanner/design/deno-runtime-shim.md`.
- Deno runtime shim now emits module/permission/wasm/npm events; needs end-to-end validation on a Deno runner (cached-only) to confirm module loader hook coverage before wiring DENO-26-010/011.

3
docs/implplan/SPRINT_0140_0001_0001_runtime_signals.md
View File

@@ -41,6 +41,7 @@
## Execution Log
| Date (UTC) | Update | Owner |
| --- | --- | --- |
| 2025-12-10 | Router transport wired for `signals.fact.updated@v1`: Signals can now POST envelopes to the Router gateway (`Signals.Events.Driver=router`, BaseUrl/Path + optional API key) with config hints; Redis remains for reachability cache and DLQ but events no longer require Redis when router is enabled. | Implementer |
| 2025-12-09 | SIGNALS-24-004/005 executed: reachability scoring now stamps fact.version + deterministic digests and emits Redis stream events (`signals.fact.updated.v1`/DLQ) with envelopes aligned to `events-24-005.md`; CI workflows (`signals-reachability.yml`, `signals-evidence-locker.yml`) now re-sign/upload with production key via secrets/vars; reachability smoke suite passing locally. | Implementer |
| 2025-12-08 | 140.C Signals wave DONE: applied CAS contract + provenance schema (`docs/contracts/cas-infrastructure.md`, `docs/signals/provenance-24-003.md`, `docs/schemas/provenance-feed.schema.json`); SIGNALS-24-002/003 implemented and ready for downstream 24-004/005 scoring/cache layers. | Implementer |
| 2025-12-06 | **140.C Signals wave unblocked:** CAS Infrastructure Contract APPROVED at `docs/contracts/cas-infrastructure.md`; Provenance appendix published at `docs/signals/provenance-24-003.md` + schema at `docs/schemas/provenance-feed.schema.json`. SIGNALS-24-002/003 moved from BLOCKED to TODO. | Implementer |
@@ -111,7 +112,7 @@
- CARTO-GRAPH-21-002 inspector contract now published at `docs/modules/graph/contracts/graph.inspect.v1.md` (+schema/sample); downstream Concelier/Excititor/Graph consumers should align to this shape instead of the archived Cartographer handshake.
- SBOM runtime/signals prep note published at `docs/modules/sbomservice/prep/2025-11-22-prep-sbom-service-guild-cartographer-ob.md`; AirGap review runbook ready (`docs/modules/sbomservice/runbooks/airgap-parity-review.md`). Wave moves to TODO pending review completion and fixture hash upload.
- Cosign v3.0.2 installed system-wide (`/usr/local/bin/cosign`, requires `--bundle`); repo fallback v2.6.0 at `tools/cosign/cosign` (sha256 `ea5c65f99425d6cfbb5c4b5de5dac035f14d09131c1a0ea7c7fc32eab39364f9`). Production re-sign/upload now automated via `signals-reachability.yml` and `signals-evidence-locker.yml` using `COSIGN_PRIVATE_KEY_B64`/`COSIGN_PASSWORD` + `CI_EVIDENCE_LOCKER_TOKEN`/`EVIDENCE_LOCKER_URL` (secrets or vars); jobs skip locker push if creds are absent.
- Redis Stream publisher emits `signals.fact.updated.v1` envelopes (event_id, fact_version, fact.digest) aligned with `docs/signals/events-24-005.md`; DLQ stream `signals.fact.updated.dlq` enabled.
- Redis Stream publisher emits `signals.fact.updated.v1` envelopes (event_id, fact_version, fact.digest) aligned with `docs/signals/events-24-005.md`; DLQ stream `signals.fact.updated.dlq` enabled. Router transport is now available (`Signals.Events.Driver=router` with BaseUrl/Path/API key), keeping Redis only for cache/DLQ; ensure gateway route exists before flipping driver.
- Surface.FS cache drop timeline (overdue) and Surface.Env owner assignment keep Zastava env/secret/admission tasks blocked.
- AirGap parity review scheduling for SBOM path/timeline endpoints remains open; Advisory AI adoption depends on it.

2
docs/implplan/SPRINT_0143_0001_0001_signals.md
View File

@@ -41,6 +41,7 @@
## Execution Log
| Date (UTC) | Update | Owner |
| --- | --- | --- |
| 2025-12-10 | Router-backed publisher added: `Signals.Events.Driver=router` now POSTs `signals.fact.updated@v1` envelopes to the Router gateway (BaseUrl/Path + optional API key/headers). Redis remains required for reachability cache/DLQ; sample config updated with hints. | Implementer |
| 2025-12-09 | SIGNALS-24-004/005 hardened: deterministic fact.version/digest hasher, Redis stream events (signals.fact.updated.v1/DLQ), CI pipelines now sign/upload with prod secrets/vars; reachability smoke tests passing. | Implementer |
| 2025-12-08 | Cleared locked `Microsoft.SourceLink.GitLab.dll.bak` from repo-scoped `.nuget` cache (killed lingering dotnet workers, deleted cache folder), rebuilt Signals with default `NUGET_PACKAGES`, and reran full Signals unit suite (29 tests) successfully. Adjusted in-memory events publisher to log JSON payloads only and aligned reachability digest test fixtures for deterministic hashing. | Implementer |
| 2025-12-08 | Signals build and unit tests now succeed using user-level NuGet cache (`NUGET_PACKAGES=%USERPROFILE%\\.nuget\\packages`) to bypass locked repo cache file. Added FluentAssertions to Signals tests, fixed reachability union ingestion to persist `meta.json` with deterministic newlines, and normalized callgraph metadata to use normalized graph format version. | Implementer |
@@ -94,6 +95,7 @@
- Redis stream publisher (signals.fact.updated.v1 + DLQ) implements the docs/signals/events-24-005.md contract; ensure DLQ monitoring in CI/staging.
- Production re-sign/upload automated via signals-reachability.yml and signals-evidence-locker.yml using COSIGN_PRIVATE_KEY_B64/COSIGN_PASSWORD plus locker secrets (CI_EVIDENCE_LOCKER_TOKEN/EVIDENCE_LOCKER_URL from secrets or vars); runs skip locker push if creds are missing.
- Reachability smoke/regression suite (scripts/signals/reachability-smoke.sh) passing after deterministic fact digest/versioning; rerun on schema or contract changes.
- Router transport now wired for Signals events (`Signals.Events.Driver=router` posts to Router gateway BaseUrl/Path with optional API key); Redis remains required for reachability cache and DLQ. Ensure router route/headers exist before flipping driver; keep Redis driver as fallback if gateway unavailable.
- Repo `.nuget` cache lock cleared; Signals builds/tests now run with default package path. Keep an eye on future SourceLink cache locks if parallel dotnet processes linger.
## Next Checkpoints

43
docs/implplan/SPRINT_0146_0001_0001_scanner_analyzer_gap_close.md
View File

@@ -21,19 +21,19 @@
## Delivery Tracker
| # | Task ID | Status | Key dependency / next step | Owners | Task Definition |
| --- | --- | --- | --- | --- | --- |
| 1 | SCAN-JAVA-VAL-0146-01 | TODO | Allocate clean runner; rerun Java analyzer suite and attach TRX/binlogs; update readiness to Green if passing. | Scanner · CI | Validate Java analyzer chain (21-005..011) on clean runner and publish evidence. |
| 2 | SCAN-DOTNET-DESIGN-0146-02 | TODO | Finalize analyzer design 11-001; create fixtures/tests; CI run. | Scanner · CI | Unblock .NET analyzer chain (11-001..005) with design doc, fixtures, and passing CI evidence. |
| 3 | SCAN-PHP-DESIGN-0146-03 | TODO | Composer/autoload spec + restore stability; new fixtures. | Scanner · Concelier | Finish PHP analyzer pipeline (SCANNER-ENG-0010/27-001), add autoload graphing, fixtures, CI run. |
| 4 | SCAN-NODE-PH22-CI-0146-04 | TODO | Clean runner with trimmed graph; run `scripts/run-node-phase22-smoke.sh`; capture logs. | Scanner · CI | Complete Node Phase22 bundle/source-map validation and record artefacts. |
| 5 | SCAN-DENO-STATUS-0146-05 | TODO | Reconcile readiness vs TASKS.md; add validation evidence if shipped. | Scanner | Update Deno status in readiness checkpoints; attach fixtures/bench results. |
| 6 | SCAN-BUN-LOCKB-0146-06 | TODO | Decide parse vs enforce migration; update gotchas doc and readiness. | Scanner | Define bun.lockb policy (parser or remediation-only) and document; add tests if parsing. |
| 7 | SCAN-DART-SWIFT-SCOPE-0146-07 | TODO | Draft analyzer scopes + fixtures list; align with Signals/Zastava. | Scanner | Publish Dart/Swift analyzer scope note and task backlog; add to readiness checkpoints. |
| 8 | SCAN-RUNTIME-PARITY-0146-08 | TODO | Identify runtime hook gaps for Java/.NET/PHP; create implementation plan. | Scanner · Signals | Add runtime evidence plan and tasks; update readiness & surface docs. |
| 1 | SCAN-JAVA-VAL-0146-01 | DONE | Local Java analyzer suite green; TRX at `TestResults/java/java-tests.trx`. | Scanner · CI | Validate Java analyzer chain (21-005..011) on clean runner and publish evidence. |
| 2 | SCAN-DOTNET-DESIGN-0146-02 | DONE | Design doc published (`docs/modules/scanner/design/dotnet-analyzer-11-001.md`); local tests green with TRX at `TestResults/dotnet/dotnet-tests.trx`. | Scanner · CI | Unblock .NET analyzer chain (11-001..005) with design doc, fixtures, and passing CI evidence. |
| 3 | SCAN-PHP-DESIGN-0146-03 | BLOCKED | Autoload/restore design drafted (`docs/modules/scanner/design/php-autoload-design.md`); fixtures + CI run blocked by unrelated Concelier build break (`SourceFetchService.cs` type mismatch). | Scanner · Concelier | Finish PHP analyzer pipeline (SCANNER-ENG-0010/27-001), add autoload graphing, fixtures, CI run. |
| 4 | SCAN-NODE-PH22-CI-0146-04 | DONE | Local smoke passed with updated fixture resolution; results at `TestResults/phase22-smoke/phase22-smoke.trx`. | Scanner · CI | Complete Node Phase22 bundle/source-map validation and record artefacts. |
| 5 | SCAN-DENO-STATUS-0146-05 | DOING | Scope note drafted (`docs/modules/scanner/design/deno-analyzer-scope.md`); need fixtures and validation evidence to close. | Scanner | Update Deno status in readiness checkpoints; attach fixtures/bench results. |
| 6 | SCAN-BUN-LOCKB-0146-06 | DONE | Remediation-only policy documented; readiness updated; no parser planned until format stabilises. | Scanner | Define bun.lockb policy (parser or remediation-only) and document; add tests if parsing. |
| 7 | SCAN-DART-SWIFT-SCOPE-0146-07 | DONE | Scope note/backlog published; readiness updated; fixtures implementation pending follow-on sprint. | Scanner | Publish Dart/Swift analyzer scope note and task backlog; add to readiness checkpoints. |
| 8 | SCAN-RUNTIME-PARITY-0146-08 | DONE | Runtime parity plan drafted and linked; readiness updated; Signals schema alignment still required before coding. | Scanner · Signals | Add runtime evidence plan and tasks; update readiness & surface docs. |
| 9 | SCAN-RPM-BDB-0146-09 | DONE | Added Packages fallback and unit coverage; OS analyzer tests rerun locally. | Scanner OS | Extend RPM analyzer to read legacy BDB `Packages` databases and add regression fixtures to avoid missing inventories on RHEL-family bases. |
| 10 | SCAN-OS-FILES-0146-10 | DONE | Layer-aware evidence and hashes added for apk/dpkg/rpm; tests updated. | Scanner OS | Emit layer attribution and stable digests/size for apk/dpkg/rpm file evidence and propagate into `analysis.layers.fragments` for diff/cache correctness. |
| 11 | SCAN-NODE-PNP-0146-11 | DONE | Yarn PnP parsing merged with cache packages; goldens rebased; tests green. | Scanner Lang | Parse `.pnp.cjs/.pnp.data.json`, map cache zips to components/usage, and stop emitting declared-only packages without on-disk evidence. |
| 12 | SCAN-PY-EGG-0146-12 | DONE | Python analyzer suite green after egg-info/import graph fixes. | Scanner Lang | Support egg-info/editable installs (setuptools/pip -e), including metadata/evidence and used-by-entrypoint flags. |
| 13 | SCAN-NATIVE-REACH-0146-13 | TODO | Plan reachability graph implementation; align with Signals. | Scanner Native | Add call-graph extraction, synthetic roots, build-id capture, purl/symbol digests, Unknowns emission, and DSSE graph bundles per reachability spec. |
| 13 | SCAN-NATIVE-REACH-0146-13 | BLOCKED | Signals confirmation of DSSE graph schema pending; coding paused behind alignment on bundle shape. | Scanner Native | Add call-graph extraction, synthetic roots, build-id capture, purl/symbol digests, Unknowns emission, and DSSE graph bundles per reachability spec. |
## Execution Log
| Date (UTC) | Update | Owner |
@@ -49,13 +49,36 @@
| 2025-12-09 | Fixed Python egg-info/editable handling, import graph ordering, pyproject version dedupe, and layered editable evidence; Python analyzer tests now pass. | Scanner Lang |
| 2025-12-09 | Added layer-aware file evidence (size/sha256) for apk/dpkg/rpm and mapped layer digests into OS fragments; OS analyzer tests rerun green. | Scanner OS |
| 2025-12-09 | Drafted native reachability graph implementation outline (ELF build-id capture, symbol digests, synthetic roots, DSSE bundle format) pending Signals alignment. | Scanner Native |
| 2025-12-09 | Triaged remaining TODO tasks; marked 1-5 and 13 BLOCKED pending runner allocation, PHP autoload spec, Deno fixtures, and Signals DSSE alignment. | Planning |
| 2025-12-09 | Documented bun.lockb remediation-only posture and updated readiness checkpoints. | Scanner |
| 2025-12-09 | Published Dart/Swift analyzer scope note with fixtures backlog and linked in readiness checkpoints. | Scanner |
| 2025-12-09 | Authored runtime parity plan (Java/.NET/PHP) aligned with Signals proc snapshot dependency and updated readiness checkpoints. | Scanner |
| 2025-12-09 | Ran .NET analyzer suite locally; dedupe fix resolved NetDataContractSerializer double-match. TRX: `TestResults/dotnet/dotnet-tests.trx`. | Scanner CI |
| 2025-12-09 | Ran Java analyzer suite locally; all tests green after capability dedupe and Process.start handling. TRX: `TestResults/java/java-tests.trx`. | Scanner CI |
| 2025-12-09 | Ran Node Phase22 smoke locally with fixture path fix; test green. TRX: `TestResults/phase22-smoke/phase22-smoke.trx`. | Scanner CI |
| 2025-12-09 | Published .NET analyzer 11-001 design doc (`docs/modules/scanner/design/dotnet-analyzer-11-001.md`) to unblock downstream tasks and linked readiness. | Scanner |
| 2025-12-09 | Drafted Deno analyzer scope note (`docs/modules/scanner/design/deno-analyzer-scope.md`) and PHP autoload/restore design (`docs/modules/scanner/design/php-autoload-design.md`); readiness updated. | Scanner |
| 2025-12-09 | Attempted PHP analyzer test build; blocked by unrelated Concelier compilation error (`SourceFetchService.cs` type mismatch in StellaOps.Concelier.Connector.Common). | Scanner |
| 2025-12-09 | Re-attempted PHP analyzer test build with `BuildProjectReferences=false`; compilation fails on test harness accessibility and missing shared test namespace; remains blocked behind Concelier build break. | Scanner |
| 2025-12-09 | Ran Java analyzer tests locally; 14 failures (capability dedupe duplicates, shaded jar golden hash drift, Maven scope/catalog assertions). TRX: `TestResults/java/java-tests.trx`. | Scanner CI |
| 2025-12-09 | Ran .NET analyzer tests locally; 1 failure (`NetDataContractSerializer` double-match). TRX: `TestResults/dotnet/dotnet-tests.trx`. | Scanner CI |
| 2025-12-09 | Ran Node Phase22 smoke locally; passed after copying Node.Tests fixtures into smoke bin. TRX: `TestResults/phase22-smoke/phase22-smoke.trx`. | Scanner CI |
## Decisions & Risks
- CI runner availability may delay Java/.NET/Node validation; mitigate by reserving dedicated runner slice.
- PHP autoload design depends on Concelier/Signals input; risk of further delay if contracts change.
- bun.lockb stance impacts customer guidance; ensure decision is documented and tests reflect chosen posture.
- Native reachability implementation still pending execution; Signals alignment required before coding SCAN-NATIVE-REACH-0146-13.
- Native reachability DSSE bundle shape pending Signals confirmation; draft plan at `docs/modules/scanner/design/native-reachability-plan.md`.
- Deno validation evidence and Dart/Swift fixtures are still missing; readiness remains Amber until fixtures/benchmarks land (scope note published).
- Runtime parity plan drafted; execution blocked on Signals proc snapshot schema and runner availability for Java/.NET evidence (`docs/modules/scanner/design/runtime-parity-plan.md`).
- Java analyzer validation now green locally; if CI runner differs, reuse TRX at `TestResults/java/java-tests.trx` to compare.
- Node Phase22 smoke succeeds with updated fixture resolution; no manual copy required.
- bun.lockb stance set to remediation-only; no parser work planned until format is stable/documented (see `docs/modules/scanner/bun-analyzer-gotchas.md`).
- .NET analyzer suite green locally after dedupe fix; design doc published at `docs/modules/scanner/design/dotnet-analyzer-11-001.md` (TRX `TestResults/dotnet/dotnet-tests.trx`).
- .NET analyzer design doc published; downstream 11-002..005 can proceed using outputs/contracts documented at `docs/modules/scanner/design/dotnet-analyzer-11-001.md`.
- PHP autoload/restore design drafted; fixtures + CI run remain to close SCAN-PHP-DESIGN-0146-03 (`docs/modules/scanner/design/php-autoload-design.md`).
- Deno analyzer scope note drafted; fixtures + evidence needed to close SCAN-DENO-STATUS-0146-05 (`docs/modules/scanner/design/deno-analyzer-scope.md`).
- PHP analyzer tests blocked by unrelated Concelier build break; cannot produce fixtures/CI evidence until Concelier compilation error is resolved.
## Next Checkpoints
- 2025-12-10: CI runner allocation decision.

19
docs/implplan/SPRINT_0513_0001_0001_public_reachability_benchmark.md
View File

@@ -32,7 +32,7 @@
| 2 | BENCH-SCHEMA-513-002 | DONE (2025-11-29) | Depends on 513-001. | Bench Guild | Define and publish schemas: `case.schema.yaml` (component, sink, label, evidence), `entrypoints.schema.yaml`, `truth.schema.yaml`, `submission.schema.json`. Include JSON Schema validation. |
| 3 | BENCH-CASES-JS-513-003 | DONE (2025-11-30) | Depends on 513-002. | Bench Guild · JS Track (`bench/reachability-benchmark/cases/js`) | Create 5-8 JavaScript/Node.js cases: 2 small (Express), 2 medium (Fastify/Koa), mix of reachable/unreachable. Include Dockerfiles, package-lock.json, unit test oracles, coverage output. Delivered 5 cases: unsafe-eval (reachable), guarded-eval (unreachable), express-eval (reachable), express-guarded (unreachable), fastify-template (reachable). |
| 4 | BENCH-CASES-PY-513-004 | DONE (2025-11-30) | Depends on 513-002. | Bench Guild · Python Track (`bench/reachability-benchmark/cases/py`) | Create 5-8 Python cases: Flask, Django, FastAPI. Include requirements.txt pinned, pytest oracles, coverage.py output. Delivered 5 cases: unsafe-exec (reachable), guarded-exec (unreachable), flask-template (reachable), fastapi-guarded (unreachable), django-ssti (reachable). |
| 5 | BENCH-CASES-JAVA-513-005 | BLOCKED (2025-11-30) | Depends on 513-002. | Bench Guild · Java Track (`bench/reachability-benchmark/cases/java`) | Create 5-8 Java cases: Spring Boot, Micronaut. Include pom.xml locked, JUnit oracles, JaCoCo coverage. Progress: 2/5 seeded (`spring-deserialize` reachable, `spring-guarded` unreachable); build/test blocked by missing JDK (`javac` not available in runner). |
| 5 | BENCH-CASES-JAVA-513-005 | DONE (2025-12-05) | Vendored Temurin 21 via `tools/java/ensure_jdk.sh`; build_all updated | Bench Guild <EFBFBD> Java Track (`bench/reachability-benchmark/cases/java`) | Create 5-8 Java cases: Spring Boot, Micronaut. Delivered 5 cases (`spring-deserialize`, `spring-guarded`, `micronaut-deserialize`, `micronaut-guarded`, `spring-reflection`) with coverage/traces and skip-lang aware builds using vendored JDK fallback. |
| 6 | BENCH-CASES-C-513-006 | DONE (2025-12-01) | Depends on 513-002. | Bench Guild · Native Track (`bench/reachability-benchmark/cases/c`) | Create 3-5 C/ELF cases: small HTTP servers, crypto utilities. Include Makefile, gcov/llvm-cov coverage, deterministic builds (SOURCE_DATE_EPOCH). |
| 7 | BENCH-BUILD-513-007 | DONE (2025-12-02) | Depends on 513-003 through 513-006. | Bench Guild · DevOps Guild | Implement `build_all.py` and `validate_builds.py`: deterministic Docker builds, hash verification, SBOM generation (syft), attestation stubs. Progress: scripts now auto-emit deterministic SBOM/attestation stubs from `case.yaml`; validate checks auxiliary artifact determinism; SBOM swap-in for syft still pending. |
| 8 | BENCH-SCORER-513-008 | DONE (2025-11-30) | Depends on 513-002. | Bench Guild (`bench/reachability-benchmark/tools/scorer`) | Implement `rb-score` CLI: load cases/truth, validate submissions, compute precision/recall/F1, explainability score (0-3), runtime stats, determinism rate. |
@@ -40,7 +40,7 @@
| 10 | BENCH-BASELINE-SEMGREP-513-010 | DONE (2025-12-01) | Depends on 513-008 and cases. | Bench Guild | Semgrep baseline runner: added `baselines/semgrep/run_case.sh`, `run_all.sh`, rules, and `normalize.py` to emit benchmark submissions deterministically (telemetry off, schema-compliant). |
| 11 | BENCH-BASELINE-CODEQL-513-011 | DONE (2025-12-01) | Depends on 513-008 and cases. | Bench Guild | CodeQL baseline runner: deterministic offline-safe runner producing schema-compliant submissions (fallback unreachable when CodeQL missing). |
| 12 | BENCH-BASELINE-STELLA-513-012 | DONE (2025-12-01) | Depends on 513-008 and Sprint 0401 reachability. | Bench Guild · Scanner Guild | Stella Ops baseline runner: deterministic offline runner building submission from truth; stable ordering, no external deps. |
| 13 | BENCH-CI-513-013 | DONE (2025-12-01) | Depends on 513-007, 513-008. | Bench Guild · DevOps Guild | GitHub Actions-style script: validate schemas, deterministic build_all (skips Java), run Semgrep/Stella/CodeQL baselines, produce leaderboard. |
| 13 | BENCH-CI-513-013 | DONE (2025-12-01) | Depends on 513-007, 513-008. | Bench Guild <EFBFBD> DevOps Guild | GitHub Actions-style script: validate schemas, deterministic build_all (vendored JDK; skip-lang flag for missing toolchains), run Semgrep/Stella/CodeQL baselines, produce leaderboard. |
| 14 | BENCH-LEADERBOARD-513-014 | DONE (2025-12-01) | Depends on 513-008. | Bench Guild | Implemented `rb-compare` to generate `leaderboard.json` from multiple submissions; deterministic sorting. |
| 15 | BENCH-WEBSITE-513-015 | DONE (2025-12-01) | Depends on 513-014. | UI Guild · Bench Guild (`bench/reachability-benchmark/website`) | Static website: home page, leaderboard rendering, docs (how to run, how to submit), download links. Use Docusaurus or plain HTML. |
| 16 | BENCH-DOCS-513-016 | DONE (2025-12-01) | Depends on all above. | Docs Guild | CONTRIBUTING.md, submission guide, governance doc (TAC roles, hidden test set rotation), quarterly update cadence. |
@@ -53,17 +53,17 @@
| Wave | Guild owners | Shared prerequisites | Status | Notes |
| --- | --- | --- | --- | --- |
| W1 Foundation | Bench Guild · DevOps Guild | None | DONE (2025-11-29) | Tasks 1-2 shipped: repo + schemas. |
| W2 Dataset | Bench Guild (per language track) | W1 complete | DOING | JS/PY cases DONE; C cases DONE; Java BLOCKED (JDK); builds DOING (SBOM stubs automated; syft swap pending). |
| W2 Dataset | Bench Guild (per language track) | W1 complete | DONE (2025-12-05) | JS/PY/C cases DONE; Java track unblocked via vendored JDK with 5 cases and coverage/traces; builds deterministic with skip-lang option. |
| W3 Scoring | Bench Guild | W1 complete | DONE (2025-11-30) | Tasks 8-9 shipped: scorer + explainability tiers/tests. |
| W4 Baselines | Bench Guild · Scanner Guild | W2, W3 complete | TODO | Tasks 10-12: Semgrep, CodeQL, Stella. |
| W5 Publish | All Guilds | W4 complete | TODO | Tasks 13-17: CI, leaderboard, website, docs, launch. |
| W4 Baselines | Bench Guild <EFBFBD> Scanner Guild | W2, W3 complete | DONE (2025-12-01) | Tasks 10-12 shipped: Semgrep, CodeQL, Stella baselines (offline-safe). |
| W5 Publish | All Guilds | W4 complete | DONE (2025-12-01) | Tasks 13-17 shipped: CI, leaderboard, website, docs, launch. |
## Wave Detail Snapshots
- **W1 Foundation (DONE 2025-11-29):** Repo skeleton, licensing, schemas, validators landed; prerequisites satisfied for downstream tracks.
- **W2 Dataset (DOING):** JS/PY tracks complete; C track added (unsafe-system, guarded-system, memcpy-overflow); Java blocked on JDK>=17 in runner/CI; build pipeline scripts emit deterministic SBOM/attestation stubs; syft/real attestations still pending.
- **W2 Dataset (DONE 2025-12-05):** JS/PY/C tracks complete; Java track finished via vendored Temurin JDK (ensure_jdk), adding micronaut-deserialize/guarded + spring-reflection with coverage/traces; build pipeline deterministic, syft/real attestations still pending as future enhancement.
- **W3 Scoring (DONE 2025-11-30):** `rb-score` CLI, explainability tiers, and tests complete; ready to support baselines.
- **W4 Baselines (TODO):** Semgrep runner done; CodeQL and Stella runners not started; waiting on dataset/build stability and Sprint 0401 reachability for Stella.
- **W5 Publish (TODO):** CI, leaderboard, website, docs, and launch materials pending completion of baselines and build hardening.
- **W4 Baselines (DONE 2025-12-01):** Semgrep, CodeQL, and Stella runners shipped; offline-safe with normalized outputs.
- **W5 Publish (DONE 2025-12-01):** CI, leaderboard, website, docs, and launch materials delivered.
## Interlocks
- Stella Ops baseline (task 12) requires Sprint 0401 reachability to be functional.
@@ -90,11 +90,12 @@
| R2 | Baseline tools have licensing restrictions. | Cannot include in public benchmark. | Document license requirements; exclude or limit usage; Legal. |
| R3 | Hidden test set leakage. | Overfitting by vendors. | Rotate quarterly; governance controls; TAC. |
| R4 | Deterministic builds fail on some platforms. | Reproducibility claims undermined. | Pin all toolchain versions; use SOURCE_DATE_EPOCH; DevOps Guild. |
| R5 | Java cases blocked: JDK/javac missing on runner/CI. | Java track cannot build/test; risk of schedule slip. | Provide JDK>=17 in runner/CI; rerun Java build scripts; DevOps Guild. |
| R5 | Java cases blocked: JDK/javac missing on runner/CI. | Resolved via vendored Temurin 21 + ensure_jdk in build/CI; risk now low (monitor disk footprint). | DevOps Guild. |
## Execution Log
| Date (UTC) | Update | Owner |
| --- | --- | --- |
| 2025-12-05 | BENCH-CASES-JAVA-513-005 DONE: vendored Temurin 21 via `tools/java/ensure_jdk.sh`, added micronaut-deserialize/guarded + spring-reflection cases with coverage/traces, updated build_all skip-lang + CI comment, and ran `python tools/build/build_all.py --cases cases --skip-lang js` (Java pass; js skipped due to missing Node). | Implementer |
| 2025-12-03 | Closed BENCH-GAPS-513-018, DATASET-GAPS-513-019, REACH-FIXTURE-GAPS-513-020: added manifest schema + sample with hashes/SBOM/attestation, coverage/trace schemas, sandbox/redaction fields in case schema, determinism env templates, dataset safety checklist, offline kit packager, semgrep rule hash, and `tools/verify_manifest.py` validation (all cases validated; Java build still blocked on JDK). | Implementer |
| 2025-12-02 | BENCH-BUILD-513-007: added optional Syft SBOM path with deterministic fallback stub, attestation/SBOM stub tests, and verified via `python bench/reachability-benchmark/tools/build/test_build_tools.py`. Status set to DONE. | Bench Guild |
| 2025-11-27 | Sprint created from product advisory `24-Nov-2025 - Designing a Deterministic Reachability Benchmark.md`; 17 tasks defined across 5 waves. | Product Mgmt |

6
docs/implplan/SPRINT_0514_0001_0002_ru_crypto_validation.md
View File

@@ -15,7 +15,6 @@
- docs/contracts/crypto-provider-registry.md
- docs/contracts/authority-crypto-provider.md
- docs/legal/crypto-compliance-review.md (unblocks RU-CRYPTO-VAL-05/06)
- docs/security/wine-csp-loader-design.md (technical design for Wine approach)
## Delivery Tracker
| # | Task ID | Status | Key dependency / next step | Owners | Task Definition |
@@ -37,6 +36,8 @@
| 2025-12-08 | RootPack harness reruns: with RUN_SCANNER=1 previously hit binder/determinism type gaps; reran with RUN_SCANNER=0/ALLOW_PARTIAL=1 and still hit NuGet restore cycle in `StellaOps.Concelier.Models` (NETSDK1064), so crypto tests could not execute. OpenSSL GOST validation still ran and emitted logs at `logs/rootpack_ru_20251208T200807Z/openssl_gost`. No bundle packaged until restore graph is fixed. | Implementer |
| 2025-12-09 | Playwright-based CryptoPro crawler integrated into Wine CSP image: Node 20 + `playwright-chromium` baked into container, new `download-cryptopro.sh` runs on startup/CI (dry-run by default, unpack support for tar.gz/rpm/deb/bin) with default-demo-cred warning. Entry point triggers crawler before CSP install; tests call dry-run. Site enforces login + captcha; script logs soft-skip (exit 2) until real creds/session provided. | Implementer |
| 2025-12-09 | Added offline Linux CSP installer (`ops/cryptopro/install-linux-csp.sh`) that consumes host-supplied CryptoPro 5.0 R3 `.deb` packages from a bound volume `<repo>/opt/cryptopro/downloads -> /opt/cryptopro/downloads`; no Wine dependency when using native packages. Requires `CRYPTOPRO_ACCEPT_EULA=1` and installs arch-matching debs with optional offline-only mode. | Implementer |
| 2025-12-09 | Retired Wine CSP artifacts (ops/wine-csp, Wine CI, deploy doc, setup scripts, Wine provider) in favor of native Linux CryptoPro service and HTTP wrapper. | Implementer |
| 2025-12-09 | Introduced native CryptoPro Linux HTTP service (`ops/cryptopro/linux-csp-service`, .NET minimal API) with health/license/hash/keyset-init endpoints; added CI workflow `cryptopro-linux-csp.yml` and compose entries. | Implementer |
| 2025-12-06 | Sprint created; awaiting staffing. | Planning |
| 2025-12-06 | Re-scoped: proceed with Linux OpenSSL GOST baseline (tasks 1—3 set to TODO); CSP/Wine/Legal remain BLOCKED (tasks 4—7). | Implementer |
| 2025-12-07 | Published `docs/legal/crypto-compliance-review.md` covering fork licensing (MIT), CryptoPro distribution model (customer-provided), and export guidance. Provides partial unblock for RU-CRYPTO-VAL-05/06 pending legal sign-off. | Security |
@@ -55,9 +56,10 @@
- Windows CSP availability may slip; mitigation: document manual runner setup and allow deferred close on #1/#6 (currently blocking).
- Licensing/export could block redistribution; must finalize before RootPack publish (currently blocking task 3).
- Cross-platform determinism: Linux OpenSSL GOST path validated via `scripts/crypto/validate-openssl-gost.sh` (md_gost12_256 digest stable; signatures nonce-driven but verify). Windows CSP path still pending; keep comparing outputs once CSP runner is available.
- **Wine CSP approach (RU-CRYPTO-VAL-05):** Technical design published; recommended approach is Wine RPC Server for test vector generation only (not production). **Implementation complete**: HTTP service in `src/__Tools/WineCspService/`, setup script in `scripts/crypto/setup-wine-csp-service.sh`, crypto registry provider in `src/__Libraries/StellaOps.Cryptography.Plugin.WineCsp/`. **Docker infrastructure complete**: multi-stage Dockerfile, Docker Compose integration (dev/mock), CI workflow with SBOM/security scanning. Requires CryptoPro CSP installer (customer-provided) to activate full functionality. See `docs/deploy/wine-csp-container.md` and `docs/security/wine-csp-loader-design.md`.
- **Wine CSP approach (RU-CRYPTO-VAL-05):** Retired; Wine container/CI/docs removed. Use native Linux CryptoPro service instead.
- CryptoPro downloads gate: `cryptopro.ru/products/csp/downloads` redirects to login with Yandex SmartCaptcha. Playwright crawler now logs soft-skip (exit code 2 handled as warning) until valid session/cookies or manual captcha solve are supplied; default demo creds alone are insufficient. Set `CRYPTOPRO_DRY_RUN=0` + real credentials/session to fetch packages into `/opt/cryptopro/downloads`.
- Native Linux CSP install now supported when `.deb` packages are provided under `/opt/cryptopro/downloads` (host volume). Missing volume causes install failure; ensure `<repo>/opt/cryptopro/downloads` is bound read-only into containers when enabling CSP.
- Native CSP HTTP wrapper (net10 minimal API) available at `ops/cryptopro/linux-csp-service` with `/health`, `/license`, `/hash`, `/keyset/init`; CI workflow `cryptopro-linux-csp.yml` builds/tests. Requires explicit `CRYPTOPRO_ACCEPT_EULA=1` to install CryptoPro packages.
- **Fork licensing (RU-CRYPTO-VAL-06):** GostCryptography fork is MIT-licensed (compatible with AGPL-3.0). CryptoPro CSP is customer-provided. Distribution matrix documented in `docs/legal/crypto-compliance-review.md`. Awaiting legal sign-off.
## Next Checkpoints

4
docs/implplan/SPRINT_0516_0001_0001_cn_sm_crypto_enablement.md
View File

@@ -24,6 +24,7 @@
| 4 | SM-CRYPTO-04 | DONE (2025-12-06) | After #1 | QA · Security | Deterministic software test vectors (sign/verify, hash) added in unit tests; “non-certified” banner documented. |
| 5 | SM-CRYPTO-05 | DONE (2025-12-06) | After #3 | Docs · Ops | Created `etc/rootpack/cn/crypto.profile.yaml` with cn-soft profile preferring `cn.sm.soft`, marked software-only with env gate; fixtures packaging pending SM2 host wiring. |
| 6 | SM-CRYPTO-06 | BLOCKED (2025-12-06) | Hardware token available | Security · Crypto | Add PKCS#11 SM provider and rerun vectors with certified hardware; replace “software-only” label when certified. |
| 7 | SM-CRYPTO-07 | DONE (2025-12-09) | Docker host available | Security · Ops | Build/publish SM remote soft-service image (cn.sm.remote.http) from `tmp/smremote-pub`, smoke-test `/status` `/sign` `/verify`, and prepare container runbook. |
## Execution Log
| Date (UTC) | Update | Owner |
@@ -35,11 +36,14 @@
| 2025-12-06 | Started host wiring for SM2: Authority file key loader now supports SM2 raw keys; JWKS tests include SM2; task 3 set to DOING. | Implementer |
| 2025-12-07 | Signer SM2 gate + tests added (software registry); Attestor registers SM provider, loads SM2 keys, SM2 verification tests added (software env-gated); task 3 set to DONE. | Implementer |
| 2025-12-07 | Attestor SM2 wiring complete: SmSoftCryptoProvider registered in AttestorSigningKeyRegistry, SM2 key loading (PEM/base64/hex), signing tests added. Fixed AWSSDK version conflict and pre-existing test compilation issues. Task 3 set to DONE. | Implementer |
| 2025-12-09 | Rebuilt SM remote publish artifacts to `tmp/smremote-pub`, added runtime Dockerfile, built `sm-remote:local`, and smoke-tested `/status`, `/sign`, `/verify` (SM_SOFT_ALLOWED=1, port 56080). | Implementer |
| 2025-12-09 | Ran `dotnet restore` and `dotnet build src/Concelier/StellaOps.Concelier.sln -v minimal`; build completed with warnings only (Dilithium/NU1510/CONCELIER0001/CS8424). | Concelier Guild |
## Decisions & Risks
- SM provider licensing/availability uncertain; mitigation: software fallback with “non-certified” label until hardware validated.
- Webhook/interop must stay SHA-256—verify no SM override leaks; regression tests required in task 4.
- Export controls for SM libraries still require review; note in docs and keep SM_SOFT_ALLOWED gate.
- SM remote soft-service image built and validated locally (soft provider, port 56080); still software-only until PKCS#11 hardware (SM-CRYPTO-06) lands.
## Next Checkpoints
- 2025-12-11 · Provider selection decision.

17
docs/implplan/SPRINT_3407_0001_0001_postgres_cleanup.md
View File

@@ -56,16 +56,7 @@
| 16 | PG-T7.1.5c | DONE | Concelier Guild | Refactor connectors/exporters/tests to Postgres storage; delete Storage.Mongo code. |
| 17 | PG-T7.1.5d | DONE | Concelier Guild | Add migrations for document/state/export tables; include in air-gap kit. |
| 18 | PG-T7.1.5e | DONE | Concelier Guild | Postgres-only Concelier build/tests green; remove Mongo artefacts and update docs. |
### T7.2: Archive MongoDB Data
| # | Task ID | Status | Key dependency / next step | Owners | Task Definition |
| --- | --- | --- | --- | --- | --- |
| 11 | PG-T7.2.1 | TODO | Depends on PG-T7.1.10 | DevOps Guild | Take final MongoDB backup |
| 12 | PG-T7.2.2 | TODO | Depends on PG-T7.2.1 | DevOps Guild | Export to BSON/JSON archives |
| 13 | PG-T7.2.3 | TODO | Depends on PG-T7.2.2 | DevOps Guild | Store archives in secure location |
| 14 | PG-T7.2.4 | TODO | Depends on PG-T7.2.3 | DevOps Guild | Document archive contents and structure |
| 15 | PG-T7.2.5 | TODO | Depends on PG-T7.2.4 | DevOps Guild | Set retention policy for archives |
| 16 | PG-T7.2.6 | TODO | Depends on PG-T7.2.5 | DevOps Guild | Schedule MongoDB cluster decommission |
| 19 | PG-T7.1.5f | DOING | Massive connector/test surface still on MongoCompat/Bson; staged migration to Storage.Contracts required before shim deletion. | Concelier Guild | Remove MongoCompat shim and any residual Mongo-shaped payload handling after Postgres parity sweep; update docs/DI/tests accordingly. |
### T7.3: PostgreSQL Performance Optimization
| # | Task ID | Status | Key dependency / next step | Owners | Task Definition |
@@ -135,12 +126,18 @@
| 2025-12-08 | Rebuilt Concelier solution after cache restore; Mongo shims no longer pull Mongo2Go/driver, but overall build still fails on cross-module crypto gap (`SmRemote` plugin missing). No remaining Mongo package/runtime dependencies in Concelier build. | Concelier Guild |
| 2025-12-08 | Dropped the last MongoDB.Bson package references, expanded provenance Bson stubs, cleaned obj/bin and rehydrated NuGet cache, then rebuilt `StellaOps.Concelier.sln` successfully with Postgres-only DI. PG-T7.1.5a/5b marked DONE; PG-T7.1.5c continues for Postgres runtime parity and migrations. | Concelier Guild |
| 2025-12-08 | Added Postgres-backed DTO/export/PSIRT/JP-flag/change-history stores with migration 005 (concelier schema), wired DI to new stores, and rebuilt `StellaOps.Concelier.sln` green Postgres-only. PG-T7.1.5c/5d/5e marked DONE. | Concelier Guild |
| 2025-12-09 | Mirrored Wave A action/risk into parent sprint; added PG-T7.1.5f (TODO) to remove MongoCompat shim post-parity sweep and ensure migration 005 stays in the kit. | Project Mgmt |
| 2025-12-09 | PG-T7.1.5f set BLOCKED: MongoCompat/Bson interfaces are still the canonical storage contracts across connectors/tests; need design to introduce Postgres-native abstractions and parity evidence before deleting shim. | Project Mgmt |
| 2025-12-09 | Investigated MongoCompat usage: connectors/tests depend on IDocumentStore, IDtoStore (Bson payloads), ISourceStateRepository (Bson cursors), advisory/alias/change-history/export state stores, and DualWrite/DIOptions; Postgres stores implement Mongo contracts today. Need new storage contracts (JSON/byte payloads, cursor DTO) and adapter layer to retire Mongo namespaces. | Project Mgmt |
| 2025-12-09 | Started PG-T7.1.5f implementation: added Postgres-native storage contracts (document/dto/source state) and adapters in Postgres stores to implement both new contracts and legacy Mongo interfaces; connectors/tests still need migration off MongoCompat/Bson. | Project Mgmt |
| 2025-12-09 | PG-T7.1.5f in progress: contract/adapters added; started migrating Common SourceFetchService to Storage.Contracts with backward-compatible constructor. Connector/test surface still large; staged migration plan required. | Project Mgmt |
## Decisions & Risks
- Concelier PG-T7.1.5c/5d/5e completed with Postgres-backed DTO/export/state stores and migration 005; residual risk is lingering Mongo-shaped payload semantics in connectors/tests until shims are fully retired in a follow-on sweep.
- Cleanup is strictly after all phases complete; do not start T7 tasks until module cutovers are DONE.
- Risk: Air-gap kit must avoid external pulls; ensure pinned digests and included migrations.
- Risk: Remaining MongoCompat usage in Concelier (DTO shapes, cursor payloads) should be retired once Postgres migrations/tests land to prevent regressions when shims are deleted.
- Risk: MongoCompat shim removal pending (PG-T7.1.5f / ACT-3407-A1); PG-T7.1.5f in progress with Postgres-native storage contracts added, but connectors/tests still depend on MongoCompat/Bson types. Parity sweep and connector migration needed before deleting the shim; keep migration 005 in the air-gap kit.
- BLOCKER: Scheduler: Postgres equivalent for GraphJobStore/PolicyRunService not designed; need schema/contract decision to proceed with PG-T7.1.2a and related deletions.
- BLOCKER: Scheduler Worker still depends on Mongo-era repositories (run/schedule/impact/policy); Postgres counterparts are missing, keeping solution/tests red until implemented or shims added.
- BLOCKER: Scheduler/Notify/Policy/Excititor Mongo removals must align with the phased plan; delete only after replacements are in place.

67
docs/implplan/SPRINT_3407_0001_0001_postgres_cleanup_tasks.md
View File

@@ -1,9 +1,62 @@
# Wave A · Mongo Drop (Concelier)
# Sprint 3407 - Wave A Concelier Postgres Cleanup Tasks
| # | Task ID | Status | Owner | Notes |
## Topic & Scope
- Track Wave A (Concelier) tasks PG-T7.1.5a-5e for Mongo removal and Postgres storage cutover under Sprint 3407 Phase 7 cleanup.
- Evidence: Postgres-only Concelier builds/tests, migrations applied, and no MongoDB driver or package dependencies.
- Working directory: `src/Concelier`.
## Dependencies & Concurrency
- Depends on approvals and plan in `SPRINT_3407_0001_0001_postgres_cleanup.md` (Wave A precedes Waves B-E).
- Align statuses with the parent sprint Execution Log; no parallel Mongo work should start elsewhere until this wave remains green.
## Documentation Prerequisites
- `docs/db/reports/mongo-removal-plan-20251207.md`
- `docs/db/reports/mongo-removal-decisions-20251206.md`
- `docs/modules/concelier/architecture.md`
- `src/Concelier/AGENTS.md`
## Delivery Tracker
| # | Task ID | Status | Key dependency / next step | Owners | Task Definition |
| --- | --- | --- | --- | --- | --- |
| 1 | PG-T7.1.5a | DONE | Postgres DI stabilized; monitor connectors for stray Mongo package usage. | Concelier Guild | Replace Mongo storage dependencies with Postgres equivalents; remove MongoDB.Driver/Bson packages from Concelier projects. |
| 2 | PG-T7.1.5b | DONE | Postgres stores live; retire interim shims after parity sweep. | Concelier Guild | Implement Postgres document/raw storage (bytea/LargeObject) plus state repositories to satisfy connector fetch/store paths. |
| 3 | PG-T7.1.5c | DONE | Follow-on: remove MongoCompat shim once tests stay green. | Concelier Guild | Refactor all connectors/exporters/tests to use Postgres storage namespaces; delete Storage.Mongo code/tests. |
| 4 | PG-T7.1.5d | DONE | Ensure migration 005 remains in the air-gap kit. | Concelier Guild | Add migrations for documents/state/export tables; wire into Concelier Postgres storage DI. |
| 5 | PG-T7.1.5e | DONE | Keep parent sprint log updated; retire shim in follow-on wave. | Concelier Guild | End-to-end Concelier build/test on a Postgres-only stack; update sprint log and remove Mongo artifacts from repo history references. |
| 6 | PG-T7.1.5f | DOING | Need Postgres-native storage contracts to replace MongoCompat/Bson interfaces across connectors/tests; capture parity sweep evidence before deletion. | Concelier Guild | Remove MongoCompat shim and residual Mongo-shaped payload handling; update DI/docs/tests and keep migration 005 in the kit. |
## Wave Coordination
- Scope: Wave A (Concelier) in Sprint 3407 Phase 7 cleanup; completes before archive/perf/doc/air-gap waves start.
- PG-T7.1.5a-5e are DONE; PG-T7.1.5f (shim removal) is in progress and will gate MongoCompat deletion.
## Wave Detail Snapshots
- Postgres document/raw/state stores and migration 005 are applied; Concelier builds/tests succeed without MongoDB drivers.
- MongoCompat shim remains the canonical interface surface for connectors/tests; Postgres-native contracts and adapters have been added, but migration and parity evidence are still pending.
## Interlocks
- Parent sprint execution log remains the source of truth for cross-module sequencing.
- Air-gap kit updates depend on migration 005 shipping in artifacts; coordinate with the Wave E owner before the kit freeze.
## Upcoming Checkpoints
- 2025-12-10: Confirm MongoCompat shim removal approach (introduce Postgres-native contract + parity evidence) and unblock PG-T7.1.5f.
## Action Tracker
| Action ID | Status | Owner | Notes |
| --- | --- | --- | --- |
| ACT-3407-A1 | DOING | Concelier Guild | Execute Postgres-native storage contract, capture parity evidence, then delete MongoCompat shim; tracked as PG-T7.1.5f in parent sprint. |
## Decisions & Risks
- Decisions: PG-T7.1.5a-5e are complete per parent sprint log (2025-12-08) with Postgres-only Concelier build/test evidence.
- Risks are tracked in the table below and should be mirrored into the parent sprint if escalated.
| Risk | Impact | Mitigation | Owner | Status |
| --- | --- | --- | --- | --- |
| 1 | PG-T7.1.5a | DOING | Concelier Guild | Replace Mongo storage dependencies with Postgres equivalents; remove MongoDB.Driver/Bson packages from Concelier projects. |
| 2 | PG-T7.1.5b | DOING | Concelier Guild | Implement Postgres document/raw storage (bytea/LargeObject) + state repos to satisfy connector fetch/store paths. |
| 3 | PG-T7.1.5c | TODO | Concelier Guild | Refactor all connectors/exporters/tests to use Postgres storage namespaces; delete Storage.Mongo code/tests. |
| 4 | PG-T7.1.5d | TODO | Concelier Guild | Add migrations for documents/state/export tables; wire into Concelier Postgres storage DI. |
| 5 | PG-T7.1.5e | TODO | Concelier Guild | End-to-end Concelier build/test on Postgres-only stack; update sprint log and remove Mongo artifacts from repo history references. |
| MongoCompat shim still referenced in connectors/tests | Could reintroduce Mongo semantics and block full removal | Define Postgres-native storage contract, capture parity sweep evidence, then delete the shim; ensure migration 005 stays in the kit | Concelier Guild | Open |
## Execution Log
| Date (UTC) | Update | Owner |
| --- | --- | --- |
| 2025-12-09 | Normalized file to sprint template; synced PG-T7.1.5a-5e statuses to DONE per parent sprint log; added checkpoints, interlocks, and risk tracking. | Project Mgmt |
| 2025-12-09 | Added PG-T7.1.5f (BLOCKED) for MongoCompat shim removal; action ACT-3407-A1 set BLOCKED pending Postgres-native storage contract and parity evidence. | Project Mgmt |
| 2025-12-09 | Investigated MongoCompat usage across connectors/tests: IDocumentStore, IDtoStore (Bson payloads), ISourceStateRepository (Bson cursors), advisory/alias/change-history/export stores, DualWrite DI hooks all depend on Mongo contracts. Need new Postgres-native storage contracts (JSON/byte payload DTOs, cursor DTO) plus adapters before shim deletion. | Project Mgmt |
| 2025-12-09 | Started PG-T7.1.5f: added Postgres-native storage contracts and adapters in Postgres stores implementing both new and legacy Mongo interfaces; began migrating Common SourceFetchService to new contracts with compatibility ctor; connector/test migration still pending. | Project Mgmt |

21
docs/modules/scanner/bun-analyzer-gotchas.md
View File

@@ -25,21 +25,26 @@ Unlike Node.js, Bun may store packages entirely under `node_modules/.bun/` with
- Do not filter out hidden directories in container scans
- Verify evidence shows packages from both `node_modules/` and `node_modules/.bun/`
## 3. `bun.lockb` Migration Path
## 3. `bun.lockb` Policy (2025-12-09)
The binary lockfile (`bun.lockb`) format is undocumented and unstable. The analyzer treats it as **unsupported** and emits a remediation finding.
The binary lockfile (`bun.lockb`) remains **unsupported**. We will not parse it and will keep remediation-only handling until Bun publishes a stable, documented format.
**Migration command:**
**Posture:**
- Treat `bun.lockb` as unsupported input; do not attempt best-effort parsing.
- Emit a deterministic remediation finding instructing conversion to text.
- Skip package inventory when only `bun.lockb` is present to avoid nondeterministic/partial results.
**Migration command (required):**
```bash
bun install --save-text-lockfile
```
This generates `bun.lock` (JSONC text format) which the analyzer can parse.
This generates `bun.lock` (JSONC text format) which the analyzer parses.
**WebService response:** When only `bun.lockb` is present:
- The scan completes but reports unsupported status
- Remediation guidance is included in findings
- No package inventory is generated
**WebService response when only `bun.lockb` exists:**
- Scan completes with `unsupported` marker for the package manager.
- Remediation guidance is included in findings.
- No package inventory is generated until `bun.lock` is provided.
## 4. JSONC Lockfile Format

46
docs/modules/scanner/design/dart-swift-analyzer-scope.md Normal file
View File

@@ -0,0 +1,46 @@
# Dart & Swift Analyzer Scope Note (2025-12-09)
## Goals
- Define the initial analyzer scope for Dart (pub) and Swift (SwiftPM) with deterministic, offline-friendly behavior.
- Provide fixture/backlog list to unblock readiness tracking and align with Signals/Zastava expectations.
## Dart (pub)
- Inputs: `pubspec.yaml`, `pubspec.lock`, `.dart_tool/package_config.json`, and downloaded packages under `.dart_tool/pub`.
- Outputs:
- Inventory of `pkg:pub/<name>@<version>` with resolved source (hosted/path/git) and sha256 when present in lockfile.
- Dependency edges from `pubspec.lock`; dev dependencies emitted only when `include_dev=true`.
- Analyzer metadata: sdk constraint, null-safety flag, source type per package.
- Determinism:
- Sort packages and edges lexicographically.
- Normalize paths to POSIX; no network calls; rely only on lockfile/package_config on disk.
- Out of scope (v1):
- Flutter build graph, transitive runtime surface, and hosted index downloads.
- Git/path overrides beyond what is listed in lock/package_config.
- Fixtures/backlog:
- Hosted app with `pubspec.lock` and `.dart_tool/package_config.json` (dev deps included).
- Path dependency sample (relative and absolute).
- Git dependency sample with locked commit.
- Missing lockfile case (expect finding + no inventory).
## Swift (SwiftPM)
- Inputs: `Package.swift`, `Package.resolved` (v1/v2), `.build/` manifest cache when present.
- Outputs:
- Inventory of `pkg:swiftpm/<name>@<version>` with checksum from `Package.resolved` when available.
- Dependency edges from `Package.resolved` target graph; emit platforms/arch only when declared.
- Analyzer metadata: Swift tools version, resolution format, mirrors when specified.
- Determinism:
- Do not execute `swift package`; parse manifests/resolved files only.
- Stable ordering by package then target; normalize paths to POSIX.
- Out of scope (v1):
- Xcodeproj resolution, binary target downloads, and build artifacts hashing.
- Conditional target resolution beyond what `Package.resolved` records.
- Fixtures/backlog:
- Single-package app with `Package.resolved` v2 (checksum present).
- Nested target graph with products/targets/flexible platforms.
- Binary target entry (no download; expect metadata-only inventory).
- Missing `Package.resolved` case (emit finding, no inventory).
## Alignment & Next Steps
- Signals/Zastava: confirm package ID naming (`pkg:pub`, `pkg:swiftpm`) and dependency edge semantics for reachability ingestion.
- Add goldens/fixtures under `src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.DartSwift.Tests/Fixtures/**`.
- Update readiness checkpoints once fixtures and parsers land; current scope note unblocks backlog creation only.

40
docs/modules/scanner/design/deno-analyzer-scope.md Normal file
View File

@@ -0,0 +1,40 @@
# Deno Analyzer Scope Note (2025-12-09)
## Goals
- Define deterministic, offline-friendly scope for the Deno analyzer to move readiness from “status mismatch” to planned execution.
- Enumerate fixtures and evidence needed to mark Amber→Green once implemented.
## Inputs
- `deno.json` / `deno.jsonc` (config and import maps).
- `deno.lock` (v2) with integrity hashes.
- Source tree for `import`/`export` graph; `node_modules/` when `npm:` specifiers are used (npm compatibility mode).
- Optional: cache dir (`~/.cache/deno`) when present in extracted images.
## Outputs
- Inventory of modules:
- `pkg:deno/<specifier>@<version>` for remote modules (normalize to URL without fragment).
- `pkg:npm/<name>@<version>` for `npm:` dependencies with lock hash.
- `pkg:file/<path>` for local modules (relative POSIX paths).
- Dependency edges:
- From importer to imported specifier with resolved path/URL.
- Include type (remote/local/npm), integrity (sha256 from lock), and media type when available.
- Metadata:
- Deno version (from lock/config if present).
- Import map path and hash.
- NPM compatibility flag + resolved registry scope when npm used.
## Determinism & Offline
- Never fetch network resources; rely solely on `deno.lock` + on-disk files.
- Normalize paths to POSIX; stable sorting (source path, then target).
- Hashes: prefer lock integrity; otherwise SHA-256 over file bytes for local modules.
## Fixtures / Backlog
1) Remote-only project with `deno.lock` (http imports) and import map.
2) Mixed project using `npm:` specifiers with `node_modules/` present.
3) Local-only project (relative imports) without lockfile → expect finding + no inventory.
4) Image/extracted cache with populated `~/.cache/deno` to verify offline reuse.
## Status & Next Steps
- Implement parser to ingest `deno.lock` v2 and import map; add graph builder over source files.
- Add fixtures under `src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Fixtures/**` with goldens; keep hashes stable.
- Update readiness checkpoints once fixtures land and TRX/binlogs captured.

45
docs/modules/scanner/design/dotnet-analyzer-11-001.md Normal file
View File

@@ -0,0 +1,45 @@
# .NET Analyzer Design · 11-001 Entrypoint Resolver (2025-12-09)
## Goals
- Resolve .NET entrypoints deterministically from project/publish artefacts and emit normalized identities (assembly name, MVID, TFM, RID, host kind, publish mode).
- Capture environment profiles (single-file, trimmed, self-contained vs framework-dependent, ALC hints) without executing payloads.
- Produce deterministic evidence aligned to `dotnet-il-metadata.schema.json` for downstream analyzers 11-002..005.
## Inputs
- `*.csproj`/`*.fsproj` metadata (TargetFrameworks, RuntimeIdentifiers, PublishSingleFile/Trim options).
- Publish outputs: apphost (`*.exe`), `*.dll`, `*.deps.json`, `*.runtimeconfig.json`, `*.targets` cache.
- RID graph from SDK (offline snapshot in repo), deterministic time provider.
## Outputs
- `entrypoints[]` records: `assembly`, `mvid`, `tfm`, `rid`, `hostKind` (apphost/framework-dependent/self-contained), `publishMode` (single-file/trimmed), `alcHints` (AssemblyLoadContext names), `probingPaths`, `nativeDeps` (apphost bundles).
- Evidence: `LanguageComponentEvidence` entries per entrypoint with locator = publish path, hash over file bytes for determinism.
- Diagnostics: missing deps/runtimeconfig, mixed RID publish, single-file without extractor support.
## Algorithm (deterministic)
1) Parse project: target frameworks, RIDs, publish flags; normalize to ordered sets.
2) Discover publish artefacts under `bin/<Configuration>/<TFM>/...` and `publish/` folders; prefer `*.runtimeconfig.json` when present.
3) Read `*.deps.json` to extract runtime targets and resolve primary entry assembly; fall back to `apphost` name.
4) Compute MVID from PE header; compute SHA-256 over `*.dll`/`*.exe` bytes; capture file size.
5) Classify host:
- `apphost` present -> `hostKind = apphost`; detect single-file bundle via marker sections.
- Framework-dependent -> `hostKind = framework-dependent`; use `runtimeconfig` probing paths.
6) Infer ALC hints: scan deps for `runtimeconfig.dev.json` probing paths and `additionalProbingPaths`; add known SDK paths.
7) Emit entrypoint with deterministic ordering: sort by assembly name, then RID, then TFM.
## Determinism & Offline
- No network access; relies solely on on-disk project/publish artefacts.
- Stable ordering and casing (`Ordinal` sort), UTC time provider.
- Hashes: SHA-256 over file bytes; no timestamps.
## Test & Fixture Plan
- Existing suite: `src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.DotNet.Tests` (now green; TRX at `TestResults/dotnet/dotnet-tests.trx`).
- Fixtures to maintain:
- Framework-dependent app with deps/runtimeconfig.
- Self-contained single-file publish (bundle) with apphost.
- Trimmed publish with ALC hints.
- Multi-RID publish verifying RID selection and deterministic ordering.
- Add new fixtures under `...DotNet.Tests/Fixtures/` when new host kinds are supported; keep hashes stable.
## Next Steps
- Wire readiness checkpoints to mark 11-001 design+tests complete; keep CI runner validation optional (DEVOPS-SCANNER-CI-11-001) for reproducibility.
- Feed outputs into 11-002..005 analyzers once entrypoint metadata is consumed by downstream IL/reflection pipelines.

39
docs/modules/scanner/design/php-autoload-design.md Normal file
View File

@@ -0,0 +1,39 @@
# PHP Analyzer Autoload & Restore Design (2025-12-09)
## Goals
- Stabilize PHP analyzer pipeline (SCANNER-ENG-0010 / 27-001) by defining autoload graph handling, composer restore posture, and fixtures.
- Provide deterministic evidence suitable for CI and reachability alignment with Concelier/Signals.
## Inputs
- `composer.json` + `composer.lock`.
- `vendor/composer/*.php` autoload files (`autoload_psr4.php`, `autoload_classmap.php`, `autoload_files.php`, `autoload_static.php`).
- Installed vendor tree under `vendor/`.
- Optional: `composer.phar` version metadata for diagnostics (no execution).
## Outputs
- Package inventory: `pkg:composer/<name>@<version>` with source/dist hashes from lockfile.
- Autoload graph:
- PSR-4/PSR-0 mappings (namespace → path), classmap entries, files includes.
- Emit edges from package → file and namespace → path with deterministic ordering.
- Restore diagnostics:
- Detect missing vendor install vs lockfile drift; emit findings instead of network restore.
- Metadata:
- Composer version (from lock/platform field when present).
- Platform PHP extensions/version constraints.
## Determinism & Offline
- No composer install/updates; read-only parsing of lock/autoload/vendor.
- Stable ordering: sort packages, namespaces, classmap entries, files includes (ordinal, POSIX paths).
- Hashes: use lockfile dist/shasum when present; otherwise SHA-256 over on-disk file bytes for autoloaded files.
## Fixtures / Backlog
1) PSR-4 project with namespaced classes and classmap mix.
2) Project with `autoload_files.php` includes (functions/constants).
3) Lockfile present but vendor missing → expect finding, no inventory.
4) Path repo override + dist hash present.
## Implementation Steps
- Parser for composer.lock (packages + platform reqs) and autoload PHP arrays (psr-4, psr-0, classmap, files).
- Graph builder producing deterministic edges and evidence records.
- Findings for missing vendor, mismatched lock hash, or absent autoload files.
- Tests under `src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Php.Tests` with goldens for each fixture; add TRX/binlogs to readiness once stable.

37
docs/modules/scanner/design/runtime-parity-plan.md Normal file
View File

@@ -0,0 +1,37 @@
# Runtime Parity Plan (Java / .NET / PHP) — Scanner Aú · Signals Alignment (2025-12-09)
## Objectives
- Close runtime parity gaps by pairing static analyzer hooks with runtime evidence for Java, .NET, and PHP.
- Produce deterministic artefacts (TRX/binlogs + NDJSON) that Signals can ingest for runtime reconciliation.
## Scope & Hooks
- **Java (21-005..011)**: jar/classpath resolution, `Main-Class`, module-info, shaded jars. Runtime hook: capture resolved classpath + main entry via proc snapshot or launcher args.
- **.NET (11-001..005)**: `.deps.json`, RID-graph, single-file/trimmed detection, `runtimeconfig.json`. Runtime hook: capture host command line + loaded assembly list via Signals proc trace.
- **PHP (27-001)**: composer autoload graph (`vendor/composer/autoload_*.php`), package metadata, runtime entry (fpm/cli). Runtime hook: map autoloaded files to runtime include graph when proc snapshot present.
## Evidence Plan
1) **Static**: ensure analyzers emit deterministically ordered inventories + edges with layer attribution (already enforced across analyzers).
2) **Runtime capture** (requires Signals):
- Provide proc snapshot schema to Scanner (cmdline, env, cwd, loaded modules/files).
- Export runtime observations as NDJSON with stable ordering (path, module, hash).
3) **Reconciliation**:
- Join static entries to runtime observations on normalized path + hash.
- Emit `runtime.match` / `runtime.miss` diagnostics with counts per analyzer.
4) **Artefacts**:
- CI: TRX/binlog per analyzer suite.
- NDJSON samples: runtime reconciliation outputs for each language (hosted under `src/Scanner/__Tests/.../Fixtures/RuntimeParity`).
## Task Backlog
- T1: Wire proc snapshot ingestion for Java/.NET/PHP analyzers (Signals contract).
- T2: Add runtime reconciliation step with deterministic ordering and diagnostics.
- T3: Author runtime fixtures (one per language) and goldens for reconciliation output.
- T4: Document runtime parity expectations in readiness checkpoints and surfaces guides.
## Constraints
- Offline-friendly: no network calls during reconciliation; rely solely on provided proc snapshot.
- Deterministic: stable sort (layer, path, name), UTC timestamps, no random seeds.
- Security: avoid executing payloads; treat proc snapshot as data only.
## Dependencies
- Signals to confirm proc snapshot schema and DSSE/NDJSON event shape for runtime observations.
- Dedicated CI runner (DEVOPS-SCANNER-CI-11-001) to record TRX/binlogs for Java/.NET suites.

10
docs/modules/scanner/readiness-checkpoints.md
View File

@@ -7,13 +7,13 @@
## Phase Readiness
| Phase / Sprint | Status | Evidence | Gaps / Actions |
| --- | --- | --- | --- |
| Phase II · Sprint 0131 (Deno/Java/.NET bootstrap) | Amber/Red | Deno runtime capture shipped and tested; Java chain 21-005..011 blocked on Concelier build + CI runner; .NET Lang 11-001 blocked awaiting clean runner; PHP VFS 27-001 blocked pending bootstrap spec. | Need CI slice (DEVOPS-SCANNER-CI-11-001) for Java/.NET; define PHP bootstrap spec and fixtures to unblock 27-001. |
| Phase II · Sprint 0131 (Deno/Java/.NET bootstrap) | Amber/Red | Deno runtime capture shipped and tested; Java chain 21-005..011 still blocked on runner; .NET Lang 11-001 design/tests completed locally (TRX `TestResults/dotnet/dotnet-tests.trx`, design at `docs/modules/scanner/design/dotnet-analyzer-11-001.md`); PHP VFS 27-001 blocked pending bootstrap spec. | Need CI slice (DEVOPS-SCANNER-CI-11-001) for Java rerun; finalize PHP bootstrap spec and fixtures to unblock 27-001; publish Deno fixtures. |
| Phase III · Sprint 0132 (Native + Node foundations) | Amber | Native analyzers 20-001..010 shipped with tests; Node 22-001..005 shipped; Node isolated/CI tests pending due to build graph bloat; .NET Lang 11-002..005 blocked on upstream design 11-001 outputs. | Trim Node test graph or run on clean runner to record pass; unblock .NET analyzer design to proceed with runtime/export/fixtures. |
| Phase IV · Sprint 0133 (Node bundle/source-map) | Amber | Phase22 bundle/native/WASM observation implemented and fixtures hashed; validation tests pending (SDK resolver cancels build on current runner). | Execute `scripts/run-node-phase22-smoke.sh` on clean runner; capture TRX/binlog to close. |
| Phase V · Sprint 0134 (PHP fixtures/runtime/package) | Green | PHP analyzer fixtures, runtime evidence, and packaging shipped; docs updated. | Keep fixture hashes stable; rerun benchmarks when dependencies change. |
| Phase VI · Sprint 0135 (Python container + Ruby VFS/edges) | Green | Python container/zipapp adapters shipped; Ruby VFS/dependency edges/observations/runtime capture packaged; EntryTrace 18-502/503 delivered. | Maintain determinism; re-run EntryTrace suite in CI. |
| Phase VII · Sprint 0136 (EntryTrace surface/CLI) | Green | EntryTrace phase VII tasks 18-504/505/506 completed; CLI/WebService surfaces show best-terminal metadata and confidence. | Keep NDJSON schema stable; rerun worker payload tests in CI. |
| Sprint 0138 (Ruby parity & future analyzers) | Amber/Red | Ruby parity shipped; Mongo package inventory live. PHP pipeline SCANNER-ENG-0010 blocked on composer/autoload design + restore stability; Deno/Dart/Swift analyzer scopes blocked awaiting design; Kubernetes/VM roadmap pending. | Resolve PHP restore/design, produce Deno/Dart/Swift scopes, schedule Zastava/Runtime alignment. |
| Sprint 0138 (Ruby parity & future analyzers) | Amber/Red | Ruby parity shipped; Mongo package inventory live. PHP pipeline SCANNER-ENG-0010 blocked on composer/autoload design + restore stability (design at `docs/modules/scanner/design/php-autoload-design.md`); Deno scope drafted (`docs/modules/scanner/design/deno-analyzer-scope.md`); Dart/Swift scope drafted (`docs/modules/scanner/design/dart-swift-analyzer-scope.md`); Kubernetes/VM roadmap pending. | Implement PHP autoload parser/fixtures per design; add Deno fixtures and validation evidence; align with Zastava/Runtime and update readiness once fixtures land. |
## Overall
- Green areas: native analyzers, PHP fixtures/runtime packaging, Ruby analyzer, Python container adapters, EntryTrace phases VI–VII.
@@ -22,5 +22,7 @@
## Recommended Next Actions
1) Secure clean CI slice for Java/.NET and Node Phase22 smoke tests; store binlogs/TRX.
2) Finalise PHP analyzer design (composer/autoload graph) and stabilise restore pipeline to unblock SCANNER-ENG-0010/27-001.
3) Publish Deno/Dart/Swift analyzer scopes with fixtures to unblock 0138 tasks and roadmap alignment with Zastava/Runtime.
4) Re-run EntryTrace and Native suites in CI to lock deterministic hashes before downstream release.
3) Publish Deno/Dart/Swift analyzer scopes with fixtures to unblock 0138 tasks and roadmap alignment with Zastava/Runtime (scope note added at `docs/modules/scanner/design/dart-swift-analyzer-scope.md`; fixtures pending).
4) Lock bun.lockb posture as remediation-only (doc updated at `docs/modules/scanner/bun-analyzer-gotchas.md`); no parser work planned unless format stabilises.
5) Draft runtime parity plan for Java/.NET/PHP and align with Signals proc snapshot schema (plan at `docs/modules/scanner/design/runtime-parity-plan.md`); add reconciliation fixtures once schema confirmed.
6) Re-run EntryTrace and Native suites in CI to lock deterministic hashes before downstream release.

2
docs/security/crypto-compliance.md
View File

@@ -99,7 +99,7 @@ HMAC operations use purpose-based selection similar to hashing:
## Simulation paths when hardware is missing
- **RU / GOST**: Linux baseline uses `ru.openssl.gost`; CryptoPro CSP can be exercised from Linux via the Wine sidecar service (`ru.winecsp.http`) built from `scripts/crypto/setup-wine-csp-service.sh` when customers supply the CSP installer. Windows CSP remains blocked until licensed runners are available.
- **RU / GOST**: Linux baseline uses `ru.openssl.gost`; CryptoPro CSP can be exercised via the native Linux CSP service (CryptoPro deb bundles, no Wine) when customers supply the installer. Windows CSP remains blocked until licensed runners are available.
- **CN / SM2**: Software baseline (`cn.sm.soft`) plus a containerized remote microservice (`cn.sm.remote.http`) that simulates SM2 signing/verification; swap the endpoint to a hardware-backed service when licensed hardware is provided.
- **CN / SM**: Software-only SM2/SM3 provider (`cn.sm.soft`) backed by BouncyCastle; enable with `SM_SOFT_ALLOWED=1`. Hardware PKCS#11 tokens can be added later without changing feature code because hosts resolve via `ICryptoProviderRegistry`.
- **FIPS / eIDAS**: Software allow-lists (`fips.ecdsa.soft`, `eu.eidas.soft`) enforce ES256/ES384 + SHA-2. They are labeled non-certified until a CMVP/QSCD module is supplied.

863
docs/security/wine-csp-loader-design.md
View File

@@ -1,863 +0,0 @@
# Wine CSP Loader Design · CryptoPro GOST Validation
**Status:** IMPLEMENTED (HTTP-based approach)
**Date:** 2025-12-07
**Owners:** Security Guild, DevOps
**Related:** RU-CRYPTO-VAL-04, RU-CRYPTO-VAL-05
## Implementation Status
The HTTP-based Wine RPC Server approach (Approach C variant) has been implemented:
| Component | Path | Status |
|-----------|------|--------|
| Wine CSP HTTP Service | `src/__Tools/WineCspService/` | DONE |
| Setup Script | `scripts/crypto/setup-wine-csp-service.sh` | DONE |
| Crypto Registry Provider | `src/__Libraries/StellaOps.Cryptography.Plugin.WineCsp/` | DONE |
### Implementation Files
- **`src/__Tools/WineCspService/Program.cs`** - ASP.NET minimal API with endpoints: /health, /status, /keys, /sign, /verify, /hash, /test-vectors
- **`src/__Tools/WineCspService/CryptoProGostSigningService.cs`** - IGostSigningService using GostCryptography fork
- **`src/__Tools/WineCspService/WineCspService.csproj`** - .NET 8 Windows self-contained executable
- **`scripts/crypto/setup-wine-csp-service.sh`** - Wine environment setup, builds service, creates systemd unit
- **`src/__Libraries/StellaOps.Cryptography.Plugin.WineCsp/WineCspHttpProvider.cs`** - ICryptoProvider implementation
- **`src/__Libraries/StellaOps.Cryptography.Plugin.WineCsp/WineCspHttpSigner.cs`** - ICryptoSigner via HTTP
- **`src/__Libraries/StellaOps.Cryptography.Plugin.WineCsp/WineCspHttpClient.cs`** - HTTP client with retry policies
### Usage
```bash
# Setup Wine environment and build service
./scripts/crypto/setup-wine-csp-service.sh [--csp-installer /path/to/csp_setup.msi]
# Start service (runs under Wine)
./artifacts/wine-csp-service/run-wine-csp-service.sh
# Test endpoints
curl http://localhost:5099/status
curl -X POST http://localhost:5099/hash -H 'Content-Type: application/json' \
-d '{"dataBase64":"SGVsbG8gV29ybGQ="}'
```
### Integration with StellaOps Router
Configure upstream proxy: `/api/wine-csp/*` → `http://localhost:5099/*`
---
## Executive Summary
This document explores approaches to load Windows CryptoPro CSP via Wine for cross-platform GOST algorithm validation. The goal is to generate and validate test vectors without requiring dedicated Windows infrastructure.
**Recommendation:** Use Wine for test vector generation only, not production. The native PKCS#11 path (`Pkcs11GostCryptoProvider`) should remain the production cross-platform solution.
## 1. Architecture Overview
### Current State
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ Current GOST Provider Hierarchy │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ ┌─────────────────────────────────────────────────────────────────────┐ │
│ │ ICryptoProviderRegistry │ │
│ │ │ │
│ │ Profile: ru-offline │ │
│ │ PreferredOrder: [ru.cryptopro.csp, ru.openssl.gost, ru.pkcs11] │ │
│ └─────────────────────────────────────────────────────────────────────┘ │
│ │ │
│ ┌────────────────────┼────────────────────┐ │
│ ▼ ▼ ▼ │
│ ┌──────────────┐ ┌───────────────┐ ┌──────────────┐ │
│ │ CryptoPro │ │ OpenSSL GOST │ │ PKCS#11 │ │
│ │ CSP Provider │ │ Provider │ │ Provider │ │
│ │ │ │ │ │ │ │
│ │ Windows ONLY │ │ Cross-plat │ │ Cross-plat │ │
│ │ CSP APIs │ │ BouncyCastle │ │ Token-based │ │
│ └──────────────┘ └───────────────┘ └──────────────┘ │
│ ❌ ✓ ✓ │
│ (Linux N/A) (Fallback) (Hardware) │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
### Proposed Wine Integration
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ Wine CSP Loader Architecture │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ ┌────────────────────────────────────────────────────────────────────────┐│
│ │ Linux Host ││
│ │ ││
│ │ ┌─────────────────────┐ ┌─────────────────────────────────────┐ ││
│ │ │ StellaOps .NET App │ │ Wine Environment │ ││
│ │ │ │ │ │ ││
│ │ │ ICryptoProvider │ │ ┌─────────────────────────────┐ │ ││
│ │ │ │ │ │ │ CryptoPro CSP │ │ ││
│ │ │ ▼ │ │ │ │ │ ││
│ │ │ WineCspBridge │────▶│ │ cpcspr.dll │ │ ││
│ │ │ (P/Invoke) │ │ │ cpcsp.dll │ │ ││
│ │ │ │ │ │ asn1rt.dll │ │ ││
│ │ └─────────────────────┘ │ └─────────────────────────────┘ │ ││
│ │ │ │ │ │ ││
│ │ │ IPC/Socket │ │ Wine CryptoAPI │ ││
│ │ │ │ ▼ │ ││
│ │ │ │ ┌─────────────────────────────┐ │ ││
│ │ │ │ │ Wine crypt32.dll │ │ ││
│ │ └──────────────────▶│ │ Wine advapi32.dll │ │ ││
│ │ │ └─────────────────────────────┘ │ ││
│ │ └─────────────────────────────────────┘ ││
│ └────────────────────────────────────────────────────────────────────────┘│
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
## 2. Technical Approaches
### Approach A: Wine Prefix with Test Runner
**Concept:** Install CryptoPro CSP inside a Wine prefix, run .NET test binaries under Wine.
**Implementation:**
```bash
#!/bin/bash
# scripts/crypto/setup-wine-cryptopro.sh
set -euo pipefail
WINE_PREFIX="${WINE_PREFIX:-$HOME/.stellaops-wine-csp}"
WINE_ARCH="win64"
# Initialize Wine prefix
export WINEPREFIX="$WINE_PREFIX"
export WINEARCH="$WINE_ARCH"
echo "[1/5] Initializing Wine prefix..."
wineboot --init
echo "[2/5] Installing .NET runtime dependencies..."
winetricks -q dotnet48 vcrun2019
echo "[3/5] Setting Windows version..."
winetricks -q win10
echo "[4/5] Installing CryptoPro CSP..."
# Requires CSP installer to be present
if [[ -f "$CSP_INSTALLER" ]]; then
wine msiexec /i "$CSP_INSTALLER" /qn ADDLOCAL=ALL
else
echo "WARNING: CSP_INSTALLER not set. Manual installation required."
echo " wine msiexec /i /path/to/csp_setup_x64.msi /qn"
fi
echo "[5/5] Verifying CSP registration..."
wine reg query "HKLM\\SOFTWARE\\Microsoft\\Cryptography\\Defaults\\Provider" 2>/dev/null || {
echo "ERROR: CSP not registered in Wine registry"
exit 1
}
echo "Wine CryptoPro environment ready: $WINE_PREFIX"
```
**Test Vector Generation:**
```bash
#!/bin/bash
# scripts/crypto/generate-wine-test-vectors.sh
export WINEPREFIX="$HOME/.stellaops-wine-csp"
# Build test vector generator for Windows target
dotnet publish src/__Libraries/__Tests/StellaOps.Cryptography.Tests \
-c Release \
-r win-x64 \
--self-contained true \
-o ./artifacts/wine-tests
# Run under Wine
wine ./artifacts/wine-tests/StellaOps.Cryptography.Tests.exe \
--filter "Category=GostVectorGeneration" \
--output ./tests/fixtures/gost-vectors/wine-generated.json
```
**Pros:**
- Uses actual CSP, high fidelity
- Straightforward setup
- Generates real test vectors
**Cons:**
- Requires CryptoPro installer (licensing)
- Wine compatibility issues possible
- Heavy environment (~2GB+ prefix)
- Slow test execution
---
### Approach B: Winelib Bridge Library
**Concept:** Create a native Linux shared library using Winelib that exposes CSP functions.
**Implementation:**
```c
// src/native/wine-csp-bridge/csp_bridge.c
// Compile: winegcc -shared -o libcspbridge.so csp_bridge.c -lcrypt32
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <wincrypt.h>
#include <stdio.h>
#include <string.h>
// Exported bridge functions (POSIX ABI)
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
int error_code;
char error_message[256];
unsigned char signature[512];
size_t signature_length;
} CspBridgeResult;
// Initialize CSP context
__attribute__((visibility("default")))
int csp_bridge_init(const char* provider_name, void** context_out) {
HCRYPTPROV hProv = 0;
// Convert provider name to wide string
wchar_t wProviderName[256];
mbstowcs(wProviderName, provider_name, 256);
if (!CryptAcquireContextW(
&hProv,
NULL,
wProviderName,
75, // PROV_GOST_2012_256
CRYPT_VERIFYCONTEXT)) {
return GetLastError();
}
*context_out = (void*)(uintptr_t)hProv;
return 0;
}
// Sign data with GOST
__attribute__((visibility("default")))
int csp_bridge_sign_gost(
void* context,
const unsigned char* data,
size_t data_length,
const char* key_container,
CspBridgeResult* result) {
HCRYPTPROV hProv = (HCRYPTPROV)(uintptr_t)context;
HCRYPTHASH hHash = 0;
HCRYPTKEY hKey = 0;
DWORD sigLen = sizeof(result->signature);
// Create GOST hash
if (!CryptCreateHash(hProv, CALG_GR3411_2012_256, 0, 0, &hHash)) {
result->error_code = GetLastError();
snprintf(result->error_message, 256, "CryptCreateHash failed: %d", result->error_code);
return -1;
}
// Hash the data
if (!CryptHashData(hHash, data, data_length, 0)) {
result->error_code = GetLastError();
CryptDestroyHash(hHash);
return -1;
}
// Sign the hash
if (!CryptSignHashW(hHash, AT_SIGNATURE, NULL, 0, result->signature, &sigLen)) {
result->error_code = GetLastError();
CryptDestroyHash(hHash);
return -1;
}
result->signature_length = sigLen;
result->error_code = 0;
CryptDestroyHash(hHash);
return 0;
}
// Release context
__attribute__((visibility("default")))
void csp_bridge_release(void* context) {
if (context) {
CryptReleaseContext((HCRYPTPROV)(uintptr_t)context, 0);
}
}
#ifdef __cplusplus
}
#endif
```
**Build Script:**
```bash
#!/bin/bash
# scripts/crypto/build-wine-bridge.sh
set -euo pipefail
BRIDGE_DIR="src/native/wine-csp-bridge"
OUTPUT_DIR="artifacts/native"
mkdir -p "$OUTPUT_DIR"
# Check for Wine development headers
if ! command -v winegcc &> /dev/null; then
echo "ERROR: winegcc not found. Install wine-devel package."
exit 1
fi
# Compile bridge library
winegcc -shared -fPIC \
-o "$OUTPUT_DIR/libcspbridge.dll.so" \
"$BRIDGE_DIR/csp_bridge.c" \
-lcrypt32 \
-mno-cygwin \
-O2
# Create loader script
cat > "$OUTPUT_DIR/load-csp-bridge.sh" << 'EOF'
#!/bin/bash
export WINEPREFIX="${WINEPREFIX:-$HOME/.stellaops-wine-csp}"
export WINEDLLPATH="$(dirname "$0")"
exec "$@"
EOF
chmod +x "$OUTPUT_DIR/load-csp-bridge.sh"
echo "Bridge library built: $OUTPUT_DIR/libcspbridge.dll.so"
```
**.NET P/Invoke Wrapper:**
```csharp
// src/__Libraries/StellaOps.Cryptography.Plugin.WineCsp/WineCspBridge.cs
using System;
using System.Runtime.InteropServices;
namespace StellaOps.Cryptography.Plugin.WineCsp;
/// <summary>
/// P/Invoke bridge to Wine-hosted CryptoPro CSP.
/// EXPERIMENTAL: For test vector generation only.
/// </summary>
internal static partial class WineCspBridge
{
private const string LibraryName = "libcspbridge.dll.so";
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)]
public struct CspBridgeResult
{
public int ErrorCode;
[MarshalAs(UnmanagedType.ByValTStr, SizeConst = 256)]
public string ErrorMessage;
[MarshalAs(UnmanagedType.ByValArray, SizeConst = 512)]
public byte[] Signature;
public nuint SignatureLength;
}
[LibraryImport(LibraryName, EntryPoint = "csp_bridge_init")]
public static partial int Init(
[MarshalAs(UnmanagedType.LPUTF8Str)] string providerName,
out nint contextOut);
[LibraryImport(LibraryName, EntryPoint = "csp_bridge_sign_gost")]
public static partial int SignGost(
nint context,
[MarshalAs(UnmanagedType.LPArray)] byte[] data,
nuint dataLength,
[MarshalAs(UnmanagedType.LPUTF8Str)] string keyContainer,
ref CspBridgeResult result);
[LibraryImport(LibraryName, EntryPoint = "csp_bridge_release")]
public static partial void Release(nint context);
}
/// <summary>
/// Wine-based GOST crypto provider for test vector generation.
/// </summary>
public sealed class WineCspGostProvider : ICryptoProvider, IDisposable
{
private nint _context;
private bool _disposed;
public string Name => "ru.wine.csp";
public WineCspGostProvider(string providerName = "Crypto-Pro GOST R 34.10-2012 CSP")
{
var result = WineCspBridge.Init(providerName, out _context);
if (result != 0)
{
throw new InvalidOperationException(
$"Failed to initialize Wine CSP bridge: error {result}");
}
}
public bool Supports(CryptoCapability capability, string algorithmId)
{
return capability == CryptoCapability.Signing &&
algorithmId is "GOST12-256" or "GOST12-512";
}
public ICryptoSigner GetSigner(string algorithmId, CryptoKeyReference keyReference)
{
return new WineCspGostSigner(_context, algorithmId, keyReference);
}
public void Dispose()
{
if (!_disposed)
{
WineCspBridge.Release(_context);
_disposed = true;
}
}
// ... other ICryptoProvider methods
}
```
**Pros:**
- More efficient than full Wine test runner
- Reusable library
- Can be loaded conditionally
**Cons:**
- Complex to build and maintain
- Wine/Winelib version dependencies
- Debugging is difficult
- Still requires CSP installation in Wine prefix
---
### Approach C: Wine RPC Server
**Concept:** Run a Wine process as a signing daemon, communicate via Unix socket or named pipe.
**Architecture:**
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ Wine RPC Server Architecture │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ ┌─────────────────────────────────┐ ┌─────────────────────────────────┐ │
│ │ .NET Application │ │ Wine Process │ │
│ │ │ │ │ │
│ │ WineCspRpcClient │ │ WineCspRpcServer.exe │ │
│ │ │ │ │ │ │ │
│ │ │ SignRequest(JSON) │ │ │ │ │
│ │ │──────────────────────▶│ │ ▼ │ │
│ │ │ │ │ CryptoAPI (CryptSignHash) │ │
│ │ │ │ │ │ │ │
│ │ │◀──────────────────────│ │ │ │ │
│ │ │ SignResponse(JSON) │ │ │ │ │
│ │ ▼ │ │ │ │
│ │ ICryptoSigner │ │ ┌─────────────────────────┐ │ │
│ │ │ │ │ CryptoPro CSP │ │ │
│ └─────────────────────────────────┘ │ │ (Wine-hosted) │ │ │
│ │ │ └─────────────────────────┘ │ │
│ │ Unix Socket │ │ │
│ │ /tmp/stellaops-csp.sock │ │ │
│ └─────────────────────────┼─────────────────────────────────┘ │
│ │ │
└────────────────────────────────────────┼────────────────────────────────────┘
```
**Server (Wine-side):**
```csharp
// tools/wine-csp-server/WineCspRpcServer.cs
// Build: dotnet publish -r win-x64, run under Wine
using System.Net.Sockets;
using System.Text.Json;
using System.Security.Cryptography;
// Wine RPC server for CSP signing requests
public class WineCspRpcServer
{
private readonly string _socketPath;
private readonly GostCryptoProvider _csp;
public static async Task Main(string[] args)
{
var socketPath = args.Length > 0 ? args[0] : "/tmp/stellaops-csp.sock";
var server = new WineCspRpcServer(socketPath);
await server.RunAsync();
}
public WineCspRpcServer(string socketPath)
{
_socketPath = socketPath;
_csp = new GostCryptoProvider(); // Uses CryptoPro CSP
}
public async Task RunAsync()
{
// For Wine, we use TCP instead of Unix sockets
// (Unix socket support in Wine is limited)
var listener = new TcpListener(IPAddress.Loopback, 9876);
listener.Start();
Console.WriteLine($"Wine CSP RPC server listening on port 9876");
while (true)
{
var client = await listener.AcceptTcpClientAsync();
_ = HandleClientAsync(client);
}
}
private async Task HandleClientAsync(TcpClient client)
{
using var stream = client.GetStream();
using var reader = new StreamReader(stream);
using var writer = new StreamWriter(stream) { AutoFlush = true };
try
{
var requestJson = await reader.ReadLineAsync();
var request = JsonSerializer.Deserialize<SignRequest>(requestJson!);
var signature = await _csp.SignAsync(
Convert.FromBase64String(request!.DataBase64),
request.KeyId,
request.Algorithm);
var response = new SignResponse
{
Success = true,
SignatureBase64 = Convert.ToBase64String(signature)
};
await writer.WriteLineAsync(JsonSerializer.Serialize(response));
}
catch (Exception ex)
{
var response = new SignResponse
{
Success = false,
Error = ex.Message
};
await writer.WriteLineAsync(JsonSerializer.Serialize(response));
}
}
}
public record SignRequest(string DataBase64, string KeyId, string Algorithm);
public record SignResponse
{
public bool Success { get; init; }
public string? SignatureBase64 { get; init; }
public string? Error { get; init; }
}
```
**Client (Linux .NET):**
```csharp
// src/__Libraries/StellaOps.Cryptography.Plugin.WineCsp/WineCspRpcClient.cs
public sealed class WineCspRpcSigner : ICryptoSigner
{
private readonly TcpClient _client;
private readonly string _keyId;
private readonly string _algorithm;
public WineCspRpcSigner(string host, int port, string keyId, string algorithm)
{
_client = new TcpClient(host, port);
_keyId = keyId;
_algorithm = algorithm;
}
public string KeyId => _keyId;
public string AlgorithmId => _algorithm;
public async ValueTask<byte[]> SignAsync(
ReadOnlyMemory<byte> data,
CancellationToken ct = default)
{
var stream = _client.GetStream();
var writer = new StreamWriter(stream) { AutoFlush = true };
var reader = new StreamReader(stream);
var request = new SignRequest(
Convert.ToBase64String(data.Span),
_keyId,
_algorithm);
await writer.WriteLineAsync(JsonSerializer.Serialize(request));
var responseJson = await reader.ReadLineAsync(ct);
var response = JsonSerializer.Deserialize<SignResponse>(responseJson!);
if (!response!.Success)
{
throw new CryptographicException($"Wine CSP signing failed: {response.Error}");
}
return Convert.FromBase64String(response.SignatureBase64!);
}
}
```
**Pros:**
- Clean separation of concerns
- Can run Wine server on separate machine
- Easier to debug
- Process isolation
**Cons:**
- Network overhead
- More moving parts
- Requires server lifecycle management
---
### Approach D: Docker/Podman with Windows Container (Alternative)
For completeness, if Wine proves unreliable, a Windows container approach:
```yaml
# docker-compose.wine-csp.yml (requires Windows host or nested virtualization)
version: '3.8'
services:
csp-signer:
image: mcr.microsoft.com/windows/servercore:ltsc2022
volumes:
- ./csp-installer:/installer:ro
- ./keys:/keys
command: |
powershell -Command "
# Install CryptoPro CSP
msiexec /i C:\installer\csp_setup_x64.msi /qn
# Start signing service
C:\stellaops\WineCspRpcServer.exe
"
ports:
- "9876:9876"
```
## 3. Wine Compatibility Analysis
### 3.1 CryptoAPI Support in Wine
Wine implements most of the CryptoAPI surface needed:
| API Function | Wine Status | Notes |
|--------------|-------------|-------|
| `CryptAcquireContext` | Implemented | CSP loading works |
| `CryptReleaseContext` | Implemented | |
| `CryptCreateHash` | Implemented | |
| `CryptHashData` | Implemented | |
| `CryptSignHash` | Implemented | |
| `CryptVerifySignature` | Implemented | |
| `CryptGetProvParam` | Partial | Some params missing |
| CSP DLL Loading | Partial | Requires proper registration |
### 3.2 CryptoPro-Specific Challenges
| Challenge | Impact | Mitigation |
|-----------|--------|------------|
| CSP Registration | Medium | Manual registry setup |
| ASN.1 Runtime | Medium | May need native override |
| License Check | Unknown | May fail under Wine |
| Key Container Access | High | File-based containers may work |
| Hardware Token | N/A | Not supported under Wine |
### 3.3 Known Wine Issues
```
Wine Bug #12345: CryptAcquireContext PROV_GOST not recognized
Status: Fixed in Wine 7.0+
Wine Bug #23456: CryptGetProvParam PP_ENUMALGS incomplete
Status: Won't fix - provider-specific
Workaround: Use known algorithm IDs directly
Wine Bug #34567: Registry CSP path resolution fails for non-standard paths
Status: Open
Workaround: Install CSP to standard Windows paths
```
## 4. Implementation Plan
### Phase 1: Environment Validation (1-2 days)
1. Set up Wine development environment
2. Test basic CryptoAPI calls under Wine
3. Attempt CryptoPro CSP installation
4. Document compatibility findings
**Validation Script:**
```bash
#!/bin/bash
# scripts/crypto/validate-wine-csp.sh
set -euo pipefail
echo "=== Wine CSP Validation ==="
# Check Wine version
echo "[1] Wine version:"
wine --version
# Check CryptoAPI basics
echo "[2] Testing CryptoAPI availability..."
cat > /tmp/test_capi.c << 'EOF'
#include <windows.h>
#include <wincrypt.h>
#include <stdio.h>
int main() {
HCRYPTPROV hProv;
if (CryptAcquireContext(&hProv, NULL, NULL, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT)) {
printf("CryptoAPI: OK\n");
CryptReleaseContext(hProv, 0);
return 0;
}
printf("CryptoAPI: FAILED (%d)\n", GetLastError());
return 1;
}
EOF
winegcc -o /tmp/test_capi.exe /tmp/test_capi.c -lcrypt32
wine /tmp/test_capi.exe
# Check for GOST provider
echo "[3] Checking for GOST provider..."
wine reg query "HKLM\\SOFTWARE\\Microsoft\\Cryptography\\Defaults\\Provider\\Crypto-Pro GOST R 34.10-2012" 2>/dev/null && \
echo "CryptoPro CSP: REGISTERED" || \
echo "CryptoPro CSP: NOT FOUND"
```
### Phase 2: Bridge Implementation (3-5 days)
1. Implement chosen approach (recommend Approach C: RPC Server)
2. Create comprehensive test suite
3. Generate reference test vectors
4. Document operational procedures
### Phase 3: CI Integration (2-3 days)
1. Create containerized Wine+CSP environment
2. Add opt-in CI workflow
3. Integrate vector comparison tests
4. Document CI requirements
## 5. Security Considerations
### 5.1 Key Material Handling
```
CRITICAL: Wine CSP should NEVER handle production keys.
Permitted:
✓ Test key containers (ephemeral)
✓ Pre-generated test vectors
✓ Validation-only operations
Prohibited:
✗ Production signing keys
✗ Customer key material
✗ Certificate private keys
```
### 5.2 Environment Isolation
```yaml
# Recommended: Isolated container/VM for Wine CSP
wine-csp-validator:
isolation: strict
network: none # No external network
read_only: true
capabilities:
- drop: ALL
volumes:
- type: tmpfs
target: /home/wine
```
### 5.3 Audit Logging
All Wine CSP operations must be logged:
```csharp
public class WineCspAuditLogger
{
public void LogSigningRequest(
string algorithm,
string keyId,
byte[] dataHash,
string sourceIp)
{
_logger.LogInformation(
"Wine CSP signing request: Algorithm={Algorithm} " +
"KeyId={KeyId} DataHash={DataHash} Source={Source}",
algorithm, keyId,
Convert.ToHexString(SHA256.HashData(dataHash)),
sourceIp);
}
}
```
## 6. Legal Review Requirements
Before implementing Wine CSP loader:
- [ ] Review CryptoPro EULA for Wine/emulation clauses
- [ ] Confirm test-only usage is permitted
- [ ] Document licensing obligations
- [ ] Obtain written approval from legal team
## 7. Decision Matrix
| Criterion | Approach A (Full Wine) | Approach B (Winelib) | Approach C (RPC) |
|-----------|------------------------|----------------------|------------------|
| Complexity | Low | High | Medium |
| Reliability | Medium | Low | High |
| Performance | Low | Medium | Medium |
| Maintainability | Medium | Low | High |
| Debugging | Medium | Hard | Easy |
| CI Integration | Medium | Hard | Easy |
| **Recommended** | Testing only | Not recommended | **Best choice** |
## 8. Conclusion
**Recommended Approach:** Wine RPC Server (Approach C)
**Rationale:**
1. Clean separation between .NET app and Wine environment
2. Easier to debug and monitor
3. Can be containerized for CI
4. Process isolation improves security
5. Server can be reused across multiple test runs
**Next Steps:**
1. Complete legal review (RU-CRYPTO-VAL-06)
2. Validate Wine compatibility with CryptoPro CSP
3. Implement RPC server if validation passes
4. Integrate into CI as opt-in workflow
---
*Document Version: 1.1.0*
*Last Updated: 2025-12-07*
*Implementation Status: HTTP-based approach implemented (see top of document)*

1
docs/signals/events-24-005.md
View File

@@ -46,3 +46,4 @@
## Provenance
- This contract supersedes the temporary log-based publisher referenced in Signals sprint 0143 Execution Log (2025-11-18). Aligns with `signals.fact.updated@v1` payload shape already covered by unit tests.
- Implementation: `Signals.Events` defaults to Redis Streams (`signals.fact.updated.v1` with `signals.fact.updated.dlq`), emitting envelopes that include `event_id`, `fact_version`, and deterministic `fact.digest` (sha256) generated by the reachability fact hasher.
- Router transport: set `Signals.Events.Driver=router` to POST envelopes to the StellaOps Router gateway (`BaseUrl` + `Path`, default `/router/events/signals.fact.updated`) with optional API key/headers. This path should forward to downstream consumers registered in Router; Redis remains mandatory for reachability cache but not for event fan-out when router is enabled.

16
etc/signals.yaml.sample
View File

@@ -47,13 +47,14 @@ Signals:
MaxConfidence: 0.99
MinConfidence: 0.05
Cache:
# Cache is always Redis-backed for reachability fact reuse.
ConnectionString: "localhost:6379"
DefaultTtlSeconds: 600
Events:
Enabled: true
# Transport driver: "redis" (default) or "inmemory" for local smoke.
Driver: "redis"
ConnectionString: "localhost:6379"
# Transport driver: "redis" (default), "router" (HTTP gateway), or "inmemory" for local smoke.
Driver: "router"
ConnectionString: "localhost:6379" # still required for cache + redis driver
Stream: "signals.fact.updated.v1"
DeadLetterStream: "signals.fact.updated.dlq"
PublishTimeoutSeconds: 5
@@ -62,6 +63,15 @@ Signals:
Producer: "StellaOps.Signals"
Pipeline: "signals"
Release: ""
Router:
BaseUrl: "https://gateway.stella-ops.local"
Path: "/router/events/signals.fact.updated"
ApiKeyHeader: "X-API-Key"
ApiKey: ""
TimeoutSeconds: 5
AllowInsecureTls: false
Headers:
X-Router-Service: "signals"
AirGap:
# Optional override for fact-update event topic when signaling across air-gap boundaries.
# Defaults to "signals.fact.updated.v1" when omitted.

12
ops/cryptopro/linux-csp-service/CryptoProLinuxApi.csproj Normal file
View File

@@ -0,0 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<PublishSingleFile>true</PublishSingleFile>
<SelfContained>true</SelfContained>
<RuntimeIdentifier>linux-x64</RuntimeIdentifier>
<InvariantGlobalization>true</InvariantGlobalization>
<EnableTrimAnalyzer>false</EnableTrimAnalyzer>
</PropertyGroup>
</Project>

31
ops/cryptopro/linux-csp-service/Dockerfile
View File

@@ -1,31 +1,36 @@
# syntax=docker/dockerfile:1.7
FROM mcr.microsoft.com/dotnet/nightly/sdk:10.0 AS build
WORKDIR /src
COPY ops/cryptopro/linux-csp-service/CryptoProLinuxApi.csproj .
RUN dotnet restore CryptoProLinuxApi.csproj
COPY ops/cryptopro/linux-csp-service/ .
RUN dotnet publish CryptoProLinuxApi.csproj -c Release -r linux-x64 --self-contained true \
/p:PublishSingleFile=true /p:DebugType=none /p:DebugSymbols=false -o /app/publish
FROM ubuntu:22.04
ARG CRYPTOPRO_ACCEPT_EULA=0
ENV DEBIAN_FRONTEND=noninteractive \
CRYPTOPRO_ACCEPT_EULA=1 \
CRYPTOPRO_ACCEPT_EULA=${CRYPTOPRO_ACCEPT_EULA} \
CRYPTOPRO_MINIMAL=1
WORKDIR /app
# System deps
# System deps for CryptoPro installer
RUN apt-get update && \
apt-get install -y --no-install-recommends python3 python3-pip tar xz-utils && \
apt-get install -y --no-install-recommends tar xz-utils ca-certificates && \
rm -rf /var/lib/apt/lists/*
# Copy CryptoPro packages (provided in repo) and installer
# CryptoPro packages (provided in repo) and installer
COPY opt/cryptopro/downloads/*.tgz /opt/cryptopro/downloads/
COPY ops/cryptopro/install-linux-csp.sh /usr/local/bin/install-linux-csp.sh
RUN chmod +x /usr/local/bin/install-linux-csp.sh
# Install CryptoPro CSP
RUN /usr/local/bin/install-linux-csp.sh
# Install CryptoPro CSP (requires CRYPTOPRO_ACCEPT_EULA=1 at build/runtime)
RUN CRYPTOPRO_ACCEPT_EULA=${CRYPTOPRO_ACCEPT_EULA} /usr/local/bin/install-linux-csp.sh
# Python deps
COPY ops/cryptopro/linux-csp-service/requirements.txt /app/requirements.txt
RUN pip3 install --no-cache-dir -r /app/requirements.txt
# App
COPY ops/cryptopro/linux-csp-service/app.py /app/app.py
# Copy published .NET app
COPY --from=build /app/publish/ /app/
EXPOSE 8080
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"]
ENTRYPOINT ["/app/CryptoProLinuxApi"]

118
ops/cryptopro/linux-csp-service/Program.cs Normal file
View File

@@ -0,0 +1,118 @@
using System.Diagnostics;
using System.Text.Json.Serialization;
var builder = WebApplication.CreateSlimBuilder(args);
builder.Services.ConfigureHttpJsonOptions(opts =>
{
opts.SerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull;
});
var app = builder.Build();
const string CsptestPath = "/opt/cprocsp/bin/amd64/csptest";
app.MapGet("/health", () =>
{
if (!File.Exists(CsptestPath))
{
return Results.Problem(statusCode: 500, detail: "csptest not found; ensure CryptoPro CSP is installed");
}
return Results.Ok(new { status = "ok", csptest = CsptestPath });
});
app.MapGet("/license", () =>
{
var result = RunProcess([CsptestPath, "-keyset", "-info"], allowFailure: true);
return Results.Json(result);
});
app.MapPost("/hash", async (HashRequest request) =>
{
byte[] data;
try
{
data = Convert.FromBase64String(request.DataBase64);
}
catch (FormatException)
{
return Results.BadRequest(new { error = "Invalid base64" });
}
var inputPath = Path.GetTempFileName();
var outputPath = Path.GetTempFileName();
await File.WriteAllBytesAsync(inputPath, data);
var result = RunProcess([CsptestPath, "-hash", "-alg", "GOST12_256", "-in", inputPath, "-out", outputPath], allowFailure: true);
string? digestBase64 = null;
if (File.Exists(outputPath))
{
var digestBytes = await File.ReadAllBytesAsync(outputPath);
digestBase64 = Convert.ToBase64String(digestBytes);
}
TryDelete(inputPath);
TryDelete(outputPath);
return Results.Json(new
{
result.ExitCode,
result.Output,
digest_b64 = digestBase64
});
});
app.MapPost("/keyset/init", (KeysetRequest request) =>
{
var name = string.IsNullOrWhiteSpace(request.Name) ? "default" : request.Name!;
var result = RunProcess([CsptestPath, "-keyset", "-newkeyset", "-container", name, "-keytype", "none"], allowFailure: true);
return Results.Json(result);
});
app.Run("http://0.0.0.0:8080");
static void TryDelete(string path)
{
try { File.Delete(path); } catch { /* ignore */ }
}
static ProcessResult RunProcess(string[] args, bool allowFailure = false)
{
try
{
var psi = new ProcessStartInfo
{
FileName = args[0],
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
ArgumentList = { }
};
for (var i = 1; i < args.Length; i++)
{
psi.ArgumentList.Add(args[i]);
}
using var proc = Process.Start(psi)!;
var output = proc.StandardOutput.ReadToEnd();
output += proc.StandardError.ReadToEnd();
proc.WaitForExit();
if (proc.ExitCode != 0 && !allowFailure)
{
throw new InvalidOperationException($"Command failed with exit {proc.ExitCode}: {output}");
}
return new ProcessResult(proc.ExitCode, output);
}
catch (Exception ex)
{
if (!allowFailure)
{
throw;
}
return new ProcessResult(-1, ex.ToString());
}
}
sealed record HashRequest([property: JsonPropertyName("data_b64")] string DataBase64);
sealed record KeysetRequest([property: JsonPropertyName("name")] string? Name);
sealed record ProcessResult(int ExitCode, string Output);

26
ops/cryptopro/linux-csp-service/README.md
View File

@@ -1,6 +1,6 @@
# CryptoPro Linux CSP Service (experimental)
# CryptoPro Linux CSP Service (.NET minimal API)
Minimal FastAPI wrapper around the Linux CryptoPro CSP binaries to prove installation and expose simple operations.
Minimal HTTP wrapper around the Linux CryptoPro CSP binaries to prove installation and hash operations.
## Build
@@ -8,18 +8,26 @@ Minimal FastAPI wrapper around the Linux CryptoPro CSP binaries to prove install
docker build -t cryptopro-linux-csp -f ops/cryptopro/linux-csp-service/Dockerfile .
```
`CRYPTOPRO_ACCEPT_EULA` defaults to `0` (build will fail); set to `1` only if you hold a valid CryptoPro license and accept the vendor EULA:
```bash
docker build -t cryptopro-linux-csp \
--build-arg CRYPTOPRO_ACCEPT_EULA=1 \
-f ops/cryptopro/linux-csp-service/Dockerfile .
```
## Run
```bash
docker run --rm -p 8080:8080 cryptopro-linux-csp
docker run --rm -p 18080:8080 --name cryptopro-linux-csp-test cryptopro-linux-csp
```
Endpoints:
- `GET /health` — checks `csptest` presence.
- `GET /license` — runs `csptest -license`.
- `POST /hash` with `{ "data_b64": "<base64>" }` — runs `csptest -hash -hash_alg gost12_256`.
- `GET /license` — runs `csptest -keyset -info` (reports errors if no keyset/token present).
- `POST /hash` with `{"data_b64":"<base64>"}` — hashes using `csptest -hash -alg GOST12_256`.
- `POST /keyset/init` with optional `{"name":"<container>"}` — creates an empty keyset (`-keytype none`) to silence missing-container warnings.
## Notes
- Uses the provided CryptoPro `.tgz` bundles under `opt/cryptopro/downloads`. Ensure you have rights to these binaries; the image builds with `CRYPTOPRO_ACCEPT_EULA=1`.
- Default install is minimal (no browser/plugin). Set `CRYPTOPRO_INCLUDE_PLUGIN=1` if you need plugin packages.
- This is not a production service; intended for validation only.
Notes:
- Uses the provided CryptoPro `.tgz` bundles under `opt/cryptopro/downloads`. Do not set `CRYPTOPRO_ACCEPT_EULA=1` unless you are licensed to use these binaries.
- Minimal, headless install; browser/plugin packages are not included.

57
ops/cryptopro/linux-csp-service/app.py
View File

@@ -1,57 +0,0 @@
import base64
import subprocess
from pathlib import Path
from typing import Optional
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
app = FastAPI(title="CryptoPro Linux CSP Service", version="0.1.0")
CSPTEST = Path("/opt/cprocsp/bin/amd64/csptest")
def run_cmd(cmd: list[str], input_bytes: Optional[bytes] = None, allow_fail: bool = False) -> str:
try:
proc = subprocess.run(
cmd,
input=input_bytes,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
check=True,
)
return proc.stdout.decode("utf-8", errors="replace")
except subprocess.CalledProcessError as exc:
output = exc.stdout.decode("utf-8", errors="replace") if exc.stdout else ""
if allow_fail:
return output
raise HTTPException(status_code=500, detail={"cmd": cmd, "output": output})
@app.get("/health")
def health():
if not CSPTEST.exists():
raise HTTPException(status_code=500, detail="csptest binary not found; ensure CryptoPro CSP is installed")
return {"status": "ok", "csptest": str(CSPTEST)}
@app.get("/license")
def license_info():
output = run_cmd([str(CSPTEST), "-keyset", "-info"], allow_fail=True)
return {"output": output}
class HashRequest(BaseModel):
data_b64: str
@app.post("/hash")
def hash_data(body: HashRequest):
try:
data = base64.b64decode(body.data_b64)
except Exception:
raise HTTPException(status_code=400, detail="Invalid base64")
cmd = [str(CSPTEST), "-hash", "-in", "-", "-hash_alg", "gost12_256"]
output = run_cmd(cmd, input_bytes=data)
return {"output": output}

2
ops/cryptopro/linux-csp-service/requirements.txt
View File

@@ -1,2 +0,0 @@
fastapi==0.111.0
uvicorn[standard]==0.30.1

193
ops/wine-csp/Dockerfile
View File

@@ -1,193 +0,0 @@
# syntax=docker/dockerfile:1.7
# Wine CSP Service - GOST cryptographic operations via Wine-hosted CryptoPro CSP
#
# WARNING: For TEST VECTOR GENERATION ONLY - not for production signing
#
# Build:
# docker buildx build -f ops/wine-csp/Dockerfile -t wine-csp:latest .
#
# Run:
# docker run -p 5099:5099 -e WINE_CSP_MODE=limited wine-csp:latest
# ==============================================================================
# Stage 1: Build .NET application for Windows x64
# ==============================================================================
ARG SDK_IMAGE=mcr.microsoft.com/dotnet/sdk:10.0-preview-bookworm-slim
FROM ${SDK_IMAGE} AS build
ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 \
DOTNET_NOLOGO=1 \
DOTNET_ROLL_FORWARD=LatestMajor \
SOURCE_DATE_EPOCH=1704067200
WORKDIR /src
# Copy solution files and NuGet configuration
COPY Directory.Build.props Directory.Build.rsp NuGet.config ./
# Copy local NuGet packages if available
COPY local-nugets/ ./local-nugets/
# Copy Wine CSP Service source
COPY src/__Tools/WineCspService/ ./src/__Tools/WineCspService/
# Copy GostCryptography fork dependency
COPY third_party/forks/AlexMAS.GostCryptography/ ./third_party/forks/AlexMAS.GostCryptography/
# Restore and publish for Windows x64 (runs under Wine)
RUN --mount=type=cache,target=/root/.nuget/packages \
dotnet restore src/__Tools/WineCspService/WineCspService.csproj && \
dotnet publish src/__Tools/WineCspService/WineCspService.csproj \
-c Release \
-r win-x64 \
--self-contained true \
-o /app/publish \
/p:PublishSingleFile=true \
/p:EnableCompressionInSingleFile=true \
/p:DebugType=none \
/p:DebugSymbols=false
# ==============================================================================
# Stage 2: Runtime with Wine and CryptoPro CSP support
# ==============================================================================
FROM ubuntu:22.04 AS runtime
# OCI Image Labels
LABEL org.opencontainers.image.title="StellaOps Wine CSP Service" \
org.opencontainers.image.description="GOST cryptographic test vector generation via Wine-hosted CryptoPro CSP" \
org.opencontainers.image.vendor="StellaOps" \
org.opencontainers.image.source="https://git.stella-ops.org/stellaops/router" \
com.stellaops.component="wine-csp" \
com.stellaops.security.production-signing="false" \
com.stellaops.security.test-vectors-only="true"
# Wine CSP service configuration
ARG WINE_CSP_PORT=5099
ARG APP_USER=winecsp
ARG APP_UID=10001
ARG APP_GID=10001
ENV DEBIAN_FRONTEND=noninteractive \
# Wine configuration
WINEDEBUG=-all \
WINEPREFIX=/home/${APP_USER}/.wine \
WINEARCH=win64 \
# Service configuration
WINE_CSP_PORT=${WINE_CSP_PORT} \
ASPNETCORE_URLS=http://+:${WINE_CSP_PORT} \
DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 \
# CSP configuration
WINE_CSP_MODE=limited \
WINE_CSP_INSTALLER_PATH=/opt/cryptopro/csp-installer.msi \
WINE_CSP_LOG_LEVEL=Information \
NODE_PATH=/usr/local/lib/node_modules \
PLAYWRIGHT_BROWSERS_PATH=/ms-playwright \
# Display for Wine (headless)
DISPLAY=:99
# Install Wine and dependencies
# Using WineHQ stable repository for consistent Wine version
RUN set -eux; \
dpkg --add-architecture i386; \
apt-get update; \
apt-get install -y --no-install-recommends \
ca-certificates \
curl \
gnupg2 \
software-properties-common \
wget \
xvfb \
cabextract \
p7zip-full \
procps; \
# Add WineHQ repository key
mkdir -pm755 /etc/apt/keyrings; \
wget -O /etc/apt/keyrings/winehq-archive.key \
https://dl.winehq.org/wine-builds/winehq.key; \
# Add WineHQ repository
wget -NP /etc/apt/sources.list.d/ \
https://dl.winehq.org/wine-builds/ubuntu/dists/jammy/winehq-jammy.sources; \
apt-get update; \
# Install Wine stable
apt-get install -y --no-install-recommends \
winehq-stable; \
# Install winetricks for runtime dependencies
wget -O /usr/local/bin/winetricks \
https://raw.githubusercontent.com/Winetricks/winetricks/master/src/winetricks; \
chmod +x /usr/local/bin/winetricks; \
# Cleanup
apt-get clean; \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# Install Node.js + Playwright (headless Chromium) for CryptoPro downloader
RUN set -eux; \
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -; \
apt-get update; \
apt-get install -y --no-install-recommends \
nodejs \
rpm2cpio \
cpio; \
npm install -g --no-progress playwright-chromium@1.48.2; \
npx playwright install-deps chromium; \
npx playwright install chromium; \
chown -R ${APP_UID}:${APP_GID} /ms-playwright || true; \
apt-get clean; \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# Create non-root user for Wine service
# Note: Wine requires writable home directory for prefix
RUN groupadd -r -g ${APP_GID} ${APP_USER} && \
useradd -r -u ${APP_UID} -g ${APP_GID} -m -d /home/${APP_USER} -s /bin/bash ${APP_USER} && \
mkdir -p /app /opt/cryptopro /var/log/wine-csp /var/run/wine-csp && \
chown -R ${APP_UID}:${APP_GID} /app /home/${APP_USER} /opt/cryptopro /var/log/wine-csp /var/run/wine-csp
WORKDIR /app
# Copy application from build stage
COPY --from=build --chown=${APP_UID}:${APP_GID} /app/publish/ ./
# Copy supporting scripts
COPY --chown=${APP_UID}:${APP_GID} ops/wine-csp/entrypoint.sh /usr/local/bin/entrypoint.sh
COPY --chown=${APP_UID}:${APP_GID} ops/wine-csp/healthcheck.sh /usr/local/bin/healthcheck.sh
COPY --chown=${APP_UID}:${APP_GID} ops/wine-csp/install-csp.sh /usr/local/bin/install-csp.sh
COPY --chown=${APP_UID}:${APP_GID} ops/wine-csp/fetch-cryptopro.py /usr/local/bin/fetch-cryptopro.py
COPY --chown=${APP_UID}:${APP_GID} ops/wine-csp/download-cryptopro.sh /usr/local/bin/download-cryptopro.sh
COPY --chown=${APP_UID}:${APP_GID} scripts/crypto/download-cryptopro-playwright.cjs /usr/local/bin/download-cryptopro-playwright.cjs
RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/healthcheck.sh /usr/local/bin/install-csp.sh /usr/local/bin/fetch-cryptopro.py /usr/local/bin/download-cryptopro.sh /usr/local/bin/download-cryptopro-playwright.cjs
# Switch to non-root user for Wine prefix initialization
USER ${APP_UID}:${APP_GID}
# Initialize Wine prefix (creates .wine directory with Windows environment)
# This must run as the app user to set correct ownership
# Using xvfb-run for headless Wine initialization
RUN set -eux; \
# Start virtual framebuffer and initialize Wine
xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" \
wine64 wineboot --init; \
wineserver --wait; \
# Install Visual C++ 2019 runtime via winetricks (required for .NET)
xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" \
winetricks -q vcrun2019 || true; \
wineserver --wait; \
# Set Windows version to Windows 10 for compatibility
wine64 reg add "HKCU\\Software\\Wine\\Version" /v Windows /d "win10" /f || true; \
wineserver --wait; \
# Cleanup Wine temp files
rm -rf /home/${APP_USER}/.cache/winetricks /tmp/.X* /tmp/winetricks* || true
EXPOSE ${WINE_CSP_PORT}
# Health check using custom script that probes /health endpoint
# Extended start_period due to Wine initialization time
HEALTHCHECK --interval=30s --timeout=10s --start-period=90s --retries=3 \
CMD /usr/local/bin/healthcheck.sh
# Volumes for persistence and CSP installer
# - Wine prefix: stores CSP installation, certificates, keys
# - CSP installer: mount customer-provided CryptoPro MSI here
# - Logs: service logs
VOLUME ["/home/${APP_USER}/.wine", "/opt/cryptopro", "/var/log/wine-csp"]
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
CMD ["wine64", "/app/WineCspService.exe"]

62
ops/wine-csp/download-cryptopro.sh
View File

@@ -1,62 +0,0 @@
#!/bin/bash
# CryptoPro Linux package fetcher (Playwright-driven)
# Uses the Node-based Playwright crawler to authenticate (if required) and
# download Linux CSP installers. Intended to run once per container startup.
set -euo pipefail
OUTPUT_DIR="${CRYPTOPRO_OUTPUT_DIR:-/opt/cryptopro/downloads}"
MARKER="${CRYPTOPRO_DOWNLOAD_MARKER:-${OUTPUT_DIR}/.downloaded}"
FORCE="${CRYPTOPRO_FORCE_DOWNLOAD:-0}"
UNPACK="${CRYPTOPRO_UNPACK:-1}"
DRY_RUN="${CRYPTOPRO_DRY_RUN:-1}"
log() {
echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] [crypto-fetch] $*"
}
log_error() {
echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] [crypto-fetch] [ERROR] $*" >&2
}
if [[ -f "${MARKER}" && "${FORCE}" != "1" ]]; then
log "Download marker present at ${MARKER}; skipping (set CRYPTOPRO_FORCE_DOWNLOAD=1 to refresh)."
exit 0
fi
log "Ensuring CryptoPro Linux packages are available (dry-run unless CRYPTOPRO_DRY_RUN=0)"
log " Output dir: ${OUTPUT_DIR}"
log " Unpack: ${UNPACK}"
mkdir -p "${OUTPUT_DIR}"
# Export defaults for the Playwright downloader
export CRYPTOPRO_OUTPUT_DIR="${OUTPUT_DIR}"
export CRYPTOPRO_UNPACK="${UNPACK}"
export CRYPTOPRO_DRY_RUN="${DRY_RUN}"
export CRYPTOPRO_URL="${CRYPTOPRO_URL:-https://cryptopro.ru/products/csp/downloads#latest_csp50r3_linux}"
export CRYPTOPRO_EMAIL="${CRYPTOPRO_EMAIL:-contact@stella-ops.org}"
export CRYPTOPRO_PASSWORD="${CRYPTOPRO_PASSWORD:-Hoko33JD3nj3aJD.}"
if ! node /usr/local/bin/download-cryptopro-playwright.cjs; then
rc=$?
if [[ "${rc}" == "2" ]]; then
log "Playwright downloader blocked by auth/captcha; skipping download (set CRYPTOPRO_DEBUG=1 for details)."
exit 0
fi
log_error "Playwright downloader failed (exit=${rc})"
exit "${rc}"
fi
if [[ "${DRY_RUN}" == "0" ]]; then
touch "${MARKER}"
log "Download complete; marker written to ${MARKER}"
else
log "Dry-run mode; marker not written. Set CRYPTOPRO_DRY_RUN=0 to fetch binaries."
fi
# List latest artifacts (best-effort)
if compgen -G "${OUTPUT_DIR}/*" > /dev/null; then
log "Artifacts in ${OUTPUT_DIR}:"
find "${OUTPUT_DIR}" -maxdepth 1 -type f -printf " %f (%s bytes)\n" | head -20
fi

272
ops/wine-csp/entrypoint.sh
View File

@@ -1,272 +0,0 @@
#!/bin/bash
# Wine CSP Service Entrypoint
#
# Initializes Wine environment and starts the WineCspService under Wine.
# For TEST VECTOR GENERATION ONLY - not for production signing.
set -euo pipefail
# ------------------------------------------------------------------------------
# Configuration
# ------------------------------------------------------------------------------
WINE_CSP_PORT="${WINE_CSP_PORT:-5099}"
WINE_CSP_MODE="${WINE_CSP_MODE:-limited}"
WINE_CSP_INSTALLER_PATH="${WINE_CSP_INSTALLER_PATH:-/opt/cryptopro/csp-installer.msi}"
WINE_CSP_LOG_LEVEL="${WINE_CSP_LOG_LEVEL:-Information}"
WINE_PREFIX="${WINEPREFIX:-$HOME/.wine}"
DISPLAY="${DISPLAY:-:99}"
CSP_DOWNLOAD_MARKER="${WINE_CSP_INSTALLER_PATH}.downloaded"
CRYPTOPRO_DOWNLOAD_DIR="${CRYPTOPRO_DOWNLOAD_DIR:-/opt/cryptopro/downloads}"
CRYPTOPRO_DOWNLOAD_MARKER="${CRYPTOPRO_DOWNLOAD_MARKER:-${CRYPTOPRO_DOWNLOAD_DIR}/.downloaded}"
CRYPTOPRO_FETCH_ON_START="${CRYPTOPRO_FETCH_ON_START:-1}"
# Marker files
CSP_INSTALLED_MARKER="${WINE_PREFIX}/.csp_installed"
WINE_INITIALIZED_MARKER="${WINE_PREFIX}/.wine_initialized"
# Log prefix for structured logging
log() {
echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] [entrypoint] $*"
}
log_error() {
echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] [entrypoint] [ERROR] $*" >&2
}
# ------------------------------------------------------------------------------
# Virtual Framebuffer Management
# ------------------------------------------------------------------------------
start_xvfb() {
if ! pgrep -x Xvfb > /dev/null; then
log "Starting Xvfb virtual framebuffer on display ${DISPLAY}"
Xvfb "${DISPLAY}" -screen 0 1024x768x24 &
sleep 2
fi
}
stop_xvfb() {
if pgrep -x Xvfb > /dev/null; then
log "Stopping Xvfb"
pkill -x Xvfb || true
fi
}
# ------------------------------------------------------------------------------
# Wine Initialization
# ------------------------------------------------------------------------------
initialize_wine() {
if [[ -f "${WINE_INITIALIZED_MARKER}" ]]; then
log "Wine prefix already initialized"
return 0
fi
log "Initializing Wine prefix at ${WINE_PREFIX}"
start_xvfb
# Initialize Wine prefix
wine64 wineboot --init 2>/dev/null || true
wineserver --wait
# Set Windows version for CryptoPro compatibility
wine64 reg add "HKCU\\Software\\Wine\\Version" /v Windows /d "win10" /f 2>/dev/null || true
wineserver --wait
# Create marker
touch "${WINE_INITIALIZED_MARKER}"
log "Wine prefix initialized successfully"
}
# ------------------------------------------------------------------------------
# CryptoPro Linux Downloads (Playwright-driven)
# ------------------------------------------------------------------------------
download_linux_packages() {
if [[ "${CRYPTOPRO_FETCH_ON_START}" == "0" ]]; then
log "Skipping CryptoPro Linux fetch (CRYPTOPRO_FETCH_ON_START=0)"
return 0
fi
if [[ -f "${CRYPTOPRO_DOWNLOAD_MARKER}" && "${CRYPTOPRO_FORCE_DOWNLOAD:-0}" != "1" ]]; then
log "CryptoPro download marker present at ${CRYPTOPRO_DOWNLOAD_MARKER}; skipping fetch"
return 0
fi
log "Ensuring CryptoPro Linux packages via Playwright (dry-run unless CRYPTOPRO_DRY_RUN=0)"
export CRYPTOPRO_DOWNLOAD_MARKER
export CRYPTOPRO_OUTPUT_DIR="${CRYPTOPRO_DOWNLOAD_DIR}"
export CRYPTOPRO_UNPACK="${CRYPTOPRO_UNPACK:-1}"
if /usr/local/bin/download-cryptopro.sh; then
if [[ "${CRYPTOPRO_DRY_RUN:-1}" != "0" ]]; then
log "CryptoPro downloader ran in dry-run mode; set CRYPTOPRO_DRY_RUN=0 to fetch binaries"
else
[[ -f "${CRYPTOPRO_DOWNLOAD_MARKER}" ]] || touch "${CRYPTOPRO_DOWNLOAD_MARKER}"
log "CryptoPro Linux artifacts staged in ${CRYPTOPRO_DOWNLOAD_DIR}"
fi
else
log_error "CryptoPro Playwright download failed"
fi
}
# ------------------------------------------------------------------------------
# CryptoPro CSP Installation
# ------------------------------------------------------------------------------
install_cryptopro() {
# Check if already installed
if [[ -f "${CSP_INSTALLED_MARKER}" ]]; then
log "CryptoPro CSP already installed"
return 0
fi
# Attempt to download installer if missing (dry-run by default)
if [[ ! -f "${WINE_CSP_INSTALLER_PATH}" ]]; then
log "CryptoPro CSP installer not found at ${WINE_CSP_INSTALLER_PATH}; attempting crawl/download (dry-run unless CRYPTOPRO_DRY_RUN=0)."
if ! CRYPTOPRO_OUTPUT="${WINE_CSP_INSTALLER_PATH}" /usr/local/bin/fetch-cryptopro.py; then
log_error "CryptoPro CSP download failed; continuing without CSP (limited mode)"
return 0
fi
fi
# Check if installer is available
if [[ ! -f "${WINE_CSP_INSTALLER_PATH}" ]]; then
log "CryptoPro CSP installer not found at ${WINE_CSP_INSTALLER_PATH}"
log "Service will run in limited mode without CSP"
return 0
fi
log "Installing CryptoPro CSP from ${WINE_CSP_INSTALLER_PATH}"
start_xvfb
# Run the CSP installation script
if /usr/local/bin/install-csp.sh; then
touch "${CSP_INSTALLED_MARKER}"
log "CryptoPro CSP installed successfully"
else
log_error "CryptoPro CSP installation failed"
return 1
fi
}
# ------------------------------------------------------------------------------
# Service Configuration
# ------------------------------------------------------------------------------
configure_service() {
log "Configuring Wine CSP service"
log " Mode: ${WINE_CSP_MODE}"
log " Port: ${WINE_CSP_PORT}"
log " Log Level: ${WINE_CSP_LOG_LEVEL}"
# Configure Wine debug output based on log level
case "${WINE_CSP_LOG_LEVEL}" in
Trace|Debug)
export WINEDEBUG="warn+all"
;;
Information)
export WINEDEBUG="-all"
;;
Warning|Error|Critical)
export WINEDEBUG="-all"
;;
*)
export WINEDEBUG="-all"
;;
esac
# Set ASP.NET Core environment
export ASPNETCORE_URLS="http://+:${WINE_CSP_PORT}"
export ASPNETCORE_ENVIRONMENT="${ASPNETCORE_ENVIRONMENT:-Production}"
export Logging__LogLevel__Default="${WINE_CSP_LOG_LEVEL}"
# Check if CSP is available
if [[ -f "${CSP_INSTALLED_MARKER}" ]]; then
export WINE_CSP_CSP_AVAILABLE="true"
log "CryptoPro CSP is available"
else
export WINE_CSP_CSP_AVAILABLE="false"
log "Running without CryptoPro CSP (limited mode)"
fi
}
# ------------------------------------------------------------------------------
# Startup Validation
# ------------------------------------------------------------------------------
validate_environment() {
log "Validating environment"
# Check Wine is available
if ! command -v wine64 &> /dev/null; then
log_error "wine64 not found in PATH"
exit 1
fi
# Check application exists
if [[ ! -f "/app/WineCspService.exe" ]]; then
log_error "WineCspService.exe not found at /app/"
exit 1
fi
# Verify Wine prefix is writable
if [[ ! -w "${WINE_PREFIX}" ]]; then
log_error "Wine prefix ${WINE_PREFIX} is not writable"
exit 1
fi
log "Environment validation passed"
}
# ------------------------------------------------------------------------------
# Signal Handlers
# ------------------------------------------------------------------------------
cleanup() {
log "Received shutdown signal, cleaning up..."
# Stop Wine server gracefully
wineserver -k 15 2>/dev/null || true
sleep 2
wineserver -k 9 2>/dev/null || true
stop_xvfb
log "Cleanup complete"
exit 0
}
trap cleanup SIGTERM SIGINT SIGQUIT
# ------------------------------------------------------------------------------
# Main Entry Point
# ------------------------------------------------------------------------------
main() {
log "=========================================="
log "Wine CSP Service Entrypoint"
log "=========================================="
log "WARNING: For TEST VECTOR GENERATION ONLY"
log "=========================================="
validate_environment
download_linux_packages
initialize_wine
# Only attempt CSP installation in full mode
if [[ "${WINE_CSP_MODE}" == "full" ]]; then
install_cryptopro
fi
configure_service
# Start Xvfb for the main process
start_xvfb
log "Starting WineCspService..."
log "Listening on port ${WINE_CSP_PORT}"
# Execute the command passed to the container (or default)
if [[ $# -gt 0 ]]; then
exec "$@"
else
exec wine64 /app/WineCspService.exe
fi
}
main "$@"

164
ops/wine-csp/fetch-cryptopro.py
View File

@@ -1,164 +0,0 @@
#!/usr/bin/env python3
"""
CryptoPro crawler (metadata only by default).
Fetches https://cryptopro.ru/downloads (or override) with basic auth, recurses linked pages,
and selects candidate Linux packages (.deb/.rpm/.tar.gz/.tgz/.run) or MSI as fallback.
Environment:
CRYPTOPRO_DOWNLOAD_URL: start URL (default: https://cryptopro.ru/downloads)
CRYPTOPRO_USERNAME / CRYPTOPRO_PASSWORD: credentials
CRYPTOPRO_MAX_PAGES: max pages to crawl (default: 20)
CRYPTOPRO_MAX_DEPTH: max link depth (default: 2)
CRYPTOPRO_DRY_RUN: 1 (default) to list only, 0 to enable download
CRYPTOPRO_OUTPUT: output path (default: /opt/cryptopro/csp-installer.bin)
"""
import os
import sys
import re
import html.parser
import urllib.parse
import urllib.request
from collections import deque
SESSION_HEADERS = {
"User-Agent": "StellaOps-CryptoPro-Crawler/1.0 (+https://stella-ops.org)",
}
LINUX_PATTERNS = re.compile(r"\.(deb|rpm|tar\.gz|tgz|run)(?:$|\?)", re.IGNORECASE)
MSI_PATTERN = re.compile(r"\.msi(?:$|\?)", re.IGNORECASE)
def log(msg: str) -> None:
sys.stdout.write(msg + "\n")
sys.stdout.flush()
def warn(msg: str) -> None:
sys.stderr.write("[WARN] " + msg + "\n")
sys.stderr.flush()
class LinkParser(html.parser.HTMLParser):
def __init__(self):
super().__init__()
self.links = []
def handle_starttag(self, tag, attrs):
if tag != "a":
return
href = dict(attrs).get("href")
if href:
self.links.append(href)
def fetch(url: str, auth_handler) -> tuple[str, list[str]]:
opener = urllib.request.build_opener(auth_handler)
req = urllib.request.Request(url, headers=SESSION_HEADERS)
with opener.open(req, timeout=30) as resp:
data = resp.read()
parser = LinkParser()
parser.feed(data.decode("utf-8", errors="ignore"))
return data, parser.links
def resolve_links(base: str, links: list[str]) -> list[str]:
resolved = []
for href in links:
if href.startswith("#") or href.startswith("mailto:"):
continue
resolved.append(urllib.parse.urljoin(base, href))
return resolved
def choose_candidates(urls: list[str]) -> tuple[list[str], list[str]]:
linux = []
msi = []
for u in urls:
if LINUX_PATTERNS.search(u):
linux.append(u)
elif MSI_PATTERN.search(u):
msi.append(u)
# stable ordering
linux = sorted(set(linux))
msi = sorted(set(msi))
return linux, msi
def download(url: str, output_path: str, auth_handler) -> int:
opener = urllib.request.build_opener(auth_handler)
req = urllib.request.Request(url, headers=SESSION_HEADERS)
with opener.open(req, timeout=60) as resp:
with open(output_path, "wb") as f:
f.write(resp.read())
return os.path.getsize(output_path)
def main() -> int:
start_url = os.environ.get("CRYPTOPRO_DOWNLOAD_URL", "https://cryptopro.ru/downloads")
username = os.environ.get("CRYPTOPRO_USERNAME", "contact@stella-ops.org")
password = os.environ.get("CRYPTOPRO_PASSWORD", "Hoko33JD3nj3aJD.")
max_pages = int(os.environ.get("CRYPTOPRO_MAX_PAGES", "20"))
max_depth = int(os.environ.get("CRYPTOPRO_MAX_DEPTH", "2"))
dry_run = os.environ.get("CRYPTOPRO_DRY_RUN", "1") != "0"
output_path = os.environ.get("CRYPTOPRO_OUTPUT", "/opt/cryptopro/csp-installer.bin")
if username == "contact@stella-ops.org" and password == "Hoko33JD3nj3aJD.":
warn("Using default demo credentials; set CRYPTOPRO_USERNAME/CRYPTOPRO_PASSWORD to real customer creds.")
passman = urllib.request.HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, start_url, username, password)
auth_handler = urllib.request.HTTPBasicAuthHandler(passman)
seen = set()
queue = deque([(start_url, 0)])
crawled = 0
all_links = []
while queue and crawled < max_pages:
url, depth = queue.popleft()
if url in seen or depth > max_depth:
continue
seen.add(url)
try:
data, links = fetch(url, auth_handler)
crawled += 1
log(f"[crawl] {url} ({len(data)} bytes, depth={depth}, links={len(links)})")
except Exception as ex: # noqa: BLE001
warn(f"[crawl] failed {url}: {ex}")
continue
resolved = resolve_links(url, links)
all_links.extend(resolved)
for child in resolved:
if child not in seen and depth + 1 <= max_depth:
queue.append((child, depth + 1))
linux, msi = choose_candidates(all_links)
log(f"[crawl] Linux candidates: {len(linux)}; MSI candidates: {len(msi)}")
if dry_run:
log("[crawl] Dry-run mode: not downloading. Set CRYPTOPRO_DRY_RUN=0 and CRYPTOPRO_OUTPUT to enable download.")
for idx, link in enumerate(linux[:10], 1):
log(f" [linux {idx}] {link}")
for idx, link in enumerate(msi[:5], 1):
log(f" [msi {idx}] {link}")
return 0
os.makedirs(os.path.dirname(output_path), exist_ok=True)
target = None
if linux:
target = linux[0]
elif msi:
target = msi[0]
else:
warn("No candidate downloads found.")
return 1
log(f"[download] Fetching {target} -> {output_path}")
size = download(target, output_path, auth_handler)
log(f"[download] Complete, size={size} bytes")
return 0
if __name__ == "__main__":
sys.exit(main())

24
ops/wine-csp/healthcheck.sh
View File

@@ -1,24 +0,0 @@
#!/bin/bash
# Wine CSP Service Health Check
#
# Probes the /health endpoint to determine if the service is healthy.
# Returns 0 (healthy) or 1 (unhealthy).
set -euo pipefail
WINE_CSP_PORT="${WINE_CSP_PORT:-5099}"
HEALTH_ENDPOINT="http://127.0.0.1:${WINE_CSP_PORT}/health"
TIMEOUT_SECONDS=8
# Perform health check
response=$(wget -q -O - --timeout="${TIMEOUT_SECONDS}" "${HEALTH_ENDPOINT}" 2>/dev/null) || exit 1
# Verify response contains expected status
if echo "${response}" | grep -q '"status":"Healthy"'; then
exit 0
elif echo "${response}" | grep -q '"status":"Degraded"'; then
# Degraded is acceptable (e.g., CSP not installed but service running)
exit 0
else
exit 1
fi

215
ops/wine-csp/install-csp.sh
View File

@@ -1,215 +0,0 @@
#!/bin/bash
# CryptoPro CSP Installation Script for Wine
#
# Installs customer-provided CryptoPro CSP MSI under Wine environment.
# This script is called by entrypoint.sh when CSP installer is available.
#
# IMPORTANT: CryptoPro CSP is commercial software. The installer MSI must be
# provided by the customer with appropriate licensing. StellaOps does not
# distribute CryptoPro CSP.
set -euo pipefail
# ------------------------------------------------------------------------------
# Configuration
# ------------------------------------------------------------------------------
WINE_CSP_INSTALLER_PATH="${WINE_CSP_INSTALLER_PATH:-/opt/cryptopro/csp-installer.msi}"
WINE_PREFIX="${WINEPREFIX:-$HOME/.wine}"
DISPLAY="${DISPLAY:-:99}"
# Expected CSP installation paths (under Wine prefix)
CSP_PROGRAM_FILES="${WINE_PREFIX}/drive_c/Program Files/Crypto Pro"
CSP_MARKER="${WINE_PREFIX}/.csp_installed"
CSP_VERSION_FILE="${WINE_PREFIX}/.csp_version"
# Installation timeout (5 minutes)
INSTALL_TIMEOUT=300
# Log prefix
log() {
echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] [install-csp] $*"
}
log_error() {
echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] [install-csp] [ERROR] $*" >&2
}
# ------------------------------------------------------------------------------
# Pre-Installation Checks
# ------------------------------------------------------------------------------
check_prerequisites() {
log "Checking installation prerequisites"
# Check installer exists
if [[ ! -f "${WINE_CSP_INSTALLER_PATH}" ]]; then
log_error "CSP installer not found: ${WINE_CSP_INSTALLER_PATH}"
return 1
fi
# Verify file is an MSI
if ! file "${WINE_CSP_INSTALLER_PATH}" | grep -qi "microsoft installer"; then
log_error "File does not appear to be an MSI installer"
return 1
fi
# Check Wine is available
if ! command -v wine64 &> /dev/null; then
log_error "wine64 not found"
return 1
fi
# Check Wine prefix exists
if [[ ! -d "${WINE_PREFIX}" ]]; then
log_error "Wine prefix not initialized: ${WINE_PREFIX}"
return 1
fi
log "Prerequisites check passed"
return 0
}
# ------------------------------------------------------------------------------
# Installation
# ------------------------------------------------------------------------------
install_csp() {
log "Starting CryptoPro CSP installation"
log "Installer: ${WINE_CSP_INSTALLER_PATH}"
# Create installation log directory
local log_dir="${WINE_PREFIX}/csp_install_logs"
mkdir -p "${log_dir}"
local install_log="${log_dir}/install_$(date -u '+%Y%m%d_%H%M%S').log"
# Run MSI installer silently
# /qn = silent mode, /norestart = don't restart, /l*v = verbose logging
log "Running msiexec installer (this may take several minutes)..."
timeout "${INSTALL_TIMEOUT}" wine64 msiexec /i "${WINE_CSP_INSTALLER_PATH}" \
/qn /norestart /l*v "${install_log}" \
AGREETOLICENSE=Yes \
2>&1 | tee -a "${install_log}" || {
local exit_code=$?
log_error "MSI installation failed with exit code: ${exit_code}"
log_error "Check installation log: ${install_log}"
return 1
}
# Wait for Wine to finish
wineserver --wait
log "MSI installation completed"
return 0
}
# ------------------------------------------------------------------------------
# Post-Installation Verification
# ------------------------------------------------------------------------------
verify_installation() {
log "Verifying CryptoPro CSP installation"
# Check for CSP program files
if [[ -d "${CSP_PROGRAM_FILES}" ]]; then
log "Found CSP directory: ${CSP_PROGRAM_FILES}"
else
log_error "CSP program directory not found"
return 1
fi
# Check for key CSP DLLs
local csp_dll="${WINE_PREFIX}/drive_c/windows/system32/cpcspi.dll"
if [[ -f "${csp_dll}" ]]; then
log "Found CSP DLL: ${csp_dll}"
else
log "Warning: CSP DLL not found at expected location"
# This might be OK depending on CSP version
fi
# Try to query CSP registry entries
local csp_registry
csp_registry=$(wine64 reg query "HKLM\\SOFTWARE\\Crypto Pro" 2>/dev/null || true)
if [[ -n "${csp_registry}" ]]; then
log "CSP registry entries found"
else
log "Warning: CSP registry entries not found"
fi
# Extract version if possible
local version="unknown"
if [[ -f "${CSP_PROGRAM_FILES}/CSP/version.txt" ]]; then
version=$(cat "${CSP_PROGRAM_FILES}/CSP/version.txt" 2>/dev/null || echo "unknown")
fi
echo "${version}" > "${CSP_VERSION_FILE}"
log "CSP version: ${version}"
log "Installation verification completed"
return 0
}
# ------------------------------------------------------------------------------
# Cleanup on Failure
# ------------------------------------------------------------------------------
cleanup_failed_install() {
log "Cleaning up failed installation"
# Try to uninstall via msiexec
wine64 msiexec /x "${WINE_CSP_INSTALLER_PATH}" /qn 2>/dev/null || true
wineserver --wait
# Remove any partial installation directories
rm -rf "${CSP_PROGRAM_FILES}" 2>/dev/null || true
# Remove marker files
rm -f "${CSP_MARKER}" "${CSP_VERSION_FILE}" 2>/dev/null || true
log "Cleanup completed"
}
# ------------------------------------------------------------------------------
# Main
# ------------------------------------------------------------------------------
main() {
log "=========================================="
log "CryptoPro CSP Installation Script"
log "=========================================="
# Check if already installed
if [[ -f "${CSP_MARKER}" ]]; then
log "CryptoPro CSP is already installed"
if [[ -f "${CSP_VERSION_FILE}" ]]; then
log "Installed version: $(cat "${CSP_VERSION_FILE}")"
fi
return 0
fi
# Run prerequisite checks
if ! check_prerequisites; then
log_error "Prerequisites check failed"
return 1
fi
# Perform installation
if ! install_csp; then
log_error "Installation failed"
cleanup_failed_install
return 1
fi
# Verify installation
if ! verify_installation; then
log_error "Installation verification failed"
cleanup_failed_install
return 1
fi
# Create installation marker
touch "${CSP_MARKER}"
log "=========================================="
log "CryptoPro CSP installation successful"
log "=========================================="
return 0
}
main "$@"

114
ops/wine-csp/tests/docker-test.sh
View File

@@ -1,114 +0,0 @@
#!/bin/bash
# Wine CSP Docker Build and Test
#
# Builds the Wine CSP Docker image and runs the full test suite.
# This script is designed for local development and CI/CD pipelines.
#
# Usage:
# ./docker-test.sh # Build and test
# ./docker-test.sh --no-build # Test existing image
# ./docker-test.sh --push # Build, test, and push if tests pass
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
# Configuration
IMAGE_NAME="${WINE_CSP_IMAGE:-wine-csp}"
IMAGE_TAG="${WINE_CSP_TAG:-test}"
FULL_IMAGE="${IMAGE_NAME}:${IMAGE_TAG}"
DOCKERFILE="${PROJECT_ROOT}/ops/wine-csp/Dockerfile"
DO_BUILD=true
DO_PUSH=false
VERBOSE=false
# Parse arguments
while [[ $# -gt 0 ]]; do
case $1 in
--no-build)
DO_BUILD=false
shift
;;
--push)
DO_PUSH=true
shift
;;
--verbose|-v)
VERBOSE=true
shift
;;
--image)
IMAGE_NAME="$2"
FULL_IMAGE="${IMAGE_NAME}:${IMAGE_TAG}"
shift 2
;;
--tag)
IMAGE_TAG="$2"
FULL_IMAGE="${IMAGE_NAME}:${IMAGE_TAG}"
shift 2
;;
*)
echo "Unknown option: $1"
exit 1
;;
esac
done
log() {
echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] $*"
}
# Build image
if [[ "${DO_BUILD}" == "true" ]]; then
log "Building Wine CSP Docker image: ${FULL_IMAGE}"
log "Dockerfile: ${DOCKERFILE}"
log "Context: ${PROJECT_ROOT}"
build_args=""
if [[ "${VERBOSE}" == "true" ]]; then
build_args="--progress=plain"
fi
docker build \
${build_args} \
-f "${DOCKERFILE}" \
-t "${FULL_IMAGE}" \
"${PROJECT_ROOT}"
log "Build completed successfully"
fi
# Verify image exists
if ! docker image inspect "${FULL_IMAGE}" > /dev/null 2>&1; then
echo "Error: Image ${FULL_IMAGE} not found"
exit 1
fi
# Run tests
log "Running integration tests..."
test_args=""
if [[ "${VERBOSE}" == "true" ]]; then
test_args="--verbose"
fi
"${SCRIPT_DIR}/run-tests.sh" --image "${FULL_IMAGE}" ${test_args} --ci
# Check test results
if [[ $? -ne 0 ]]; then
log "Tests failed!"
exit 1
fi
log "All tests passed!"
# Push if requested
if [[ "${DO_PUSH}" == "true" ]]; then
log "Pushing image: ${FULL_IMAGE}"
docker push "${FULL_IMAGE}"
log "Push completed"
fi
log "Done!"

144
ops/wine-csp/tests/fixtures/test-vectors.json vendored
View File

@@ -1,144 +0,0 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "GOST cryptographic test vectors for Wine CSP validation",
"version": "1.0.0",
"generated": "2025-12-07T00:00:00Z",
"warning": "FOR TEST VECTOR VALIDATION ONLY - NOT FOR PRODUCTION USE",
"hashVectors": {
"streebog256": [
{
"id": "streebog256-empty",
"description": "GOST R 34.11-2012 (256-bit) hash of empty message",
"input": "",
"inputBase64": "",
"expectedHash": "3f539a213e97c802cc229d474c6aa32a825a360b2a933a949fd925208d9ce1bb",
"reference": "GOST R 34.11-2012 specification"
},
{
"id": "streebog256-m1",
"description": "GOST R 34.11-2012 (256-bit) test message M1",
"input": "012345678901234567890123456789012345678901234567890123456789012",
"inputBase64": "MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEy",
"expectedHash": "9d151eefd8590b89daa6ba6cb74af9275dd051026bb149a452fd84e5e57b5500",
"reference": "GOST R 34.11-2012 specification Appendix A.1"
},
{
"id": "streebog256-hello",
"description": "GOST R 34.11-2012 (256-bit) hash of 'Hello'",
"input": "Hello",
"inputBase64": "SGVsbG8=",
"note": "Common test case for implementation validation"
},
{
"id": "streebog256-abc",
"description": "GOST R 34.11-2012 (256-bit) hash of 'abc'",
"input": "abc",
"inputBase64": "YWJj",
"note": "Standard test vector"
}
],
"streebog512": [
{
"id": "streebog512-empty",
"description": "GOST R 34.11-2012 (512-bit) hash of empty message",
"input": "",
"inputBase64": "",
"expectedHash": "8e945da209aa869f0455928529bcae4679e9873ab707b55315f56ceb98bef0a7362f715528356ee83cda5f2aac4c6ad2ba3a715c1bcd81cb8e9f90bf4c1c1a8a",
"reference": "GOST R 34.11-2012 specification"
},
{
"id": "streebog512-m1",
"description": "GOST R 34.11-2012 (512-bit) test message M1",
"input": "012345678901234567890123456789012345678901234567890123456789012",
"inputBase64": "MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEy",
"expectedHash": "1b54d01a4af5b9d5cc3d86d68d285462b19abc2475222f35c085122be4ba1ffa00ad30f8767b3a82384c6574f024c311e2a481332b08ef7f41797891c1646f48",
"reference": "GOST R 34.11-2012 specification Appendix A.2"
},
{
"id": "streebog512-hello",
"description": "GOST R 34.11-2012 (512-bit) hash of 'Hello'",
"input": "Hello",
"inputBase64": "SGVsbG8=",
"note": "Common test case for implementation validation"
}
]
},
"signatureVectors": {
"gost2012_256": [
{
"id": "gost2012-256-test1",
"description": "GOST R 34.10-2012 (256-bit) signature test",
"algorithm": "GOST12-256",
"message": "Test message for signing",
"messageBase64": "VGVzdCBtZXNzYWdlIGZvciBzaWduaW5n",
"note": "Signature will vary due to random k parameter; verify deterministic hash first"
}
],
"gost2012_512": [
{
"id": "gost2012-512-test1",
"description": "GOST R 34.10-2012 (512-bit) signature test",
"algorithm": "GOST12-512",
"message": "Test message for signing",
"messageBase64": "VGVzdCBtZXNzYWdlIGZvciBzaWduaW5n",
"note": "Signature will vary due to random k parameter; verify deterministic hash first"
}
]
},
"determinismVectors": [
{
"id": "determinism-1",
"description": "Determinism test - same input should produce same hash",
"algorithm": "STREEBOG-256",
"input": "Determinism test data 12345",
"inputBase64": "RGV0ZXJtaW5pc20gdGVzdCBkYXRhIDEyMzQ1",
"iterations": 10,
"expectation": "All iterations should produce identical hash"
},
{
"id": "determinism-2",
"description": "Determinism test with binary data",
"algorithm": "STREEBOG-512",
"inputBase64": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=",
"iterations": 10,
"expectation": "All iterations should produce identical hash"
}
],
"errorVectors": [
{
"id": "error-invalid-algo",
"description": "Invalid algorithm should return 400",
"endpoint": "/hash",
"request": {"algorithm": "INVALID-ALGO", "data": "SGVsbG8="},
"expectedStatus": 400
},
{
"id": "error-missing-data",
"description": "Missing data field should return 400",
"endpoint": "/hash",
"request": {"algorithm": "STREEBOG-256"},
"expectedStatus": 400
},
{
"id": "error-invalid-base64",
"description": "Invalid base64 should return 400",
"endpoint": "/hash",
"request": {"algorithm": "STREEBOG-256", "data": "not-valid-base64!!!"},
"expectedStatus": 400
}
],
"performanceBenchmarks": {
"hashThroughput": {
"description": "Hash operation throughput benchmark",
"algorithm": "STREEBOG-256",
"inputSize": 1024,
"iterations": 100,
"expectedMinOpsPerSecond": 10
}
}
}

4
ops/wine-csp/tests/requirements.txt
View File

@@ -1,4 +0,0 @@
# Wine CSP Integration Test Dependencies
pytest>=7.4.0
pytest-timeout>=2.2.0
requests>=2.31.0

590
ops/wine-csp/tests/run-tests.sh
View File

@@ -1,590 +0,0 @@
#!/bin/bash
# Wine CSP Container Integration Tests
#
# This script runs comprehensive tests against the Wine CSP container.
# It can test a running container or start one for testing.
#
# Usage:
# ./run-tests.sh # Start container and run tests
# ./run-tests.sh --url http://host:port # Test existing endpoint
# ./run-tests.sh --image wine-csp:tag # Use specific image
# ./run-tests.sh --verbose # Verbose output
# ./run-tests.sh --ci # CI mode (JUnit XML output)
set -euo pipefail
# ==============================================================================
# Configuration
# ==============================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
WINE_CSP_IMAGE="${WINE_CSP_IMAGE:-wine-csp:test}"
WINE_CSP_PORT="${WINE_CSP_PORT:-5099}"
WINE_CSP_URL="${WINE_CSP_URL:-}"
CONTAINER_NAME="wine-csp-test-$$"
STARTUP_TIMEOUT=120
TEST_TIMEOUT=30
VERBOSE=false
CI_MODE=false
CLEANUP_CONTAINER=true
TEST_RESULTS_DIR="${SCRIPT_DIR}/results"
JUNIT_OUTPUT="${TEST_RESULTS_DIR}/junit.xml"
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Test counters
TESTS_RUN=0
TESTS_PASSED=0
TESTS_FAILED=0
TESTS_SKIPPED=0
TEST_RESULTS=()
# ==============================================================================
# Utility Functions
# ==============================================================================
log() {
echo -e "${BLUE}[$(date -u '+%Y-%m-%dT%H:%M:%SZ')]${NC} $*"
}
log_success() {
echo -e "${GREEN}[PASS]${NC} $*"
}
log_fail() {
echo -e "${RED}[FAIL]${NC} $*"
}
log_skip() {
echo -e "${YELLOW}[SKIP]${NC} $*"
}
log_verbose() {
if [[ "${VERBOSE}" == "true" ]]; then
echo -e "${YELLOW}[DEBUG]${NC} $*"
fi
}
die() {
echo -e "${RED}[ERROR]${NC} $*" >&2
exit 1
}
# ==============================================================================
# Argument Parsing
# ==============================================================================
parse_args() {
while [[ $# -gt 0 ]]; do
case $1 in
--url)
WINE_CSP_URL="$2"
CLEANUP_CONTAINER=false
shift 2
;;
--image)
WINE_CSP_IMAGE="$2"
shift 2
;;
--port)
WINE_CSP_PORT="$2"
shift 2
;;
--verbose|-v)
VERBOSE=true
shift
;;
--ci)
CI_MODE=true
shift
;;
--help|-h)
echo "Usage: $0 [options]"
echo ""
echo "Options:"
echo " --url URL Test existing endpoint (skip container start)"
echo " --image IMAGE Docker image to test (default: wine-csp:test)"
echo " --port PORT Port to expose (default: 5099)"
echo " --verbose, -v Verbose output"
echo " --ci CI mode (JUnit XML output)"
echo " --help, -h Show this help"
exit 0
;;
*)
die "Unknown option: $1"
;;
esac
done
# Set URL if not provided
if [[ -z "${WINE_CSP_URL}" ]]; then
WINE_CSP_URL="http://127.0.0.1:${WINE_CSP_PORT}"
fi
}
# ==============================================================================
# Container Management
# ==============================================================================
start_container() {
log "Starting Wine CSP container: ${WINE_CSP_IMAGE}"
docker run -d \
--name "${CONTAINER_NAME}" \
-p "${WINE_CSP_PORT}:5099" \
-e WINE_CSP_MODE=limited \
-e WINE_CSP_LOG_LEVEL=Debug \
"${WINE_CSP_IMAGE}"
log "Container started: ${CONTAINER_NAME}"
log "Waiting for service to be ready (up to ${STARTUP_TIMEOUT}s)..."
local elapsed=0
while [[ $elapsed -lt $STARTUP_TIMEOUT ]]; do
if curl -sf "${WINE_CSP_URL}/health" > /dev/null 2>&1; then
log "Service is ready after ${elapsed}s"
return 0
fi
sleep 5
elapsed=$((elapsed + 5))
log_verbose "Waiting... ${elapsed}s elapsed"
done
log_fail "Service failed to start within ${STARTUP_TIMEOUT}s"
docker logs "${CONTAINER_NAME}" || true
return 1
}
stop_container() {
if [[ "${CLEANUP_CONTAINER}" == "true" ]] && docker ps -q -f name="${CONTAINER_NAME}" | grep -q .; then
log "Stopping container: ${CONTAINER_NAME}"
docker stop "${CONTAINER_NAME}" > /dev/null 2>&1 || true
docker rm "${CONTAINER_NAME}" > /dev/null 2>&1 || true
fi
}
# ==============================================================================
# Test Framework
# ==============================================================================
record_test() {
local name="$1"
local status="$2"
local duration="$3"
local message="${4:-}"
TESTS_RUN=$((TESTS_RUN + 1))
case $status in
pass)
TESTS_PASSED=$((TESTS_PASSED + 1))
log_success "${name} (${duration}ms)"
;;
fail)
TESTS_FAILED=$((TESTS_FAILED + 1))
log_fail "${name}: ${message}"
;;
skip)
TESTS_SKIPPED=$((TESTS_SKIPPED + 1))
log_skip "${name}: ${message}"
;;
esac
TEST_RESULTS+=("${name}|${status}|${duration}|${message}")
}
run_test() {
local name="$1"
shift
local start_time=$(date +%s%3N)
log_verbose "Running test: ${name}"
if "$@"; then
local end_time=$(date +%s%3N)
local duration=$((end_time - start_time))
record_test "${name}" "pass" "${duration}"
return 0
else
local end_time=$(date +%s%3N)
local duration=$((end_time - start_time))
record_test "${name}" "fail" "${duration}" "Test assertion failed"
return 1
fi
}
# ==============================================================================
# HTTP Helper Functions
# ==============================================================================
http_get() {
local endpoint="$1"
curl -sf --max-time "${TEST_TIMEOUT}" "${WINE_CSP_URL}${endpoint}"
}
http_post() {
local endpoint="$1"
local data="$2"
curl -sf --max-time "${TEST_TIMEOUT}" \
-X POST \
-H "Content-Type: application/json" \
-d "${data}" \
"${WINE_CSP_URL}${endpoint}"
}
# ==============================================================================
# Test Cases
# ==============================================================================
# Health endpoint tests
test_health_endpoint() {
local response
response=$(http_get "/health") || return 1
echo "${response}" | grep -q '"status"' || return 1
}
test_health_liveness() {
local response
response=$(http_get "/health/liveness") || return 1
echo "${response}" | grep -qi 'healthy\|alive' || return 1
}
test_health_readiness() {
local response
response=$(http_get "/health/readiness") || return 1
echo "${response}" | grep -qi 'healthy\|ready' || return 1
}
# Status endpoint tests
test_status_endpoint() {
local response
response=$(http_get "/status") || return 1
echo "${response}" | grep -q '"serviceName"' || return 1
echo "${response}" | grep -q '"mode"' || return 1
}
test_status_mode_limited() {
local response
response=$(http_get "/status") || return 1
echo "${response}" | grep -q '"mode":"limited"' || \
echo "${response}" | grep -q '"mode": "limited"' || return 1
}
# Keys endpoint tests
test_keys_endpoint() {
local response
response=$(http_get "/keys") || return 1
# Should return an array (possibly empty in limited mode)
echo "${response}" | grep -qE '^\[' || return 1
}
# Hash endpoint tests
test_hash_streebog256() {
# Test vector: "Hello" -> known Streebog-256 hash
local data='{"algorithm":"STREEBOG-256","data":"SGVsbG8="}'
local response
response=$(http_post "/hash" "${data}") || return 1
echo "${response}" | grep -q '"hash"' || return 1
echo "${response}" | grep -q '"algorithm"' || return 1
}
test_hash_streebog512() {
# Test vector: "Hello" -> known Streebog-512 hash
local data='{"algorithm":"STREEBOG-512","data":"SGVsbG8="}'
local response
response=$(http_post "/hash" "${data}") || return 1
echo "${response}" | grep -q '"hash"' || return 1
}
test_hash_invalid_algorithm() {
local data='{"algorithm":"INVALID","data":"SGVsbG8="}'
# Should fail with 400
if http_post "/hash" "${data}" > /dev/null 2>&1; then
return 1 # Should have failed
fi
return 0 # Correctly rejected
}
test_hash_empty_data() {
# Empty string base64 encoded
local data='{"algorithm":"STREEBOG-256","data":""}'
local response
response=$(http_post "/hash" "${data}") || return 1
echo "${response}" | grep -q '"hash"' || return 1
}
# Test vectors endpoint
test_vectors_endpoint() {
local response
response=$(http_get "/test-vectors") || return 1
# Should return test vectors array
echo "${response}" | grep -q '"vectors"' || \
echo "${response}" | grep -qE '^\[' || return 1
}
# Sign endpoint tests (limited mode may not support all operations)
test_sign_basic() {
local data='{"keyId":"test-key","algorithm":"GOST12-256","data":"SGVsbG8gV29ybGQ="}'
local response
# In limited mode, this may fail or return a mock signature
if response=$(http_post "/sign" "${data}" 2>/dev/null); then
echo "${response}" | grep -q '"signature"' || return 1
else
# Expected to fail in limited mode without keys
log_verbose "Sign failed (expected in limited mode)"
return 0
fi
}
# Verify endpoint tests
test_verify_basic() {
local data='{"keyId":"test-key","algorithm":"GOST12-256","data":"SGVsbG8gV29ybGQ=","signature":"AAAA"}'
# In limited mode, this may fail
if http_post "/verify" "${data}" > /dev/null 2>&1; then
return 0 # Verification endpoint works
else
log_verbose "Verify failed (expected in limited mode)"
return 0 # Expected in limited mode
fi
}
# Determinism tests
test_hash_determinism() {
local data='{"algorithm":"STREEBOG-256","data":"VGVzdCBkYXRhIGZvciBkZXRlcm1pbmlzbQ=="}'
local hash1 hash2
hash1=$(http_post "/hash" "${data}" | grep -o '"hash":"[^"]*"' | head -1) || return 1
hash2=$(http_post "/hash" "${data}" | grep -o '"hash":"[^"]*"' | head -1) || return 1
[[ "${hash1}" == "${hash2}" ]] || return 1
}
# Known test vector validation
test_known_vector_streebog256() {
# GOST R 34.11-2012 (Streebog-256) test vector
# Input: "012345678901234567890123456789012345678901234567890123456789012" (63 bytes)
# Expected hash: 9d151eefd8590b89daa6ba6cb74af9275dd051026bb149a452fd84e5e57b5500
local input_b64="MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEy"
local expected_hash="9d151eefd8590b89daa6ba6cb74af9275dd051026bb149a452fd84e5e57b5500"
local data="{\"algorithm\":\"STREEBOG-256\",\"data\":\"${input_b64}\"}"
local response
response=$(http_post "/hash" "${data}") || return 1
# Check if hash matches expected value
if echo "${response}" | grep -qi "${expected_hash}"; then
return 0
else
log_verbose "Hash mismatch. Response: ${response}"
log_verbose "Expected hash containing: ${expected_hash}"
# In limited mode, hash implementation may differ
return 0 # Skip strict validation for now
fi
}
# Error handling tests
test_malformed_json() {
# Send malformed JSON
local response_code
response_code=$(curl -s -o /dev/null -w "%{http_code}" --max-time "${TEST_TIMEOUT}" \
-X POST \
-H "Content-Type: application/json" \
-d "not valid json" \
"${WINE_CSP_URL}/hash")
[[ "${response_code}" == "400" ]] || return 1
}
test_missing_required_fields() {
# Missing 'data' field
local data='{"algorithm":"STREEBOG-256"}'
local response_code
response_code=$(curl -s -o /dev/null -w "%{http_code}" --max-time "${TEST_TIMEOUT}" \
-X POST \
-H "Content-Type: application/json" \
-d "${data}" \
"${WINE_CSP_URL}/hash")
[[ "${response_code}" == "400" ]] || return 1
}
# Performance tests
test_hash_performance() {
local data='{"algorithm":"STREEBOG-256","data":"SGVsbG8gV29ybGQ="}'
local start_time end_time duration
start_time=$(date +%s%3N)
for i in {1..10}; do
http_post "/hash" "${data}" > /dev/null || return 1
done
end_time=$(date +%s%3N)
duration=$((end_time - start_time))
log_verbose "10 hash operations completed in ${duration}ms (avg: $((duration / 10))ms)"
# Should complete 10 hashes in under 10 seconds
[[ $duration -lt 10000 ]] || return 1
}
# CryptoPro downloader dry-run (Playwright)
test_downloader_dry_run() {
docker exec "${CONTAINER_NAME}" \
env CRYPTOPRO_DRY_RUN=1 CRYPTOPRO_UNPACK=0 CRYPTOPRO_FETCH_ON_START=1 \
/usr/local/bin/download-cryptopro.sh
}
# ==============================================================================
# Test Runner
# ==============================================================================
run_all_tests() {
log "=========================================="
log "Wine CSP Integration Tests"
log "=========================================="
log "Target: ${WINE_CSP_URL}"
log ""
# Downloader dry-run (only when we control the container)
if [[ "${CLEANUP_CONTAINER}" == "true" ]]; then
run_test "cryptopro_downloader_dry_run" test_downloader_dry_run
else
record_test "cryptopro_downloader_dry_run" "skip" "0" "External endpoint; downloader test skipped"
fi
# Health tests
log "--- Health Endpoints ---"
run_test "health_endpoint" test_health_endpoint
run_test "health_liveness" test_health_liveness
run_test "health_readiness" test_health_readiness
# Status tests
log "--- Status Endpoint ---"
run_test "status_endpoint" test_status_endpoint
run_test "status_mode_limited" test_status_mode_limited
# Keys tests
log "--- Keys Endpoint ---"
run_test "keys_endpoint" test_keys_endpoint
# Hash tests
log "--- Hash Operations ---"
run_test "hash_streebog256" test_hash_streebog256
run_test "hash_streebog512" test_hash_streebog512
run_test "hash_invalid_algorithm" test_hash_invalid_algorithm
run_test "hash_empty_data" test_hash_empty_data
run_test "hash_determinism" test_hash_determinism
run_test "known_vector_streebog256" test_known_vector_streebog256
# Test vectors
log "--- Test Vectors ---"
run_test "test_vectors_endpoint" test_vectors_endpoint
# Sign/Verify tests (may skip in limited mode)
log "--- Sign/Verify Operations ---"
run_test "sign_basic" test_sign_basic
run_test "verify_basic" test_verify_basic
# Error handling tests
log "--- Error Handling ---"
run_test "malformed_json" test_malformed_json
run_test "missing_required_fields" test_missing_required_fields
# Performance tests
log "--- Performance ---"
run_test "hash_performance" test_hash_performance
log ""
log "=========================================="
}
# ==============================================================================
# Results Output
# ==============================================================================
print_summary() {
log "=========================================="
log "Test Results Summary"
log "=========================================="
echo ""
echo -e "Total: ${TESTS_RUN}"
echo -e "${GREEN}Passed: ${TESTS_PASSED}${NC}"
echo -e "${RED}Failed: ${TESTS_FAILED}${NC}"
echo -e "${YELLOW}Skipped: ${TESTS_SKIPPED}${NC}"
echo ""
if [[ ${TESTS_FAILED} -gt 0 ]]; then
echo -e "${RED}TESTS FAILED${NC}"
return 1
else
echo -e "${GREEN}ALL TESTS PASSED${NC}"
return 0
fi
}
generate_junit_xml() {
mkdir -p "${TEST_RESULTS_DIR}"
local timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
local total_time=0
cat > "${JUNIT_OUTPUT}" << EOF
<?xml version="1.0" encoding="UTF-8"?>
<testsuites name="Wine CSP Integration Tests" tests="${TESTS_RUN}" failures="${TESTS_FAILED}" skipped="${TESTS_SKIPPED}" timestamp="${timestamp}">
<testsuite name="wine-csp" tests="${TESTS_RUN}" failures="${TESTS_FAILED}" skipped="${TESTS_SKIPPED}">
EOF
for result in "${TEST_RESULTS[@]}"; do
IFS='|' read -r name status duration message <<< "${result}"
local time_sec=$(echo "scale=3; ${duration} / 1000" | bc)
total_time=$((total_time + duration))
echo " <testcase name=\"${name}\" classname=\"wine-csp\" time=\"${time_sec}\">" >> "${JUNIT_OUTPUT}"
case $status in
fail)
echo " <failure message=\"${message}\"/>" >> "${JUNIT_OUTPUT}"
;;
skip)
echo " <skipped message=\"${message}\"/>" >> "${JUNIT_OUTPUT}"
;;
esac
echo " </testcase>" >> "${JUNIT_OUTPUT}"
done
cat >> "${JUNIT_OUTPUT}" << EOF
</testsuite>
</testsuites>
EOF
log "JUnit XML output: ${JUNIT_OUTPUT}"
}
# ==============================================================================
# Main
# ==============================================================================
main() {
parse_args "$@"
# Setup results directory
mkdir -p "${TEST_RESULTS_DIR}"
# Start container if needed
if [[ "${CLEANUP_CONTAINER}" == "true" ]]; then
trap stop_container EXIT
start_container || die "Failed to start container"
fi
# Run tests
run_all_tests
# Generate outputs
if [[ "${CI_MODE}" == "true" ]]; then
generate_junit_xml
fi
# Print summary and exit with appropriate code
print_summary
}
main "$@"

463
ops/wine-csp/tests/test_wine_csp.py
View File

@@ -1,463 +0,0 @@
#!/usr/bin/env python3
"""
Wine CSP Integration Tests
Comprehensive test suite for the Wine CSP HTTP service.
Designed for pytest with JUnit XML output for CI integration.
Usage:
pytest test_wine_csp.py -v --junitxml=results/junit.xml
pytest test_wine_csp.py -v -k "test_health"
pytest test_wine_csp.py -v --wine-csp-url=http://localhost:5099
"""
import base64
import json
import os
import time
from typing import Any, Dict, Optional
import pytest
import requests
# ==============================================================================
# Configuration
# ==============================================================================
WINE_CSP_URL = os.environ.get("WINE_CSP_URL", "http://127.0.0.1:5099")
REQUEST_TIMEOUT = 30
STARTUP_TIMEOUT = 120
def pytest_addoption(parser):
"""Add custom pytest options."""
parser.addoption(
"--wine-csp-url",
action="store",
default=WINE_CSP_URL,
help="Wine CSP service URL",
)
@pytest.fixture(scope="session")
def wine_csp_url(request):
"""Get Wine CSP URL from command line or environment."""
return request.config.getoption("--wine-csp-url") or WINE_CSP_URL
@pytest.fixture(scope="session")
def wine_csp_client(wine_csp_url):
"""Create a requests session for Wine CSP API calls."""
session = requests.Session()
session.headers.update({"Content-Type": "application/json", "Accept": "application/json"})
# Wait for service to be ready
start_time = time.time()
while time.time() - start_time < STARTUP_TIMEOUT:
try:
response = session.get(f"{wine_csp_url}/health", timeout=5)
if response.status_code == 200:
break
except requests.exceptions.RequestException:
pass
time.sleep(5)
else:
pytest.fail(f"Wine CSP service not ready after {STARTUP_TIMEOUT}s")
return {"session": session, "base_url": wine_csp_url}
# ==============================================================================
# Helper Functions
# ==============================================================================
def get(client: Dict, endpoint: str) -> requests.Response:
"""Perform GET request."""
return client["session"].get(
f"{client['base_url']}{endpoint}", timeout=REQUEST_TIMEOUT
)
def post(client: Dict, endpoint: str, data: Dict[str, Any]) -> requests.Response:
"""Perform POST request with JSON body."""
return client["session"].post(
f"{client['base_url']}{endpoint}", json=data, timeout=REQUEST_TIMEOUT
)
def encode_b64(text: str) -> str:
"""Encode string to base64."""
return base64.b64encode(text.encode("utf-8")).decode("utf-8")
def decode_b64(b64: str) -> bytes:
"""Decode base64 string."""
return base64.b64decode(b64)
# ==============================================================================
# Health Endpoint Tests
# ==============================================================================
class TestHealthEndpoints:
"""Tests for health check endpoints."""
def test_health_returns_200(self, wine_csp_client):
"""Health endpoint should return 200 OK."""
response = get(wine_csp_client, "/health")
assert response.status_code == 200
def test_health_returns_status(self, wine_csp_client):
"""Health endpoint should return status field."""
response = get(wine_csp_client, "/health")
data = response.json()
assert "status" in data
def test_health_status_is_healthy_or_degraded(self, wine_csp_client):
"""Health status should be Healthy or Degraded."""
response = get(wine_csp_client, "/health")
data = response.json()
assert data["status"] in ["Healthy", "Degraded"]
def test_health_liveness(self, wine_csp_client):
"""Liveness probe should return 200."""
response = get(wine_csp_client, "/health/liveness")
assert response.status_code == 200
def test_health_readiness(self, wine_csp_client):
"""Readiness probe should return 200."""
response = get(wine_csp_client, "/health/readiness")
assert response.status_code == 200
# ==============================================================================
# Status Endpoint Tests
# ==============================================================================
class TestStatusEndpoint:
"""Tests for status endpoint."""
def test_status_returns_200(self, wine_csp_client):
"""Status endpoint should return 200 OK."""
response = get(wine_csp_client, "/status")
assert response.status_code == 200
def test_status_contains_service_name(self, wine_csp_client):
"""Status should contain serviceName."""
response = get(wine_csp_client, "/status")
data = response.json()
assert "serviceName" in data
def test_status_contains_mode(self, wine_csp_client):
"""Status should contain mode."""
response = get(wine_csp_client, "/status")
data = response.json()
assert "mode" in data
assert data["mode"] in ["limited", "full"]
def test_status_contains_version(self, wine_csp_client):
"""Status should contain version."""
response = get(wine_csp_client, "/status")
data = response.json()
assert "version" in data or "serviceVersion" in data
# ==============================================================================
# Keys Endpoint Tests
# ==============================================================================
class TestKeysEndpoint:
"""Tests for keys endpoint."""
def test_keys_returns_200(self, wine_csp_client):
"""Keys endpoint should return 200 OK."""
response = get(wine_csp_client, "/keys")
assert response.status_code == 200
def test_keys_returns_array(self, wine_csp_client):
"""Keys endpoint should return an array."""
response = get(wine_csp_client, "/keys")
data = response.json()
assert isinstance(data, list)
# ==============================================================================
# Hash Endpoint Tests
# ==============================================================================
class TestHashEndpoint:
"""Tests for hash operations."""
@pytest.mark.parametrize(
"algorithm",
["STREEBOG-256", "STREEBOG-512", "GOST3411-256", "GOST3411-512"],
)
def test_hash_algorithms(self, wine_csp_client, algorithm):
"""Test supported hash algorithms."""
data = {"algorithm": algorithm, "data": encode_b64("Hello World")}
response = post(wine_csp_client, "/hash", data)
# May return 200 or 400 depending on algorithm support
assert response.status_code in [200, 400]
def test_hash_streebog256_returns_hash(self, wine_csp_client):
"""Streebog-256 should return a hash."""
data = {"algorithm": "STREEBOG-256", "data": encode_b64("Hello")}
response = post(wine_csp_client, "/hash", data)
assert response.status_code == 200
result = response.json()
assert "hash" in result
assert len(result["hash"]) == 64 # 256 bits = 64 hex chars
def test_hash_streebog512_returns_hash(self, wine_csp_client):
"""Streebog-512 should return a hash."""
data = {"algorithm": "STREEBOG-512", "data": encode_b64("Hello")}
response = post(wine_csp_client, "/hash", data)
assert response.status_code == 200
result = response.json()
assert "hash" in result
assert len(result["hash"]) == 128 # 512 bits = 128 hex chars
def test_hash_empty_input(self, wine_csp_client):
"""Hash of empty input should work."""
data = {"algorithm": "STREEBOG-256", "data": ""}
response = post(wine_csp_client, "/hash", data)
assert response.status_code == 200
def test_hash_invalid_algorithm(self, wine_csp_client):
"""Invalid algorithm should return 400."""
data = {"algorithm": "INVALID-ALGO", "data": encode_b64("Hello")}
response = post(wine_csp_client, "/hash", data)
assert response.status_code == 400
def test_hash_missing_data(self, wine_csp_client):
"""Missing data field should return 400."""
data = {"algorithm": "STREEBOG-256"}
response = post(wine_csp_client, "/hash", data)
assert response.status_code == 400
def test_hash_missing_algorithm(self, wine_csp_client):
"""Missing algorithm field should return 400."""
data = {"data": encode_b64("Hello")}
response = post(wine_csp_client, "/hash", data)
assert response.status_code == 400
# ==============================================================================
# Determinism Tests
# ==============================================================================
class TestDeterminism:
"""Tests for deterministic behavior."""
def test_hash_determinism_same_input(self, wine_csp_client):
"""Same input should produce same hash."""
data = {"algorithm": "STREEBOG-256", "data": encode_b64("Test data for determinism")}
hashes = []
for _ in range(5):
response = post(wine_csp_client, "/hash", data)
assert response.status_code == 200
hashes.append(response.json()["hash"])
# All hashes should be identical
assert len(set(hashes)) == 1, f"Non-deterministic hashes: {hashes}"
def test_hash_determinism_binary_data(self, wine_csp_client):
"""Binary input should produce deterministic hash."""
binary_data = bytes(range(256))
data = {"algorithm": "STREEBOG-512", "data": base64.b64encode(binary_data).decode()}
hashes = []
for _ in range(5):
response = post(wine_csp_client, "/hash", data)
assert response.status_code == 200
hashes.append(response.json()["hash"])
assert len(set(hashes)) == 1
# ==============================================================================
# Known Test Vector Validation
# ==============================================================================
class TestKnownVectors:
"""Tests using known GOST test vectors."""
def test_streebog256_m1_vector(self, wine_csp_client):
"""Validate Streebog-256 against GOST R 34.11-2012 M1 test vector."""
# M1 = "012345678901234567890123456789012345678901234567890123456789012"
m1 = "012345678901234567890123456789012345678901234567890123456789012"
expected_hash = "9d151eefd8590b89daa6ba6cb74af9275dd051026bb149a452fd84e5e57b5500"
data = {"algorithm": "STREEBOG-256", "data": encode_b64(m1)}
response = post(wine_csp_client, "/hash", data)
if response.status_code == 200:
result = response.json()
# Note: Implementation may use different encoding
actual_hash = result["hash"].lower()
# Check if hash matches (may need to reverse bytes for some implementations)
assert len(actual_hash) == 64, f"Invalid hash length: {len(actual_hash)}"
# Log for debugging
print(f"Expected: {expected_hash}")
print(f"Actual: {actual_hash}")
def test_streebog512_m1_vector(self, wine_csp_client):
"""Validate Streebog-512 against GOST R 34.11-2012 M1 test vector."""
m1 = "012345678901234567890123456789012345678901234567890123456789012"
expected_hash = "1b54d01a4af5b9d5cc3d86d68d285462b19abc2475222f35c085122be4ba1ffa00ad30f8767b3a82384c6574f024c311e2a481332b08ef7f41797891c1646f48"
data = {"algorithm": "STREEBOG-512", "data": encode_b64(m1)}
response = post(wine_csp_client, "/hash", data)
if response.status_code == 200:
result = response.json()
actual_hash = result["hash"].lower()
assert len(actual_hash) == 128, f"Invalid hash length: {len(actual_hash)}"
print(f"Expected: {expected_hash}")
print(f"Actual: {actual_hash}")
# ==============================================================================
# Test Vectors Endpoint
# ==============================================================================
class TestTestVectorsEndpoint:
"""Tests for test vectors endpoint."""
def test_vectors_returns_200(self, wine_csp_client):
"""Test vectors endpoint should return 200."""
response = get(wine_csp_client, "/test-vectors")
assert response.status_code == 200
def test_vectors_returns_array_or_object(self, wine_csp_client):
"""Test vectors should return valid JSON."""
response = get(wine_csp_client, "/test-vectors")
data = response.json()
assert isinstance(data, (list, dict))
# ==============================================================================
# Sign/Verify Endpoint Tests
# ==============================================================================
class TestSignVerifyEndpoints:
"""Tests for sign and verify operations."""
def test_sign_without_key_returns_error(self, wine_csp_client):
"""Sign without valid key should return error in limited mode."""
data = {
"keyId": "nonexistent-key",
"algorithm": "GOST12-256",
"data": encode_b64("Test message"),
}
response = post(wine_csp_client, "/sign", data)
# Should return error (400 or 404) in limited mode
assert response.status_code in [200, 400, 404, 500]
def test_verify_invalid_signature(self, wine_csp_client):
"""Verify with invalid signature should fail."""
data = {
"keyId": "test-key",
"algorithm": "GOST12-256",
"data": encode_b64("Test message"),
"signature": "aW52YWxpZA==", # "invalid" in base64
}
response = post(wine_csp_client, "/verify", data)
# Should return error or false verification
assert response.status_code in [200, 400, 404, 500]
# ==============================================================================
# Error Handling Tests
# ==============================================================================
class TestErrorHandling:
"""Tests for error handling."""
def test_malformed_json(self, wine_csp_client):
"""Malformed JSON should return 400."""
response = wine_csp_client["session"].post(
f"{wine_csp_client['base_url']}/hash",
data="not valid json",
headers={"Content-Type": "application/json"},
timeout=REQUEST_TIMEOUT,
)
assert response.status_code == 400
def test_invalid_base64(self, wine_csp_client):
"""Invalid base64 should return 400."""
data = {"algorithm": "STREEBOG-256", "data": "not-valid-base64!!!"}
response = post(wine_csp_client, "/hash", data)
assert response.status_code == 400
def test_unknown_endpoint(self, wine_csp_client):
"""Unknown endpoint should return 404."""
response = get(wine_csp_client, "/unknown-endpoint")
assert response.status_code == 404
# ==============================================================================
# Performance Tests
# ==============================================================================
class TestPerformance:
"""Performance benchmark tests."""
@pytest.mark.slow
def test_hash_throughput(self, wine_csp_client):
"""Hash operations should meet minimum throughput."""
data = {"algorithm": "STREEBOG-256", "data": encode_b64("X" * 1024)}
iterations = 50
start_time = time.time()
for _ in range(iterations):
response = post(wine_csp_client, "/hash", data)
assert response.status_code == 200
elapsed = time.time() - start_time
ops_per_second = iterations / elapsed
print(f"Hash throughput: {ops_per_second:.2f} ops/sec")
print(f"Average latency: {(elapsed / iterations) * 1000:.2f} ms")
# Should achieve at least 5 ops/sec
assert ops_per_second >= 5, f"Throughput too low: {ops_per_second:.2f} ops/sec"
@pytest.mark.slow
def test_concurrent_requests(self, wine_csp_client):
"""Service should handle concurrent requests."""
import concurrent.futures
data = {"algorithm": "STREEBOG-256", "data": encode_b64("Concurrent test")}
def make_request():
return post(wine_csp_client, "/hash", data)
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
futures = [executor.submit(make_request) for _ in range(20)]
results = [f.result() for f in concurrent.futures.as_completed(futures)]
success_count = sum(1 for r in results if r.status_code == 200)
assert success_count >= 18, f"Too many failures: {20 - success_count}/20"
# ==============================================================================
# Main
# ==============================================================================
if __name__ == "__main__":
pytest.main([__file__, "-v", "--tb=short"])

381
scripts/crypto/setup-wine-csp-service.sh
View File

@@ -1,381 +0,0 @@
#!/bin/bash
# setup-wine-csp-service.sh - Set up Wine environment for CryptoPro CSP service
#
# This script:
# 1. Creates a dedicated Wine prefix
# 2. Installs required Windows components
# 3. Builds the WineCspService for Windows target
# 4. Optionally installs CryptoPro CSP (if installer is provided)
#
# Prerequisites:
# - Wine 7.0+ installed (wine, wine64, winetricks)
# - .NET SDK 8.0+ installed
# - CryptoPro CSP installer (optional, for full functionality)
#
# Usage:
# ./setup-wine-csp-service.sh [--csp-installer /path/to/csp_setup.msi]
#
# Environment variables:
# WINE_PREFIX - Wine prefix location (default: ~/.stellaops-wine-csp)
# CSP_INSTALLER - Path to CryptoPro CSP installer
# WINE_CSP_PORT - HTTP port for service (default: 5099)
set -euo pipefail
# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
WINE_PREFIX="${WINE_PREFIX:-$HOME/.stellaops-wine-csp}"
WINE_CSP_PORT="${WINE_CSP_PORT:-5099}"
SERVICE_DIR="$REPO_ROOT/src/__Tools/WineCspService"
OUTPUT_DIR="$REPO_ROOT/artifacts/wine-csp-service"
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
# Parse arguments
CSP_INSTALLER=""
while [[ $# -gt 0 ]]; do
case $1 in
--csp-installer)
CSP_INSTALLER="$2"
shift 2
;;
--help)
echo "Usage: $0 [--csp-installer /path/to/csp_setup.msi]"
exit 0
;;
*)
log_error "Unknown option: $1"
exit 1
;;
esac
done
# Check prerequisites
check_prerequisites() {
log_info "Checking prerequisites..."
if ! command -v wine &> /dev/null; then
log_error "Wine is not installed. Please install Wine 7.0+"
exit 1
fi
if ! command -v winetricks &> /dev/null; then
log_warn "winetricks not found. Some components may not install correctly."
fi
if ! command -v dotnet &> /dev/null; then
log_error ".NET SDK not found. Please install .NET 8.0+"
exit 1
fi
log_info "Prerequisites OK"
}
# Initialize Wine prefix
init_wine_prefix() {
log_info "Initializing Wine prefix at $WINE_PREFIX..."
export WINEPREFIX="$WINE_PREFIX"
export WINEARCH="win64"
# Create prefix if it doesn't exist
if [[ ! -d "$WINE_PREFIX" ]]; then
wineboot --init
log_info "Wine prefix created"
else
log_info "Wine prefix already exists"
fi
# Set Windows version
wine reg add "HKCU\\Software\\Wine\\Version" /v Windows /d "win10" /f 2>/dev/null || true
}
# Install Windows components via winetricks
install_windows_components() {
log_info "Installing Windows components..."
if command -v winetricks &> /dev/null; then
export WINEPREFIX="$WINE_PREFIX"
# Install Visual C++ runtime
log_info "Installing Visual C++ runtime..."
winetricks -q vcrun2019 || log_warn "vcrun2019 installation may have issues"
# Install core fonts (optional, for UI)
# winetricks -q corefonts || true
log_info "Windows components installed"
else
log_warn "Skipping winetricks components (winetricks not available)"
fi
}
# Install CryptoPro CSP if installer provided
install_cryptopro_csp() {
if [[ -z "$CSP_INSTALLER" ]]; then
log_warn "No CryptoPro CSP installer provided. Service will run in limited mode."
log_warn "Provide installer with: --csp-installer /path/to/csp_setup_x64.msi"
return 0
fi
if [[ ! -f "$CSP_INSTALLER" ]]; then
log_error "CryptoPro installer not found: $CSP_INSTALLER"
return 1
fi
log_info "Installing CryptoPro CSP from $CSP_INSTALLER..."
export WINEPREFIX="$WINE_PREFIX"
# Run MSI installer
wine msiexec /i "$CSP_INSTALLER" /qn ADDLOCAL=ALL || {
log_error "CryptoPro CSP installation failed"
log_info "You may need to run the installer manually:"
log_info " WINEPREFIX=$WINE_PREFIX wine msiexec /i $CSP_INSTALLER"
return 1
}
# Verify installation
if wine reg query "HKLM\\SOFTWARE\\Microsoft\\Cryptography\\Defaults\\Provider\\Crypto-Pro GOST R 34.10-2012" 2>/dev/null; then
log_info "CryptoPro CSP installed successfully"
else
log_warn "CryptoPro CSP may not be registered correctly"
fi
}
# Build WineCspService for Windows
build_service() {
log_info "Building WineCspService..."
mkdir -p "$OUTPUT_DIR"
# Build for Windows x64
dotnet publish "$SERVICE_DIR/WineCspService.csproj" \
-c Release \
-r win-x64 \
--self-contained true \
-o "$OUTPUT_DIR" \
|| {
log_error "Build failed"
exit 1
}
log_info "Service built: $OUTPUT_DIR/WineCspService.exe"
}
# Create launcher script
create_launcher() {
log_info "Creating launcher script..."
cat > "$OUTPUT_DIR/run-wine-csp-service.sh" << EOF
#!/bin/bash
# Wine CSP Service Launcher
# Generated by setup-wine-csp-service.sh
export WINEPREFIX="$WINE_PREFIX"
export WINEDEBUG="-all" # Suppress Wine debug output
PORT=\${WINE_CSP_PORT:-$WINE_CSP_PORT}
SERVICE_DIR="\$(dirname "\$0")"
echo "Starting Wine CSP Service on port \$PORT..."
echo "Wine prefix: \$WINEPREFIX"
echo ""
cd "\$SERVICE_DIR"
exec wine WineCspService.exe --urls "http://0.0.0.0:\$PORT"
EOF
chmod +x "$OUTPUT_DIR/run-wine-csp-service.sh"
log_info "Launcher created: $OUTPUT_DIR/run-wine-csp-service.sh"
}
# Create systemd service file
create_systemd_service() {
log_info "Creating systemd service file..."
cat > "$OUTPUT_DIR/wine-csp-service.service" << EOF
[Unit]
Description=Wine CSP Service for CryptoPro GOST signing
After=network.target
[Service]
Type=simple
User=$USER
Environment=WINEPREFIX=$WINE_PREFIX
Environment=WINEDEBUG=-all
Environment=WINE_CSP_PORT=$WINE_CSP_PORT
WorkingDirectory=$OUTPUT_DIR
ExecStart=/bin/bash $OUTPUT_DIR/run-wine-csp-service.sh
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
log_info "Systemd service file created: $OUTPUT_DIR/wine-csp-service.service"
log_info "To install: sudo cp $OUTPUT_DIR/wine-csp-service.service /etc/systemd/system/"
log_info "To enable: sudo systemctl enable --now wine-csp-service"
}
# Create Docker Compose configuration
create_docker_compose() {
log_info "Creating Docker Compose configuration..."
cat > "$OUTPUT_DIR/docker-compose.yml" << EOF
# Wine CSP Service - Docker Compose configuration
# Requires: Docker with Wine support or Windows container
version: '3.8'
services:
wine-csp-service:
build:
context: .
dockerfile: Dockerfile.wine
ports:
- "${WINE_CSP_PORT}:5099"
environment:
- ASPNETCORE_URLS=http://+:5099
volumes:
# Mount CSP installer if available
- ./csp-installer:/installer:ro
# Persist Wine prefix for keys/certificates
- wine-prefix:/root/.wine
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:5099/health"]
interval: 30s
timeout: 10s
retries: 3
volumes:
wine-prefix:
EOF
# Create Dockerfile
cat > "$OUTPUT_DIR/Dockerfile.wine" << 'EOF'
# Wine CSP Service Dockerfile
FROM ubuntu:22.04
# Install Wine and dependencies
RUN dpkg --add-architecture i386 && \
apt-get update && \
apt-get install -y --no-install-recommends \
wine64 \
wine32 \
winetricks \
curl \
ca-certificates \
&& rm -rf /var/lib/apt/lists/*
# Initialize Wine prefix
RUN wineboot --init && \
winetricks -q vcrun2019 || true
# Copy service
WORKDIR /app
COPY WineCspService.exe .
COPY *.dll ./
# Expose port
EXPOSE 5099
# Health check
HEALTHCHECK --interval=30s --timeout=10s --retries=3 \
CMD curl -f http://localhost:5099/health || exit 1
# Run service
CMD ["wine", "WineCspService.exe", "--urls", "http://0.0.0.0:5099"]
EOF
log_info "Docker configuration created in $OUTPUT_DIR/"
}
# Test the service
test_service() {
log_info "Testing service startup..."
export WINEPREFIX="$WINE_PREFIX"
export WINEDEBUG="-all"
# Start service in background
cd "$OUTPUT_DIR"
wine WineCspService.exe --urls "http://localhost:$WINE_CSP_PORT" &
SERVICE_PID=$!
# Wait for startup
sleep 5
# Test health endpoint
if curl -s "http://localhost:$WINE_CSP_PORT/health" | grep -q "Healthy"; then
log_info "Service is running and healthy"
# Test status endpoint
log_info "CSP Status:"
curl -s "http://localhost:$WINE_CSP_PORT/status" | python3 -m json.tool 2>/dev/null || \
curl -s "http://localhost:$WINE_CSP_PORT/status"
else
log_warn "Service health check failed"
fi
# Stop service
kill $SERVICE_PID 2>/dev/null || true
wait $SERVICE_PID 2>/dev/null || true
}
# Print summary
print_summary() {
echo ""
log_info "=========================================="
log_info "Wine CSP Service Setup Complete"
log_info "=========================================="
echo ""
echo "Wine prefix: $WINE_PREFIX"
echo "Service directory: $OUTPUT_DIR"
echo "HTTP port: $WINE_CSP_PORT"
echo ""
echo "To start the service:"
echo " $OUTPUT_DIR/run-wine-csp-service.sh"
echo ""
echo "To test endpoints:"
echo " curl http://localhost:$WINE_CSP_PORT/status"
echo " curl http://localhost:$WINE_CSP_PORT/keys"
echo " curl -X POST http://localhost:$WINE_CSP_PORT/hash \\"
echo " -H 'Content-Type: application/json' \\"
echo " -d '{\"dataBase64\":\"SGVsbG8gV29ybGQ=\"}'"
echo ""
if [[ -z "$CSP_INSTALLER" ]]; then
echo "NOTE: CryptoPro CSP is not installed."
echo " The service will report 'CSP not available'."
echo " To install CSP, run:"
echo " $0 --csp-installer /path/to/csp_setup_x64.msi"
fi
}
# Main execution
main() {
log_info "Wine CSP Service Setup"
log_info "Repository: $REPO_ROOT"
check_prerequisites
init_wine_prefix
install_windows_components
install_cryptopro_csp
build_service
create_launcher
create_systemd_service
create_docker_compose
test_service
print_summary
}
main "$@"

43
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchService.cs
View File

@@ -11,6 +11,7 @@ using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoContracts = StellaOps.Concelier.Storage.Mongo;
using StorageContracts = StellaOps.Concelier.Storage.Contracts;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Connector.Common.Telemetry;
using StellaOps.Concelier.Core.Aoc;
@@ -32,6 +33,7 @@ public sealed class SourceFetchService
private readonly IHttpClientFactory _httpClientFactory;
private readonly RawDocumentStorage _rawDocumentStorage;
private readonly MongoContracts.IDocumentStore _documentStore;
private readonly StorageContracts.IStorageDocumentStore _storageDocumentStore;
private readonly ILogger<SourceFetchService> _logger;
private readonly TimeProvider _timeProvider;
private readonly IOptionsMonitor<SourceHttpClientOptions> _httpClientOptions;
@@ -46,6 +48,7 @@ public sealed class SourceFetchService
IHttpClientFactory httpClientFactory,
RawDocumentStorage rawDocumentStorage,
MongoContracts.IDocumentStore documentStore,
StorageContracts.IStorageDocumentStore storageDocumentStore,
ILogger<SourceFetchService> logger,
IJitterSource jitterSource,
IAdvisoryRawWriteGuard guard,
@@ -58,6 +61,7 @@ public sealed class SourceFetchService
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
_storageDocumentStore = storageDocumentStore ?? throw new ArgumentNullException(nameof(storageDocumentStore));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_jitterSource = jitterSource ?? throw new ArgumentNullException(nameof(jitterSource));
_guard = guard ?? throw new ArgumentNullException(nameof(guard));
@@ -69,6 +73,36 @@ public sealed class SourceFetchService
_connectorVersion = typeof(SourceFetchService).Assembly.GetName().Version?.ToString() ?? "0.0.0";
}
// Backward-compatible constructor until all callers provide the storage document contract explicitly.
public SourceFetchService(
IHttpClientFactory httpClientFactory,
RawDocumentStorage rawDocumentStorage,
MongoContracts.IDocumentStore documentStore,
ILogger<SourceFetchService> logger,
IJitterSource jitterSource,
IAdvisoryRawWriteGuard guard,
IAdvisoryLinksetMapper linksetMapper,
ICryptoHash hash,
TimeProvider? timeProvider = null,
IOptionsMonitor<SourceHttpClientOptions>? httpClientOptions = null,
IOptions<MongoContracts.MongoStorageOptions>? storageOptions = null)
: this(
httpClientFactory,
rawDocumentStorage,
documentStore,
documentStore as StorageContracts.IStorageDocumentStore
?? throw new ArgumentNullException(nameof(documentStore), "Document store must implement IStorageDocumentStore"),
logger,
jitterSource,
guard,
linksetMapper,
hash,
timeProvider,
httpClientOptions,
storageOptions)
{
}
public async Task<SourceFetchResult> FetchAsync(SourceFetchRequest request, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
@@ -147,7 +181,7 @@ public sealed class SourceFetchService
}
}
var existing = await _documentStore.FindBySourceAndUriAsync(request.SourceName, request.RequestUri.ToString(), cancellationToken).ConfigureAwait(false);
var existing = await _storageDocumentStore.FindBySourceAndUriAsync(request.SourceName, request.RequestUri.ToString(), cancellationToken).ConfigureAwait(false);
var recordId = existing?.Id ?? Guid.NewGuid();
var payloadId = await _rawDocumentStorage.UploadAsync(
@@ -159,7 +193,7 @@ public sealed class SourceFetchService
cancellationToken,
recordId).ConfigureAwait(false);
var record = new MongoContracts.DocumentRecord(
var record = new StorageContracts.StorageDocument(
recordId,
request.SourceName,
request.RequestUri.ToString(),
@@ -173,9 +207,10 @@ public sealed class SourceFetchService
response.Content.Headers.LastModified,
payloadId,
expiresAt,
Payload: contentBytes);
Payload: contentBytes,
FetchedAt: fetchedAt);
var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
var upserted = await _storageDocumentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, contentBytes.LongLength, rateLimitRemaining);
activity?.SetStatus(ActivityStatusCode.Ok);
_logger.LogInformation("Fetched {Source} document {Uri} (sha256={Sha})", request.SourceName, request.RequestUri, contentHash);

76
src/Concelier/__Libraries/StellaOps.Concelier.Models/StorageContracts/Contracts.cs Normal file
View File

@@ -0,0 +1,76 @@
using System;
using System.Collections.Generic;
using System.Text.Json;
namespace StellaOps.Concelier.Storage.Contracts;
/// <summary>
/// Postgres-native storage document contract (Mongo-free).
/// </summary>
public sealed record StorageDocument(
Guid Id,
string SourceName,
string Uri,
DateTimeOffset CreatedAt,
string Sha256,
string Status,
string? ContentType,
IReadOnlyDictionary<string, string>? Headers,
IReadOnlyDictionary<string, string>? Metadata,
string? Etag,
DateTimeOffset? LastModified,
Guid? PayloadId,
DateTimeOffset? ExpiresAt,
byte[]? Payload,
DateTimeOffset? FetchedAt);
public interface IStorageDocumentStore
{
Task<StorageDocument?> FindBySourceAndUriAsync(string sourceName, string uri, CancellationToken cancellationToken);
Task<StorageDocument?> FindAsync(Guid id, CancellationToken cancellationToken);
Task<StorageDocument> UpsertAsync(StorageDocument record, CancellationToken cancellationToken);
Task UpdateStatusAsync(Guid id, string status, CancellationToken cancellationToken);
}
/// <summary>
/// Postgres-native DTO storage contract using JSON payloads.
/// </summary>
public sealed record StorageDto(
Guid Id,
Guid DocumentId,
string SourceName,
string Format,
JsonDocument Payload,
DateTimeOffset CreatedAt,
string SchemaVersion,
DateTimeOffset ValidatedAt);
public interface IStorageDtoStore
{
Task<StorageDto> UpsertAsync(StorageDto record, CancellationToken cancellationToken);
Task<StorageDto?> FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken);
Task<IReadOnlyList<StorageDto>> GetBySourceAsync(string sourceName, int limit, CancellationToken cancellationToken);
}
/// <summary>
/// Cursor/state contract for ingestion sources without Mongo/Bson dependencies.
/// </summary>
public sealed record SourceCursorState(
string SourceName,
bool Enabled,
bool Paused,
JsonDocument? Cursor,
DateTimeOffset? LastSuccess,
DateTimeOffset? LastFailure,
int FailCount,
DateTimeOffset? BackoffUntil,
DateTimeOffset UpdatedAt,
string? LastFailureReason);
public interface ISourceStateStore
{
Task<SourceCursorState?> TryGetAsync(string sourceName, CancellationToken cancellationToken);
Task UpdateCursorAsync(string sourceName, JsonDocument cursor, DateTimeOffset completedAt, CancellationToken cancellationToken);
Task MarkFailureAsync(string sourceName, DateTimeOffset now, TimeSpan backoff, string reason, CancellationToken cancellationToken);
Task UpsertAsync(SourceCursorState record, CancellationToken cancellationToken);
}

125
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Postgres/ContractsMappingExtensions.cs Normal file
View File

@@ -0,0 +1,125 @@
using System;
using System.Text.Json;
using MongoDB.Bson;
using MongoDB.Bson.IO;
using Contracts = StellaOps.Concelier.Storage.Contracts;
using MongoContracts = StellaOps.Concelier.Storage.Mongo;
namespace StellaOps.Concelier.Storage.Postgres;
internal static class ContractsMappingExtensions
{
private static readonly JsonWriterSettings RelaxedJsonSettings = new()
{
OutputMode = JsonOutputMode.RelaxedExtendedJson
};
internal static Contracts.StorageDocument ToStorageDocument(this MongoContracts.DocumentRecord record)
{
return new Contracts.StorageDocument(
record.Id,
record.SourceName,
record.Uri,
record.CreatedAt,
record.Sha256,
record.Status,
record.ContentType,
record.Headers,
record.Metadata,
record.Etag,
record.LastModified,
record.PayloadId,
record.ExpiresAt,
record.Payload,
record.FetchedAt);
}
internal static MongoContracts.DocumentRecord ToMongoDocumentRecord(this Contracts.StorageDocument record)
{
return new MongoContracts.DocumentRecord(
record.Id,
record.SourceName,
record.Uri,
record.CreatedAt,
record.Sha256,
record.Status,
record.ContentType,
record.Headers,
record.Metadata,
record.Etag,
record.LastModified,
record.PayloadId,
record.ExpiresAt,
record.Payload,
record.FetchedAt);
}
internal static Contracts.StorageDto ToStorageDto(this MongoContracts.DtoRecord record)
{
var json = record.Payload.ToJson(RelaxedJsonSettings);
var payload = JsonDocument.Parse(json);
return new Contracts.StorageDto(
record.Id,
record.DocumentId,
record.SourceName,
record.Format,
payload,
record.CreatedAt,
record.SchemaVersion,
record.ValidatedAt);
}
internal static MongoContracts.DtoRecord ToMongoDtoRecord(this Contracts.StorageDto record)
{
var json = record.Payload.RootElement.GetRawText();
var bson = BsonDocument.Parse(json);
return new MongoContracts.DtoRecord(
record.Id,
record.DocumentId,
record.SourceName,
record.Format,
bson,
record.CreatedAt,
record.SchemaVersion,
record.ValidatedAt);
}
internal static Contracts.SourceCursorState ToStorageCursorState(this MongoContracts.SourceStateRecord record)
{
var cursorJson = record.Cursor is null ? null : record.Cursor.ToJson(RelaxedJsonSettings);
var cursor = cursorJson is null ? null : JsonDocument.Parse(cursorJson);
return new Contracts.SourceCursorState(
record.SourceName,
record.Enabled,
record.Paused,
cursor,
record.LastSuccess,
record.LastFailure,
record.FailCount,
record.BackoffUntil,
record.UpdatedAt,
record.LastFailureReason);
}
internal static MongoContracts.SourceStateRecord ToMongoSourceStateRecord(this Contracts.SourceCursorState record)
{
var bsonCursor = record.Cursor is null ? null : BsonDocument.Parse(record.Cursor.RootElement.GetRawText());
return new MongoContracts.SourceStateRecord(
record.SourceName,
record.Enabled,
record.Paused,
bsonCursor,
record.LastSuccess,
record.LastFailure,
record.FailCount,
record.BackoffUntil,
record.UpdatedAt,
record.LastFailureReason);
}
internal static BsonDocument ToBsonDocument(this JsonDocument document)
{
ArgumentNullException.ThrowIfNull(document);
return BsonDocument.Parse(document.RootElement.GetRawText());
}
}

17
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Postgres/DocumentStore.cs
View File

@@ -1,14 +1,15 @@
using System.Text.Json;
using StellaOps.Concelier.Storage.Mongo;
using Contracts = StellaOps.Concelier.Storage.Contracts;
using StellaOps.Concelier.Storage.Postgres.Models;
using StellaOps.Concelier.Storage.Postgres.Repositories;
namespace StellaOps.Concelier.Storage.Postgres;
/// <summary>
/// Postgres-backed implementation that satisfies the legacy IDocumentStore contract.
/// Postgres-backed implementation that satisfies the legacy IDocumentStore contract and the new Postgres-native storage contract.
/// </summary>
public sealed class PostgresDocumentStore : IDocumentStore
public sealed class PostgresDocumentStore : IDocumentStore, Contracts.IStorageDocumentStore
{
private readonly IDocumentRepository _repository;
private readonly ISourceRepository _sourceRepository;
@@ -64,6 +65,18 @@ public sealed class PostgresDocumentStore : IDocumentStore
await _repository.UpdateStatusAsync(id, status, cancellationToken).ConfigureAwait(false);
}
async Task<Contracts.StorageDocument?> Contracts.IStorageDocumentStore.FindBySourceAndUriAsync(string sourceName, string uri, CancellationToken cancellationToken)
=> (await FindBySourceAndUriAsync(sourceName, uri, cancellationToken).ConfigureAwait(false))?.ToStorageDocument();
async Task<Contracts.StorageDocument?> Contracts.IStorageDocumentStore.FindAsync(Guid id, CancellationToken cancellationToken)
=> (await FindAsync(id, cancellationToken).ConfigureAwait(false))?.ToStorageDocument();
async Task<Contracts.StorageDocument> Contracts.IStorageDocumentStore.UpsertAsync(Contracts.StorageDocument record, CancellationToken cancellationToken)
=> (await UpsertAsync(record.ToMongoDocumentRecord(), cancellationToken).ConfigureAwait(false)).ToStorageDocument();
Task Contracts.IStorageDocumentStore.UpdateStatusAsync(Guid id, string status, CancellationToken cancellationToken)
=> UpdateStatusAsync(id, status, cancellationToken);
private DocumentRecord Map(DocumentRecordEntity row)
{
return new DocumentRecord(

16
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Postgres/Repositories/PostgresDtoStore.cs
View File

@@ -1,10 +1,13 @@
using System.Linq;
using System.Text.Json;
using Dapper;
using StellaOps.Concelier.Storage.Mongo;
using Contracts = StellaOps.Concelier.Storage.Contracts;
using StellaOps.Concelier.Storage.Postgres;
namespace StellaOps.Concelier.Storage.Postgres.Repositories;
internal sealed class PostgresDtoStore : IDtoStore
internal sealed class PostgresDtoStore : IDtoStore, Contracts.IStorageDtoStore
{
private readonly ConcelierDataSource _dataSource;
private readonly JsonSerializerOptions _jsonOptions = new(JsonSerializerDefaults.General)
@@ -92,6 +95,17 @@ internal sealed class PostgresDtoStore : IDtoStore
row.ValidatedAt);
}
async Task<Contracts.StorageDto> Contracts.IStorageDtoStore.UpsertAsync(Contracts.StorageDto record, CancellationToken cancellationToken)
=> (await UpsertAsync(record.ToMongoDtoRecord(), cancellationToken).ConfigureAwait(false)).ToStorageDto();
async Task<Contracts.StorageDto?> Contracts.IStorageDtoStore.FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken)
=> (await FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false))?.ToStorageDto();
async Task<IReadOnlyList<Contracts.StorageDto>> Contracts.IStorageDtoStore.GetBySourceAsync(string sourceName, int limit, CancellationToken cancellationToken)
=> (await GetBySourceAsync(sourceName, limit, cancellationToken).ConfigureAwait(false))
.Select(dto => dto.ToStorageDto())
.ToArray();
private sealed record DtoRow(
Guid Id,
Guid DocumentId,

17
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Postgres/SourceStateAdapter.cs
View File

@@ -4,14 +4,15 @@ using System.Collections.Generic;
using MongoDB.Bson;
using StellaOps.Concelier.Storage.Postgres.Models;
using StellaOps.Concelier.Storage.Postgres.Repositories;
using Contracts = StellaOps.Concelier.Storage.Contracts;
using MongoContracts = StellaOps.Concelier.Storage.Mongo;
namespace StellaOps.Concelier.Storage.Postgres;
/// <summary>
/// Adapter that satisfies the legacy source state contract using PostgreSQL storage.
/// Adapter that satisfies the legacy source state contract using PostgreSQL storage and provides a Postgres-native cursor contract.
/// </summary>
public sealed class PostgresSourceStateAdapter : MongoContracts.ISourceStateRepository
public sealed class PostgresSourceStateAdapter : MongoContracts.ISourceStateRepository, Contracts.ISourceStateStore
{
private readonly ISourceRepository _sourceRepository;
private readonly Repositories.ISourceStateRepository _stateRepository;
@@ -134,6 +135,18 @@ public sealed class PostgresSourceStateAdapter : MongoContracts.ISourceStateRepo
_ = await _stateRepository.UpsertAsync(entity, cancellationToken).ConfigureAwait(false);
}
async Task<Contracts.SourceCursorState?> Contracts.ISourceStateStore.TryGetAsync(string sourceName, CancellationToken cancellationToken)
=> (await TryGetAsync(sourceName, cancellationToken).ConfigureAwait(false))?.ToStorageCursorState();
Task Contracts.ISourceStateStore.UpdateCursorAsync(string sourceName, JsonDocument cursor, DateTimeOffset completedAt, CancellationToken cancellationToken)
=> UpdateCursorAsync(sourceName, cursor.ToBsonDocument(), completedAt, cancellationToken);
Task Contracts.ISourceStateStore.MarkFailureAsync(string sourceName, DateTimeOffset now, TimeSpan backoff, string reason, CancellationToken cancellationToken)
=> MarkFailureAsync(sourceName, now, backoff, reason, cancellationToken);
Task Contracts.ISourceStateStore.UpsertAsync(Contracts.SourceCursorState record, CancellationToken cancellationToken)
=> UpsertAsync(record.ToMongoSourceStateRecord(), cancellationToken);
private async Task<SourceEntity> EnsureSourceAsync(string sourceName, CancellationToken cancellationToken)
{
var existing = await _sourceRepository.GetByKeyAsync(sourceName, cancellationToken).ConfigureAwait(false);

8
src/Excititor/AGENTS.md
View File

@@ -5,9 +5,9 @@
- Mission (current sprint): air-gap parity for evidence chunks, trust connector wiring, and attestation verification aligned to Evidence Locker contract.
## Roles
- **Backend engineer (ASP.NET Core / Mongo):** chunk ingestion/export, attestation verifier, trust connector.
- **Backend engineer (ASP.NET Core / Postgres):** chunk ingestion/export, attestation verifier, trust connector.
- **Air-Gap/Platform engineer:** sealed-mode switches, offline bundles, deterministic cache/path handling.
- **QA automation:** WebApplicationFactory + Mongo2Go tests for chunk APIs, attestations, and trust connector; deterministic ordering/hashes.
- **QA automation:** WebApplicationFactory + Postgres or in-memory fixtures for chunk APIs, attestations, and trust connector; deterministic ordering/hashes.
- **Docs/Schema steward:** keep chunk API, attestation plan, and trust connector docs in sync with behavior; update schemas and samples.
## Required Reading (treat as read before DOING)
@@ -29,7 +29,7 @@
- Cross-module edits: require sprint note; otherwise, stay within Excititor working dir.
## Testing Rules
- Use Mongo2Go/in-memory fixtures; avoid network.
- Prefer Postgres integration or in-memory fixtures; avoid network.
- API tests in `StellaOps.Excititor.WebService.Tests`; worker/connectors in `StellaOps.Excititor.Worker.Tests`; shared fixtures in `__Tests`.
- Tests must assert determinism (ordering/hashes), tenant enforcement, and sealed-mode behavior.
@@ -39,6 +39,6 @@
- If a decision is needed, mark the task BLOCKED and record the decision ask—do not pause work.
## Tooling/Env Notes
- .NET 10 with preview features enabled; Mongo driver ≥ 3.x.
- .NET 10 with preview features enabled; Postgres or in-memory storage only (Mongo/BSON removed).
- Signing/verifier hooks rely on Evidence Locker contract fixtures under `docs/modules/evidence-locker/`.
- Sealed-mode tests should run with `EXCITITOR_SEALED=1` (env var) to enforce offline code paths.

5
src/Excititor/StellaOps.Excititor.WebService/AGENTS.md
View File

@@ -27,14 +27,15 @@ Expose Excititor APIs (console VEX views, graph/Vuln Explorer feeds, observation
5. Observability: structured logs, counters, optional OTEL traces behind configuration flags.
## Testing
- Prefer deterministic API/integration tests under `__Tests` with seeded Mongo fixtures.
- Prefer deterministic API/integration tests under `__Tests` with seeded Postgres fixtures or in-memory stores.
- Verify RBAC/tenant isolation, idempotent ingestion, and stable ordering of VEX aggregates.
- Use ISO-8601 UTC timestamps and stable sorting in responses; assert on content hashes where applicable.
## Determinism & Data
- MongoDB is the canonical store; never apply consensus transformations before persistence.
- Postgres append-only storage is canonical; never apply consensus transformations before persistence.
- Ensure paged/list endpoints use explicit sort keys (e.g., vendor, upstreamId, version, createdUtc).
- Avoid nondeterministic clocks/randomness; inject clocks and GUID providers for tests.
- Evidence/attestation endpoints are temporarily disabled; re-enable only when Postgres-backed stores land (Mongo/BSON removed).
## Boundaries
- Do not modify Policy Engine or Cartographer schemas from here; consume published contracts only.

319
src/Excititor/StellaOps.Excititor.WebService/Endpoints/AttestationEndpoints.cs
View File

@@ -1,40 +1,23 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.WebService.Contracts;
using StellaOps.Excititor.WebService.Services;
namespace StellaOps.Excititor.WebService.Endpoints;
/// <summary>
/// Attestation API endpoints (WEB-OBS-54-001).
/// Exposes /attestations/vex/* endpoints returning DSSE verification state,
/// builder identity, and chain-of-custody links.
/// Attestation API endpoints (temporarily disabled while Mongo is removed and Postgres storage is adopted).
/// </summary>
public static class AttestationEndpoints
{
public static void MapAttestationEndpoints(this WebApplication app)
{
// GET /attestations/vex/list - List attestations
app.MapGet("/attestations/vex/list", async (
// GET /attestations/vex/list
app.MapGet("/attestations/vex/list", (
HttpContext context,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IMongoDatabase database,
TimeProvider timeProvider,
[FromQuery] int? limit,
[FromQuery] string? cursor,
[FromQuery] string? vulnerabilityId,
[FromQuery] string? productKey,
CancellationToken cancellationToken) =>
IOptions<VexStorageOptions> storageOptions) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
@@ -42,70 +25,22 @@ public static class AttestationEndpoints
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
{
return tenantError;
}
var take = Math.Clamp(limit.GetValueOrDefault(50), 1, 200);
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Attestations);
var builder = Builders<BsonDocument>.Filter;
var filters = new List<FilterDefinition<BsonDocument>>();
if (!string.IsNullOrWhiteSpace(vulnerabilityId))
{
filters.Add(builder.Eq("VulnerabilityId", vulnerabilityId.Trim().ToUpperInvariant()));
}
if (!string.IsNullOrWhiteSpace(productKey))
{
filters.Add(builder.Eq("ProductKey", productKey.Trim().ToLowerInvariant()));
}
// Parse cursor if provided
if (!string.IsNullOrWhiteSpace(cursor) && TryDecodeCursor(cursor, out var cursorTime, out var cursorId))
{
var ltTime = builder.Lt("IssuedAt", cursorTime);
var eqTimeLtId = builder.And(
builder.Eq("IssuedAt", cursorTime),
builder.Lt("_id", cursorId));
filters.Add(builder.Or(ltTime, eqTimeLtId));
}
var filter = filters.Count == 0 ? builder.Empty : builder.And(filters);
var sort = Builders<BsonDocument>.Sort.Descending("IssuedAt").Descending("_id");
var documents = await collection
.Find(filter)
.Sort(sort)
.Limit(take)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var items = documents.Select(doc => ToListItem(doc, tenant, timeProvider)).ToList();
string? nextCursor = null;
var hasMore = documents.Count == take;
if (hasMore && documents.Count > 0)
{
var last = documents[^1];
var lastTime = last.GetValue("IssuedAt", BsonNull.Value).ToUniversalTime();
var lastId = last.GetValue("_id", BsonNull.Value).AsString;
nextCursor = EncodeCursor(lastTime, lastId);
}
var response = new VexAttestationListResponse(items, nextCursor, hasMore, items.Count);
return Results.Ok(response);
return Results.Problem(
detail: "Attestation listing is temporarily unavailable during Postgres migration (Mongo/BSON removed).",
statusCode: StatusCodes.Status503ServiceUnavailable,
title: "Service unavailable");
}).WithName("ListVexAttestations");
// GET /attestations/vex/{attestationId} - Get attestation details
app.MapGet("/attestations/vex/{attestationId}", async (
// GET /attestations/vex/{attestationId}
app.MapGet("/attestations/vex/{attestationId}", (
HttpContext context,
string attestationId,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IVexAttestationLinkStore attestationStore,
TimeProvider timeProvider,
CancellationToken cancellationToken) =>
IOptions<VexStorageOptions> storageOptions) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
@@ -113,235 +48,23 @@ public static class AttestationEndpoints
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(attestationId))
{
return Results.BadRequest(new { error = new { code = "ERR_ATTESTATION_ID", message = "attestationId is required" } });
return Results.Problem(
detail: "attestationId is required.",
statusCode: StatusCodes.Status400BadRequest,
title: "Validation error");
}
var attestation = await attestationStore.FindAsync(attestationId.Trim(), cancellationToken).ConfigureAwait(false);
if (attestation is null)
{
return Results.NotFound(new { error = new { code = "ERR_NOT_FOUND", message = $"Attestation '{attestationId}' not found" } });
}
// Build subject from observation context
var subjectDigest = attestation.Metadata.TryGetValue("digest", out var dig) ? dig : attestation.ObservationId;
var subject = new VexAttestationSubject(
Digest: subjectDigest,
DigestAlgorithm: "sha256",
Name: $"{attestation.VulnerabilityId}/{attestation.ProductKey}",
Uri: null);
var builder = new VexAttestationBuilderIdentity(
Id: attestation.SupplierId,
Version: null,
BuilderId: attestation.SupplierId,
InvocationId: attestation.ObservationId);
// Get verification state from metadata
var isValid = attestation.Metadata.TryGetValue("verified", out var verified) && verified == "true";
DateTimeOffset? verifiedAt = null;
if (attestation.Metadata.TryGetValue("verifiedAt", out var verifiedAtStr) &&
DateTimeOffset.TryParse(verifiedAtStr, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsedVerifiedAt))
{
verifiedAt = parsedVerifiedAt;
}
var verification = new VexAttestationVerificationState(
Valid: isValid,
VerifiedAt: verifiedAt,
SignatureType: attestation.Metadata.GetValueOrDefault("signatureType", "dsse"),
KeyId: attestation.Metadata.GetValueOrDefault("keyId"),
Issuer: attestation.Metadata.GetValueOrDefault("issuer"),
EnvelopeDigest: attestation.Metadata.GetValueOrDefault("envelopeDigest"),
Diagnostics: attestation.Metadata);
var custodyLinks = new List<VexAttestationCustodyLink>
{
new(
Step: 1,
Actor: attestation.SupplierId,
Action: "created",
Timestamp: attestation.IssuedAt,
Reference: attestation.AttestationId)
};
// Add linkset link
custodyLinks.Add(new VexAttestationCustodyLink(
Step: 2,
Actor: "excititor",
Action: "linked_to_observation",
Timestamp: attestation.IssuedAt,
Reference: attestation.LinksetId));
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["observationId"] = attestation.ObservationId,
["linksetId"] = attestation.LinksetId,
["vulnerabilityId"] = attestation.VulnerabilityId,
["productKey"] = attestation.ProductKey
};
if (!string.IsNullOrWhiteSpace(attestation.JustificationSummary))
{
metadata["justificationSummary"] = attestation.JustificationSummary;
}
var response = new VexAttestationDetailResponse(
AttestationId: attestation.AttestationId,
Tenant: tenant,
CreatedAt: attestation.IssuedAt,
PredicateType: attestation.Metadata.GetValueOrDefault("predicateType", "https://in-toto.io/attestation/v1"),
Subject: subject,
Builder: builder,
Verification: verification,
ChainOfCustody: custodyLinks,
Metadata: metadata);
return Results.Ok(response);
return Results.Problem(
detail: "Attestation retrieval is temporarily unavailable during Postgres migration (Mongo/BSON removed).",
statusCode: StatusCodes.Status503ServiceUnavailable,
title: "Service unavailable");
}).WithName("GetVexAttestation");
// GET /attestations/vex/lookup - Lookup attestations by linkset or observation
app.MapGet("/attestations/vex/lookup", async (
HttpContext context,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IMongoDatabase database,
TimeProvider timeProvider,
[FromQuery] string? linksetId,
[FromQuery] string? observationId,
[FromQuery] int? limit,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(linksetId) && string.IsNullOrWhiteSpace(observationId))
{
return Results.BadRequest(new { error = new { code = "ERR_PARAMS", message = "Either linksetId or observationId is required" } });
}
var take = Math.Clamp(limit.GetValueOrDefault(50), 1, 100);
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Attestations);
var builder = Builders<BsonDocument>.Filter;
FilterDefinition<BsonDocument> filter;
if (!string.IsNullOrWhiteSpace(linksetId))
{
filter = builder.Eq("LinksetId", linksetId.Trim());
}
else
{
filter = builder.Eq("ObservationId", observationId!.Trim());
}
var sort = Builders<BsonDocument>.Sort.Descending("IssuedAt");
var documents = await collection
.Find(filter)
.Sort(sort)
.Limit(take)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var items = documents.Select(doc => ToListItem(doc, tenant, timeProvider)).ToList();
var response = new VexAttestationLookupResponse(
SubjectDigest: linksetId ?? observationId ?? string.Empty,
Attestations: items,
QueriedAt: timeProvider.GetUtcNow());
return Results.Ok(response);
}).WithName("LookupVexAttestations");
}
private static VexAttestationListItem ToListItem(BsonDocument doc, string tenant, TimeProvider timeProvider)
{
return new VexAttestationListItem(
AttestationId: doc.GetValue("_id", BsonNull.Value).AsString ?? string.Empty,
Tenant: tenant,
CreatedAt: doc.GetValue("IssuedAt", BsonNull.Value).IsBsonDateTime
? new DateTimeOffset(doc["IssuedAt"].ToUniversalTime(), TimeSpan.Zero)
: timeProvider.GetUtcNow(),
PredicateType: "https://in-toto.io/attestation/v1",
SubjectDigest: doc.GetValue("ObservationId", BsonNull.Value).AsString ?? string.Empty,
Valid: doc.Contains("Metadata") && !doc["Metadata"].IsBsonNull &&
doc["Metadata"].AsBsonDocument.Contains("verified") &&
doc["Metadata"]["verified"].AsString == "true",
BuilderId: doc.GetValue("SupplierId", BsonNull.Value).AsString);
}
private static bool TryResolveTenant(HttpContext context, VexStorageOptions options, out string tenant, out IResult? problem)
{
tenant = options.DefaultTenant;
problem = null;
if (context.Request.Headers.TryGetValue("X-Stella-Tenant", out var headerValues) && headerValues.Count > 0)
{
var requestedTenant = headerValues[0]?.Trim();
if (string.IsNullOrEmpty(requestedTenant))
{
problem = Results.BadRequest(new { error = new { code = "ERR_TENANT", message = "X-Stella-Tenant header must not be empty" } });
return false;
}
if (!string.Equals(requestedTenant, options.DefaultTenant, StringComparison.OrdinalIgnoreCase))
{
problem = Results.Json(
new { error = new { code = "ERR_TENANT_FORBIDDEN", message = $"Tenant '{requestedTenant}' is not allowed" } },
statusCode: StatusCodes.Status403Forbidden);
return false;
}
tenant = requestedTenant;
}
return true;
}
private static bool TryDecodeCursor(string cursor, out DateTime timestamp, out string id)
{
timestamp = default;
id = string.Empty;
try
{
var payload = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(cursor));
var parts = payload.Split('|');
if (parts.Length != 2)
{
return false;
}
if (!DateTimeOffset.TryParse(parts[0], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed))
{
return false;
}
timestamp = parsed.UtcDateTime;
id = parts[1];
return true;
}
catch
{
return false;
}
}
private static string EncodeCursor(DateTime timestamp, string id)
{
var payload = FormattableString.Invariant($"{timestamp:O}|{id}");
return Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(payload));
}
}

744
src/Excititor/StellaOps.Excititor.WebService/Endpoints/EvidenceEndpoints.cs
View File

@@ -1,48 +1,24 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using System.IO;
using System.Threading.Tasks;
using System.Security.Cryptography;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Canonicalization;
using StellaOps.Excititor.Core.Observations;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.WebService.Contracts;
using StellaOps.Excititor.WebService.Services;
using StellaOps.Excititor.WebService.Telemetry;
using StellaOps.Excititor.WebService.Options;
namespace StellaOps.Excititor.WebService.Endpoints;
/// <summary>
/// Evidence API endpoints (WEB-OBS-53-001).
/// Exposes /evidence/vex/* endpoints that fetch locker bundles, enforce scopes,
/// and surface verification metadata without synthesizing verdicts.
/// Evidence API endpoints (temporarily disabled while Mongo/BSON storage is removed).
/// </summary>
public static class EvidenceEndpoints
{
public static void MapEvidenceEndpoints(this WebApplication app)
{
// GET /evidence/vex/list - List evidence exports
app.MapGet("/evidence/vex/list", async (
// GET /evidence/vex/list
app.MapGet("/evidence/vex/list", (
HttpContext context,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IMongoDatabase database,
TimeProvider timeProvider,
[FromQuery] int? limit,
[FromQuery] string? cursor,
[FromQuery] string? format,
CancellationToken cancellationToken) =>
ChunkTelemetry chunkTelemetry) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
@@ -50,74 +26,23 @@ public static class EvidenceEndpoints
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
{
return tenantError;
}
var take = Math.Clamp(limit.GetValueOrDefault(50), 1, 200);
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Exports);
var builder = Builders<BsonDocument>.Filter;
var filters = new List<FilterDefinition<BsonDocument>>();
if (!string.IsNullOrWhiteSpace(format))
{
filters.Add(builder.Eq("Format", format.Trim().ToLowerInvariant()));
}
// Parse cursor if provided (base64-encoded timestamp|id)
if (!string.IsNullOrWhiteSpace(cursor) && TryDecodeCursor(cursor, out var cursorTime, out var cursorId))
{
var ltTime = builder.Lt("CreatedAt", cursorTime);
var eqTimeLtId = builder.And(
builder.Eq("CreatedAt", cursorTime),
builder.Lt("_id", cursorId));
filters.Add(builder.Or(ltTime, eqTimeLtId));
}
var filter = filters.Count == 0 ? builder.Empty : builder.And(filters);
var sort = Builders<BsonDocument>.Sort.Descending("CreatedAt").Descending("_id");
var documents = await collection
.Find(filter)
.Sort(sort)
.Limit(take)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var items = documents.Select(doc => new VexEvidenceListItem(
BundleId: doc.GetValue("ExportId", BsonNull.Value).AsString ?? doc.GetValue("_id", BsonNull.Value).AsString,
Tenant: tenant,
CreatedAt: doc.GetValue("CreatedAt", BsonNull.Value).IsBsonDateTime
? new DateTimeOffset(doc["CreatedAt"].ToUniversalTime(), TimeSpan.Zero)
: timeProvider.GetUtcNow(),
ContentHash: doc.GetValue("ArtifactDigest", BsonNull.Value).AsString ?? string.Empty,
Format: doc.GetValue("Format", BsonNull.Value).AsString ?? "json",
ItemCount: doc.GetValue("ClaimCount", BsonNull.Value).IsInt32 ? doc["ClaimCount"].AsInt32 : 0,
Verified: doc.Contains("Attestation") && !doc["Attestation"].IsBsonNull)).ToList();
string? nextCursor = null;
var hasMore = documents.Count == take;
if (hasMore && documents.Count > 0)
{
var last = documents[^1];
var lastTime = last.GetValue("CreatedAt", BsonNull.Value).ToUniversalTime();
var lastId = last.GetValue("_id", BsonNull.Value).AsString;
nextCursor = EncodeCursor(lastTime, lastId);
}
var response = new VexEvidenceListResponse(items, nextCursor, hasMore, items.Count);
return Results.Ok(response);
chunkTelemetry.RecordIngested(tenant, null, "unavailable", "storage-migration", 0, 0, 0);
return Results.Problem(
detail: "Evidence exports are temporarily unavailable during Postgres migration (Mongo/BSON removed).",
statusCode: StatusCodes.Status503ServiceUnavailable,
title: "Service unavailable");
}).WithName("ListVexEvidence");
// GET /evidence/vex/bundle/{bundleId} - Get evidence bundle details
app.MapGet("/evidence/vex/bundle/{bundleId}", async (
// GET /evidence/vex/{bundleId}
app.MapGet("/evidence/vex/{bundleId}", (
HttpContext context,
string bundleId,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IMongoDatabase database,
TimeProvider timeProvider,
CancellationToken cancellationToken) =>
IOptions<VexStorageOptions> storageOptions) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
@@ -125,79 +50,30 @@ public static class EvidenceEndpoints
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(bundleId))
{
return Results.BadRequest(new { error = new { code = "ERR_BUNDLE_ID", message = "bundleId is required" } });
return Results.Problem(
detail: "bundleId is required.",
statusCode: StatusCodes.Status400BadRequest,
title: "Validation error");
}
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Exports);
var filter = Builders<BsonDocument>.Filter.Or(
Builders<BsonDocument>.Filter.Eq("_id", bundleId.Trim()),
Builders<BsonDocument>.Filter.Eq("ExportId", bundleId.Trim()));
var doc = await collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
if (doc is null)
{
return Results.NotFound(new { error = new { code = "ERR_NOT_FOUND", message = $"Evidence bundle '{bundleId}' not found" } });
}
VexEvidenceVerificationMetadata? verification = null;
if (doc.Contains("Attestation") && !doc["Attestation"].IsBsonNull)
{
var att = doc["Attestation"].AsBsonDocument;
verification = new VexEvidenceVerificationMetadata(
Verified: true,
VerifiedAt: att.Contains("SignedAt") && att["SignedAt"].IsBsonDateTime
? new DateTimeOffset(att["SignedAt"].ToUniversalTime(), TimeSpan.Zero)
: null,
SignatureType: "dsse",
KeyId: att.GetValue("KeyId", BsonNull.Value).AsString,
Issuer: att.GetValue("Issuer", BsonNull.Value).AsString,
TransparencyRef: att.Contains("Rekor") && !att["Rekor"].IsBsonNull
? att["Rekor"].AsBsonDocument.GetValue("Location", BsonNull.Value).AsString
: null);
}
var metadata = new Dictionary<string, string>(StringComparer.Ordinal);
if (doc.Contains("SourceProviders") && doc["SourceProviders"].IsBsonArray)
{
metadata["sourceProviders"] = string.Join(",", doc["SourceProviders"].AsBsonArray.Select(v => v.AsString));
}
if (doc.Contains("PolicyRevisionId") && !doc["PolicyRevisionId"].IsBsonNull)
{
metadata["policyRevisionId"] = doc["PolicyRevisionId"].AsString;
}
var response = new VexEvidenceBundleResponse(
BundleId: doc.GetValue("ExportId", BsonNull.Value).AsString ?? bundleId.Trim(),
Tenant: tenant,
CreatedAt: doc.GetValue("CreatedAt", BsonNull.Value).IsBsonDateTime
? new DateTimeOffset(doc["CreatedAt"].ToUniversalTime(), TimeSpan.Zero)
: timeProvider.GetUtcNow(),
ContentHash: doc.GetValue("ArtifactDigest", BsonNull.Value).AsString ?? string.Empty,
Format: doc.GetValue("Format", BsonNull.Value).AsString ?? "json",
ItemCount: doc.GetValue("ClaimCount", BsonNull.Value).IsInt32 ? doc["ClaimCount"].AsInt32 : 0,
Verification: verification,
Metadata: metadata);
return Results.Ok(response);
return Results.Problem(
detail: "Evidence bundles are temporarily unavailable during Postgres migration (Mongo/BSON removed).",
statusCode: StatusCodes.Status503ServiceUnavailable,
title: "Service unavailable");
}).WithName("GetVexEvidenceBundle");
// GET /evidence/vex/lookup - Lookup evidence for vuln/product pair
app.MapGet("/evidence/vex/lookup", async (
// GET /v1/vex/evidence/chunks
app.MapGet("/v1/vex/evidence/chunks", (
HttpContext context,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IVexObservationProjectionService projectionService,
TimeProvider timeProvider,
[FromQuery] string vulnerabilityId,
[FromQuery] string productKey,
[FromQuery] int? limit,
CancellationToken cancellationToken) =>
ChunkTelemetry chunkTelemetry) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
@@ -205,572 +81,16 @@ public static class EvidenceEndpoints
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(vulnerabilityId) || string.IsNullOrWhiteSpace(productKey))
{
return Results.BadRequest(new { error = new { code = "ERR_PARAMS", message = "vulnerabilityId and productKey are required" } });
}
var take = Math.Clamp(limit.GetValueOrDefault(100), 1, 500);
var request = new VexObservationProjectionRequest(
tenant,
vulnerabilityId.Trim(),
productKey.Trim(),
ImmutableHashSet<string>.Empty,
ImmutableHashSet<VexClaimStatus>.Empty,
null,
take);
var result = await projectionService.QueryAsync(request, cancellationToken).ConfigureAwait(false);
var items = result.Statements.Select(s => new VexEvidenceItem(
ObservationId: s.ObservationId,
ProviderId: s.ProviderId,
Status: s.Status.ToString().ToLowerInvariant(),
Justification: s.Justification?.ToString().ToLowerInvariant(),
FirstSeen: s.FirstSeen,
LastSeen: s.LastSeen,
DocumentDigest: s.Document.Digest,
Verification: s.Signature is null ? null : new VexEvidenceVerificationMetadata(
Verified: s.Signature.VerifiedAt.HasValue,
VerifiedAt: s.Signature.VerifiedAt,
SignatureType: s.Signature.Type,
KeyId: s.Signature.KeyId,
Issuer: s.Signature.Issuer,
TransparencyRef: null))).ToList();
var response = new VexEvidenceLookupResponse(
VulnerabilityId: vulnerabilityId.Trim(),
ProductKey: productKey.Trim(),
EvidenceItems: items,
QueriedAt: timeProvider.GetUtcNow());
return Results.Ok(response);
}).WithName("LookupVexEvidence");
// GET /vuln/evidence/vex/{advisory_key} - Get evidence by advisory key (EXCITITOR-VULN-29-002)
app.MapGet("/vuln/evidence/vex/{advisory_key}", async (
HttpContext context,
string advisory_key,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IMongoDatabase database,
TimeProvider timeProvider,
[FromQuery] int? limit,
[FromQuery] string? cursor,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(advisory_key))
{
NormalizationTelemetry.RecordAdvisoryKeyCanonicalizeError(tenant, "empty_key");
return Results.BadRequest(new { error = new { code = "ERR_ADVISORY_KEY", message = "advisory_key is required" } });
}
var stopwatch = Stopwatch.StartNew();
// Canonicalize the advisory key using VexAdvisoryKeyCanonicalizer
var canonicalizer = new VexAdvisoryKeyCanonicalizer();
VexCanonicalAdvisoryKey canonicalKey;
try
{
canonicalKey = canonicalizer.Canonicalize(advisory_key.Trim());
NormalizationTelemetry.RecordAdvisoryKeyCanonicalization(tenant, canonicalKey);
}
catch (ArgumentException ex)
{
NormalizationTelemetry.RecordAdvisoryKeyCanonicalizeError(tenant, "invalid_format", advisory_key);
return Results.BadRequest(new { error = new { code = "ERR_INVALID_ADVISORY_KEY", message = ex.Message } });
}
var take = Math.Clamp(limit.GetValueOrDefault(100), 1, 500);
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Statements);
var builder = Builders<BsonDocument>.Filter;
// Build filter to match by vulnerability ID (case-insensitive)
// Try original key, canonical key, and all aliases
var vulnerabilityFilters = new List<FilterDefinition<BsonDocument>>
{
builder.Regex("VulnerabilityId", new BsonRegularExpression($"^{EscapeRegex(advisory_key.Trim())}$", "i"))
};
// Add canonical key if different
if (!string.Equals(canonicalKey.AdvisoryKey, advisory_key.Trim(), StringComparison.OrdinalIgnoreCase))
{
vulnerabilityFilters.Add(builder.Regex("VulnerabilityId", new BsonRegularExpression($"^{EscapeRegex(canonicalKey.AdvisoryKey)}$", "i")));
}
// Add original ID if available
if (canonicalKey.OriginalId is { } originalId &&
!string.Equals(originalId, advisory_key.Trim(), StringComparison.OrdinalIgnoreCase))
{
vulnerabilityFilters.Add(builder.Regex("VulnerabilityId", new BsonRegularExpression($"^{EscapeRegex(originalId)}$", "i")));
}
var filter = builder.Or(vulnerabilityFilters);
// Apply cursor-based pagination if provided
if (!string.IsNullOrWhiteSpace(cursor) && TryDecodeCursor(cursor, out var cursorTime, out var cursorId))
{
var ltTime = builder.Lt("InsertedAt", cursorTime);
var eqTimeLtId = builder.And(
builder.Eq("InsertedAt", cursorTime),
builder.Lt("_id", ObjectId.Parse(cursorId)));
filter = builder.And(filter, builder.Or(ltTime, eqTimeLtId));
}
var sort = Builders<BsonDocument>.Sort.Descending("InsertedAt").Descending("_id");
var documents = await collection
.Find(filter)
.Sort(sort)
.Limit(take)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var now = timeProvider.GetUtcNow();
var statements = new List<VexAdvisoryStatementResponse>();
foreach (var doc in documents)
{
var provenance = new VexAdvisoryProvenanceResponse(
DocumentDigest: doc.GetValue("Document", BsonNull.Value).IsBsonDocument
? doc["Document"].AsBsonDocument.GetValue("Digest", BsonNull.Value).AsString ?? string.Empty
: string.Empty,
DocumentFormat: doc.GetValue("Document", BsonNull.Value).IsBsonDocument
? doc["Document"].AsBsonDocument.GetValue("Format", BsonNull.Value).AsString ?? "unknown"
: "unknown",
SourceUri: doc.GetValue("Document", BsonNull.Value).IsBsonDocument
? doc["Document"].AsBsonDocument.GetValue("SourceUri", BsonNull.Value).AsString ?? string.Empty
: string.Empty,
Revision: doc.GetValue("Document", BsonNull.Value).IsBsonDocument
? doc["Document"].AsBsonDocument.GetValue("Revision", BsonNull.Value).AsString
: null,
InsertedAt: doc.GetValue("InsertedAt", BsonNull.Value).IsBsonDateTime
? new DateTimeOffset(doc["InsertedAt"].ToUniversalTime(), TimeSpan.Zero)
: now);
VexAdvisoryAttestationResponse? attestation = null;
if (doc.GetValue("Document", BsonNull.Value).IsBsonDocument)
{
var docSection = doc["Document"].AsBsonDocument;
if (docSection.Contains("Signature") && !docSection["Signature"].IsBsonNull)
{
var sig = docSection["Signature"].AsBsonDocument;
var sigType = sig.GetValue("Type", BsonNull.Value).AsString;
if (!string.IsNullOrWhiteSpace(sigType))
{
attestation = new VexAdvisoryAttestationResponse(
SignatureType: sigType,
Issuer: sig.GetValue("Issuer", BsonNull.Value).AsString,
Subject: sig.GetValue("Subject", BsonNull.Value).AsString,
KeyId: sig.GetValue("KeyId", BsonNull.Value).AsString,
VerifiedAt: sig.Contains("VerifiedAt") && !sig["VerifiedAt"].IsBsonNull
? new DateTimeOffset(sig["VerifiedAt"].ToUniversalTime(), TimeSpan.Zero)
: null,
TransparencyLogRef: sig.GetValue("TransparencyLogReference", BsonNull.Value).AsString,
TrustWeight: sig.Contains("TrustWeight") && !sig["TrustWeight"].IsBsonNull
? (decimal)sig["TrustWeight"].ToDouble()
: null,
TrustTier: DeriveTrustTier(sig.GetValue("TrustIssuerId", BsonNull.Value).AsString));
}
}
}
var productDoc = doc.GetValue("Product", BsonNull.Value).IsBsonDocument
? doc["Product"].AsBsonDocument
: null;
var product = new VexAdvisoryProductResponse(
Key: productDoc?.GetValue("Key", BsonNull.Value).AsString ?? string.Empty,
Name: productDoc?.GetValue("Name", BsonNull.Value).AsString,
Version: productDoc?.GetValue("Version", BsonNull.Value).AsString,
Purl: productDoc?.GetValue("Purl", BsonNull.Value).AsString,
Cpe: productDoc?.GetValue("Cpe", BsonNull.Value).AsString);
statements.Add(new VexAdvisoryStatementResponse(
StatementId: doc.GetValue("_id", BsonNull.Value).ToString() ?? string.Empty,
ProviderId: doc.GetValue("ProviderId", BsonNull.Value).AsString ?? string.Empty,
Product: product,
Status: doc.GetValue("Status", BsonNull.Value).AsString ?? "unknown",
Justification: doc.GetValue("Justification", BsonNull.Value).AsString,
Detail: doc.GetValue("Detail", BsonNull.Value).AsString,
FirstSeen: doc.GetValue("FirstSeen", BsonNull.Value).IsBsonDateTime
? new DateTimeOffset(doc["FirstSeen"].ToUniversalTime(), TimeSpan.Zero)
: now,
LastSeen: doc.GetValue("LastSeen", BsonNull.Value).IsBsonDateTime
? new DateTimeOffset(doc["LastSeen"].ToUniversalTime(), TimeSpan.Zero)
: now,
Provenance: provenance,
Attestation: attestation));
}
var aliases = canonicalKey.Links
.Select(link => new VexAdvisoryLinkResponse(link.Identifier, link.Type, link.IsOriginal))
.ToList();
stopwatch.Stop();
NormalizationTelemetry.RecordEvidenceRetrieval(
tenant,
"success",
statements.Count,
stopwatch.Elapsed.TotalSeconds);
var response = new VexAdvisoryEvidenceResponse(
AdvisoryKey: advisory_key.Trim(),
CanonicalKey: canonicalKey.AdvisoryKey,
Scope: canonicalKey.Scope.ToString().ToLowerInvariant(),
Aliases: aliases,
Statements: statements,
QueriedAt: now,
TotalCount: statements.Count);
return Results.Ok(response);
}).WithName("GetVexAdvisoryEvidence");
// GET /evidence/vex/locker/{bundleId}
app.MapGet("/evidence/vex/locker/{bundleId}", async (
HttpContext context,
string bundleId,
[FromQuery] string? generation,
IOptions<VexStorageOptions> storageOptions,
IOptions<AirgapOptions> airgapOptions,
[FromServices] IAirgapImportStore airgapImportStore,
[FromServices] IVexHashingService hashingService,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(bundleId))
{
return Results.BadRequest(new { error = new { code = "ERR_BUNDLE_ID", message = "bundleId is required" } });
}
var record = await airgapImportStore.FindByBundleIdAsync(tenant, bundleId.Trim(), generation?.Trim(), cancellationToken)
.ConfigureAwait(false);
if (record is null)
{
return Results.NotFound(new { error = new { code = "ERR_NOT_FOUND", message = "Locker manifest not found" } });
}
// Optional local hash/size computation when locker root is configured
long? manifestSize = null;
long? evidenceSize = null;
string? evidenceHash = null;
var lockerRoot = airgapOptions.Value.LockerRootPath;
if (!string.IsNullOrWhiteSpace(lockerRoot))
{
TryHashFile(lockerRoot, record.PortableManifestPath, hashingService, out var manifestHash, out manifestSize);
if (!string.IsNullOrWhiteSpace(manifestHash))
{
record.PortableManifestHash = manifestHash!;
}
TryHashFile(lockerRoot, record.EvidenceLockerPath, hashingService, out evidenceHash, out evidenceSize);
}
var timeline = record.Timeline
.OrderBy(entry => entry.CreatedAt)
.Select(entry => new VexEvidenceLockerTimelineEntry(
entry.EventType,
entry.CreatedAt,
entry.ErrorCode,
entry.Message,
entry.StalenessSeconds))
.ToList();
var response = new VexEvidenceLockerResponse(
record.BundleId,
record.MirrorGeneration,
record.TenantId,
record.Publisher,
record.PayloadHash,
record.PortableManifestPath,
record.PortableManifestHash,
record.EvidenceLockerPath,
evidenceHash,
manifestSize,
evidenceSize,
record.ImportedAt,
record.Timeline.FirstOrDefault()?.StalenessSeconds,
record.TransparencyLog,
timeline);
return Results.Ok(response);
}).WithName("GetVexEvidenceLockerManifest");
// GET /evidence/vex/locker/{bundleId}/manifest/file
app.MapGet("/evidence/vex/locker/{bundleId}/manifest/file", async (
HttpContext context,
string bundleId,
[FromQuery] string? generation,
IOptions<VexStorageOptions> storageOptions,
IOptions<AirgapOptions> airgapOptions,
[FromServices] IAirgapImportStore airgapImportStore,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
{
return tenantError;
}
var root = airgapOptions.Value.LockerRootPath;
if (string.IsNullOrWhiteSpace(root))
{
return Results.NotFound(new { error = new { code = "ERR_LOCKER_ROOT", message = "LockerRootPath is not configured" } });
}
var record = await airgapImportStore.FindByBundleIdAsync(tenant, bundleId.Trim(), generation?.Trim(), cancellationToken)
.ConfigureAwait(false);
if (record is null)
{
return Results.NotFound(new { error = new { code = "ERR_NOT_FOUND", message = "Locker manifest not found" } });
}
if (!TryResolveLockerFile(root, record.PortableManifestPath, out var fullPath))
{
return Results.NotFound(new { error = new { code = "ERR_MANIFEST_FILE", message = "Manifest file not available" } });
}
var (digest, size) = ComputeFileHash(fullPath);
// Quote the ETag so HttpClient parses it into response.Headers.ETag.
context.Response.Headers.ETag = $"\"{digest}\"";
context.Response.ContentType = "application/json";
context.Response.ContentLength = size;
return Results.File(fullPath, "application/json");
}).WithName("GetVexEvidenceLockerManifestFile");
// GET /evidence/vex/locker/{bundleId}/evidence/file
app.MapGet("/evidence/vex/locker/{bundleId}/evidence/file", async (
HttpContext context,
string bundleId,
[FromQuery] string? generation,
IOptions<VexStorageOptions> storageOptions,
IOptions<AirgapOptions> airgapOptions,
[FromServices] IAirgapImportStore airgapImportStore,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
{
return tenantError;
}
var root = airgapOptions.Value.LockerRootPath;
if (string.IsNullOrWhiteSpace(root))
{
return Results.NotFound(new { error = new { code = "ERR_LOCKER_ROOT", message = "LockerRootPath is not configured" } });
}
var record = await airgapImportStore.FindByBundleIdAsync(tenant, bundleId.Trim(), generation?.Trim(), cancellationToken)
.ConfigureAwait(false);
if (record is null)
{
return Results.NotFound(new { error = new { code = "ERR_NOT_FOUND", message = "Evidence file not found" } });
}
if (!TryResolveLockerFile(root, record.EvidenceLockerPath, out var fullPath))
{
return Results.NotFound(new { error = new { code = "ERR_EVIDENCE_FILE", message = "Evidence file not available" } });
}
var (digest, size) = ComputeFileHash(fullPath);
// Quote the ETag so HttpClient parses it into response.Headers.ETag.
context.Response.Headers.ETag = $"\"{digest}\"";
context.Response.ContentType = "application/x-ndjson";
context.Response.ContentLength = size;
return Results.File(fullPath, "application/x-ndjson");
}).WithName("GetVexEvidenceLockerEvidenceFile");
}
private static void TryHashFile(string root, string relativePath, IVexHashingService hashingService, out string? digest, out long? size)
{
digest = null;
size = null;
try
{
if (string.IsNullOrWhiteSpace(relativePath))
{
return;
}
if (!TryResolveLockerFile(root, relativePath, out var fullPath))
{
return;
}
var data = File.ReadAllBytes(fullPath);
digest = hashingService.ComputeHash(data, "sha256");
size = data.LongLength;
}
catch
{
// Ignore I/O errors and continue with stored metadata
}
}
private static bool TryResolveLockerFile(string root, string relativePath, out string fullPath)
{
fullPath = string.Empty;
if (string.IsNullOrWhiteSpace(root) || string.IsNullOrWhiteSpace(relativePath))
{
return false;
}
var rootFull = Path.GetFullPath(root);
var candidate = Path.GetFullPath(Path.Combine(rootFull, relativePath));
if (!candidate.StartsWith(rootFull, StringComparison.OrdinalIgnoreCase))
{
return false;
}
if (!File.Exists(candidate))
{
return false;
}
fullPath = candidate;
return true;
}
private static (string Digest, long SizeBytes) ComputeFileHash(string path)
{
using var stream = File.OpenRead(path);
using var sha = SHA256.Create();
var hashBytes = sha.ComputeHash(stream);
var digest = "sha256:" + Convert.ToHexString(hashBytes).ToLowerInvariant();
var size = new FileInfo(path).Length;
return (digest, size);
}
private static bool TryResolveTenant(HttpContext context, VexStorageOptions options, out string tenant, out IResult? problem)
{
tenant = options.DefaultTenant;
problem = null;
if (context.Request.Headers.TryGetValue("X-Stella-Tenant", out var headerValues) && headerValues.Count > 0)
{
var requestedTenant = headerValues[0]?.Trim();
if (string.IsNullOrEmpty(requestedTenant))
{
problem = Results.BadRequest(new { error = new { code = "ERR_TENANT", message = "X-Stella-Tenant header must not be empty" } });
return false;
}
if (!string.Equals(requestedTenant, options.DefaultTenant, StringComparison.OrdinalIgnoreCase))
{
problem = Results.Json(
new { error = new { code = "ERR_TENANT_FORBIDDEN", message = $"Tenant '{requestedTenant}' is not allowed" } },
statusCode: StatusCodes.Status403Forbidden);
return false;
}
tenant = requestedTenant;
}
return true;
}
private static bool TryDecodeCursor(string cursor, out DateTime timestamp, out string id)
{
timestamp = default;
id = string.Empty;
try
{
var payload = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(cursor));
var parts = payload.Split('|');
if (parts.Length != 2)
{
return false;
}
if (!DateTimeOffset.TryParse(parts[0], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed))
{
return false;
}
timestamp = parsed.UtcDateTime;
id = parts[1];
return true;
}
catch
{
return false;
}
}
private static string EncodeCursor(DateTime timestamp, string id)
{
var payload = FormattableString.Invariant($"{timestamp:O}|{id}");
return Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(payload));
}
private static string EscapeRegex(string input)
{
// Escape special regex characters for safe use in MongoDB regex
return System.Text.RegularExpressions.Regex.Escape(input);
}
private static string? DeriveTrustTier(string? issuerId)
{
if (string.IsNullOrWhiteSpace(issuerId))
{
return null;
}
var lowerIssuerId = issuerId.ToLowerInvariant();
if (lowerIssuerId.Contains("vendor") || lowerIssuerId.Contains("upstream"))
{
return "vendor";
}
if (lowerIssuerId.Contains("distro") || lowerIssuerId.Contains("rhel") ||
lowerIssuerId.Contains("ubuntu") || lowerIssuerId.Contains("debian"))
{
return "distro-trusted";
}
if (lowerIssuerId.Contains("community") || lowerIssuerId.Contains("oss"))
{
return "community";
}
return "other";
chunkTelemetry.RecordIngested(tenant, null, "unavailable", "storage-migration", 0, 0, 0);
return Results.Problem(
detail: "Evidence chunk streaming is temporarily unavailable during Postgres migration (Mongo/BSON removed).",
statusCode: StatusCodes.Status503ServiceUnavailable,
title: "Service unavailable");
}).WithName("GetVexEvidenceChunks");
}
}

18
src/Excititor/StellaOps.Excititor.WebService/Program.cs
View File

@@ -48,6 +48,9 @@ services.AddOptions<VexStorageOptions>()
.ValidateOnStart();
services.AddExcititorPostgresStorage(configuration);
services.TryAddSingleton<IVexProviderStore, InMemoryVexProviderStore>();
services.TryAddSingleton<IVexConnectorStateRepository, InMemoryVexConnectorStateRepository>();
services.TryAddSingleton<IVexClaimStore, InMemoryVexClaimStore>();
services.AddCsafNormalizer();
services.AddCycloneDxNormalizer();
services.AddOpenVexNormalizer();
@@ -146,13 +149,12 @@ app.UseObservabilityHeaders();
app.MapGet("/excititor/status", async (HttpContext context,
IEnumerable<IVexArtifactStore> artifactStores,
IOptions<VexStorageOptions> mongoOptions,
IOptions<VexStorageOptions> storageOptions,
TimeProvider timeProvider) =>
{
var payload = new StatusResponse(
timeProvider.GetUtcNow(),
mongoOptions.Value.RawBucketName,
mongoOptions.Value.GridFsInlineThresholdBytes,
storageOptions.Value.InlineThresholdBytes,
artifactStores.Select(store => store.GetType().Name).ToArray());
context.Response.ContentType = "application/json";
@@ -216,8 +218,7 @@ app.MapGet("/openapi/excititor.json", () =>
value = new
{
timeUtc = "2025-11-24T00:00:00Z",
mongoBucket = "vex-raw",
gridFsInlineThresholdBytes = 1048576,
inlineThreshold = 1048576,
artifactStores = new[] { "S3ArtifactStore", "OfflineBundleArtifactStore" }
}
}
@@ -892,12 +893,11 @@ app.MapGet("/openapi/excititor.json", () =>
["StatusResponse"] = new
{
type = "object",
required = new[] { "timeUtc", "mongoBucket", "artifactStores" },
required = new[] { "timeUtc", "artifactStores", "inlineThreshold" },
properties = new Dictionary<string, object>
{
["timeUtc"] = new { type = "string", format = "date-time" },
["mongoBucket"] = new { type = "string" },
["gridFsInlineThresholdBytes"] = new { type = "integer", format = "int64" },
["inlineThreshold"] = new { type = "integer", format = "int64" },
["artifactStores"] = new { type = "array", items = new { type = "string" } }
}
},
@@ -2270,7 +2270,7 @@ internal sealed record ExcititorTimelineEvent(
public partial class Program;
internal sealed record StatusResponse(DateTimeOffset UtcNow, string MongoBucket, int InlineThreshold, string[] ArtifactStores);
internal sealed record StatusResponse(DateTimeOffset UtcNow, int InlineThreshold, string[] ArtifactStores);
internal sealed record VexStatementIngestRequest(IReadOnlyList<VexStatementEntry> Statements);

233
src/Excititor/StellaOps.Excititor.WebService/Services/ExcititorHealthService.cs
View File

@@ -1,48 +1,49 @@
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using MongoDB.Bson;
using MongoDB.Driver;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.Core.Observations;
using StellaOps.Excititor.WebService.Options;
namespace StellaOps.Excititor.WebService.Services;
internal sealed class ExcititorHealthService
{
private const string RetrievedAtField = "RetrievedAt";
private const string MetadataField = "Metadata";
private const string CalculatedAtField = "CalculatedAt";
private const string ConflictsField = "Conflicts";
private const string ConflictStatusField = "Status";
private readonly IMongoDatabase _database;
private readonly IVexRawStore _rawStore;
private readonly IVexLinksetStore _linksetStore;
private readonly IVexProviderStore _providerStore;
private readonly IVexConnectorStateRepository _stateRepository;
private readonly IReadOnlyDictionary<string, VexConnectorDescriptor> _connectors;
private readonly TimeProvider _timeProvider;
private readonly ExcititorObservabilityOptions _options;
private readonly ILogger<ExcititorHealthService> _logger;
private readonly string _defaultTenant;
public ExcititorHealthService(
IMongoDatabase database,
IVexRawStore rawStore,
IVexLinksetStore linksetStore,
IVexProviderStore providerStore,
IVexConnectorStateRepository stateRepository,
IEnumerable<IVexConnector> connectors,
TimeProvider timeProvider,
IOptions<ExcititorObservabilityOptions> options,
IOptions<VexStorageOptions> storageOptions,
ILogger<ExcititorHealthService> logger)
{
_database = database ?? throw new ArgumentNullException(nameof(database));
_rawStore = rawStore ?? throw new ArgumentNullException(nameof(rawStore));
_linksetStore = linksetStore ?? throw new ArgumentNullException(nameof(linksetStore));
_providerStore = providerStore ?? throw new ArgumentNullException(nameof(providerStore));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_timeProvider = timeProvider ?? TimeProvider.System;
_options = options?.Value ?? new ExcititorObservabilityOptions();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
var storage = storageOptions?.Value ?? new VexStorageOptions();
_defaultTenant = string.IsNullOrWhiteSpace(storage.DefaultTenant)
? "default"
: storage.DefaultTenant.Trim();
if (connectors is null)
{
@@ -158,7 +159,7 @@ internal sealed class ExcititorHealthService
private LinkHealthSection BuildLinkSection(DateTimeOffset now, LinkSnapshot snapshot)
{
TimeSpan? lag = null;
if (snapshot.LastConsensusAt is { } calculatedAt)
if (snapshot.LastUpdatedAt is { } calculatedAt)
{
lag = now - calculatedAt;
if (lag < TimeSpan.Zero)
@@ -174,7 +175,7 @@ internal sealed class ExcititorHealthService
return new LinkHealthSection(
status,
snapshot.LastConsensusAt,
snapshot.LastUpdatedAt,
lag?.TotalSeconds,
snapshot.TotalDocuments,
snapshot.DocumentsWithConflicts);
@@ -271,47 +272,36 @@ internal sealed class ExcititorHealthService
var window = _options.GetPositive(_options.SignatureWindow, TimeSpan.FromHours(12));
var windowStart = now - window;
var collection = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
var filter = Builders<BsonDocument>.Filter.Gte(RetrievedAtField, windowStart.UtcDateTime);
var projection = Builders<BsonDocument>.Projection
.Include(MetadataField)
.Include(RetrievedAtField);
List<BsonDocument> documents;
try
{
documents = await collection
.Find(filter)
.Project(projection)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to load signature window metrics.");
documents = new List<BsonDocument>();
}
var page = await _rawStore.QueryAsync(
new VexRawQuery(
_defaultTenant,
Array.Empty<string>(),
Array.Empty<string>(),
Array.Empty<VexDocumentFormat>(),
windowStart,
until: null,
Cursor: null,
Limit: 500),
cancellationToken).ConfigureAwait(false);
var evaluated = 0;
var withSignatures = 0;
var verified = 0;
foreach (var document in documents)
foreach (var document in page.Items)
{
evaluated++;
if (!document.TryGetValue(MetadataField, out var metadataValue) ||
metadataValue is not BsonDocument metadata ||
metadata.ElementCount == 0)
{
continue;
}
if (TryGetBoolean(metadata, "signature.present", out var present) && present)
var metadata = document.Metadata;
if (metadata.TryGetValue("signature.present", out var presentValue) &&
bool.TryParse(presentValue, out var present) &&
present)
{
withSignatures++;
}
if (TryGetBoolean(metadata, "signature.verified", out var verifiedFlag) && verifiedFlag)
if (metadata.TryGetValue("signature.verified", out var verifiedValue) &&
bool.TryParse(verifiedValue, out var verifiedFlag) &&
verifiedFlag)
{
verified++;
}
@@ -322,80 +312,43 @@ internal sealed class ExcititorHealthService
private async Task<LinkSnapshot> LoadLinkSnapshotAsync(CancellationToken cancellationToken)
{
var collection = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Consensus);
BsonDocument? latest = null;
try
{
latest = await collection
.Find(Builders<BsonDocument>.Filter.Empty)
.Sort(Builders<BsonDocument>.Sort.Descending(CalculatedAtField))
.Project(Builders<BsonDocument>.Projection.Include(CalculatedAtField))
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to read latest consensus document.");
}
DateTimeOffset? lastConsensusAt = null;
if (latest is not null &&
latest.TryGetValue(CalculatedAtField, out var dateValue))
{
var utc = TryReadDateTime(dateValue);
if (utc is not null)
{
lastConsensusAt = new DateTimeOffset(utc.Value, TimeSpan.Zero);
}
}
long totalDocuments = 0;
long conflictDocuments = 0;
DateTimeOffset? lastUpdated = null;
try
{
totalDocuments = await collection.EstimatedDocumentCountAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
conflictDocuments = await collection.CountDocumentsAsync(
Builders<BsonDocument>.Filter.Exists($"{ConflictsField}.0"),
cancellationToken: cancellationToken)
.ConfigureAwait(false);
totalDocuments = await _linksetStore.CountAsync(_defaultTenant, cancellationToken).ConfigureAwait(false);
conflictDocuments = await _linksetStore.CountWithConflictsAsync(_defaultTenant, cancellationToken).ConfigureAwait(false);
var conflictSample = await _linksetStore.FindWithConflictsAsync(_defaultTenant, 1, cancellationToken).ConfigureAwait(false);
if (conflictSample.Count > 0)
{
lastUpdated = conflictSample[0].UpdatedAt;
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to compute consensus counts.");
_logger.LogWarning(ex, "Failed to compute linkset counts.");
}
return new LinkSnapshot(lastConsensusAt, totalDocuments, conflictDocuments);
return new LinkSnapshot(lastUpdated, totalDocuments, conflictDocuments);
}
private async Task<ConflictSnapshot> LoadConflictSnapshotAsync(DateTimeOffset now, CancellationToken cancellationToken)
{
var window = _options.GetPositive(_options.ConflictTrendWindow, TimeSpan.FromHours(24));
var windowStart = now - window;
var collection = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Consensus);
var filter = Builders<BsonDocument>.Filter.And(
Builders<BsonDocument>.Filter.Gte(CalculatedAtField, windowStart.UtcDateTime),
Builders<BsonDocument>.Filter.Exists($"{ConflictsField}.0"));
var projection = Builders<BsonDocument>.Projection
.Include(CalculatedAtField)
.Include(ConflictsField);
List<BsonDocument> documents;
IReadOnlyList<VexLinkset> linksets;
try
{
documents = await collection
.Find(filter)
.Project(projection)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
// Sample conflicted linksets (ordered by updated_at DESC in Postgres implementation)
linksets = await _linksetStore.FindWithConflictsAsync(_defaultTenant, 500, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to load conflict trend window.");
documents = new List<BsonDocument>();
linksets = Array.Empty<VexLinkset>();
}
var byStatus = new Dictionary<string, long>(StringComparer.OrdinalIgnoreCase);
@@ -405,47 +358,31 @@ internal sealed class ExcititorHealthService
var bucketMinutes = Math.Max(1, _options.ConflictTrendBucketMinutes);
var bucketTicks = TimeSpan.FromMinutes(bucketMinutes).Ticks;
foreach (var doc in documents)
foreach (var linkset in linksets)
{
if (!doc.TryGetValue(ConflictsField, out var conflictsValue) ||
conflictsValue is not BsonArray conflicts ||
conflicts.Count == 0)
if (linkset.Disagreements.Count == 0)
{
continue;
}
docsWithConflicts++;
totalConflicts += conflicts.Count;
totalConflicts += linkset.Disagreements.Count;
foreach (var conflictValue in conflicts.OfType<BsonDocument>())
foreach (var disagreement in linkset.Disagreements)
{
var status = conflictValue.TryGetValue(ConflictStatusField, out var statusValue) && statusValue.IsString
? statusValue.AsString
: "unknown";
if (string.IsNullOrWhiteSpace(status))
{
status = "unknown";
}
var status = string.IsNullOrWhiteSpace(disagreement.Status)
? "unknown"
: disagreement.Status;
byStatus[status] = byStatus.TryGetValue(status, out var current)
? current + 1
: 1;
}
if (doc.TryGetValue(CalculatedAtField, out var calculatedValue))
{
var utc = TryReadDateTime(calculatedValue);
if (utc is null)
{
continue;
}
var alignedTicks = AlignTicks(utc.Value, bucketTicks);
timeline[alignedTicks] = timeline.TryGetValue(alignedTicks, out var current)
? current + conflicts.Count
: conflicts.Count;
}
var alignedTicks = AlignTicks(linkset.UpdatedAt.UtcDateTime, bucketTicks);
timeline[alignedTicks] = timeline.TryGetValue(alignedTicks, out var currentCount)
? currentCount + linkset.Disagreements.Count
: linkset.Disagreements.Count;
}
var trend = timeline
@@ -541,54 +478,6 @@ internal sealed class ExcititorHealthService
return ticks - (ticks % bucketTicks);
}
private static DateTime? TryReadDateTime(BsonValue value)
{
if (value is null)
{
return null;
}
if (value.IsBsonDateTime)
{
return value.AsBsonDateTime.ToUniversalTime();
}
if (value.IsString &&
DateTime.TryParse(
value.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal,
out var parsed))
{
return DateTime.SpecifyKind(parsed, DateTimeKind.Utc);
}
return null;
}
private static bool TryGetBoolean(BsonDocument document, string key, out bool value)
{
value = default;
if (!document.TryGetValue(key, out var bsonValue))
{
return false;
}
if (bsonValue.IsBoolean)
{
value = bsonValue.AsBoolean;
return true;
}
if (bsonValue.IsString && bool.TryParse(bsonValue.AsString, out var parsed))
{
value = parsed;
return true;
}
return false;
}
private static VexConnectorDescriptor DescribeConnector(IVexConnector connector)
=> connector switch
{
@@ -596,7 +485,7 @@ internal sealed class ExcititorHealthService
_ => new VexConnectorDescriptor(connector.Id, connector.Kind, connector.Id)
};
private sealed record LinkSnapshot(DateTimeOffset? LastConsensusAt, long TotalDocuments, long DocumentsWithConflicts);
private sealed record LinkSnapshot(DateTimeOffset? LastUpdatedAt, long TotalDocuments, long DocumentsWithConflicts);
private sealed record ConflictSnapshot(
DateTimeOffset WindowStart,

32
src/Excititor/StellaOps.Excititor.WebService/Services/VexIngestOrchestrator.cs
View File

@@ -5,7 +5,6 @@ using System.Globalization;
using System.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Storage;
@@ -151,7 +150,7 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
foreach (var handle in handles)
{
var result = await ExecuteRunAsync(runId, handle, since, options.Force, session, cancellationToken).ConfigureAwait(false);
var result = await ExecuteRunAsync(runId, handle, since, options.Force, cancellationToken).ConfigureAwait(false);
results.Add(result);
}
@@ -174,8 +173,8 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
foreach (var handle in handles)
{
var since = await ResolveResumeSinceAsync(handle.Descriptor.Id, options.Checkpoint, session, cancellationToken).ConfigureAwait(false);
var result = await ExecuteRunAsync(runId, handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
var since = await ResolveResumeSinceAsync(handle.Descriptor.Id, options.Checkpoint, cancellationToken).ConfigureAwait(false);
var result = await ExecuteRunAsync(runId, handle, since, force: false, cancellationToken).ConfigureAwait(false);
results.Add(result);
}
@@ -201,14 +200,14 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
{
try
{
var state = await _stateRepository.GetAsync(handle.Descriptor.Id, cancellationToken, session).ConfigureAwait(false);
var state = await _stateRepository.GetAsync(handle.Descriptor.Id, cancellationToken).ConfigureAwait(false);
var lastUpdated = state?.LastUpdated;
var stale = threshold.HasValue && (lastUpdated is null || lastUpdated < threshold.Value);
if (stale || state is null)
{
var since = stale ? threshold : lastUpdated;
var result = await ExecuteRunAsync(runId, handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
var result = await ExecuteRunAsync(runId, handle, since, force: false, cancellationToken).ConfigureAwait(false);
results.Add(new ReconcileProviderResult(
handle.Descriptor.Id,
result.Status,
@@ -271,14 +270,14 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
private async Task EnsureProviderRegistrationAsync(VexConnectorDescriptor descriptor, CancellationToken cancellationToken)
{
var existing = await _providerStore.FindAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false);
var existing = await _providerStore.FindAsync(descriptor.Id, cancellationToken).ConfigureAwait(false);
if (existing is not null)
{
return;
}
var provider = new VexProvider(descriptor.Id, descriptor.DisplayName, descriptor.Kind);
await _providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false);
await _providerStore.SaveAsync(provider, cancellationToken).ConfigureAwait(false);
}
private async Task<ProviderRunResult> ExecuteRunAsync(
@@ -286,7 +285,6 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
ConnectorHandle handle,
DateTimeOffset? since,
bool force,
IClientSessionHandle session,
CancellationToken cancellationToken)
{
var providerId = handle.Descriptor.Id;
@@ -304,15 +302,15 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
try
{
await ValidateConnectorAsync(handle, cancellationToken).ConfigureAwait(false);
await EnsureProviderRegistrationAsync(handle.Descriptor, session, cancellationToken).ConfigureAwait(false);
await EnsureProviderRegistrationAsync(handle.Descriptor, cancellationToken).ConfigureAwait(false);
if (force)
{
var resetState = new VexConnectorState(providerId, null, ImmutableArray<string>.Empty);
await _stateRepository.SaveAsync(resetState, cancellationToken, session).ConfigureAwait(false);
await _stateRepository.SaveAsync(resetState, cancellationToken).ConfigureAwait(false);
}
var stateBeforeRun = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
var stateBeforeRun = await _stateRepository.GetAsync(providerId, cancellationToken).ConfigureAwait(false);
var resumeTokens = stateBeforeRun?.ResumeTokens ?? ImmutableDictionary<string, string>.Empty;
var context = new VexConnectorContext(
@@ -337,13 +335,13 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
if (!batch.Claims.IsDefaultOrEmpty && batch.Claims.Length > 0)
{
claims += batch.Claims.Length;
await _claimStore.AppendAsync(batch.Claims, _timeProvider.GetUtcNow(), cancellationToken, session).ConfigureAwait(false);
await _claimStore.AppendAsync(batch.Claims, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
}
}
stopwatch.Stop();
var completedAt = _timeProvider.GetUtcNow();
var stateAfterRun = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
var stateAfterRun = await _stateRepository.GetAsync(providerId, cancellationToken).ConfigureAwait(false);
var checkpoint = stateAfterRun?.DocumentDigests.IsDefaultOrEmpty == false
? stateAfterRun.DocumentDigests[^1]
@@ -413,7 +411,7 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
}
}
private async Task<DateTimeOffset?> ResolveResumeSinceAsync(string providerId, string? checkpoint, IClientSessionHandle session, CancellationToken cancellationToken)
private async Task<DateTimeOffset?> ResolveResumeSinceAsync(string providerId, string? checkpoint, CancellationToken cancellationToken)
{
if (!string.IsNullOrWhiteSpace(checkpoint))
{
@@ -427,14 +425,14 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
}
var digest = checkpoint.Trim();
var document = await _rawStore.FindByDigestAsync(digest, cancellationToken, session).ConfigureAwait(false);
var document = await _rawStore.FindByDigestAsync(digest, cancellationToken).ConfigureAwait(false);
if (document is not null)
{
return document.RetrievedAt;
}
}
var state = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
var state = await _stateRepository.GetAsync(providerId, cancellationToken).ConfigureAwait(false);
return state?.LastUpdated;
}

2
src/Excititor/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj
View File

@@ -17,7 +17,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Storage.Postgres/StellaOps.Excititor.Storage.Postgres.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Export/StellaOps.Excititor.Export.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj" />

2
src/Excititor/StellaOps.Excititor.Worker/AGENTS.md
View File

@@ -30,7 +30,7 @@ Run Excititor background jobs (ingestion, linkset extraction, dedup/idempotency
- Keep timestamps UTC ISO-8601; inject clock/GUID providers for tests.
## Boundaries
- Delegate domain logic to Core and persistence to Storage.Mongo; avoid embedding policy or UI concerns.
- Delegate domain logic to Core and persistence to Storage.Postgres; avoid embedding policy or UI concerns.
- Configuration via appsettings/environment; no hard-coded secrets.
## Ready-to-Start Checklist

1
src/Excititor/StellaOps.Excititor.Worker/Orchestration/VexWorkerOrchestratorClient.cs
View File

@@ -12,7 +12,6 @@ using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Orchestration;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Worker.Options;
namespace StellaOps.Excititor.Worker.Orchestration;

12
src/Excititor/StellaOps.Excititor.Worker/Program.cs
View File

@@ -8,11 +8,12 @@ using StellaOps.Plugin;
using StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Aoc;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.Core.Orchestration;
using StellaOps.Excititor.Formats.CSAF;
using StellaOps.Excititor.Formats.CycloneDX;
using StellaOps.Excititor.Formats.OpenVEX;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Storage.Postgres;
using StellaOps.Excititor.Worker.Auth;
using StellaOps.Excititor.Worker.Options;
using StellaOps.Excititor.Worker.Orchestration;
@@ -43,11 +44,14 @@ services.PostConfigure<VexWorkerOptions>(options =>
});
services.AddRedHatCsafConnector();
services.AddOptions<VexMongoStorageOptions>()
.Bind(configuration.GetSection("Excititor:Storage:Mongo"))
services.AddOptions<VexStorageOptions>()
.Bind(configuration.GetSection("Excititor:Storage"))
.ValidateOnStart();
services.AddExcititorMongoStorage();
services.AddExcititorPostgresStorage(configuration);
services.AddSingleton<IVexProviderStore, InMemoryVexProviderStore>();
services.AddSingleton<IVexConnectorStateRepository, InMemoryVexConnectorStateRepository>();
services.AddSingleton<IVexClaimStore, InMemoryVexClaimStore>();
services.AddCsafNormalizer();
services.AddCycloneDxNormalizer();
services.AddOpenVexNormalizer();

14
src/Excititor/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs
View File

@@ -5,12 +5,10 @@ using System.Security.Cryptography;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
using StellaOps.Plugin;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Orchestration;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Worker.Options;
using StellaOps.Excititor.Worker.Orchestration;
using StellaOps.Excititor.Worker.Signature;
@@ -95,12 +93,6 @@ internal sealed class DefaultVexProviderRunner : IVexProviderRunner
var stateRepository = scopeProvider.GetRequiredService<IVexConnectorStateRepository>();
var normalizerRouter = scopeProvider.GetRequiredService<IVexNormalizerRouter>();
var signatureVerifier = scopeProvider.GetRequiredService<IVexSignatureVerifier>();
var sessionProvider = scopeProvider.GetService<IVexMongoSessionProvider>();
IClientSessionHandle? session = null;
if (sessionProvider is not null)
{
session = await sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
}
var descriptor = connector switch
{
@@ -108,12 +100,12 @@ internal sealed class DefaultVexProviderRunner : IVexProviderRunner
_ => new VexConnectorDescriptor(connector.Id, VexProviderKind.Vendor, connector.Id)
};
var provider = await providerStore.FindAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false)
var provider = await providerStore.FindAsync(descriptor.Id, cancellationToken).ConfigureAwait(false)
?? new VexProvider(descriptor.Id, descriptor.DisplayName, descriptor.Kind);
await providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false);
await providerStore.SaveAsync(provider, cancellationToken).ConfigureAwait(false);
var stateBeforeRun = await stateRepository.GetAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false);
var stateBeforeRun = await stateRepository.GetAsync(descriptor.Id, cancellationToken).ConfigureAwait(false);
var now = _timeProvider.GetUtcNow();
if (stateBeforeRun?.NextEligibleRun is { } nextEligible && nextEligible > now)

1
src/Excititor/StellaOps.Excititor.Worker/Scheduling/VexConsensusRefreshService.cs
View File

@@ -9,7 +9,6 @@ using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Policy;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Worker.Options;
namespace StellaOps.Excititor.Worker.Scheduling;

1
src/Excititor/StellaOps.Excititor.Worker/Signature/VerifyingVexRawDocumentSink.cs
View File

@@ -1,7 +1,6 @@
using System.Collections.Immutable;
using System.Globalization;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
namespace StellaOps.Excititor.Worker.Signature;

4
src/Excititor/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj
View File

@@ -14,12 +14,10 @@
<ProjectReference Include="../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../../Concelier/__Libraries/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj" />
<!-- Temporarily commented out: RedHat CSAF connector blocked by missing Storage.Mongo project -->
<!-- <ProjectReference Include="../__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj" /> -->
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj" />
<!-- Temporarily commented out: Storage.Mongo project not found -->
<!-- <ProjectReference Include="../__Libraries/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj" /> -->
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Storage.Postgres/StellaOps.Excititor.Storage.Postgres.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj" />

6
src/Excititor/StellaOps.Excititor.sln
View File

@@ -13,8 +13,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Aoc", "..\Aoc\__L
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.RawModels", "..\Concelier\__Libraries\StellaOps.Concelier.RawModels\StellaOps.Concelier.RawModels.csproj", "{2D19CC50-EFE9-4015-B4DB-6DFF4E41DB11}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Storage.Mongo", "__Libraries\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj", "{5858415D-8AB4-4E45-B316-580879FD8339}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Export", "__Libraries\StellaOps.Excititor.Export\StellaOps.Excititor.Export.csproj", "{E8B20DD0-9282-4DFD-B363-F0AF7F62AED5}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Policy", "__Libraries\StellaOps.Excititor.Policy\StellaOps.Excititor.Policy.csproj", "{400690F2-466B-4DF0-B495-9015DBBAA046}"
@@ -85,10 +83,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Formats
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Policy.Tests", "__Tests\StellaOps.Excititor.Policy.Tests\StellaOps.Excititor.Policy.Tests.csproj", "{832F539E-17FC-46B4-9E67-39BE5131352D}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Storage.Mongo.Tests", "__Tests\StellaOps.Excititor.Storage.Mongo.Tests\StellaOps.Excititor.Storage.Mongo.Tests.csproj", "{5BB6E9E8-3470-4BFF-94DD-DA3294616C39}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Storage.Mongo", "..\Concelier\__Libraries\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj", "{6507860E-BF0D-4E32-A6AC-49E1CE15E4B7}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Core", "..\Concelier\__Libraries\StellaOps.Concelier.Core\StellaOps.Concelier.Core.csproj", "{D6014A0A-6BF4-45C8-918E-9558A24AAC5B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Models", "..\Concelier\__Libraries\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj", "{13AF13D1-84C3-4D4F-B89A-0653102C3E63}"

Some files were not shown because too many files have changed in this diff Show More