feat: Implement BerkeleyDB reader for RPM databases

- Added BerkeleyDbReader class to read and extract RPM header blobs from BerkeleyDB hash databases. - Implemented methods to detect BerkeleyDB format and extract values, including handling of page sizes and magic numbers. - Added tests for BerkeleyDbReader to ensure correct functionality and header extraction. feat: Add Yarn PnP data tests - Created YarnPnpDataTests to validate package resolution and data loading from Yarn PnP cache. - Implemented tests for resolved keys, package presence, and loading from cache structure. test: Add egg-info package fixtures for Python tests - Created egg-info package fixtures for testing Python analyzers. - Included PKG-INFO, entry_points.txt, and installed-files.txt for comprehensive coverage. test: Enhance RPM database reader tests - Added tests for RpmDatabaseReader to validate fallback to legacy packages when SQLite is missing. - Implemented helper methods to create legacy package files and RPM headers for testing. test: Implement dual signing tests - Added DualSignTests to validate secondary signature addition when configured. - Created stub implementations for crypto providers and key resolvers to facilitate testing. chore: Update CI script for Playwright Chromium installation - Modified ci-console-exports.sh to ensure deterministic Chromium binary installation for console exports tests. - Added checks for Windows compatibility and environment variable setups for Playwright browsers.
2025-12-07 16:24:45 +02:00
parent e3f28a21ab
commit 11597679ed
199 changed files with 9809 additions and 4404 deletions
--- a/.gitea/workflows/concelier-store-aoc-19-005.yml
+++ b/.gitea/workflows/concelier-store-aoc-19-005.yml
@@ -0,0 +1,32 @@
+name: Concelier STORE-AOC-19-005 Dataset
+
+on:
+  workflow_dispatch: {}
+
+jobs:
+  build-dataset:
+    runs-on: ubuntu-22.04
+    env:
+      ARTIFACT_DIR: ${{ github.workspace }}/out/linksets
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: sudo apt-get update && sudo apt-get install -y zstd
+
+      - name: Build dataset tarball
+        run: |
+          chmod +x scripts/concelier/build-store-aoc-19-005-dataset.sh scripts/concelier/test-store-aoc-19-005-dataset.sh
+          scripts/concelier/build-store-aoc-19-005-dataset.sh "${ARTIFACT_DIR}/linksets-stage-backfill.tar.zst"
+
+      - name: Validate dataset
+        run: scripts/concelier/test-store-aoc-19-005-dataset.sh "${ARTIFACT_DIR}/linksets-stage-backfill.tar.zst"
+
+      - name: Upload dataset artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: concelier-store-aoc-19-005-dataset
+          path: |
+            ${ARTIFACT_DIR}/linksets-stage-backfill.tar.zst
+            ${ARTIFACT_DIR}/linksets-stage-backfill.tar.zst.sha256
--- a/.gitea/workflows/console-ci.yml
+++ b/.gitea/workflows/console-ci.yml
@@ -35,8 +35,8 @@ jobs:
      - name: Lint
        run: npm run lint -- --no-progress

-      - name: Unit tests
-        run: npm test -- --watch=false --browsers=ChromeHeadless --no-progress || true
+      - name: Console export specs (targeted)
+        run: bash ./scripts/ci-console-exports.sh

      - name: Build
        run: npm run build -- --configuration=production --progress=false
--- a/.gitea/workflows/wine-csp-build.yml
+++ b/.gitea/workflows/wine-csp-build.yml
@@ -22,20 +22,31 @@ on:
        description: "Version tag (e.g., 2025.10.0-edge)"
        required: false
        default: "2025.10.0-edge"
+      skip_tests:
+        description: "Skip integration tests"
+        required: false
+        default: "false"

 env:
  IMAGE_NAME: registry.stella-ops.org/stellaops/wine-csp
  DOCKERFILE: ops/wine-csp/Dockerfile
  # Wine CSP only supports linux/amd64 (Wine ARM64 has compatibility issues with Windows x64 apps)
  PLATFORMS: linux/amd64
+  PYTHON_VERSION: "3.11"

 jobs:
+  # ===========================================================================
+  # Job 1: Build Docker Image
+  # ===========================================================================
  build:
    name: Build Wine CSP Image
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
+    outputs:
+      image_tag: ${{ steps.version.outputs.tag }}
+      image_digest: ${{ steps.build.outputs.digest }}

    steps:
      - name: Checkout
@@ -48,12 +59,6 @@ jobs:
        with:
          install: true

-      - name: Install syft (SBOM generation)
-        uses: anchore/sbom-action/download-syft@v0
-
-      - name: Install cosign (attestation)
-        uses: sigstore/cosign-installer@v3.7.0
-
      - name: Set version tag
        id: version
        run: |
@@ -74,7 +79,7 @@ jobs:
            type=raw,value=${{ steps.version.outputs.tag }}
            type=sha,format=short

-      - name: Build image (no push)
+      - name: Build image
        id: build
        uses: docker/build-push-action@v6
        with:
@@ -88,66 +93,246 @@ jobs:
          cache-from: type=gha
          cache-to: type=gha,mode=max

-      - name: Test container startup
+      - name: Save image for testing
        run: |
-          set -e
-          echo "Starting Wine CSP container for health check test..."
+          mkdir -p /tmp/images
+          docker save "${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}" | gzip > /tmp/images/wine-csp.tar.gz

-          # Run container in detached mode
+      - name: Upload image artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: wine-csp-image
+          path: /tmp/images/wine-csp.tar.gz
+          retention-days: 1
+
+  # ===========================================================================
+  # Job 2: Integration Tests
+  # ===========================================================================
+  test:
+    name: Integration Tests
+    runs-on: ubuntu-latest
+    needs: build
+    if: ${{ github.event.inputs.skip_tests != 'true' }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Download image artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: wine-csp-image
+          path: /tmp/images
+
+      - name: Load Docker image
+        run: |
+          gunzip -c /tmp/images/wine-csp.tar.gz | docker load
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install test dependencies
+        run: |
+          pip install -r ops/wine-csp/tests/requirements.txt
+
+      - name: Start Wine CSP container
+        id: container
+        run: |
+          echo "Starting Wine CSP container..."
          docker run -d --name wine-csp-test \
            -e WINE_CSP_MODE=limited \
            -e WINE_CSP_LOG_LEVEL=Debug \
            -p 5099:5099 \
-            "${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}"
+            "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}"

-          # Wait for container startup (Wine takes time to initialize)
-          echo "Waiting for container startup (90s max)..."
-          for i in $(seq 1 18); do
-            sleep 5
+          echo "container_id=$(docker ps -q -f name=wine-csp-test)" >> $GITHUB_OUTPUT
+
+      - name: Wait for service startup
+        run: |
+          echo "Waiting for Wine CSP service to be ready (up to 120s)..."
+          for i in $(seq 1 24); do
            if curl -sf http://127.0.0.1:5099/health > /dev/null 2>&1; then
-              echo "Health check passed after $((i * 5))s"
-              break
+              echo "Service ready after $((i * 5))s"
+              exit 0
            fi
            echo "Waiting... ($((i * 5))s elapsed)"
+            sleep 5
          done
+          echo "Service failed to start!"
+          docker logs wine-csp-test
+          exit 1

-          # Final health check
-          echo "Final health check:"
-          curl -sf http://127.0.0.1:5099/health || {
-            echo "Health check failed!"
-            docker logs wine-csp-test
-            exit 1
-          }
+      - name: Run integration tests (pytest)
+        id: pytest
+        run: |
+          mkdir -p test-results
+          export WINE_CSP_URL=http://127.0.0.1:5099

-          # Test status endpoint
-          echo "Testing /status endpoint:"
-          curl -sf http://127.0.0.1:5099/status | jq .
+          pytest ops/wine-csp/tests/test_wine_csp.py \
+            -v \
+            --tb=short \
+            --junitxml=test-results/junit.xml \
+            --timeout=60 \
+            -x \
+            2>&1 | tee test-results/pytest-output.txt

-          # Cleanup
-          docker stop wine-csp-test
-          docker rm wine-csp-test
+      - name: Run shell integration tests
+        if: always()
+        run: |
+          chmod +x ops/wine-csp/tests/run-tests.sh
+          ops/wine-csp/tests/run-tests.sh \
+            --url http://127.0.0.1:5099 \
+            --ci \
+            --verbose || true

-          echo "Container tests passed!"
+      - name: Collect container logs
+        if: always()
+        run: |
+          docker logs wine-csp-test > test-results/container.log 2>&1 || true
+
+      - name: Stop container
+        if: always()
+        run: |
+          docker stop wine-csp-test || true
+          docker rm wine-csp-test || true
+
+      - name: Upload test results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: wine-csp-test-results
+          path: test-results/
+
+      - name: Publish test results
+        uses: mikepenz/action-junit-report@v4
+        if: always()
+        with:
+          report_paths: 'test-results/junit.xml'
+          check_name: 'Wine CSP Integration Tests'
+          fail_on_failure: true
+
+  # ===========================================================================
+  # Job 3: Security Scan
+  # ===========================================================================
+  security:
+    name: Security Scan
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      security-events: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Download image artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: wine-csp-image
+          path: /tmp/images
+
+      - name: Load Docker image
+        run: |
+          gunzip -c /tmp/images/wine-csp.tar.gz | docker load
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}"
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+          severity: 'CRITICAL,HIGH'
+          ignore-unfixed: true
+
+      - name: Upload Trivy scan results
+        uses: github/codeql-action/upload-sarif@v3
+        if: always()
+        with:
+          sarif_file: 'trivy-results.sarif'
+
+      - name: Run Trivy for JSON report
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}"
+          format: 'json'
+          output: 'trivy-results.json'
+          severity: 'CRITICAL,HIGH,MEDIUM'
+
+      - name: Upload Trivy JSON report
+        uses: actions/upload-artifact@v4
+        with:
+          name: wine-csp-security-scan
+          path: trivy-results.json
+
+  # ===========================================================================
+  # Job 4: Generate SBOM
+  # ===========================================================================
+  sbom:
+    name: Generate SBOM
+    runs-on: ubuntu-latest
+    needs: build
+
+    steps:
+      - name: Download image artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: wine-csp-image
+          path: /tmp/images
+
+      - name: Load Docker image
+        run: |
+          gunzip -c /tmp/images/wine-csp.tar.gz | docker load
+
+      - name: Install syft
+        uses: anchore/sbom-action/download-syft@v0

      - name: Generate SBOM (SPDX)
        run: |
          mkdir -p out/sbom
-          syft "${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}" \
+          syft "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}" \
            -o spdx-json=out/sbom/wine-csp.spdx.json

      - name: Generate SBOM (CycloneDX)
        run: |
-          syft "${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}" \
+          syft "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}" \
            -o cyclonedx-json=out/sbom/wine-csp.cdx.json

      - name: Upload SBOM artifacts
        uses: actions/upload-artifact@v4
        with:
-          name: wine-csp-sbom-${{ steps.version.outputs.tag }}
+          name: wine-csp-sbom-${{ needs.build.outputs.image_tag }}
          path: out/sbom/

+  # ===========================================================================
+  # Job 5: Publish (only on main branch or manual trigger)
+  # ===========================================================================
+  publish:
+    name: Publish Image
+    runs-on: ubuntu-latest
+    needs: [build, test, security]
+    if: ${{ (github.event.inputs.push == 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && needs.test.result == 'success' }}
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+
+    steps:
+      - name: Download image artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: wine-csp-image
+          path: /tmp/images
+
+      - name: Load Docker image
+        run: |
+          gunzip -c /tmp/images/wine-csp.tar.gz | docker load
+
+      - name: Install cosign
+        uses: sigstore/cosign-installer@v3.7.0
+
      - name: Login to registry
-        if: ${{ github.event.inputs.push == 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
        uses: docker/login-action@v3
        with:
          registry: registry.stella-ops.org
@@ -155,57 +340,110 @@ jobs:
          password: ${{ secrets.REGISTRY_TOKEN }}

      - name: Push to registry
-        if: ${{ github.event.inputs.push == 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
        run: |
-          docker push "${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}"
-          docker push "${{ env.IMAGE_NAME }}:sha-${{ github.sha }}"
+          docker push "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}"
+
+          # Also tag as latest if on main
+          if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
+            docker tag "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}" "${{ env.IMAGE_NAME }}:latest"
+            docker push "${{ env.IMAGE_NAME }}:latest"
+          fi

      - name: Sign image with cosign
-        if: ${{ github.event.inputs.push == 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
        env:
          COSIGN_EXPERIMENTAL: "1"
        run: |
-          # Sign with keyless signing (requires OIDC)
-          cosign sign --yes "${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}" || echo "Signing skipped (no OIDC available)"
+          cosign sign --yes "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}" || echo "Signing skipped (no OIDC available)"

-      - name: Build air-gap bundle
+      - name: Create release summary
+        run: |
+          echo "## Wine CSP Image Published" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Image:** \`${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**WARNING:** This image is for TEST VECTOR GENERATION ONLY." >> $GITHUB_STEP_SUMMARY
+
+  # ===========================================================================
+  # Job 6: Air-Gap Bundle
+  # ===========================================================================
+  airgap:
+    name: Air-Gap Bundle
+    runs-on: ubuntu-latest
+    needs: [build, test]
+    if: ${{ needs.test.result == 'success' }}
+
+    steps:
+      - name: Download image artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: wine-csp-image
+          path: /tmp/images
+
+      - name: Create air-gap bundle
        run: |
          mkdir -p out/bundles
-          docker save "${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}" | gzip > out/bundles/wine-csp-${{ steps.version.outputs.tag }}.tar.gz
+
+          # Copy the image tarball
+          cp /tmp/images/wine-csp.tar.gz out/bundles/wine-csp-${{ needs.build.outputs.image_tag }}.tar.gz

          # Generate bundle manifest
-          cat > out/bundles/wine-csp-${{ steps.version.outputs.tag }}.manifest.json <<EOF
+          cat > out/bundles/wine-csp-${{ needs.build.outputs.image_tag }}.manifest.json <<EOF
          {
            "name": "wine-csp",
-            "version": "${{ steps.version.outputs.tag }}",
-            "image": "${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}",
+            "version": "${{ needs.build.outputs.image_tag }}",
+            "image": "${{ env.IMAGE_NAME }}:${{ needs.build.outputs.image_tag }}",
            "platform": "linux/amd64",
-            "sha256": "$(sha256sum out/bundles/wine-csp-${{ steps.version.outputs.tag }}.tar.gz | cut -d' ' -f1)",
+            "sha256": "$(sha256sum out/bundles/wine-csp-${{ needs.build.outputs.image_tag }}.tar.gz | cut -d' ' -f1)",
            "created": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
            "git_commit": "${{ github.sha }}",
+            "git_ref": "${{ github.ref }}",
            "warning": "FOR TEST VECTOR GENERATION ONLY - NOT FOR PRODUCTION SIGNING"
          }
          EOF

-          echo "Air-gap bundle created:"
-          ls -lh out/bundles/
+          # Create checksums file
+          cd out/bundles
+          sha256sum *.tar.gz *.json > SHA256SUMS
+
+          echo "Air-gap bundle contents:"
+          ls -lh

      - name: Upload air-gap bundle
        uses: actions/upload-artifact@v4
        with:
-          name: wine-csp-bundle-${{ steps.version.outputs.tag }}
+          name: wine-csp-bundle-${{ needs.build.outputs.image_tag }}
          path: out/bundles/

-      - name: Security scan with Trivy
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: "${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}"
-          format: 'sarif'
-          output: 'trivy-results.sarif'
-          severity: 'CRITICAL,HIGH'
+  # ===========================================================================
+  # Job 7: Test Summary
+  # ===========================================================================
+  summary:
+    name: Test Summary
+    runs-on: ubuntu-latest
+    needs: [build, test, security, sbom]
+    if: always()

-      - name: Upload Trivy scan results
-        uses: github/codeql-action/upload-sarif@v3
-        if: always()
+    steps:
+      - name: Download test results
+        uses: actions/download-artifact@v4
        with:
-          sarif_file: 'trivy-results.sarif'
+          name: wine-csp-test-results
+          path: test-results/
+        continue-on-error: true
+
+      - name: Create summary
+        run: |
+          echo "## Wine CSP Build Summary" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "| Stage | Status |" >> $GITHUB_STEP_SUMMARY
+          echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY
+          echo "| Build | ${{ needs.build.result }} |" >> $GITHUB_STEP_SUMMARY
+          echo "| Tests | ${{ needs.test.result }} |" >> $GITHUB_STEP_SUMMARY
+          echo "| Security | ${{ needs.security.result }} |" >> $GITHUB_STEP_SUMMARY
+          echo "| SBOM | ${{ needs.sbom.result }} |" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Image Tag:** \`${{ needs.build.outputs.image_tag }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "---" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**SECURITY WARNING:** Wine CSP is for TEST VECTOR GENERATION ONLY." >> $GITHUB_STEP_SUMMARY