diff --git a/deploy/README.md b/deploy/README.md index 6647d0c76..48b2f8ecd 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -23,13 +23,27 @@ This directory contains deterministic deployment bundles for the core Stella Ops `python ./ops/devops/telemetry/smoke_otel_collector.py` to verify the OTLP endpoints. 5. Commit the change alongside any documentation updates (e.g. install guide cross-links). -Maintaining the digest linkage keeps offline/air-gapped installs reproducible and avoids tag drift between environments. - -### Additional tooling - -- `deploy/tools/check-channel-alignment.py` – verifies that Helm/Compose profiles reference the exact images listed in a release manifest. Run it for each channel before promoting a release. -- `ops/devops/telemetry/generate_dev_tls.sh` – produces local CA/server/client certificates for Compose-based collector testing. -- `ops/devops/telemetry/smoke_otel_collector.py` – sends OTLP traffic and asserts the collector accepted traces, metrics, and logs. +Maintaining the digest linkage keeps offline/air-gapped installs reproducible and avoids tag drift between environments. + +### Surface.Env rollout warnings + +- Compose (`deploy/compose/env/*.env.example`) and Helm (`deploy/helm/stellaops/values-*.yaml`) now seed `SCANNER_SURFACE_*` variables so the worker and web service resolve cache roots, Surface.FS endpoints, and secrets providers through `StellaOps.Scanner.Surface.Env`. +- During rollout, watch for structured log messages (and readiness output) prefixed with `surface.env.`—for example, `surface.env.cache_root_missing`, `surface.env.endpoint_unreachable`, or `surface.env.secrets_provider_invalid`. +- Treat these warnings as deployment blockers: update the endpoint/cache/secrets values or permissions before promoting the environment, otherwise workers will fail fast at startup. +- Air-gapped bundles default the secrets provider to `file` with `/etc/stellaops/secrets`; connected clusters default to `kubernetes`. Adjust the provider/root pair if your secrets manager differs. + +### Mongo2Go OpenSSL prerequisites + +- Linux runners that execute Mongo2Go-backed suites (Excititor, Scheduler, Graph, etc.) must expose OpenSSL 1.1 (`libcrypto.so.1.1`, `libssl.so.1.1`). The canonical copies live under `tests/native/openssl-1.1/linux-x64`. +- Export `LD_LIBRARY_PATH="$(git rev-parse --show-toplevel)/tests/native/openssl-1.1/linux-x64:${LD_LIBRARY_PATH:-}"` before invoking `dotnet test`. Example:\ + `LD_LIBRARY_PATH="$(pwd)/tests/native/openssl-1.1/linux-x64" dotnet test src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj --nologo`. +- CI agents or Dockerfiles that host these tests should either mount the directory into the container or copy the two `.so` files into a directory that is already on the runtime library path. + +### Additional tooling + +- `deploy/tools/check-channel-alignment.py` – verifies that Helm/Compose profiles reference the exact images listed in a release manifest. Run it for each channel before promoting a release. +- `ops/devops/telemetry/generate_dev_tls.sh` – produces local CA/server/client certificates for Compose-based collector testing. +- `ops/devops/telemetry/smoke_otel_collector.py` – sends OTLP traffic and asserts the collector accepted traces, metrics, and logs. - `ops/devops/telemetry/package_offline_bundle.py` – packages telemetry assets (config/Helm/Compose) into a signed tarball for air-gapped installs. - `docs/modules/devops/runbooks/deployment-upgrade.md` – end-to-end instructions for upgrade, rollback, and channel promotion workflows (Helm + Compose). diff --git a/deploy/compose/env/airgap.env.example b/deploy/compose/env/airgap.env.example index 73beab11b..2a3581a77 100644 --- a/deploy/compose/env/airgap.env.example +++ b/deploy/compose/env/airgap.env.example @@ -27,6 +27,10 @@ SCANNER_EVENTS_DSN= SCANNER_EVENTS_STREAM=stella.events SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS=5 SCANNER_EVENTS_MAX_STREAM_LENGTH=10000 +SCANNER_SURFACE_FS_ENDPOINT=http://rustfs:8080/api/v1 +SCANNER_SURFACE_CACHE_ROOT=/var/lib/stellaops/surface +SCANNER_SURFACE_SECRETS_PROVIDER=file +SCANNER_SURFACE_SECRETS_ROOT=/etc/stellaops/secrets SCHEDULER_QUEUE_KIND=Nats SCHEDULER_QUEUE_NATS_URL=nats://nats:4222 SCHEDULER_STORAGE_DATABASE=stellaops_scheduler diff --git a/deploy/compose/env/dev.env.example b/deploy/compose/env/dev.env.example index ed81829cf..988070b18 100644 --- a/deploy/compose/env/dev.env.example +++ b/deploy/compose/env/dev.env.example @@ -26,6 +26,11 @@ SCANNER_EVENTS_DSN= SCANNER_EVENTS_STREAM=stella.events SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS=5 SCANNER_EVENTS_MAX_STREAM_LENGTH=10000 +# Surface.Env defaults keep worker/web service aligned with local RustFS and inline secrets. +SCANNER_SURFACE_FS_ENDPOINT=http://rustfs:8080/api/v1 +SCANNER_SURFACE_CACHE_ROOT=/var/lib/stellaops/surface +SCANNER_SURFACE_SECRETS_PROVIDER=inline +SCANNER_SURFACE_SECRETS_ROOT= SCHEDULER_QUEUE_KIND=Nats SCHEDULER_QUEUE_NATS_URL=nats://nats:4222 SCHEDULER_STORAGE_DATABASE=stellaops_scheduler diff --git a/deploy/compose/env/mirror.env.example b/deploy/compose/env/mirror.env.example index 13cea5afa..9ec687b09 100644 --- a/deploy/compose/env/mirror.env.example +++ b/deploy/compose/env/mirror.env.example @@ -6,10 +6,16 @@ MONGO_INITDB_ROOT_PASSWORD=mirror-password MINIO_ROOT_USER=stellaops-mirror MINIO_ROOT_PASSWORD=mirror-minio-secret RUSTFS_HTTP_PORT=8080 - -# Mirror HTTP listeners -MIRROR_GATEWAY_HTTP_PORT=8080 -MIRROR_GATEWAY_HTTPS_PORT=9443 + +# Scanner surface integration +SCANNER_SURFACE_FS_ENDPOINT=http://rustfs:8080/api/v1 +SCANNER_SURFACE_CACHE_ROOT=/var/lib/stellaops/surface +SCANNER_SURFACE_SECRETS_PROVIDER=file +SCANNER_SURFACE_SECRETS_ROOT=/etc/stellaops/secrets + +# Mirror HTTP listeners +MIRROR_GATEWAY_HTTP_PORT=8080 +MIRROR_GATEWAY_HTTPS_PORT=9443 # Concelier mirror configuration CONCELIER_MIRROR_LATEST_SEGMENT=latest diff --git a/deploy/compose/env/prod.env.example b/deploy/compose/env/prod.env.example index 2f43d9c14..218178c0b 100644 --- a/deploy/compose/env/prod.env.example +++ b/deploy/compose/env/prod.env.example @@ -26,9 +26,13 @@ SCANNER_EVENTS_ENABLED=true SCANNER_EVENTS_DRIVER=redis # Leave SCANNER_EVENTS_DSN empty to inherit the Redis queue DSN when SCANNER_QUEUE_BROKER uses redis://. SCANNER_EVENTS_DSN= -SCANNER_EVENTS_STREAM=stella.events -SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS=5 +SCANNER_EVENTS_STREAM=stella.events +SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS=5 SCANNER_EVENTS_MAX_STREAM_LENGTH=10000 +SCANNER_SURFACE_FS_ENDPOINT=https://surfacefs.prod.stella-ops.org/api/v1 +SCANNER_SURFACE_CACHE_ROOT=/var/lib/stellaops/surface +SCANNER_SURFACE_SECRETS_PROVIDER=kubernetes +SCANNER_SURFACE_SECRETS_ROOT=stellaops/scanner SCHEDULER_QUEUE_KIND=Nats SCHEDULER_QUEUE_NATS_URL=nats://nats:4222 SCHEDULER_STORAGE_DATABASE=stellaops_scheduler diff --git a/deploy/compose/env/stage.env.example b/deploy/compose/env/stage.env.example index e7f652daa..b3c494bd8 100644 --- a/deploy/compose/env/stage.env.example +++ b/deploy/compose/env/stage.env.example @@ -26,6 +26,10 @@ SCANNER_EVENTS_DSN= SCANNER_EVENTS_STREAM=stella.events SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS=5 SCANNER_EVENTS_MAX_STREAM_LENGTH=10000 +SCANNER_SURFACE_FS_ENDPOINT=http://rustfs:8080/api/v1 +SCANNER_SURFACE_CACHE_ROOT=/var/lib/stellaops/surface +SCANNER_SURFACE_SECRETS_PROVIDER=kubernetes +SCANNER_SURFACE_SECRETS_ROOT=stellaops/scanner SCHEDULER_QUEUE_KIND=Nats SCHEDULER_QUEUE_NATS_URL=nats://nats:4222 SCHEDULER_STORAGE_DATABASE=stellaops_scheduler diff --git a/deploy/helm/stellaops/values-airgap.yaml b/deploy/helm/stellaops/values-airgap.yaml index 0241ed8b8..8b223efde 100644 --- a/deploy/helm/stellaops/values-airgap.yaml +++ b/deploy/helm/stellaops/values-airgap.yaml @@ -1,102 +1,102 @@ -global: - profile: airgap - release: - version: "2025.09.2-airgap" - channel: airgap - manifestSha256: "b787b833dddd73960c31338279daa0b0a0dce2ef32bd32ef1aaf953d66135f94" - image: - pullPolicy: IfNotPresent - labels: - stellaops.io/channel: airgap - -configMaps: - notify-config: - data: - notify.yaml: | - storage: - driver: mongo - connectionString: "mongodb://notify-mongo.prod.svc.cluster.local:27017" - database: "stellaops_notify" - commandTimeoutSeconds: 60 - - authority: - enabled: true - issuer: "https://authority.stella-ops.org" - metadataAddress: "https://authority.stella-ops.org/.well-known/openid-configuration" - requireHttpsMetadata: true - allowAnonymousFallback: false - backchannelTimeoutSeconds: 30 - tokenClockSkewSeconds: 60 - audiences: - - notify - readScope: notify.read - adminScope: notify.admin - - api: - basePath: "/api/v1/notify" - internalBasePath: "/internal/notify" - tenantHeader: "X-StellaOps-Tenant" - - plugins: - baseDirectory: "/var/opt/stellaops" - directory: "plugins/notify" - searchPatterns: - - "StellaOps.Notify.Connectors.*.dll" - orderedPlugins: - - StellaOps.Notify.Connectors.Slack - - StellaOps.Notify.Connectors.Teams - - StellaOps.Notify.Connectors.Email - - StellaOps.Notify.Connectors.Webhook - - telemetry: - enableRequestLogging: true - minimumLogLevel: Warning -services: - authority: - image: registry.stella-ops.org/stellaops/authority@sha256:5551a3269b7008cd5aceecf45df018c67459ed519557ccbe48b093b926a39bcc - service: - port: 8440 - env: - STELLAOPS_AUTHORITY__ISSUER: "https://stellaops-authority:8440" - STELLAOPS_AUTHORITY__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" - STELLAOPS_AUTHORITY__ALLOWANONYMOUSFALLBACK: "false" - signer: - image: registry.stella-ops.org/stellaops/signer@sha256:ddbbd664a42846cea6b40fca6465bc679b30f72851158f300d01a8571c5478fc - service: - port: 8441 - env: - SIGNER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" - SIGNER__POE__INTROSPECTURL: "file:///offline/poe/introspect.json" - SIGNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" - attestor: - image: registry.stella-ops.org/stellaops/attestor@sha256:1ff0a3124d66d3a2702d8e421df40fbd98cc75cb605d95510598ebbae1433c50 - service: - port: 8442 - env: - ATTESTOR__SIGNER__BASEURL: "https://stellaops-signer:8441" - ATTESTOR__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" - concelier: - image: registry.stella-ops.org/stellaops/concelier@sha256:29e2e1a0972707e092cbd3d370701341f9fec2aa9316fb5d8100480f2a1c76b5 - service: - port: 8445 - env: - CONCELIER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" - CONCELIER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" - CONCELIER__STORAGE__S3__ACCESSKEYID: "stellaops-airgap" - CONCELIER__STORAGE__S3__SECRETACCESSKEY: "airgap-minio-secret" - CONCELIER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" - CONCELIER__AUTHORITY__RESILIENCE__ALLOWOFFLINECACHEFALLBACK: "true" - CONCELIER__AUTHORITY__RESILIENCE__OFFLINECACHETOLERANCE: "00:45:00" - volumeMounts: - - name: concelier-jobs - mountPath: /var/lib/concelier/jobs - volumeClaims: - - name: concelier-jobs - claimName: stellaops-concelier-jobs +global: + profile: airgap + release: + version: "2025.09.2-airgap" + channel: airgap + manifestSha256: "b787b833dddd73960c31338279daa0b0a0dce2ef32bd32ef1aaf953d66135f94" + image: + pullPolicy: IfNotPresent + labels: + stellaops.io/channel: airgap + +configMaps: + notify-config: + data: + notify.yaml: | + storage: + driver: mongo + connectionString: "mongodb://notify-mongo.prod.svc.cluster.local:27017" + database: "stellaops_notify" + commandTimeoutSeconds: 60 + + authority: + enabled: true + issuer: "https://authority.stella-ops.org" + metadataAddress: "https://authority.stella-ops.org/.well-known/openid-configuration" + requireHttpsMetadata: true + allowAnonymousFallback: false + backchannelTimeoutSeconds: 30 + tokenClockSkewSeconds: 60 + audiences: + - notify + readScope: notify.read + adminScope: notify.admin + + api: + basePath: "/api/v1/notify" + internalBasePath: "/internal/notify" + tenantHeader: "X-StellaOps-Tenant" + + plugins: + baseDirectory: "/var/opt/stellaops" + directory: "plugins/notify" + searchPatterns: + - "StellaOps.Notify.Connectors.*.dll" + orderedPlugins: + - StellaOps.Notify.Connectors.Slack + - StellaOps.Notify.Connectors.Teams + - StellaOps.Notify.Connectors.Email + - StellaOps.Notify.Connectors.Webhook + + telemetry: + enableRequestLogging: true + minimumLogLevel: Warning +services: + authority: + image: registry.stella-ops.org/stellaops/authority@sha256:5551a3269b7008cd5aceecf45df018c67459ed519557ccbe48b093b926a39bcc + service: + port: 8440 + env: + STELLAOPS_AUTHORITY__ISSUER: "https://stellaops-authority:8440" + STELLAOPS_AUTHORITY__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" + STELLAOPS_AUTHORITY__ALLOWANONYMOUSFALLBACK: "false" + signer: + image: registry.stella-ops.org/stellaops/signer@sha256:ddbbd664a42846cea6b40fca6465bc679b30f72851158f300d01a8571c5478fc + service: + port: 8441 + env: + SIGNER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" + SIGNER__POE__INTROSPECTURL: "file:///offline/poe/introspect.json" + SIGNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" + attestor: + image: registry.stella-ops.org/stellaops/attestor@sha256:1ff0a3124d66d3a2702d8e421df40fbd98cc75cb605d95510598ebbae1433c50 + service: + port: 8442 + env: + ATTESTOR__SIGNER__BASEURL: "https://stellaops-signer:8441" + ATTESTOR__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" + concelier: + image: registry.stella-ops.org/stellaops/concelier@sha256:29e2e1a0972707e092cbd3d370701341f9fec2aa9316fb5d8100480f2a1c76b5 + service: + port: 8445 + env: + CONCELIER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" + CONCELIER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + CONCELIER__STORAGE__S3__ACCESSKEYID: "stellaops-airgap" + CONCELIER__STORAGE__S3__SECRETACCESSKEY: "airgap-minio-secret" + CONCELIER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" + CONCELIER__AUTHORITY__RESILIENCE__ALLOWOFFLINECACHEFALLBACK: "true" + CONCELIER__AUTHORITY__RESILIENCE__OFFLINECACHETOLERANCE: "00:45:00" + volumeMounts: + - name: concelier-jobs + mountPath: /var/lib/concelier/jobs + volumeClaims: + - name: concelier-jobs + claimName: stellaops-concelier-jobs scanner-web: - image: registry.stella-ops.org/stellaops/scanner-web@sha256:3df8ca21878126758203c1a0444e39fd97f77ddacf04a69685cda9f1e5e94718 - service: - port: 8444 + image: registry.stella-ops.org/stellaops/scanner-web@sha256:3df8ca21878126758203c1a0444e39fd97f77ddacf04a69685cda9f1e5e94718 + service: + port: 8444 env: SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" @@ -110,8 +110,12 @@ services: SCANNER__EVENTS__STREAM: "stella.events" SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" + SCANNER_SURFACE_FS_ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER_SURFACE_CACHE_ROOT: "/var/lib/stellaops/surface" + SCANNER_SURFACE_SECRETS_PROVIDER: "file" + SCANNER_SURFACE_SECRETS_ROOT: "/etc/stellaops/secrets" scanner-worker: - image: registry.stella-ops.org/stellaops/scanner-worker@sha256:eea5d6cfe7835950c5ec7a735a651f2f0d727d3e470cf9027a4a402ea89c4fb5 + image: registry.stella-ops.org/stellaops/scanner-worker@sha256:eea5d6cfe7835950c5ec7a735a651f2f0d727d3e470cf9027a4a402ea89c4fb5 env: SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" @@ -125,62 +129,66 @@ services: SCANNER__EVENTS__STREAM: "stella.events" SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" - notify-web: - image: registry.stella-ops.org/stellaops/notify-web:2025.09.2 - service: - port: 8446 - env: - DOTNET_ENVIRONMENT: Production - configMounts: - - name: notify-config - mountPath: /app/etc/notify.yaml - subPath: notify.yaml - configMap: notify-config - excititor: - image: registry.stella-ops.org/stellaops/excititor@sha256:65c0ee13f773efe920d7181512349a09d363ab3f3e177d276136bd2742325a68 - env: - EXCITITOR__CONCELIER__BASEURL: "https://stellaops-concelier:8445" - EXCITITOR__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" - web-ui: - image: registry.stella-ops.org/stellaops/web-ui@sha256:bee9668011ff414572131dc777faab4da24473fe12c230893f161cabee092a1d - service: - port: 9443 - targetPort: 8443 - env: - STELLAOPS_UI__BACKEND__BASEURL: "https://stellaops-scanner-web:8444" - mongo: - class: infrastructure - image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 - service: - port: 27017 - command: - - mongod - - --bind_ip_all - env: - MONGO_INITDB_ROOT_USERNAME: stellaops-airgap - MONGO_INITDB_ROOT_PASSWORD: stellaops-airgap - volumeMounts: - - name: mongo-data - mountPath: /data/db - volumeClaims: - - name: mongo-data - claimName: stellaops-mongo-data + SCANNER_SURFACE_FS_ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER_SURFACE_CACHE_ROOT: "/var/lib/stellaops/surface" + SCANNER_SURFACE_SECRETS_PROVIDER: "file" + SCANNER_SURFACE_SECRETS_ROOT: "/etc/stellaops/secrets" + notify-web: + image: registry.stella-ops.org/stellaops/notify-web:2025.09.2 + service: + port: 8446 + env: + DOTNET_ENVIRONMENT: Production + configMounts: + - name: notify-config + mountPath: /app/etc/notify.yaml + subPath: notify.yaml + configMap: notify-config + excititor: + image: registry.stella-ops.org/stellaops/excititor@sha256:65c0ee13f773efe920d7181512349a09d363ab3f3e177d276136bd2742325a68 + env: + EXCITITOR__CONCELIER__BASEURL: "https://stellaops-concelier:8445" + EXCITITOR__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" + web-ui: + image: registry.stella-ops.org/stellaops/web-ui@sha256:bee9668011ff414572131dc777faab4da24473fe12c230893f161cabee092a1d + service: + port: 9443 + targetPort: 8443 + env: + STELLAOPS_UI__BACKEND__BASEURL: "https://stellaops-scanner-web:8444" + mongo: + class: infrastructure + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + service: + port: 27017 + command: + - mongod + - --bind_ip_all + env: + MONGO_INITDB_ROOT_USERNAME: stellaops-airgap + MONGO_INITDB_ROOT_PASSWORD: stellaops-airgap + volumeMounts: + - name: mongo-data + mountPath: /data/db + volumeClaims: + - name: mongo-data + claimName: stellaops-mongo-data minio: class: infrastructure image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e service: port: 9000 - command: - - server - - /data - - --console-address - - :9001 - env: - MINIO_ROOT_USER: stellaops-airgap - MINIO_ROOT_PASSWORD: airgap-minio-secret - volumeMounts: - - name: minio-data - mountPath: /data + command: + - server + - /data + - --console-address + - :9001 + env: + MINIO_ROOT_USER: stellaops-airgap + MINIO_ROOT_PASSWORD: airgap-minio-secret + volumeMounts: + - name: minio-data + mountPath: /data volumeClaims: - name: minio-data claimName: stellaops-minio-data @@ -204,18 +212,18 @@ services: volumeClaims: - name: rustfs-data claimName: stellaops-rustfs-data - nats: - class: infrastructure - image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e - service: - port: 4222 - command: - - -js - - -sd - - /data - volumeMounts: - - name: nats-data - mountPath: /data - volumeClaims: - - name: nats-data - claimName: stellaops-nats-data + nats: + class: infrastructure + image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e + service: + port: 4222 + command: + - -js + - -sd + - /data + volumeMounts: + - name: nats-data + mountPath: /data + volumeClaims: + - name: nats-data + claimName: stellaops-nats-data diff --git a/deploy/helm/stellaops/values-dev.yaml b/deploy/helm/stellaops/values-dev.yaml index eb99fc833..bcd64aa0d 100644 --- a/deploy/helm/stellaops/values-dev.yaml +++ b/deploy/helm/stellaops/values-dev.yaml @@ -17,92 +17,92 @@ telemetry: secretName: stellaops-otel-tls configMaps: - notify-config: - data: - notify.yaml: | - storage: - driver: mongo - connectionString: "mongodb://notify-mongo.dev.svc.cluster.local:27017" - database: "stellaops_notify_dev" - commandTimeoutSeconds: 30 - - authority: - enabled: true - issuer: "https://authority.dev.stella-ops.local" - metadataAddress: "https://authority.dev.stella-ops.local/.well-known/openid-configuration" - requireHttpsMetadata: false - allowAnonymousFallback: false - backchannelTimeoutSeconds: 30 - tokenClockSkewSeconds: 60 - audiences: - - notify.dev - readScope: notify.read - adminScope: notify.admin - - api: - basePath: "/api/v1/notify" - internalBasePath: "/internal/notify" - tenantHeader: "X-StellaOps-Tenant" - - plugins: - baseDirectory: "../" - directory: "plugins/notify" - searchPatterns: - - "StellaOps.Notify.Connectors.*.dll" - orderedPlugins: - - StellaOps.Notify.Connectors.Slack - - StellaOps.Notify.Connectors.Teams - - StellaOps.Notify.Connectors.Email - - StellaOps.Notify.Connectors.Webhook - - telemetry: - enableRequestLogging: true - minimumLogLevel: Debug + notify-config: + data: + notify.yaml: | + storage: + driver: mongo + connectionString: "mongodb://notify-mongo.dev.svc.cluster.local:27017" + database: "stellaops_notify_dev" + commandTimeoutSeconds: 30 + + authority: + enabled: true + issuer: "https://authority.dev.stella-ops.local" + metadataAddress: "https://authority.dev.stella-ops.local/.well-known/openid-configuration" + requireHttpsMetadata: false + allowAnonymousFallback: false + backchannelTimeoutSeconds: 30 + tokenClockSkewSeconds: 60 + audiences: + - notify.dev + readScope: notify.read + adminScope: notify.admin + + api: + basePath: "/api/v1/notify" + internalBasePath: "/internal/notify" + tenantHeader: "X-StellaOps-Tenant" + + plugins: + baseDirectory: "../" + directory: "plugins/notify" + searchPatterns: + - "StellaOps.Notify.Connectors.*.dll" + orderedPlugins: + - StellaOps.Notify.Connectors.Slack + - StellaOps.Notify.Connectors.Teams + - StellaOps.Notify.Connectors.Email + - StellaOps.Notify.Connectors.Webhook + + telemetry: + enableRequestLogging: true + minimumLogLevel: Debug services: - authority: - image: registry.stella-ops.org/stellaops/authority@sha256:a8e8faec44a579aa5714e58be835f25575710430b1ad2ccd1282a018cd9ffcdd - service: - port: 8440 - env: - STELLAOPS_AUTHORITY__ISSUER: "https://stellaops-authority:8440" - STELLAOPS_AUTHORITY__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" - STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0: "/app/plugins" - STELLAOPS_AUTHORITY__PLUGINS__CONFIGURATIONDIRECTORY: "/app/etc/authority.plugins" - signer: - image: registry.stella-ops.org/stellaops/signer@sha256:8bfef9a75783883d49fc18e3566553934e970b00ee090abee9cb110d2d5c3298 - service: - port: 8441 - env: - SIGNER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" - SIGNER__POE__INTROSPECTURL: "https://licensing.svc.local/introspect" - SIGNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" - attestor: - image: registry.stella-ops.org/stellaops/attestor@sha256:5cc417948c029da01dccf36e4645d961a3f6d8de7e62fe98d845f07cd2282114 - service: - port: 8442 - env: - ATTESTOR__SIGNER__BASEURL: "https://stellaops-signer:8441" - ATTESTOR__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" - concelier: - image: registry.stella-ops.org/stellaops/concelier@sha256:dafef3954eb4b837e2c424dd2d23e1e4d60fa83794840fac9cd3dea1d43bd085 - service: - port: 8445 - env: - CONCELIER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" - CONCELIER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" - CONCELIER__STORAGE__S3__ACCESSKEYID: "stellaops" - CONCELIER__STORAGE__S3__SECRETACCESSKEY: "dev-minio-secret" - CONCELIER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" - volumeMounts: - - name: concelier-jobs - mountPath: /var/lib/concelier/jobs - volumes: - - name: concelier-jobs - emptyDir: {} + authority: + image: registry.stella-ops.org/stellaops/authority@sha256:a8e8faec44a579aa5714e58be835f25575710430b1ad2ccd1282a018cd9ffcdd + service: + port: 8440 + env: + STELLAOPS_AUTHORITY__ISSUER: "https://stellaops-authority:8440" + STELLAOPS_AUTHORITY__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" + STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0: "/app/plugins" + STELLAOPS_AUTHORITY__PLUGINS__CONFIGURATIONDIRECTORY: "/app/etc/authority.plugins" + signer: + image: registry.stella-ops.org/stellaops/signer@sha256:8bfef9a75783883d49fc18e3566553934e970b00ee090abee9cb110d2d5c3298 + service: + port: 8441 + env: + SIGNER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" + SIGNER__POE__INTROSPECTURL: "https://licensing.svc.local/introspect" + SIGNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" + attestor: + image: registry.stella-ops.org/stellaops/attestor@sha256:5cc417948c029da01dccf36e4645d961a3f6d8de7e62fe98d845f07cd2282114 + service: + port: 8442 + env: + ATTESTOR__SIGNER__BASEURL: "https://stellaops-signer:8441" + ATTESTOR__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" + concelier: + image: registry.stella-ops.org/stellaops/concelier@sha256:dafef3954eb4b837e2c424dd2d23e1e4d60fa83794840fac9cd3dea1d43bd085 + service: + port: 8445 + env: + CONCELIER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" + CONCELIER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + CONCELIER__STORAGE__S3__ACCESSKEYID: "stellaops" + CONCELIER__STORAGE__S3__SECRETACCESSKEY: "dev-minio-secret" + CONCELIER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" + volumeMounts: + - name: concelier-jobs + mountPath: /var/lib/concelier/jobs + volumes: + - name: concelier-jobs + emptyDir: {} scanner-web: - image: registry.stella-ops.org/stellaops/scanner-web@sha256:e0dfdb087e330585a5953029fb4757f5abdf7610820a085bd61b457dbead9a11 - service: - port: 8444 + image: registry.stella-ops.org/stellaops/scanner-web@sha256:e0dfdb087e330585a5953029fb4757f5abdf7610820a085bd61b457dbead9a11 + service: + port: 8444 env: SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" @@ -116,8 +116,12 @@ services: SCANNER__EVENTS__STREAM: "stella.events" SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" + SCANNER_SURFACE_FS_ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER_SURFACE_CACHE_ROOT: "/var/lib/stellaops/surface" + SCANNER_SURFACE_SECRETS_PROVIDER: "inline" + SCANNER_SURFACE_SECRETS_ROOT: "" scanner-worker: - image: registry.stella-ops.org/stellaops/scanner-worker@sha256:92dda42f6f64b2d9522104a5c9ffb61d37b34dd193132b68457a259748008f37 + image: registry.stella-ops.org/stellaops/scanner-worker@sha256:92dda42f6f64b2d9522104a5c9ffb61d37b34dd193132b68457a259748008f37 env: SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" @@ -131,61 +135,65 @@ services: SCANNER__EVENTS__STREAM: "stella.events" SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" - notify-web: - image: registry.stella-ops.org/stellaops/notify-web:2025.10.0-edge - service: - port: 8446 - env: - DOTNET_ENVIRONMENT: Development - configMounts: - - name: notify-config - mountPath: /app/etc/notify.yaml - subPath: notify.yaml - configMap: notify-config - excititor: - image: registry.stella-ops.org/stellaops/excititor@sha256:d9bd5cadf1eab427447ce3df7302c30ded837239771cc6433b9befb895054285 - env: - EXCITITOR__CONCELIER__BASEURL: "https://stellaops-concelier:8445" - EXCITITOR__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" - web-ui: - image: registry.stella-ops.org/stellaops/web-ui@sha256:38b225fa7767a5b94ebae4dae8696044126aac429415e93de514d5dd95748dcf - service: - port: 8443 - env: - STELLAOPS_UI__BACKEND__BASEURL: "https://stellaops-scanner-web:8444" - mongo: - class: infrastructure - image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 - service: - port: 27017 - command: - - mongod - - --bind_ip_all - env: - MONGO_INITDB_ROOT_USERNAME: stellaops - MONGO_INITDB_ROOT_PASSWORD: stellaops - volumeMounts: - - name: mongo-data - mountPath: /data/db - volumes: - - name: mongo-data - emptyDir: {} + SCANNER_SURFACE_FS_ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER_SURFACE_CACHE_ROOT: "/var/lib/stellaops/surface" + SCANNER_SURFACE_SECRETS_PROVIDER: "inline" + SCANNER_SURFACE_SECRETS_ROOT: "" + notify-web: + image: registry.stella-ops.org/stellaops/notify-web:2025.10.0-edge + service: + port: 8446 + env: + DOTNET_ENVIRONMENT: Development + configMounts: + - name: notify-config + mountPath: /app/etc/notify.yaml + subPath: notify.yaml + configMap: notify-config + excititor: + image: registry.stella-ops.org/stellaops/excititor@sha256:d9bd5cadf1eab427447ce3df7302c30ded837239771cc6433b9befb895054285 + env: + EXCITITOR__CONCELIER__BASEURL: "https://stellaops-concelier:8445" + EXCITITOR__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" + web-ui: + image: registry.stella-ops.org/stellaops/web-ui@sha256:38b225fa7767a5b94ebae4dae8696044126aac429415e93de514d5dd95748dcf + service: + port: 8443 + env: + STELLAOPS_UI__BACKEND__BASEURL: "https://stellaops-scanner-web:8444" + mongo: + class: infrastructure + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + service: + port: 27017 + command: + - mongod + - --bind_ip_all + env: + MONGO_INITDB_ROOT_USERNAME: stellaops + MONGO_INITDB_ROOT_PASSWORD: stellaops + volumeMounts: + - name: mongo-data + mountPath: /data/db + volumes: + - name: mongo-data + emptyDir: {} minio: class: infrastructure image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e - service: - port: 9000 - command: - - server - - /data - - --console-address - - :9001 - env: - MINIO_ROOT_USER: stellaops - MINIO_ROOT_PASSWORD: dev-minio-secret - volumeMounts: - - name: minio-data - mountPath: /data + service: + port: 9000 + command: + - server + - /data + - --console-address + - :9001 + env: + MINIO_ROOT_USER: stellaops + MINIO_ROOT_PASSWORD: dev-minio-secret + volumeMounts: + - name: minio-data + mountPath: /data volumes: - name: minio-data emptyDir: {} @@ -203,18 +211,18 @@ services: volumes: - name: rustfs-data emptyDir: {} - nats: - class: infrastructure - image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e - service: - port: 4222 - command: - - -js - - -sd - - /data - volumeMounts: - - name: nats-data - mountPath: /data - volumes: - - name: nats-data - emptyDir: {} + nats: + class: infrastructure + image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e + service: + port: 4222 + command: + - -js + - -sd + - /data + volumeMounts: + - name: nats-data + mountPath: /data + volumes: + - name: nats-data + emptyDir: {} diff --git a/deploy/helm/stellaops/values-prod.yaml b/deploy/helm/stellaops/values-prod.yaml index bb1f57687..5426b76a0 100644 --- a/deploy/helm/stellaops/values-prod.yaml +++ b/deploy/helm/stellaops/values-prod.yaml @@ -1,221 +1,229 @@ -global: - profile: prod - release: - version: "2025.09.2" - channel: stable - manifestSha256: "dc3c8fe1ab83941c838ccc5a8a5862f7ddfa38c2078e580b5649db26554565b7" - image: - pullPolicy: IfNotPresent - labels: - stellaops.io/channel: stable - stellaops.io/profile: prod - -configMaps: - notify-config: - data: - notify.yaml: | - storage: - driver: mongo - connectionString: "mongodb://stellaops-mongo:27017" - database: "stellaops_notify_prod" - commandTimeoutSeconds: 45 - - authority: - enabled: true - issuer: "https://authority.prod.stella-ops.org" - metadataAddress: "https://authority.prod.stella-ops.org/.well-known/openid-configuration" - requireHttpsMetadata: true - allowAnonymousFallback: false - backchannelTimeoutSeconds: 30 - tokenClockSkewSeconds: 60 - audiences: - - notify - readScope: notify.read - adminScope: notify.admin - - api: - basePath: "/api/v1/notify" - internalBasePath: "/internal/notify" - tenantHeader: "X-StellaOps-Tenant" - - plugins: - baseDirectory: "/opt/stellaops" - directory: "plugins/notify" - searchPatterns: - - "StellaOps.Notify.Connectors.*.dll" - orderedPlugins: - - StellaOps.Notify.Connectors.Slack - - StellaOps.Notify.Connectors.Teams - - StellaOps.Notify.Connectors.Email - - StellaOps.Notify.Connectors.Webhook - - telemetry: - enableRequestLogging: true - minimumLogLevel: Information -services: - authority: - image: registry.stella-ops.org/stellaops/authority@sha256:b0348bad1d0b401cc3c71cb40ba034c8043b6c8874546f90d4783c9dbfcc0bf5 - service: - port: 8440 - env: - STELLAOPS_AUTHORITY__ISSUER: "https://authority.prod.stella-ops.org" - STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0: "/app/plugins" - STELLAOPS_AUTHORITY__PLUGINS__CONFIGURATIONDIRECTORY: "/app/etc/authority.plugins" - envFrom: - - secretRef: - name: stellaops-prod-core - signer: - image: registry.stella-ops.org/stellaops/signer@sha256:8ad574e61f3a9e9bda8a58eb2700ae46813284e35a150b1137bc7c2b92ac0f2e - service: - port: 8441 - env: - SIGNER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" - SIGNER__POE__INTROSPECTURL: "https://licensing.prod.stella-ops.org/introspect" - envFrom: - - secretRef: - name: stellaops-prod-core - attestor: - image: registry.stella-ops.org/stellaops/attestor@sha256:0534985f978b0b5d220d73c96fddd962cd9135f616811cbe3bff4666c5af568f - service: - port: 8442 - env: - ATTESTOR__SIGNER__BASEURL: "https://stellaops-signer:8441" - envFrom: - - secretRef: - name: stellaops-prod-core - concelier: - image: registry.stella-ops.org/stellaops/concelier@sha256:c58cdcaee1d266d68d498e41110a589dd204b487d37381096bd61ab345a867c5 - service: - port: 8445 - env: - CONCELIER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" - CONCELIER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" - envFrom: - - secretRef: - name: stellaops-prod-core - volumeMounts: - - name: concelier-jobs - mountPath: /var/lib/concelier/jobs - volumeClaims: - - name: concelier-jobs - claimName: stellaops-concelier-jobs - scanner-web: - image: registry.stella-ops.org/stellaops/scanner-web@sha256:14b23448c3f9586a9156370b3e8c1991b61907efa666ca37dd3aaed1e79fe3b7 - service: - port: 8444 - env: - SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" - SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1" - SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" - SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" - SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" - SCANNER__EVENTS__ENABLED: "true" - SCANNER__EVENTS__DRIVER: "redis" - SCANNER__EVENTS__DSN: "" - SCANNER__EVENTS__STREAM: "stella.events" - SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" - SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" - envFrom: - - secretRef: - name: stellaops-prod-core - scanner-worker: - image: registry.stella-ops.org/stellaops/scanner-worker@sha256:32e25e76386eb9ea8bee0a1ad546775db9a2df989fab61ac877e351881960dab - replicas: 3 - env: - SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" - SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1" - SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" - SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" - SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" - SCANNER__EVENTS__ENABLED: "true" - SCANNER__EVENTS__DRIVER: "redis" - SCANNER__EVENTS__DSN: "" - SCANNER__EVENTS__STREAM: "stella.events" - SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" - SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" - envFrom: - - secretRef: - name: stellaops-prod-core - notify-web: - image: registry.stella-ops.org/stellaops/notify-web:2025.09.2 - service: - port: 8446 - env: - DOTNET_ENVIRONMENT: Production - envFrom: - - secretRef: - name: stellaops-prod-notify - configMounts: - - name: notify-config - mountPath: /app/etc/notify.yaml - subPath: notify.yaml - configMap: notify-config - excititor: - image: registry.stella-ops.org/stellaops/excititor@sha256:59022e2016aebcef5c856d163ae705755d3f81949d41195256e935ef40a627fa - env: - EXCITITOR__CONCELIER__BASEURL: "https://stellaops-concelier:8445" - envFrom: - - secretRef: - name: stellaops-prod-core - web-ui: - image: registry.stella-ops.org/stellaops/web-ui@sha256:10d924808c48e4353e3a241da62eb7aefe727a1d6dc830eb23a8e181013b3a23 - service: - port: 8443 - env: - STELLAOPS_UI__BACKEND__BASEURL: "https://stellaops-scanner-web:8444" - mongo: - class: infrastructure - image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 - service: - port: 27017 - command: - - mongod - - --bind_ip_all - envFrom: - - secretRef: - name: stellaops-prod-mongo - volumeMounts: - - name: mongo-data - mountPath: /data/db - volumeClaims: - - name: mongo-data - claimName: stellaops-mongo-data - minio: - class: infrastructure - image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e - service: - port: 9000 - command: - - server - - /data - - --console-address - - :9001 - envFrom: - - secretRef: - name: stellaops-prod-minio - volumeMounts: - - name: minio-data - mountPath: /data - volumeClaims: - - name: minio-data - claimName: stellaops-minio-data - rustfs: - class: infrastructure - image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge - service: - port: 8080 - command: - - serve - - --listen - - 0.0.0.0:8080 - - --root - - /data - env: - RUSTFS__LOG__LEVEL: info - RUSTFS__STORAGE__PATH: /data - volumeMounts: - - name: rustfs-data - mountPath: /data - volumeClaims: - - name: rustfs-data - claimName: stellaops-rustfs-data +global: + profile: prod + release: + version: "2025.09.2" + channel: stable + manifestSha256: "dc3c8fe1ab83941c838ccc5a8a5862f7ddfa38c2078e580b5649db26554565b7" + image: + pullPolicy: IfNotPresent + labels: + stellaops.io/channel: stable + stellaops.io/profile: prod + +configMaps: + notify-config: + data: + notify.yaml: | + storage: + driver: mongo + connectionString: "mongodb://stellaops-mongo:27017" + database: "stellaops_notify_prod" + commandTimeoutSeconds: 45 + + authority: + enabled: true + issuer: "https://authority.prod.stella-ops.org" + metadataAddress: "https://authority.prod.stella-ops.org/.well-known/openid-configuration" + requireHttpsMetadata: true + allowAnonymousFallback: false + backchannelTimeoutSeconds: 30 + tokenClockSkewSeconds: 60 + audiences: + - notify + readScope: notify.read + adminScope: notify.admin + + api: + basePath: "/api/v1/notify" + internalBasePath: "/internal/notify" + tenantHeader: "X-StellaOps-Tenant" + + plugins: + baseDirectory: "/opt/stellaops" + directory: "plugins/notify" + searchPatterns: + - "StellaOps.Notify.Connectors.*.dll" + orderedPlugins: + - StellaOps.Notify.Connectors.Slack + - StellaOps.Notify.Connectors.Teams + - StellaOps.Notify.Connectors.Email + - StellaOps.Notify.Connectors.Webhook + + telemetry: + enableRequestLogging: true + minimumLogLevel: Information +services: + authority: + image: registry.stella-ops.org/stellaops/authority@sha256:b0348bad1d0b401cc3c71cb40ba034c8043b6c8874546f90d4783c9dbfcc0bf5 + service: + port: 8440 + env: + STELLAOPS_AUTHORITY__ISSUER: "https://authority.prod.stella-ops.org" + STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0: "/app/plugins" + STELLAOPS_AUTHORITY__PLUGINS__CONFIGURATIONDIRECTORY: "/app/etc/authority.plugins" + envFrom: + - secretRef: + name: stellaops-prod-core + signer: + image: registry.stella-ops.org/stellaops/signer@sha256:8ad574e61f3a9e9bda8a58eb2700ae46813284e35a150b1137bc7c2b92ac0f2e + service: + port: 8441 + env: + SIGNER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" + SIGNER__POE__INTROSPECTURL: "https://licensing.prod.stella-ops.org/introspect" + envFrom: + - secretRef: + name: stellaops-prod-core + attestor: + image: registry.stella-ops.org/stellaops/attestor@sha256:0534985f978b0b5d220d73c96fddd962cd9135f616811cbe3bff4666c5af568f + service: + port: 8442 + env: + ATTESTOR__SIGNER__BASEURL: "https://stellaops-signer:8441" + envFrom: + - secretRef: + name: stellaops-prod-core + concelier: + image: registry.stella-ops.org/stellaops/concelier@sha256:c58cdcaee1d266d68d498e41110a589dd204b487d37381096bd61ab345a867c5 + service: + port: 8445 + env: + CONCELIER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + CONCELIER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" + envFrom: + - secretRef: + name: stellaops-prod-core + volumeMounts: + - name: concelier-jobs + mountPath: /var/lib/concelier/jobs + volumeClaims: + - name: concelier-jobs + claimName: stellaops-concelier-jobs + scanner-web: + image: registry.stella-ops.org/stellaops/scanner-web@sha256:14b23448c3f9586a9156370b3e8c1991b61907efa666ca37dd3aaed1e79fe3b7 + service: + port: 8444 + env: + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" + SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + SCANNER__EVENTS__ENABLED: "true" + SCANNER__EVENTS__DRIVER: "redis" + SCANNER__EVENTS__DSN: "" + SCANNER__EVENTS__STREAM: "stella.events" + SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" + SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" + SCANNER_SURFACE_FS_ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER_SURFACE_CACHE_ROOT: "/var/lib/stellaops/surface" + SCANNER_SURFACE_SECRETS_PROVIDER: "kubernetes" + SCANNER_SURFACE_SECRETS_ROOT: "stellaops/scanner" + envFrom: + - secretRef: + name: stellaops-prod-core + scanner-worker: + image: registry.stella-ops.org/stellaops/scanner-worker@sha256:32e25e76386eb9ea8bee0a1ad546775db9a2df989fab61ac877e351881960dab + replicas: 3 + env: + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" + SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + SCANNER__EVENTS__ENABLED: "true" + SCANNER__EVENTS__DRIVER: "redis" + SCANNER__EVENTS__DSN: "" + SCANNER__EVENTS__STREAM: "stella.events" + SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" + SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" + SCANNER_SURFACE_FS_ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER_SURFACE_CACHE_ROOT: "/var/lib/stellaops/surface" + SCANNER_SURFACE_SECRETS_PROVIDER: "kubernetes" + SCANNER_SURFACE_SECRETS_ROOT: "stellaops/scanner" + envFrom: + - secretRef: + name: stellaops-prod-core + notify-web: + image: registry.stella-ops.org/stellaops/notify-web:2025.09.2 + service: + port: 8446 + env: + DOTNET_ENVIRONMENT: Production + envFrom: + - secretRef: + name: stellaops-prod-notify + configMounts: + - name: notify-config + mountPath: /app/etc/notify.yaml + subPath: notify.yaml + configMap: notify-config + excititor: + image: registry.stella-ops.org/stellaops/excititor@sha256:59022e2016aebcef5c856d163ae705755d3f81949d41195256e935ef40a627fa + env: + EXCITITOR__CONCELIER__BASEURL: "https://stellaops-concelier:8445" + envFrom: + - secretRef: + name: stellaops-prod-core + web-ui: + image: registry.stella-ops.org/stellaops/web-ui@sha256:10d924808c48e4353e3a241da62eb7aefe727a1d6dc830eb23a8e181013b3a23 + service: + port: 8443 + env: + STELLAOPS_UI__BACKEND__BASEURL: "https://stellaops-scanner-web:8444" + mongo: + class: infrastructure + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + service: + port: 27017 + command: + - mongod + - --bind_ip_all + envFrom: + - secretRef: + name: stellaops-prod-mongo + volumeMounts: + - name: mongo-data + mountPath: /data/db + volumeClaims: + - name: mongo-data + claimName: stellaops-mongo-data + minio: + class: infrastructure + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + service: + port: 9000 + command: + - server + - /data + - --console-address + - :9001 + envFrom: + - secretRef: + name: stellaops-prod-minio + volumeMounts: + - name: minio-data + mountPath: /data + volumeClaims: + - name: minio-data + claimName: stellaops-minio-data + rustfs: + class: infrastructure + image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge + service: + port: 8080 + command: + - serve + - --listen + - 0.0.0.0:8080 + - --root + - /data + env: + RUSTFS__LOG__LEVEL: info + RUSTFS__STORAGE__PATH: /data + volumeMounts: + - name: rustfs-data + mountPath: /data + volumeClaims: + - name: rustfs-data + claimName: stellaops-rustfs-data diff --git a/deploy/helm/stellaops/values-stage.yaml b/deploy/helm/stellaops/values-stage.yaml index bc4dfe178..5d164cdb8 100644 --- a/deploy/helm/stellaops/values-stage.yaml +++ b/deploy/helm/stellaops/values-stage.yaml @@ -2,10 +2,10 @@ global: profile: stage release: version: "2025.09.2" - channel: stable - manifestSha256: "dc3c8fe1ab83941c838ccc5a8a5862f7ddfa38c2078e580b5649db26554565b7" - image: - pullPolicy: IfNotPresent + channel: stable + manifestSha256: "dc3c8fe1ab83941c838ccc5a8a5862f7ddfa38c2078e580b5649db26554565b7" + image: + pullPolicy: IfNotPresent labels: stellaops.io/channel: stable @@ -15,94 +15,94 @@ telemetry: defaultTenant: stage tls: secretName: stellaops-otel-tls-stage - -configMaps: - notify-config: - data: - notify.yaml: | - storage: - driver: mongo - connectionString: "mongodb://notify-mongo.stage.svc.cluster.local:27017" - database: "stellaops_notify_stage" - commandTimeoutSeconds: 45 - - authority: - enabled: true - issuer: "https://authority.stage.stella-ops.org" - metadataAddress: "https://authority.stage.stella-ops.org/.well-known/openid-configuration" - requireHttpsMetadata: true - allowAnonymousFallback: false - backchannelTimeoutSeconds: 30 - tokenClockSkewSeconds: 60 - audiences: - - notify - readScope: notify.read - adminScope: notify.admin - - api: - basePath: "/api/v1/notify" - internalBasePath: "/internal/notify" - tenantHeader: "X-StellaOps-Tenant" - - plugins: - baseDirectory: "/opt/stellaops" - directory: "plugins/notify" - searchPatterns: - - "StellaOps.Notify.Connectors.*.dll" - orderedPlugins: - - StellaOps.Notify.Connectors.Slack - - StellaOps.Notify.Connectors.Teams - - StellaOps.Notify.Connectors.Email - - StellaOps.Notify.Connectors.Webhook - - telemetry: - enableRequestLogging: true - minimumLogLevel: Information -services: - authority: - image: registry.stella-ops.org/stellaops/authority@sha256:b0348bad1d0b401cc3c71cb40ba034c8043b6c8874546f90d4783c9dbfcc0bf5 - service: - port: 8440 - env: - STELLAOPS_AUTHORITY__ISSUER: "https://stellaops-authority:8440" - STELLAOPS_AUTHORITY__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" - STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0: "/app/plugins" - STELLAOPS_AUTHORITY__PLUGINS__CONFIGURATIONDIRECTORY: "/app/etc/authority.plugins" - signer: - image: registry.stella-ops.org/stellaops/signer@sha256:8ad574e61f3a9e9bda8a58eb2700ae46813284e35a150b1137bc7c2b92ac0f2e - service: - port: 8441 - env: - SIGNER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" - SIGNER__POE__INTROSPECTURL: "https://licensing.stage.stella-ops.internal/introspect" - SIGNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" - attestor: - image: registry.stella-ops.org/stellaops/attestor@sha256:0534985f978b0b5d220d73c96fddd962cd9135f616811cbe3bff4666c5af568f - service: - port: 8442 - env: - ATTESTOR__SIGNER__BASEURL: "https://stellaops-signer:8441" - ATTESTOR__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" - concelier: - image: registry.stella-ops.org/stellaops/concelier@sha256:c58cdcaee1d266d68d498e41110a589dd204b487d37381096bd61ab345a867c5 - service: - port: 8445 - env: - CONCELIER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" - CONCELIER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" - CONCELIER__STORAGE__S3__ACCESSKEYID: "stellaops-stage" - CONCELIER__STORAGE__S3__SECRETACCESSKEY: "stage-minio-secret" - CONCELIER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" - volumeMounts: - - name: concelier-jobs - mountPath: /var/lib/concelier/jobs - volumeClaims: - - name: concelier-jobs - claimName: stellaops-concelier-jobs + +configMaps: + notify-config: + data: + notify.yaml: | + storage: + driver: mongo + connectionString: "mongodb://notify-mongo.stage.svc.cluster.local:27017" + database: "stellaops_notify_stage" + commandTimeoutSeconds: 45 + + authority: + enabled: true + issuer: "https://authority.stage.stella-ops.org" + metadataAddress: "https://authority.stage.stella-ops.org/.well-known/openid-configuration" + requireHttpsMetadata: true + allowAnonymousFallback: false + backchannelTimeoutSeconds: 30 + tokenClockSkewSeconds: 60 + audiences: + - notify + readScope: notify.read + adminScope: notify.admin + + api: + basePath: "/api/v1/notify" + internalBasePath: "/internal/notify" + tenantHeader: "X-StellaOps-Tenant" + + plugins: + baseDirectory: "/opt/stellaops" + directory: "plugins/notify" + searchPatterns: + - "StellaOps.Notify.Connectors.*.dll" + orderedPlugins: + - StellaOps.Notify.Connectors.Slack + - StellaOps.Notify.Connectors.Teams + - StellaOps.Notify.Connectors.Email + - StellaOps.Notify.Connectors.Webhook + + telemetry: + enableRequestLogging: true + minimumLogLevel: Information +services: + authority: + image: registry.stella-ops.org/stellaops/authority@sha256:b0348bad1d0b401cc3c71cb40ba034c8043b6c8874546f90d4783c9dbfcc0bf5 + service: + port: 8440 + env: + STELLAOPS_AUTHORITY__ISSUER: "https://stellaops-authority:8440" + STELLAOPS_AUTHORITY__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" + STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0: "/app/plugins" + STELLAOPS_AUTHORITY__PLUGINS__CONFIGURATIONDIRECTORY: "/app/etc/authority.plugins" + signer: + image: registry.stella-ops.org/stellaops/signer@sha256:8ad574e61f3a9e9bda8a58eb2700ae46813284e35a150b1137bc7c2b92ac0f2e + service: + port: 8441 + env: + SIGNER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" + SIGNER__POE__INTROSPECTURL: "https://licensing.stage.stella-ops.internal/introspect" + SIGNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" + attestor: + image: registry.stella-ops.org/stellaops/attestor@sha256:0534985f978b0b5d220d73c96fddd962cd9135f616811cbe3bff4666c5af568f + service: + port: 8442 + env: + ATTESTOR__SIGNER__BASEURL: "https://stellaops-signer:8441" + ATTESTOR__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" + concelier: + image: registry.stella-ops.org/stellaops/concelier@sha256:c58cdcaee1d266d68d498e41110a589dd204b487d37381096bd61ab345a867c5 + service: + port: 8445 + env: + CONCELIER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" + CONCELIER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + CONCELIER__STORAGE__S3__ACCESSKEYID: "stellaops-stage" + CONCELIER__STORAGE__S3__SECRETACCESSKEY: "stage-minio-secret" + CONCELIER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" + volumeMounts: + - name: concelier-jobs + mountPath: /var/lib/concelier/jobs + volumeClaims: + - name: concelier-jobs + claimName: stellaops-concelier-jobs scanner-web: - image: registry.stella-ops.org/stellaops/scanner-web@sha256:14b23448c3f9586a9156370b3e8c1991b61907efa666ca37dd3aaed1e79fe3b7 - service: - port: 8444 + image: registry.stella-ops.org/stellaops/scanner-web@sha256:14b23448c3f9586a9156370b3e8c1991b61907efa666ca37dd3aaed1e79fe3b7 + service: + port: 8444 env: SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" @@ -116,9 +116,13 @@ services: SCANNER__EVENTS__STREAM: "stella.events" SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" + SCANNER_SURFACE_FS_ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER_SURFACE_CACHE_ROOT: "/var/lib/stellaops/surface" + SCANNER_SURFACE_SECRETS_PROVIDER: "kubernetes" + SCANNER_SURFACE_SECRETS_ROOT: "stellaops/scanner" scanner-worker: - image: registry.stella-ops.org/stellaops/scanner-worker@sha256:32e25e76386eb9ea8bee0a1ad546775db9a2df989fab61ac877e351881960dab - replicas: 2 + image: registry.stella-ops.org/stellaops/scanner-worker@sha256:32e25e76386eb9ea8bee0a1ad546775db9a2df989fab61ac877e351881960dab + replicas: 2 env: SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" @@ -132,61 +136,65 @@ services: SCANNER__EVENTS__STREAM: "stella.events" SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" - notify-web: - image: registry.stella-ops.org/stellaops/notify-web:2025.09.2 - service: - port: 8446 - env: - DOTNET_ENVIRONMENT: Production - configMounts: - - name: notify-config - mountPath: /app/etc/notify.yaml - subPath: notify.yaml - configMap: notify-config - excititor: - image: registry.stella-ops.org/stellaops/excititor@sha256:59022e2016aebcef5c856d163ae705755d3f81949d41195256e935ef40a627fa - env: - EXCITITOR__CONCELIER__BASEURL: "https://stellaops-concelier:8445" - EXCITITOR__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" - web-ui: - image: registry.stella-ops.org/stellaops/web-ui@sha256:10d924808c48e4353e3a241da62eb7aefe727a1d6dc830eb23a8e181013b3a23 - service: - port: 8443 - env: - STELLAOPS_UI__BACKEND__BASEURL: "https://stellaops-scanner-web:8444" - mongo: - class: infrastructure - image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 - service: - port: 27017 - command: - - mongod - - --bind_ip_all - env: - MONGO_INITDB_ROOT_USERNAME: stellaops-stage - MONGO_INITDB_ROOT_PASSWORD: stellaops-stage - volumeMounts: - - name: mongo-data - mountPath: /data/db - volumeClaims: - - name: mongo-data - claimName: stellaops-mongo-data + SCANNER_SURFACE_FS_ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER_SURFACE_CACHE_ROOT: "/var/lib/stellaops/surface" + SCANNER_SURFACE_SECRETS_PROVIDER: "kubernetes" + SCANNER_SURFACE_SECRETS_ROOT: "stellaops/scanner" + notify-web: + image: registry.stella-ops.org/stellaops/notify-web:2025.09.2 + service: + port: 8446 + env: + DOTNET_ENVIRONMENT: Production + configMounts: + - name: notify-config + mountPath: /app/etc/notify.yaml + subPath: notify.yaml + configMap: notify-config + excititor: + image: registry.stella-ops.org/stellaops/excititor@sha256:59022e2016aebcef5c856d163ae705755d3f81949d41195256e935ef40a627fa + env: + EXCITITOR__CONCELIER__BASEURL: "https://stellaops-concelier:8445" + EXCITITOR__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" + web-ui: + image: registry.stella-ops.org/stellaops/web-ui@sha256:10d924808c48e4353e3a241da62eb7aefe727a1d6dc830eb23a8e181013b3a23 + service: + port: 8443 + env: + STELLAOPS_UI__BACKEND__BASEURL: "https://stellaops-scanner-web:8444" + mongo: + class: infrastructure + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + service: + port: 27017 + command: + - mongod + - --bind_ip_all + env: + MONGO_INITDB_ROOT_USERNAME: stellaops-stage + MONGO_INITDB_ROOT_PASSWORD: stellaops-stage + volumeMounts: + - name: mongo-data + mountPath: /data/db + volumeClaims: + - name: mongo-data + claimName: stellaops-mongo-data minio: class: infrastructure image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e service: port: 9000 - command: - - server - - /data - - --console-address - - :9001 - env: - MINIO_ROOT_USER: stellaops-stage - MINIO_ROOT_PASSWORD: stage-minio-secret - volumeMounts: - - name: minio-data - mountPath: /data + command: + - server + - /data + - --console-address + - :9001 + env: + MINIO_ROOT_USER: stellaops-stage + MINIO_ROOT_PASSWORD: stage-minio-secret + volumeMounts: + - name: minio-data + mountPath: /data volumeClaims: - name: minio-data claimName: stellaops-minio-data @@ -210,18 +218,18 @@ services: volumeClaims: - name: rustfs-data claimName: stellaops-rustfs-data - nats: - class: infrastructure - image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e - service: - port: 4222 - command: - - -js - - -sd - - /data - volumeMounts: - - name: nats-data - mountPath: /data - volumeClaims: - - name: nats-data - claimName: stellaops-nats-data + nats: + class: infrastructure + image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e + service: + port: 4222 + command: + - -js + - -sd + - /data + volumeMounts: + - name: nats-data + mountPath: /data + volumeClaims: + - name: nats-data + claimName: stellaops-nats-data diff --git a/docs/implplan/SPRINT_110_ingestion_evidence.md b/docs/implplan/SPRINT_110_ingestion_evidence.md index eb2eeb7e8..10c00d90d 100644 --- a/docs/implplan/SPRINT_110_ingestion_evidence.md +++ b/docs/implplan/SPRINT_110_ingestion_evidence.md @@ -227,7 +227,7 @@ EXCITITOR-AIRGAP-56-002 `Bundle provenance` | TODO | Persist bundle metadata on EXCITITOR-AIRGAP-57-001 `Sealed-mode enforcement` | TODO | Block non-mirror connectors in sealed mode and surface remediation errors. Dependencies: EXCITITOR-AIRGAP-56-002. | Excititor Core Guild, AirGap Policy Guild (src/Excititor/__Libraries/StellaOps.Excititor.Core/TASKS.md) EXCITITOR-AIRGAP-57-002 `Staleness annotations` | TODO | Annotate VEX statements with staleness metrics and expose via API. Dependencies: EXCITITOR-AIRGAP-57-001. | Excititor Core Guild, AirGap Time Guild (src/Excititor/__Libraries/StellaOps.Excititor.Core/TASKS.md) EXCITITOR-AIRGAP-58-001 `Portable VEX evidence` | TODO | Package VEX evidence segments into portable evidence bundles linked to timeline. Dependencies: EXCITITOR-AIRGAP-57-002. | Excititor Core Guild, Evidence Locker Guild (src/Excititor/__Libraries/StellaOps.Excititor.Core/TASKS.md) -EXCITITOR-ATTEST-01-003 – Verification suite & observability | Team Excititor Attestation | DOING (2025-10-22) – Continuing implementation: build `IVexAttestationVerifier`, wire metrics/logging, and add regression tests. Draft plan in `EXCITITOR-ATTEST-01-003-plan.md` (2025-10-19) guides scope; updating with worknotes as progress lands.
2025-10-31: Verifier now tolerates duplicate source providers from AOC raw projections, downgrades offline Rekor verification to a degraded result, and enforces trusted signer registry checks with detailed diagnostics/tests.
2025-11-05 14:35Z: Resuming with diagnostics/observability deliverables (typed diagnostics record, ActivitySource wiring, metrics dimensions) before WebService/Worker integration. | EXCITITOR-ATTEST-01-002 (src/Excititor/__Libraries/StellaOps.Excititor.Attestation/TASKS.md) +EXCITITOR-ATTEST-01-003 – Verification suite & observability | Team Excititor Attestation | TODO (2025-11-06) – Continuing implementation: build `IVexAttestationVerifier`, wire metrics/logging, and add regression tests. Draft plan in `EXCITITOR-ATTEST-01-003-plan.md` (2025-10-19) guides scope; updating with worknotes as progress lands.
2025-10-31: Verifier now tolerates duplicate source providers from AOC raw projections, downgrades offline Rekor verification to a degraded result, and enforces trusted signer registry checks with detailed diagnostics/tests.
2025-11-05 14:35Z: Resuming with diagnostics/observability deliverables (typed diagnostics record, ActivitySource wiring, metrics dimensions) before WebService/Worker integration.
2025-11-06 07:12Z: Worker & web service suites pass with new diagnostics (`dotnet test` via staged libssl1.1); export envelope context exposed publicly for mirror bundle publishing.
2025-11-06 07:55Z: Paused—automation for OpenSSL shim tracked under `DEVOPS-OPENSSL-11-001/002`. | EXCITITOR-ATTEST-01-002 (src/Excititor/__Libraries/StellaOps.Excititor.Attestation/TASKS.md) EXCITITOR-ATTEST-73-001 `VEX attestation payloads` | TODO | Provide VEX statement metadata (supplier identity, justification, scope) required for VEXAttestation payloads. Dependencies: EXCITITOR-ATTEST-01-003. | Excititor Core Guild, Attestation Payloads Guild (src/Excititor/__Libraries/StellaOps.Excititor.Core/TASKS.md) EXCITITOR-ATTEST-73-002 `Chain provenance` | TODO | Expose linkage from VEX statements to subject/product for chain of custody graph. Dependencies: EXCITITOR-ATTEST-73-001. | Excititor Core Guild (src/Excititor/__Libraries/StellaOps.Excititor.Core/TASKS.md) EXCITITOR-CONN-MS-01-003 – Trust metadata & provenance hints | Team Excititor Connectors – MSRC | TODO – Emit cosign/AAD issuer metadata, attach provenance details, and document policy integration. | EXCITITOR-CONN-MS-01-002, EXCITITOR-POLICY-01-001 (src/Excititor/__Libraries/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md) diff --git a/docs/implplan/SPRINT_130_scanner_surface.md b/docs/implplan/SPRINT_130_scanner_surface.md index 71e4ee43b..a7685198e 100644 --- a/docs/implplan/SPRINT_130_scanner_surface.md +++ b/docs/implplan/SPRINT_130_scanner_surface.md @@ -134,8 +134,8 @@ Summary: Scanner & Surface focus on Scanner (phase VII). Task ID | State | Task description | Owners (Source) --- | --- | --- | --- SCANNER-ENTRYTRACE-18-504 | TODO | Emit EntryTrace AOC NDJSON (`entrytrace.entry/node/edge/target/warning/capability`) and wire CLI/service streaming outputs. Dependencies: SCANNER-ENTRYTRACE-18-503. | EntryTrace Guild (src/Scanner/__Libraries/StellaOps.Scanner.EntryTrace/TASKS.md) -SCANNER-ENV-01 | DOING (2025-11-02) | Replace ad-hoc environment reads with `StellaOps.Scanner.Surface.Env` helpers for cache roots and CAS endpoints.
2025-11-02: Env helper wiring drafted for Worker startup; initial tests validate cache root resolution.
2025-11-05 14:55Z: Continuing integration by propagating resolved settings into cache/secret services and prepping worker smoke tests + docs updates.
2025-11-05 19:18Z: Bound `SurfaceCacheOptions` root to Surface.Env settings and added configurator unit coverage.
2025-11-06 17:05Z: Documented misconfiguration warnings and updated module README to highlight Surface.Env usage. | Scanner Worker Guild (src/Scanner/StellaOps.Scanner.Worker/TASKS.md) -SCANNER-ENV-02 | DOING (2025-11-02) | Wire Surface.Env helpers into WebService hosting (cache roots, feature flags) and document configuration. Dependencies: SCANNER-ENV-01.
2025-11-02: WebService bootstrap now consumes Surface.Env helpers for cache roots and feature flag toggles; configuration doc draft pending.
2025-11-05 14:55Z: Picking up configuration/documentation work and aligning API readiness checks with Surface.Env validation outputs.
2025-11-05 19:18Z: Added unit test for Surface.Env cache root binding and ensured configurator registration.
2025-11-06 17:05Z: Surface.Env design doc expanded with warning catalogue and release notes, README refreshed. | Scanner WebService Guild, Ops Guild (src/Scanner/StellaOps.Scanner.WebService/TASKS.md) +SCANNER-ENV-01 | TODO (2025-11-06) | Replace ad-hoc environment reads with `StellaOps.Scanner.Surface.Env` helpers for cache roots and CAS endpoints.
2025-11-02: Env helper wiring drafted for Worker startup; initial tests validate cache root resolution.
2025-11-05 14:55Z: Continuing integration by propagating resolved settings into cache/secret services and prepping worker smoke tests + docs updates.
2025-11-05 19:18Z: Bound `SurfaceCacheOptions` root to Surface.Env settings and added configurator unit coverage.
2025-11-06 17:05Z: Documented misconfiguration warnings and updated module README to highlight Surface.Env usage.
2025-11-06 07:45Z: Helm/Compose env profiles (dev/stage/prod/airgap/mirror) now emit `SCANNER_SURFACE_*` defaults and ops README covers rollout warnings.
2025-11-06 07:55Z: Paused pending automation tracked under `DEVOPS-OPENSSL-11-001/002` and additional Surface.Env fixtures. | Scanner Worker Guild (src/Scanner/StellaOps.Scanner.Worker/TASKS.md) +SCANNER-ENV-02 | TODO (2025-11-06) | Wire Surface.Env helpers into WebService hosting (cache roots, feature flags) and document configuration. Dependencies: SCANNER-ENV-01.
2025-11-02: WebService bootstrap now consumes Surface.Env helpers for cache roots and feature flag toggles; configuration doc draft pending.
2025-11-05 14:55Z: Picking up configuration/documentation work and aligning API readiness checks with Surface.Env validation outputs.
2025-11-05 19:18Z: Added unit test for Surface.Env cache root binding and ensured configurator registration.
2025-11-06 17:05Z: Surface.Env design doc expanded with warning catalogue and release notes, README refreshed.
2025-11-06 07:45Z: Helm/Compose templates ship `SCANNER_SURFACE_*` defaults across dev/stage/prod/airgap/mirror profiles with rollout guidance in deploy docs.
2025-11-06 07:55Z: Paused; follow-up automation tracked under `DEVOPS-OPENSSL-11-001/002` and readiness tests outstanding. | Scanner WebService Guild, Ops Guild (src/Scanner/StellaOps.Scanner.WebService/TASKS.md) SCANNER-ENV-03 | TODO | Adopt Surface.Env helpers for plugin configuration (cache roots, CAS endpoints, feature toggles). Dependencies: SCANNER-ENV-02. | BuildX Plugin Guild (src/Scanner/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md) SCANNER-EVENTS-16-301 | BLOCKED (2025-10-26) | Emit orchestrator-compatible envelopes (`scanner.event.*`) and update integration tests to verify Notifier ingestion (no Redis queue coupling). | Scanner WebService Guild (src/Scanner/StellaOps.Scanner.WebService/TASKS.md) SCANNER-EVENTS-16-302 | DOING (2025-10-26) | Extend orchestrator event links (report/policy/attestation) once endpoints are finalised across gateway + console. Dependencies: SCANNER-EVENTS-16-301. | Scanner WebService Guild (src/Scanner/StellaOps.Scanner.WebService/TASKS.md) diff --git a/local-nuget/Google.Api.Gax.Grpc.4.7.0.nupkg b/local-nuget/Google.Api.Gax.Grpc.4.7.0.nupkg new file mode 100644 index 000000000..a531abd0e Binary files /dev/null and b/local-nuget/Google.Api.Gax.Grpc.4.7.0.nupkg differ diff --git a/local-nuget/Google.Apis.Core.1.64.0.nupkg b/local-nuget/Google.Apis.Core.1.64.0.nupkg new file mode 100644 index 000000000..b8b1e8be4 Binary files /dev/null and b/local-nuget/Google.Apis.Core.1.64.0.nupkg differ diff --git a/local-nuget/Google.Cloud.Kms.V1.3.19.0.nupkg b/local-nuget/Google.Cloud.Kms.V1.3.19.0.nupkg new file mode 100644 index 000000000..7d5fcf1aa Binary files /dev/null and b/local-nuget/Google.Cloud.Kms.V1.3.19.0.nupkg differ diff --git a/local-nuget/Google.Protobuf.3.31.1.nupkg b/local-nuget/Google.Protobuf.3.31.1.nupkg new file mode 100644 index 000000000..cba650326 Binary files /dev/null and b/local-nuget/Google.Protobuf.3.31.1.nupkg differ diff --git a/local-nuget/Grpc.Auth.2.71.0.nupkg b/local-nuget/Grpc.Auth.2.71.0.nupkg new file mode 100644 index 000000000..2773616bc Binary files /dev/null and b/local-nuget/Grpc.Auth.2.71.0.nupkg differ diff --git a/local-nuget/Grpc.Core.Api.2.71.0.nupkg b/local-nuget/Grpc.Core.Api.2.71.0.nupkg new file mode 100644 index 000000000..32dba3f32 Binary files /dev/null and b/local-nuget/Grpc.Core.Api.2.71.0.nupkg differ diff --git a/local-nuget/Grpc.Net.Client.2.71.0.nupkg b/local-nuget/Grpc.Net.Client.2.71.0.nupkg new file mode 100644 index 000000000..071cfb626 Binary files /dev/null and b/local-nuget/Grpc.Net.Client.2.71.0.nupkg differ diff --git a/local-nuget/Grpc.Net.Common.2.71.0.nupkg b/local-nuget/Grpc.Net.Common.2.71.0.nupkg new file mode 100644 index 000000000..219045bba Binary files /dev/null and b/local-nuget/Grpc.Net.Common.2.71.0.nupkg differ diff --git a/local-nuget/Grpc.Tools.2.71.0.nupkg b/local-nuget/Grpc.Tools.2.71.0.nupkg new file mode 100644 index 000000000..3de683b30 Binary files /dev/null and b/local-nuget/Grpc.Tools.2.71.0.nupkg differ diff --git a/local-nuget/Pkcs11Interop.5.3.0.nupkg b/local-nuget/Pkcs11Interop.5.3.0.nupkg new file mode 100644 index 000000000..754865202 Binary files /dev/null and b/local-nuget/Pkcs11Interop.5.3.0.nupkg differ diff --git a/ops/devops/TASKS.md b/ops/devops/TASKS.md index 7978b9d24..e079a06e4 100644 --- a/ops/devops/TASKS.md +++ b/ops/devops/TASKS.md @@ -13,6 +13,9 @@ > Docs hand-off (2025-10-26): see `docs/ingestion/aggregation-only-contract.md` §5, `docs/modules/platform/architecture-overview.md`, and `docs/modules/cli/guides/cli-reference.md` for guard + verifier expectations. | DEVOPS-AOC-19-002 | BLOCKED (2025-10-26) | DevOps Guild | CLI-AOC-19-002, CONCELIER-WEB-AOC-19-004, EXCITITOR-WEB-AOC-19-004 | Add pipeline stage executing `stella aoc verify --since` against seeded Mongo snapshots for Concelier + Excititor, publishing violation report artefacts. | Stage runs on main/nightly, fails on violations, artifacts retained, runbook documented. | > Blocked: waiting on CLI verifier command and Concelier/Excititor guard endpoints to land (CLI-AOC-19-002, CONCELIER-WEB-AOC-19-004, EXCITITOR-WEB-AOC-19-004). +| DEVOPS-OPENSSL-11-001 | TODO (2025-11-06) | DevOps Guild, Build Infra Guild | — | Package the OpenSSL 1.1 shim (`tests/native/openssl-1.1/linux-x64`) into test harness output so Mongo2Go suites discover it automatically. | Shim copied during `dotnet test`, documentation updated, redundant manual extraction removed. | +> 2025-11-06: Interim guidance published in `tests/native/openssl-1.1/README.md` and `deploy/README.md`; automation still required. +| DEVOPS-OPENSSL-11-002 | TODO (2025-11-06) | DevOps Guild, CI Guild | DEVOPS-OPENSSL-11-001 | Ensure CI runners and Docker images that execute Mongo2Go tests export `LD_LIBRARY_PATH` (or embed the shim) to unblock unattended pipelines. | CI jobs set the variable or bake the libraries; runbook documents the location; smoke builds green without manual exports. | | DEVOPS-AOC-19-003 | BLOCKED (2025-10-26) | DevOps Guild, QA Guild | CONCELIER-WEB-AOC-19-003, EXCITITOR-WEB-AOC-19-003 | Enforce unit test coverage thresholds for AOC guard suites and ensure coverage exported to dashboards. | Coverage report includes guard projects, threshold gate passes/fails as expected, dashboards refreshed with new metrics. | > Blocked: guard coverage suites and exporter hooks pending in Concelier/Excititor (CONCELIER-WEB-AOC-19-003, EXCITITOR-WEB-AOC-19-003). | DEVOPS-AOC-19-101 | TODO (2025-10-28) | DevOps Guild, Concelier Storage Guild | CONCELIER-STORE-AOC-19-002 | Draft supersedes backfill rollout (freeze window, dry-run steps, rollback) once advisory_raw idempotency index passes staging verification. | Runbook committed in `docs/deploy/containers.md` + Offline Kit notes, staging rehearsal scheduled with dependencies captured in SPRINTS. | diff --git a/src/Excititor/StellaOps.Excititor.WebService/Program.cs b/src/Excititor/StellaOps.Excititor.WebService/Program.cs index 9bdb2a7f2..2860bca32 100644 --- a/src/Excititor/StellaOps.Excititor.WebService/Program.cs +++ b/src/Excititor/StellaOps.Excititor.WebService/Program.cs @@ -18,7 +18,6 @@ using StellaOps.Excititor.Policy; using StellaOps.Excititor.Storage.Mongo; using StellaOps.Excititor.WebService.Endpoints; using StellaOps.Excititor.WebService.Services; -using StellaOps.Excititor.Core; using StellaOps.Excititor.Core.Aoc; var builder = WebApplication.CreateBuilder(args); diff --git a/src/Excititor/__Libraries/StellaOps.Excititor.Attestation/TASKS.md b/src/Excititor/__Libraries/StellaOps.Excititor.Attestation/TASKS.md index 867e6ea93..b7d212323 100644 --- a/src/Excititor/__Libraries/StellaOps.Excititor.Attestation/TASKS.md +++ b/src/Excititor/__Libraries/StellaOps.Excititor.Attestation/TASKS.md @@ -2,6 +2,8 @@ If you are working on this file you need to read docs/modules/excititor/ARCHITEC # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| -|EXCITITOR-ATTEST-01-003 – Verification suite & observability|Team Excititor Attestation|EXCITITOR-ATTEST-01-002|DOING (2025-10-22) – Continuing implementation: build `IVexAttestationVerifier`, wire metrics/logging, and add regression tests. Draft plan in `EXCITITOR-ATTEST-01-003-plan.md` (2025-10-19) guides scope; updating with worknotes as progress lands.
2025-10-31: Verifier now tolerates duplicate source providers from AOC raw projections, downgrades offline Rekor verification to a degraded result, and enforces trusted signer registry checks with detailed diagnostics/tests.
2025-11-05 14:35Z: Picking up diagnostics record/ActivitySource work and aligning metrics dimensions before wiring verifier into WebService/Worker paths.| +|EXCITITOR-ATTEST-01-003 – Verification suite & observability|Team Excititor Attestation|EXCITITOR-ATTEST-01-002|TODO (2025-11-06) – Continuing implementation: build `IVexAttestationVerifier`, wire metrics/logging, and add regression tests. Draft plan in `EXCITITOR-ATTEST-01-003-plan.md` (2025-10-19) guides scope; updating with worknotes as progress lands.
2025-10-31: Verifier now tolerates duplicate source providers from AOC raw projections, downgrades offline Rekor verification to a degraded result, and enforces trusted signer registry checks with detailed diagnostics/tests.
2025-11-05 14:35Z: Picking up diagnostics record/ActivitySource work and aligning metrics dimensions before wiring verifier into WebService/Worker paths.| > 2025-11-05 19:10Z: Worker signature verifier now emits structured diagnostics/metrics via `VexAttestationDiagnostics`; attestation verification results flow into metric labels and logs. +> 2025-11-06 07:12Z: Export verifier builds unblocked; Excititor worker + web service test suites pass with diagnostics wiring (`dotnet test` invocations succeed with staged libssl1.1). +> 2025-11-06 07:55Z: Paused after documenting OpenSSL shim usage; follow-up automation tracked under `DEVOPS-OPENSSL-11-001/002`. > Remark (2025-10-22): Added verifier implementation + metrics/tests; next steps include wiring into WebService/Worker flows and expanding negative-path coverage. diff --git a/src/Excititor/__Libraries/StellaOps.Excititor.Export/ExportEngine.cs b/src/Excititor/__Libraries/StellaOps.Excititor.Export/ExportEngine.cs index 5f37d8b4c..453d09baf 100644 --- a/src/Excititor/__Libraries/StellaOps.Excititor.Export/ExportEngine.cs +++ b/src/Excititor/__Libraries/StellaOps.Excititor.Export/ExportEngine.cs @@ -98,6 +98,7 @@ public sealed class VexExportEngine : IExportEngine cached.PolicyDigest, cached.ConsensusDigest, cached.ScoreDigest, + cached.QuietProvenance, cached.Attestation, cached.SizeBytes); } diff --git a/src/Excititor/__Libraries/StellaOps.Excititor.Export/VexExportEnvelopeBuilder.cs b/src/Excititor/__Libraries/StellaOps.Excititor.Export/VexExportEnvelopeBuilder.cs index 5ff91ab94..653fa737c 100644 --- a/src/Excititor/__Libraries/StellaOps.Excititor.Export/VexExportEnvelopeBuilder.cs +++ b/src/Excititor/__Libraries/StellaOps.Excititor.Export/VexExportEnvelopeBuilder.cs @@ -130,7 +130,7 @@ internal static class VexExportEnvelopeBuilder } } -internal sealed record VexExportEnvelopeContext( +public sealed record VexExportEnvelopeContext( ImmutableArray Consensus, string ConsensusCanonicalJson, VexContentAddress ConsensusDigest, diff --git a/src/Excititor/__Libraries/StellaOps.Excititor.Export/VexMirrorBundlePublisher.cs b/src/Excititor/__Libraries/StellaOps.Excititor.Export/VexMirrorBundlePublisher.cs index 24276e9f5..acf4eca92 100644 --- a/src/Excititor/__Libraries/StellaOps.Excititor.Export/VexMirrorBundlePublisher.cs +++ b/src/Excititor/__Libraries/StellaOps.Excititor.Export/VexMirrorBundlePublisher.cs @@ -280,7 +280,7 @@ public sealed class VexMirrorBundlePublisher : IVexMirrorBundlePublisher ToRelativePath(mirrorRoot, manifestPath), manifestBytes.LongLength, ComputeDigest(manifestBytes), - signature: null); + Signature: null); var bundleDescriptor = manifestDocument.Bundle with { @@ -298,7 +298,7 @@ public sealed class VexMirrorBundlePublisher : IVexMirrorBundlePublisher manifestDocument.DomainId, manifestDocument.DisplayName, manifestDocument.GeneratedAt, - manifestDocument.Exports.Length, + manifestDocument.Exports.Count, manifestDescriptor, bundleDescriptor, exportKeys)); @@ -474,6 +474,11 @@ public sealed class VexMirrorBundlePublisher : IVexMirrorBundlePublisher private JsonMirrorSigningContext PrepareSigningContext(MirrorSigningOptions signingOptions) { + if (_cryptoRegistry is null) + { + throw new InvalidOperationException("Mirror signing requires a crypto provider registry to be configured."); + } + var algorithm = string.IsNullOrWhiteSpace(signingOptions.Algorithm) ? SignatureAlgorithms.Es256 : signingOptions.Algorithm.Trim(); @@ -496,7 +501,7 @@ public sealed class VexMirrorBundlePublisher : IVexMirrorBundlePublisher var provider = ResolveProvider(algorithm, providerHint); var signingKey = LoadSigningKey(signingOptions, provider, algorithm); provider.UpsertSigningKey(signingKey); - resolved = _cryptoRegistry.ResolveSigner(CryptoCapability.Signing, algorithm, new CryptoKeyReference(keyId, provider.Name), provider.Name); + resolved = _cryptoRegistry!.ResolveSigner(CryptoCapability.Signing, algorithm, new CryptoKeyReference(keyId, provider.Name), provider.Name); } return new JsonMirrorSigningContext(resolved.Signer, algorithm, resolved.ProviderName, _timeProvider); diff --git a/src/Scanner/StellaOps.Scanner.WebService/TASKS.md b/src/Scanner/StellaOps.Scanner.WebService/TASKS.md index c8387266c..f9b047f87 100644 --- a/src/Scanner/StellaOps.Scanner.WebService/TASKS.md +++ b/src/Scanner/StellaOps.Scanner.WebService/TASKS.md @@ -4,7 +4,7 @@ |----|--------|----------|------------|-------------|---------------| | SCAN-REPLAY-186-001 | TODO | Scanner WebService Guild | REPLAY-CORE-185-001 | Implement scan `record` mode producing replay manifests/bundles, capture policy/feed/tool hashes, and update `docs/modules/scanner/architecture.md` referencing `docs/replay/DETERMINISTIC_REPLAY.md` Section 6. | API/worker integration tests cover record mode; docs merged; replay artifacts stored per spec. | | SCANNER-SURFACE-02 | DONE (2025-11-05) | Scanner WebService Guild | SURFACE-FS-02 | Publish Surface.FS pointers (CAS URIs, manifests) via scan/report APIs and update attestation metadata.
2025-11-05: Surface pointers projected through scan/report endpoints, orchestrator samples + DSSE fixtures refreshed with manifest block, readiness tests updated to use validator stub. | OpenAPI updated; clients regenerated; integration tests validate pointer presence and tenancy. | -| SCANNER-ENV-02 | DOING (2025-11-02) | Scanner WebService Guild, Ops Guild | SURFACE-ENV-02 | Wire Surface.Env helpers into WebService hosting (cache roots, feature flags) and document configuration.
2025-11-02: Cache root resolution switched to helper; feature flag bindings updated; Helm/Compose updates pending review.
2025-11-05 14:55Z: Aligning readiness checks, docs, and Helm/Compose templates with Surface.Env outputs and planning test coverage for configuration fallbacks.
2025-11-06 17:05Z: Surface.Env documentation/README refreshed; warning catalogue captured for ops handoff. | Service uses helper; env table documented; helm/compose templates updated. | +| SCANNER-ENV-02 | TODO (2025-11-06) | Scanner WebService Guild, Ops Guild | SURFACE-ENV-02 | Wire Surface.Env helpers into WebService hosting (cache roots, feature flags) and document configuration.
2025-11-02: Cache root resolution switched to helper; feature flag bindings updated; Helm/Compose updates pending review.
2025-11-05 14:55Z: Aligning readiness checks, docs, and Helm/Compose templates with Surface.Env outputs and planning test coverage for configuration fallbacks.
2025-11-06 17:05Z: Surface.Env documentation/README refreshed; warning catalogue captured for ops handoff.
2025-11-06 07:45Z: Helm values (dev/stage/prod/airgap/mirror) and Compose examples updated with `SCANNER_SURFACE_*` defaults plus rollout warning note in `deploy/README.md`.
2025-11-06 07:55Z: Paused; follow-up automation captured under `DEVOPS-OPENSSL-11-001/002` and pending Surface.Env readiness tests. | Service uses helper; env table documented; helm/compose templates updated. | > 2025-11-05 19:18Z: Added configurator to project wiring and unit test ensuring Surface.Env cache root is honoured. | SCANNER-SECRETS-02 | DOING (2025-11-02) | Scanner WebService Guild, Security Guild | SURFACE-SECRETS-02 | Replace ad-hoc secret wiring with Surface.Secrets for report/export operations (registry and CAS tokens).
2025-11-02: Export/report flows now depend on Surface.Secrets stub; integration tests in progress. | Secrets fetched through shared provider; unit/integration tests cover rotation + failure cases. | | SCANNER-EVENTS-16-301 | BLOCKED (2025-10-26) | Scanner WebService Guild | ORCH-SVC-38-101, NOTIFY-SVC-38-001 | Emit orchestrator-compatible envelopes (`scanner.event.*`) and update integration tests to verify Notifier ingestion (no Redis queue coupling). | Tests assert envelope schema + orchestrator publish; Notifier consumer harness passes; docs updated with new event contract. Blocked by .NET 10 preview OpenAPI/Auth dependency drift preventing `dotnet test` completion. | diff --git a/src/Scanner/StellaOps.Scanner.Worker/TASKS.md b/src/Scanner/StellaOps.Scanner.Worker/TASKS.md index 20ec406a1..9c179643a 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/TASKS.md +++ b/src/Scanner/StellaOps.Scanner.Worker/TASKS.md @@ -4,6 +4,6 @@ |----|--------|----------|------------|-------------|---------------| | SCAN-REPLAY-186-002 | TODO | Scanner Worker Guild | REPLAY-CORE-185-001 | Enforce deterministic analyzer execution when consuming replay input bundles, emit layer Merkle metadata, and author `docs/modules/scanner/deterministic-execution.md` summarising invariants from `docs/replay/DETERMINISTIC_REPLAY.md` Section 4. | Replay mode analyzers pass determinism tests; new doc merged; integration fixtures updated. | | SCANNER-SURFACE-01 | DOING (2025-11-02) | Scanner Worker Guild | SURFACE-FS-02 | Persist Surface.FS manifests after analyzer stages, including layer CAS metadata and EntryTrace fragments.
2025-11-02: Draft Surface.FS manifests emitted for sample scans; telemetry counters under review. | Integration tests prove cache entries exist; telemetry counters exported. | -| SCANNER-ENV-01 | DOING (2025-11-02) | Scanner Worker Guild | SURFACE-ENV-02 | Replace ad-hoc environment reads with `StellaOps.Scanner.Surface.Env` helpers for cache roots and CAS endpoints.
2025-11-02: Worker bootstrap now resolves cache roots via helper; warning path documented; smoke tests running.
2025-11-05 14:55Z: Extending helper usage into cache/secrets configuration, updating worker validator wiring, and drafting docs/tests for new Surface.Env outputs.
2025-11-06 17:05Z: README/design docs updated with warning catalogue; startup logging guidance captured for ops runbooks. | Worker boots with helper; misconfiguration warnings documented; smoke tests updated. | +| SCANNER-ENV-01 | TODO (2025-11-06) | Scanner Worker Guild | SURFACE-ENV-02 | Replace ad-hoc environment reads with `StellaOps.Scanner.Surface.Env` helpers for cache roots and CAS endpoints.
2025-11-02: Worker bootstrap now resolves cache roots via helper; warning path documented; smoke tests running.
2025-11-05 14:55Z: Extending helper usage into cache/secrets configuration, updating worker validator wiring, and drafting docs/tests for new Surface.Env outputs.
2025-11-06 17:05Z: README/design docs updated with warning catalogue; startup logging guidance captured for ops runbooks.
2025-11-06 07:45Z: Helm/Compose env profiles (dev/stage/prod/airgap/mirror) now seed `SCANNER_SURFACE_*` defaults to keep worker cache roots aligned with Surface.Env helpers.
2025-11-06 07:55Z: Paused; pending automation tracked via `DEVOPS-OPENSSL-11-001/002` and Surface.Env test fixtures. | Worker boots with helper; misconfiguration warnings documented; smoke tests updated. | > 2025-11-05 19:18Z: Bound `SurfaceCacheOptions` root directory to resolved Surface.Env settings and added unit coverage around the configurator. | SCANNER-SECRETS-01 | DOING (2025-11-02) | Scanner Worker Guild, Security Guild | SURFACE-SECRETS-02 | Adopt `StellaOps.Scanner.Surface.Secrets` for registry/CAS credentials during scan execution.
2025-11-02: Surface.Secrets provider wired for CAS token retrieval; integration tests added. | Secrets fetched via shared provider; legacy secret code removed; integration tests cover rotation. | diff --git a/src/Scheduler/__Libraries/StellaOps.Scheduler.ImpactIndex/Fixtures/sample.bom-index.json b/src/Scheduler/__Libraries/StellaOps.Scheduler.ImpactIndex/Fixtures/sample.bom-index.json new file mode 100644 index 000000000..8d25380fd --- /dev/null +++ b/src/Scheduler/__Libraries/StellaOps.Scheduler.ImpactIndex/Fixtures/sample.bom-index.json @@ -0,0 +1,23 @@ +{ + "schema": "scheduler-impact-index@1", + "generatedAt": "2025-10-01T00:00:00Z", + "image": { + "repository": "registry.stellaops.test/team/sample-service", + "digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "tag": "1.0.0" + }, + "components": [ + { + "purl": "pkg:docker/sample-service@1.0.0", + "usage": [ + "runtime" + ] + }, + { + "purl": "pkg:pypi/requests@2.31.0", + "usage": [ + "usedByEntrypoint" + ] + } + ] +} diff --git a/src/Scheduler/__Libraries/StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj b/src/Scheduler/__Libraries/StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj index c16068b17..0dd39ae42 100644 --- a/src/Scheduler/__Libraries/StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj +++ b/src/Scheduler/__Libraries/StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj @@ -8,6 +8,7 @@ + diff --git a/src/Scheduler/__Tests/StellaOps.Scheduler.WebService.Tests/ImpactIndexFixtureTests.cs b/src/Scheduler/__Tests/StellaOps.Scheduler.WebService.Tests/ImpactIndexFixtureTests.cs new file mode 100644 index 000000000..0cce2eb09 --- /dev/null +++ b/src/Scheduler/__Tests/StellaOps.Scheduler.WebService.Tests/ImpactIndexFixtureTests.cs @@ -0,0 +1,52 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Scheduler.ImpactIndex; +using StellaOps.Scheduler.Models; +using Xunit; + +namespace StellaOps.Scheduler.WebService.Tests; + +public sealed class ImpactIndexFixtureTests +{ + [Fact] + public void FixtureDirectoryExists() + { + var fixtureDirectory = GetFixtureDirectory(); + Assert.True(Directory.Exists(fixtureDirectory), $"Fixture directory not found: {fixtureDirectory}"); + + var files = Directory.EnumerateFiles(fixtureDirectory, "bom-index.json", SearchOption.AllDirectories).ToArray(); + Assert.NotEmpty(files); + + var sampleFile = Path.Combine(fixtureDirectory, "sample", "bom-index.json"); + Assert.Contains(sampleFile, files); + } + + [Fact] + public async Task FixtureImpactIndexLoadsSampleImage() + { + var fixtureDirectory = GetFixtureDirectory(); + var options = new ImpactIndexStubOptions + { + FixtureDirectory = fixtureDirectory, + SnapshotId = "tests/impact-index-stub" + }; + + var index = new FixtureImpactIndex(options, TimeProvider.System, NullLogger.Instance); + var selector = new Selector(SelectorScope.AllImages); + + var impactSet = await index.ResolveAllAsync(selector, usageOnly: false); + + Assert.True(impactSet.Total > 0, "Expected the fixture impact index to load at least one image."); + } + + private static string GetFixtureDirectory() + { + var assemblyLocation = typeof(SchedulerWebApplicationFactory).Assembly.Location; + var assemblyDirectory = Path.GetDirectoryName(assemblyLocation) + ?? AppContext.BaseDirectory; + + return Path.GetFullPath(Path.Combine(assemblyDirectory, "seed-data", "impact-index")); + } +} diff --git a/src/Scheduler/__Tests/StellaOps.Scheduler.WebService.Tests/RunEndpointTests.cs b/src/Scheduler/__Tests/StellaOps.Scheduler.WebService.Tests/RunEndpointTests.cs index aa2840854..db8608cad 100644 --- a/src/Scheduler/__Tests/StellaOps.Scheduler.WebService.Tests/RunEndpointTests.cs +++ b/src/Scheduler/__Tests/StellaOps.Scheduler.WebService.Tests/RunEndpointTests.cs @@ -11,16 +11,16 @@ using Microsoft.Extensions.DependencyInjection; using StellaOps.Scheduler.Models; using StellaOps.Scheduler.Queue; using StellaOps.Scheduler.Storage.Mongo.Repositories; - -namespace StellaOps.Scheduler.WebService.Tests; - -public sealed class RunEndpointTests : IClassFixture> -{ - private readonly WebApplicationFactory _factory; - - public RunEndpointTests(WebApplicationFactory factory) - { - _factory = factory; + +namespace StellaOps.Scheduler.WebService.Tests; + +public sealed class RunEndpointTests : IClassFixture> +{ + private readonly WebApplicationFactory _factory; + + public RunEndpointTests(WebApplicationFactory factory) + { + _factory = factory; } [Fact] @@ -100,13 +100,13 @@ public sealed class RunEndpointTests : IClassFixture(); Assert.True(preview.GetProperty("total").GetInt32() >= 0); diff --git a/src/Scheduler/__Tests/StellaOps.Scheduler.WebService.Tests/SchedulerWebApplicationFactory.cs b/src/Scheduler/__Tests/StellaOps.Scheduler.WebService.Tests/SchedulerWebApplicationFactory.cs index aaf77f36f..0072a96e6 100644 --- a/src/Scheduler/__Tests/StellaOps.Scheduler.WebService.Tests/SchedulerWebApplicationFactory.cs +++ b/src/Scheduler/__Tests/StellaOps.Scheduler.WebService.Tests/SchedulerWebApplicationFactory.cs @@ -1,11 +1,14 @@ using System; using System.Collections.Generic; +using System.IO; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using StellaOps.Scheduler.WebService.Options; using StellaOps.Scheduler.WebService.Runs; +using StellaOps.Scheduler.ImpactIndex; namespace StellaOps.Scheduler.WebService.Tests; @@ -15,6 +18,8 @@ public sealed class SchedulerWebApplicationFactory : WebApplicationFactory { + var fixtureDirectory = GetFixtureDirectory(); + configuration.AddInMemoryCollection(new[] { new KeyValuePair("Scheduler:Authority:Enabled", "false"), @@ -27,12 +32,22 @@ public sealed class SchedulerWebApplicationFactory : WebApplicationFactory("Scheduler:Events:Webhooks:Excitor:Enabled", "true"), new KeyValuePair("Scheduler:Events:Webhooks:Excitor:HmacSecret", "excitor-secret"), new KeyValuePair("Scheduler:Events:Webhooks:Excitor:RateLimitRequests", "20"), - new KeyValuePair("Scheduler:Events:Webhooks:Excitor:RateLimitWindowSeconds", "60") + new KeyValuePair("Scheduler:Events:Webhooks:Excitor:RateLimitWindowSeconds", "60"), + new KeyValuePair("Scheduler:ImpactIndex:FixtureDirectory", fixtureDirectory) }); }); builder.ConfigureServices(services => { + var fixtureDirectory = GetFixtureDirectory(); + + services.RemoveAll(); + services.AddSingleton(new ImpactIndexStubOptions + { + FixtureDirectory = fixtureDirectory, + SnapshotId = "tests/impact-index-stub" + }); + services.Configure(options => { options.Webhooks ??= new SchedulerInboundWebhooksOptions(); @@ -52,4 +67,14 @@ public sealed class SchedulerWebApplicationFactory : WebApplicationFactory + + + PreserveNewest + + diff --git a/src/Scheduler/__Tests/StellaOps.Scheduler.WebService.Tests/seed-data/impact-index/sample/bom-index.json b/src/Scheduler/__Tests/StellaOps.Scheduler.WebService.Tests/seed-data/impact-index/sample/bom-index.json new file mode 100644 index 000000000..8d25380fd --- /dev/null +++ b/src/Scheduler/__Tests/StellaOps.Scheduler.WebService.Tests/seed-data/impact-index/sample/bom-index.json @@ -0,0 +1,23 @@ +{ + "schema": "scheduler-impact-index@1", + "generatedAt": "2025-10-01T00:00:00Z", + "image": { + "repository": "registry.stellaops.test/team/sample-service", + "digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "tag": "1.0.0" + }, + "components": [ + { + "purl": "pkg:docker/sample-service@1.0.0", + "usage": [ + "runtime" + ] + }, + { + "purl": "pkg:pypi/requests@2.31.0", + "usage": [ + "usedByEntrypoint" + ] + } + ] +} diff --git a/src/TaskRunner/StellaOps.TaskRunner/StellaOps.TaskRunner.Infrastructure/Execution/FilesystemPackRunArtifactUploader.cs b/src/TaskRunner/StellaOps.TaskRunner/StellaOps.TaskRunner.Infrastructure/Execution/FilesystemPackRunArtifactUploader.cs new file mode 100644 index 000000000..fe6b9f84f --- /dev/null +++ b/src/TaskRunner/StellaOps.TaskRunner/StellaOps.TaskRunner.Infrastructure/Execution/FilesystemPackRunArtifactUploader.cs @@ -0,0 +1,239 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.Logging; +using StellaOps.TaskRunner.Core.Execution; +using StellaOps.TaskRunner.Core.Planning; + +namespace StellaOps.TaskRunner.Infrastructure.Execution; + +/// +/// Stores pack run artifacts on the local file system so they can be mirrored to the eventual remote store. +/// +public sealed class FilesystemPackRunArtifactUploader : IPackRunArtifactUploader +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true + }; + + private readonly string rootPath; + private readonly ILogger logger; + private readonly TimeProvider timeProvider; + + public FilesystemPackRunArtifactUploader( + string rootPath, + TimeProvider? timeProvider, + ILogger logger) + { + ArgumentException.ThrowIfNullOrWhiteSpace(rootPath); + + this.rootPath = Path.GetFullPath(rootPath); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + this.timeProvider = timeProvider ?? TimeProvider.System; + + Directory.CreateDirectory(this.rootPath); + } + + public async Task UploadAsync( + PackRunExecutionContext context, + PackRunState state, + IReadOnlyList outputs, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(state); + ArgumentNullException.ThrowIfNull(outputs); + + if (outputs.Count == 0) + { + return; + } + + var destinationRoot = Path.Combine(rootPath, SanitizeFileName(context.RunId)); + var filesRoot = Path.Combine(destinationRoot, "files"); + var expressionsRoot = Path.Combine(destinationRoot, "expressions"); + + Directory.CreateDirectory(destinationRoot); + + var manifest = new ArtifactManifest( + context.RunId, + timeProvider.GetUtcNow(), + new List(outputs.Count)); + + foreach (var output in outputs) + { + cancellationToken.ThrowIfCancellationRequested(); + + var record = await ProcessOutputAsync( + context, + output, + destinationRoot, + filesRoot, + expressionsRoot, + cancellationToken).ConfigureAwait(false); + + manifest.Outputs.Add(record); + } + + var manifestPath = Path.Combine(destinationRoot, "artifact-manifest.json"); + await using (var stream = File.Open(manifestPath, FileMode.Create, FileAccess.Write, FileShare.None)) + { + await JsonSerializer.SerializeAsync(stream, manifest, SerializerOptions, cancellationToken) + .ConfigureAwait(false); + } + + logger.LogInformation( + "Pack run {RunId} artifact manifest written to {Path} with {Count} output entries.", + context.RunId, + manifestPath, + manifest.Outputs.Count); + } + + private async Task ProcessOutputAsync( + PackRunExecutionContext context, + TaskPackPlanOutput output, + string destinationRoot, + string filesRoot, + string expressionsRoot, + CancellationToken cancellationToken) + { + var sourcePath = ResolveString(output.Path); + var expressionNode = ResolveExpression(output.Expression); + var status = "skipped"; + string? storedPath = null; + string? notes = null; + + if (IsFileOutput(output)) + { + if (string.IsNullOrWhiteSpace(sourcePath)) + { + status = "unresolved"; + notes = "Output path requires runtime value."; + } + else if (!File.Exists(sourcePath)) + { + status = "missing"; + notes = $"Source file '{sourcePath}' not found."; + logger.LogWarning( + "Pack run {RunId} output {Output} referenced missing file {Path}.", + context.RunId, + output.Name, + sourcePath); + } + else + { + Directory.CreateDirectory(filesRoot); + + var destinationPath = Path.Combine(filesRoot, DetermineDestinationFileName(output, sourcePath)); + Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!); + + await CopyFileAsync(sourcePath, destinationPath, cancellationToken).ConfigureAwait(false); + storedPath = GetRelativePath(destinationPath, destinationRoot); + status = "copied"; + + logger.LogInformation( + "Pack run {RunId} output {Output} copied to {Destination}.", + context.RunId, + output.Name, + destinationPath); + } + } + + if (expressionNode is not null) + { + Directory.CreateDirectory(expressionsRoot); + + var expressionPath = Path.Combine( + expressionsRoot, + $"{SanitizeFileName(output.Name)}.json"); + + var json = expressionNode.ToJsonString(SerializerOptions); + await File.WriteAllTextAsync(expressionPath, json, cancellationToken).ConfigureAwait(false); + + storedPath ??= GetRelativePath(expressionPath, destinationRoot); + status = status == "copied" ? "copied" : "materialized"; + } + + return new ArtifactRecord( + output.Name, + output.Type, + sourcePath, + storedPath, + status, + notes); + } + + private static async Task CopyFileAsync(string sourcePath, string destinationPath, CancellationToken cancellationToken) + { + await using var source = File.Open(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read); + await using var destination = File.Open(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None); + await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false); + } + + private static bool IsFileOutput(TaskPackPlanOutput output) + => string.Equals(output.Type, "file", StringComparison.OrdinalIgnoreCase); + + private static string DetermineDestinationFileName(TaskPackPlanOutput output, string sourcePath) + { + var extension = Path.GetExtension(sourcePath); + var baseName = SanitizeFileName(output.Name); + + if (!string.IsNullOrWhiteSpace(extension) && + !baseName.EndsWith(extension, StringComparison.OrdinalIgnoreCase)) + { + return baseName + extension; + } + + return baseName; + } + + private static string? ResolveString(TaskPackPlanParameterValue? parameter) + { + if (parameter is null || parameter.RequiresRuntimeValue || parameter.Value is null) + { + return null; + } + + if (parameter.Value is JsonValue jsonValue && jsonValue.TryGetValue(out var value)) + { + return value; + } + + return null; + } + + private static JsonNode? ResolveExpression(TaskPackPlanParameterValue? parameter) + { + if (parameter is null || parameter.RequiresRuntimeValue) + { + return null; + } + + return parameter.Value; + } + + private static string SanitizeFileName(string value) + { + var result = value; + foreach (var invalid in Path.GetInvalidFileNameChars()) + { + result = result.Replace(invalid, '_'); + } + + return string.IsNullOrWhiteSpace(result) ? "output" : result; + } + + private static string GetRelativePath(string path, string root) + => Path.GetRelativePath(root, path) + .Replace('\\', '/'); + + private sealed record ArtifactManifest(string RunId, DateTimeOffset UploadedAt, List Outputs); + + private sealed record ArtifactRecord( + string Name, + string Type, + string? SourcePath, + string? StoredPath, + string Status, + string? Notes); +} diff --git a/src/TaskRunner/StellaOps.TaskRunner/StellaOps.TaskRunner.Tests/FilesystemPackRunArtifactUploaderTests.cs b/src/TaskRunner/StellaOps.TaskRunner/StellaOps.TaskRunner.Tests/FilesystemPackRunArtifactUploaderTests.cs new file mode 100644 index 000000000..dde0f58dc --- /dev/null +++ b/src/TaskRunner/StellaOps.TaskRunner/StellaOps.TaskRunner.Tests/FilesystemPackRunArtifactUploaderTests.cs @@ -0,0 +1,138 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.TaskRunner.Core.Execution; +using StellaOps.TaskRunner.Core.Planning; +using StellaOps.TaskRunner.Infrastructure.Execution; +using Xunit; + +namespace StellaOps.TaskRunner.Tests; + +public sealed class FilesystemPackRunArtifactUploaderTests : IDisposable +{ + private readonly string artifactsRoot; + + public FilesystemPackRunArtifactUploaderTests() + { + artifactsRoot = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("n")); + } + + [Fact] + public async Task CopiesFileOutputs() + { + var sourceFile = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid():n}.txt"); + await File.WriteAllTextAsync(sourceFile, "artifact-content", TestContext.Current.CancellationToken); + + var uploader = CreateUploader(); + var output = CreateFileOutput("bundle", sourceFile); + var context = CreateContext(); + var state = CreateState(context); + + await uploader.UploadAsync(context, state, new[] { output }, TestContext.Current.CancellationToken); + + var runPath = Path.Combine(artifactsRoot, context.RunId); + var filesDirectory = Path.Combine(runPath, "files"); + var copiedFiles = Directory.GetFiles(filesDirectory); + Assert.Single(copiedFiles); + Assert.Equal("bundle.txt", Path.GetFileName(copiedFiles[0])); + Assert.Equal("artifact-content", await File.ReadAllTextAsync(copiedFiles[0], TestContext.Current.CancellationToken)); + + var manifest = await ReadManifestAsync(runPath); + Assert.Single(manifest.Outputs); + Assert.Equal("copied", manifest.Outputs[0].Status); + Assert.Equal("files/bundle.txt", manifest.Outputs[0].StoredPath); + } + + [Fact] + public async Task RecordsMissingFilesWithoutThrowing() + { + var uploader = CreateUploader(); + var output = CreateFileOutput("missing", Path.Combine(Path.GetTempPath(), "does-not-exist.txt")); + var context = CreateContext(); + var state = CreateState(context); + + await uploader.UploadAsync(context, state, new[] { output }, TestContext.Current.CancellationToken); + + var manifest = await ReadManifestAsync(Path.Combine(artifactsRoot, context.RunId)); + Assert.Equal("missing", manifest.Outputs[0].Status); + } + + [Fact] + public async Task WritesExpressionOutputsAsJson() + { + var uploader = CreateUploader(); + var output = CreateExpressionOutput("metadata", JsonNode.Parse("""{"foo":"bar"}""")!); + var context = CreateContext(); + var state = CreateState(context); + + await uploader.UploadAsync(context, state, new[] { output }, TestContext.Current.CancellationToken); + + var expressionPath = Path.Combine(artifactsRoot, context.RunId, "expressions", "metadata.json"); + Assert.True(File.Exists(expressionPath)); + + var manifest = await ReadManifestAsync(Path.Combine(artifactsRoot, context.RunId)); + Assert.Equal("materialized", manifest.Outputs[0].Status); + Assert.Equal("expressions/metadata.json", manifest.Outputs[0].StoredPath); + } + + private FilesystemPackRunArtifactUploader CreateUploader() + => new(artifactsRoot, TimeProvider.System, NullLogger.Instance); + + private static TaskPackPlanOutput CreateFileOutput(string name, string path) + => new( + name, + Type: "file", + Path: new TaskPackPlanParameterValue(JsonValue.Create(path), null, null, false), + Expression: null); + + private static TaskPackPlanOutput CreateExpressionOutput(string name, JsonNode expression) + => new( + name, + Type: "object", + Path: null, + Expression: new TaskPackPlanParameterValue(expression, null, null, false)); + + private static PackRunExecutionContext CreateContext() + => new("run-" + Guid.NewGuid().ToString("n"), CreatePlan(), DateTimeOffset.UtcNow); + + private static PackRunState CreateState(PackRunExecutionContext context) + => PackRunState.Create( + runId: context.RunId, + planHash: context.Plan.Hash, + context.Plan, + failurePolicy: new TaskPackPlanFailurePolicy(1, 1, false), + requestedAt: DateTimeOffset.UtcNow, + steps: new Dictionary(StringComparer.Ordinal), + timestamp: DateTimeOffset.UtcNow); + + private static TaskPackPlan CreatePlan() + { + return new TaskPackPlan( + new TaskPackPlanMetadata("sample-pack", "1.0.0", null, Array.Empty()), + new Dictionary(StringComparer.Ordinal), + Array.Empty(), + hash: "hash", + approvals: Array.Empty(), + secrets: Array.Empty(), + outputs: Array.Empty(), + failurePolicy: new TaskPackPlanFailurePolicy(1, 1, false)); + } + + private static async Task ReadManifestAsync(string runPath) + { + var json = await File.ReadAllTextAsync(Path.Combine(runPath, "artifact-manifest.json"), TestContext.Current.CancellationToken); + return JsonSerializer.Deserialize(json, new JsonSerializerOptions(JsonSerializerDefaults.Web))!; + } + + public void Dispose() + { + if (Directory.Exists(artifactsRoot)) + { + Directory.Delete(artifactsRoot, recursive: true); + } + } + + private sealed record ArtifactManifestModel(string RunId, DateTimeOffset UploadedAt, List Outputs); + + private sealed record ArtifactRecordModel(string Name, string Type, string? SourcePath, string? StoredPath, string Status, string? Notes); +} diff --git a/src/TaskRunner/StellaOps.TaskRunner/StellaOps.TaskRunner.Worker/Program.cs b/src/TaskRunner/StellaOps.TaskRunner/StellaOps.TaskRunner.Worker/Program.cs index 2aaaa771c..06ec2fd14 100644 --- a/src/TaskRunner/StellaOps.TaskRunner/StellaOps.TaskRunner.Worker/Program.cs +++ b/src/TaskRunner/StellaOps.TaskRunner/StellaOps.TaskRunner.Worker/Program.cs @@ -51,7 +51,13 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); -builder.Services.AddSingleton(); +builder.Services.AddSingleton(sp => +{ + var options = sp.GetRequiredService>().Value; + var timeProvider = sp.GetService(); + var logger = sp.GetRequiredService>(); + return new FilesystemPackRunArtifactUploader(options.ArtifactsPath, timeProvider, logger); +}); builder.Services.AddHostedService(); var host = builder.Build(); diff --git a/src/TaskRunner/StellaOps.TaskRunner/StellaOps.TaskRunner.Worker/Services/PackRunWorkerOptions.cs b/src/TaskRunner/StellaOps.TaskRunner/StellaOps.TaskRunner.Worker/Services/PackRunWorkerOptions.cs index fc1c31ab9..2d976285c 100644 --- a/src/TaskRunner/StellaOps.TaskRunner/StellaOps.TaskRunner.Worker/Services/PackRunWorkerOptions.cs +++ b/src/TaskRunner/StellaOps.TaskRunner/StellaOps.TaskRunner.Worker/Services/PackRunWorkerOptions.cs @@ -4,11 +4,13 @@ public sealed class PackRunWorkerOptions { public TimeSpan IdleDelay { get; set; } = TimeSpan.FromSeconds(1); - public string QueuePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue"); - - public string ArchivePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue", "archive"); - + public string QueuePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue"); + + public string ArchivePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue", "archive"); + public string ApprovalStorePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "approvals"); public string RunStatePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "state", "runs"); + + public string ArtifactsPath { get; set; } = Path.Combine(AppContext.BaseDirectory, "artifacts"); } diff --git a/src/__Libraries/StellaOps.Cryptography.Kms/GcpKmsFacade.cs b/src/__Libraries/StellaOps.Cryptography.Kms/GcpKmsFacade.cs index d724de4a4..a8bc19aec 100644 --- a/src/__Libraries/StellaOps.Cryptography.Kms/GcpKmsFacade.cs +++ b/src/__Libraries/StellaOps.Cryptography.Kms/GcpKmsFacade.cs @@ -1,5 +1,6 @@ using Google.Cloud.Kms.V1; using Google.Protobuf; +using Google.Protobuf.WellKnownTypes; namespace StellaOps.Cryptography.Kms; diff --git a/src/__Libraries/StellaOps.Cryptography.Kms/Pkcs11Facade.cs b/src/__Libraries/StellaOps.Cryptography.Kms/Pkcs11Facade.cs index 951207242..fb3fd3705 100644 --- a/src/__Libraries/StellaOps.Cryptography.Kms/Pkcs11Facade.cs +++ b/src/__Libraries/StellaOps.Cryptography.Kms/Pkcs11Facade.cs @@ -271,7 +271,7 @@ internal sealed class Pkcs11InteropFacade : IPkcs11Facade } catch { - # ignore logout failures + // ignore logout failures } } diff --git a/src/__Libraries/StellaOps.Cryptography.Kms/Pkcs11Options.cs b/src/__Libraries/StellaOps.Cryptography.Kms/Pkcs11Options.cs index 265fb3dd6..bea3f867b 100644 --- a/src/__Libraries/StellaOps.Cryptography.Kms/Pkcs11Options.cs +++ b/src/__Libraries/StellaOps.Cryptography.Kms/Pkcs11Options.cs @@ -64,9 +64,8 @@ public sealed class Pkcs11Options /// /// Gets or sets an optional factory for advanced facade injection (testing, custom providers). /// - public Func? FacadeFactory { get; set; } + internal Func? FacadeFactory { get; set; } private static TimeSpan EnsurePositive(TimeSpan value, TimeSpan fallback) => value <= TimeSpan.Zero ? fallback : value; } - diff --git a/tests/native/openssl-1.1/README.md b/tests/native/openssl-1.1/README.md index b82f693d5..97f530318 100644 --- a/tests/native/openssl-1.1/README.md +++ b/tests/native/openssl-1.1/README.md @@ -5,3 +5,22 @@ These binaries (libcrypto.so.1.1 and libssl.so.1.1) are bundled for Mongo2Go-bas Source package: https://launchpad.net/ubuntu/+archive/primary/+files/libssl1.1_1.1.1f-1ubuntu2_amd64.deb Licensing follows the OpenSSL and SSLeay licenses that accompany the upstream package. + +## Usage + +1. Point `LD_LIBRARY_PATH` at this directory before running any test suite that spins up Mongo2Go: + + ```bash + export LD_LIBRARY_PATH="$(git rev-parse --show-toplevel)/tests/native/openssl-1.1/linux-x64:${LD_LIBRARY_PATH:-}" + ``` + + The helper in `tests/shared/OpenSslLegacyShim.cs` will append the path automatically when the tests run, but exporting the variable up-front avoids surprises when using custom harnesses (e.g., `dotnet test` filters). + +2. Example one-shot command: + + ```bash + LD_LIBRARY_PATH="$(pwd)/tests/native/openssl-1.1/linux-x64" \ + dotnet test src/Excititor/__Tests/StellaOps.Excititor.Worker.Tests/StellaOps.Excititor.Worker.Tests.csproj --nologo + ``` + +3. CI runners should add the same directory to their environment or place the binaries on a globally accessible library path.