up
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled
Mirror Thin Bundle Sign & Verify / mirror-sign (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-11-29 02:40:21 +02:00
parent 887b0a1c67
commit 7e7be4d2fd
54 changed files with 4907 additions and 3 deletions

124
ops/orchestrator/Dockerfile Normal file
View File

@@ -0,0 +1,124 @@
# syntax=docker/dockerfile:1.7-labs
# Orchestrator Service Dockerfile
# Multi-stage build for deterministic, reproducible container images.
# Supports air-gapped deployment via digest-pinned base images.
ARG SDK_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:10.0
ARG RUNTIME_IMAGE=mcr.microsoft.com/dotnet/nightly/aspnet:10.0
ARG VERSION=0.0.0
ARG CHANNEL=dev
ARG GIT_SHA=0000000
ARG SOURCE_DATE_EPOCH=0
# ==============================================================================
# Stage 1: Build
# ==============================================================================
FROM ${SDK_IMAGE} AS build
ARG GIT_SHA
ARG SOURCE_DATE_EPOCH
WORKDIR /src
ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 \
DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 \
NUGET_XMLDOC_MODE=skip \
SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
# Copy solution and project files for restore
COPY src/Orchestrator/StellaOps.Orchestrator/StellaOps.Orchestrator.sln ./
COPY src/Orchestrator/StellaOps.Orchestrator/StellaOps.Orchestrator.Core/StellaOps.Orchestrator.Core.csproj StellaOps.Orchestrator.Core/
COPY src/Orchestrator/StellaOps.Orchestrator/StellaOps.Orchestrator.Infrastructure/StellaOps.Orchestrator.Infrastructure.csproj StellaOps.Orchestrator.Infrastructure/
COPY src/Orchestrator/StellaOps.Orchestrator/StellaOps.Orchestrator.WebService/StellaOps.Orchestrator.WebService.csproj StellaOps.Orchestrator.WebService/
COPY src/Orchestrator/StellaOps.Orchestrator/StellaOps.Orchestrator.Worker/StellaOps.Orchestrator.Worker.csproj StellaOps.Orchestrator.Worker/
COPY Directory.Build.props Directory.Packages.props ./
# Restore dependencies with cache mount
RUN --mount=type=cache,target=/root/.nuget/packages \
dotnet restore StellaOps.Orchestrator.sln
# Copy source files
COPY src/Orchestrator/StellaOps.Orchestrator/ ./
# Publish WebService
RUN --mount=type=cache,target=/root/.nuget/packages \
dotnet publish StellaOps.Orchestrator.WebService/StellaOps.Orchestrator.WebService.csproj \
-c Release \
-o /app/publish/webservice \
/p:UseAppHost=false \
/p:ContinuousIntegrationBuild=true \
/p:SourceRevisionId=${GIT_SHA} \
/p:Deterministic=true \
/p:TreatWarningsAsErrors=true
# Publish Worker (optional, for hybrid deployments)
RUN --mount=type=cache,target=/root/.nuget/packages \
dotnet publish StellaOps.Orchestrator.Worker/StellaOps.Orchestrator.Worker.csproj \
-c Release \
-o /app/publish/worker \
/p:UseAppHost=false \
/p:ContinuousIntegrationBuild=true \
/p:SourceRevisionId=${GIT_SHA} \
/p:Deterministic=true \
/p:TreatWarningsAsErrors=true
# ==============================================================================
# Stage 2: Runtime (WebService)
# ==============================================================================
FROM ${RUNTIME_IMAGE} AS orchestrator-web
WORKDIR /app
ARG VERSION
ARG CHANNEL
ARG GIT_SHA
ENV DOTNET_EnableDiagnostics=0 \
ASPNETCORE_URLS=http://0.0.0.0:8080 \
ASPNETCORE_ENVIRONMENT=Production \
ORCHESTRATOR__TELEMETRY__MINIMUMLOGLEVEL=Information
COPY --from=build /app/publish/webservice/ ./
# Health check endpoints
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
CMD wget --no-verbose --tries=1 --spider http://localhost:8080/healthz || exit 1
EXPOSE 8080
LABEL org.opencontainers.image.title="StellaOps Orchestrator WebService" \
org.opencontainers.image.description="Job scheduling, DAG planning, and worker coordination service" \
org.opencontainers.image.version="${VERSION}" \
org.opencontainers.image.revision="${GIT_SHA}" \
org.opencontainers.image.source="https://git.stella-ops.org/stella-ops/stellaops" \
org.opencontainers.image.vendor="StellaOps" \
org.opencontainers.image.licenses="AGPL-3.0-or-later" \
org.stellaops.release.channel="${CHANNEL}" \
org.stellaops.component="orchestrator-web"
ENTRYPOINT ["dotnet", "StellaOps.Orchestrator.WebService.dll"]
# ==============================================================================
# Stage 3: Runtime (Worker)
# ==============================================================================
FROM ${RUNTIME_IMAGE} AS orchestrator-worker
WORKDIR /app
ARG VERSION
ARG CHANNEL
ARG GIT_SHA
ENV DOTNET_EnableDiagnostics=0 \
ASPNETCORE_ENVIRONMENT=Production \
ORCHESTRATOR__TELEMETRY__MINIMUMLOGLEVEL=Information
COPY --from=build /app/publish/worker/ ./
LABEL org.opencontainers.image.title="StellaOps Orchestrator Worker" \
org.opencontainers.image.description="Background worker for job execution and orchestration tasks" \
org.opencontainers.image.version="${VERSION}" \
org.opencontainers.image.revision="${GIT_SHA}" \
org.opencontainers.image.source="https://git.stella-ops.org/stella-ops/stellaops" \
org.opencontainers.image.vendor="StellaOps" \
org.opencontainers.image.licenses="AGPL-3.0-or-later" \
org.stellaops.release.channel="${CHANNEL}" \
org.stellaops.component="orchestrator-worker"
ENTRYPOINT ["dotnet", "StellaOps.Orchestrator.Worker.dll"]

View File

@@ -0,0 +1,108 @@
# Orchestrator Service GA Checklist
> Pre-release validation checklist for StellaOps Orchestrator Service.
> All items must be verified before promoting to `stable` channel.
## Build & Packaging
- [ ] Container images build successfully for all target architectures (amd64, arm64)
- [ ] Multi-stage Dockerfile produces minimal runtime images (<100MB compressed)
- [ ] OCI labels include version, git SHA, and license metadata
- [ ] HEALTHCHECK directive validates endpoint availability
- [ ] Build is reproducible (same inputs produce byte-identical outputs)
- [ ] SBOM generated and attached to container images (SPDX 3.0.1 or CycloneDX 1.6)
- [ ] Provenance attestation generated per SLSA v1 specification
- [ ] Air-gap bundle script creates valid offline deployment package
## Security
- [ ] Container runs as non-root user (UID 1000+)
- [ ] No secrets baked into container image layers
- [ ] Base image digest-pinned to known-good version
- [ ] Vulnerability scan passes with no HIGH/CRITICAL unfixed CVEs
- [ ] TLS 1.3 enforced for all external endpoints
- [ ] Authority JWT validation enabled and tested
- [ ] Tenant isolation enforced at API and storage layers
- [ ] Sensitive configuration loaded from Kubernetes secrets only
## Functional
- [ ] Job scheduling CRUD operations work correctly
- [ ] Cron expression parsing handles edge cases (DST, leap years)
- [ ] DAG planning respects dependency ordering
- [ ] Dead letter queue captures failed jobs with full context
- [ ] Backfill API handles large date ranges without OOM
- [ ] Worker heartbeat detection marks stale jobs correctly
- [ ] Rate limiting and concurrency limits enforced per tenant
## Performance & Scale
- [ ] System tracks 10,000+ pending jobs without degradation
- [ ] Dispatch latency P95 < 150ms under normal load
- [ ] Queue depth metrics exposed for autoscaling (KEDA/HPA)
- [ ] Load shedding activates at configured thresholds
- [ ] Database connection pooling sized appropriately
- [ ] Memory usage stable under sustained load (no leaks)
## Observability
- [ ] Structured logging with correlation IDs enabled
- [ ] OpenTelemetry traces exported to configured endpoint
- [ ] Prometheus metrics exposed at `/metrics` endpoint
- [ ] Health probes respond correctly:
- `/healthz` - basic liveness
- `/livez` - deep liveness with dependency checks
- `/readyz` - readiness for traffic
- `/startupz` - startup completion check
- [ ] Autoscaling metrics endpoint returns valid JSON
## Deployment
- [ ] Helm values overlay tested with production-like configuration
- [ ] PostgreSQL schema migrations run idempotently
- [ ] Rolling update strategy configured (maxSurge/maxUnavailable)
- [ ] Pod disruption budget prevents full outage
- [ ] Resource requests/limits appropriate for target workload
- [ ] Network policies restrict traffic to required paths only
- [ ] Service mesh (Istio/Linkerd) integration tested if applicable
## Documentation
- [ ] Architecture document updated in `docs/modules/orchestrator/`
- [ ] API reference generated from OpenAPI spec
- [ ] Runbook for common operations (restart, scale, failover)
- [ ] Troubleshooting guide for known issues
- [ ] Upgrade path documented from previous versions
## Testing
- [ ] Unit tests pass (100% of Core, 80%+ of Infrastructure)
- [ ] Integration tests pass against real PostgreSQL
- [ ] Performance benchmarks meet targets
- [ ] Chaos testing validates graceful degradation
- [ ] E2E tests cover critical user journeys
## Compliance
- [ ] AGPL-3.0-or-later license headers in all source files
- [ ] Third-party license notices collected and bundled
- [ ] Attestation chain verifiable via `stella attest verify`
- [ ] Air-gap deployment tested in isolated network
- [ ] CryptoProfile compatibility verified (FIPS/eIDAS if required)
---
## Sign-off
| Role | Name | Date | Signature |
|------|------|------|-----------|
| Engineering Lead | | | |
| QA Lead | | | |
| Security Review | | | |
| Release Manager | | | |
**Release Version:** ________________
**Release Channel:** [ ] edge [ ] stable [ ] lts
**Notes:**

View File

@@ -0,0 +1,276 @@
#!/usr/bin/env bash
set -euo pipefail
# ORCH-SVC-34-004: Build air-gap bundle for Orchestrator service
# Packages container images, configs, and manifests for offline deployment.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
VERSION="${VERSION:-2025.10.0-edge}"
CHANNEL="${CHANNEL:-edge}"
BUNDLE_DIR="${BUNDLE_DIR:-$REPO_ROOT/out/bundles/orchestrator-${VERSION}}"
SRC_DIR="${SRC_DIR:-$REPO_ROOT/out/buildx/orchestrator}"
usage() {
cat <<EOF
Usage: $0 [options]
Build an air-gap bundle for StellaOps Orchestrator service.
Options:
--version VERSION Bundle version (default: $VERSION)
--channel CHANNEL Release channel (default: $CHANNEL)
--output DIR Output bundle directory (default: $BUNDLE_DIR)
--source DIR Source buildx directory (default: $SRC_DIR)
--skip-images Skip OCI image export (use existing)
--help Show this help
Environment variables:
VERSION, CHANNEL, BUNDLE_DIR, SRC_DIR
Examples:
$0 --version 2025.10.0 --channel stable
VERSION=2025.10.0 CHANNEL=stable $0
EOF
exit "${1:-0}"
}
SKIP_IMAGES=false
while [[ $# -gt 0 ]]; do
case "$1" in
--version) VERSION="$2"; shift 2 ;;
--channel) CHANNEL="$2"; shift 2 ;;
--output) BUNDLE_DIR="$2"; shift 2 ;;
--source) SRC_DIR="$2"; shift 2 ;;
--skip-images) SKIP_IMAGES=true; shift ;;
--help) usage 0 ;;
*) echo "Unknown option: $1" >&2; usage 64 ;;
esac
done
BUNDLE_DIR="${BUNDLE_DIR:-$REPO_ROOT/out/bundles/orchestrator-${VERSION}}"
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
echo "[orchestrator-airgap] Building bundle v${VERSION} (${CHANNEL})"
echo "[orchestrator-airgap] Output: ${BUNDLE_DIR}"
mkdir -p "$BUNDLE_DIR"/{images,configs,manifests,docs}
# ------------------------------------------------------------------------------
# Stage 1: Export container images as OCI archives
# ------------------------------------------------------------------------------
if [[ "$SKIP_IMAGES" == "false" ]]; then
echo "[orchestrator-airgap] Exporting container images..."
IMAGES=(
"orchestrator-web:${VERSION}"
"orchestrator-worker:${VERSION}"
)
for img in "${IMAGES[@]}"; do
img_name="${img%%:*}"
img_file="${BUNDLE_DIR}/images/${img_name}.oci.tar.gz"
if [[ -f "${SRC_DIR}/${img_name}/image.oci" ]]; then
echo "[orchestrator-airgap] Packaging ${img_name} from buildx output..."
gzip -c "${SRC_DIR}/${img_name}/image.oci" > "$img_file"
else
echo "[orchestrator-airgap] Exporting ${img_name} via docker save..."
docker save "registry.stella-ops.org/stellaops/${img}" | gzip > "$img_file"
fi
# Generate checksum
sha256sum "$img_file" | cut -d' ' -f1 > "${img_file}.sha256"
# Copy SBOM if available
if [[ -f "${SRC_DIR}/${img_name}/sbom.syft.json" ]]; then
cp "${SRC_DIR}/${img_name}/sbom.syft.json" "${BUNDLE_DIR}/manifests/${img_name}.sbom.json"
fi
done
else
echo "[orchestrator-airgap] Skipping image export (--skip-images)"
fi
# ------------------------------------------------------------------------------
# Stage 2: Copy configuration templates
# ------------------------------------------------------------------------------
echo "[orchestrator-airgap] Copying configuration templates..."
# Helm values overlay
if [[ -f "$REPO_ROOT/deploy/helm/stellaops/values-orchestrator.yaml" ]]; then
cp "$REPO_ROOT/deploy/helm/stellaops/values-orchestrator.yaml" \
"${BUNDLE_DIR}/configs/values-orchestrator.yaml"
fi
# Sample configuration
if [[ -f "$REPO_ROOT/etc/orchestrator.yaml.sample" ]]; then
cp "$REPO_ROOT/etc/orchestrator.yaml.sample" \
"${BUNDLE_DIR}/configs/orchestrator.yaml.sample"
fi
# PostgreSQL migration scripts
if [[ -d "$REPO_ROOT/src/Orchestrator/StellaOps.Orchestrator/migrations" ]]; then
mkdir -p "${BUNDLE_DIR}/configs/migrations"
cp "$REPO_ROOT/src/Orchestrator/StellaOps.Orchestrator/migrations/"*.sql \
"${BUNDLE_DIR}/configs/migrations/" 2>/dev/null || true
fi
# Bootstrap secrets template
cat > "${BUNDLE_DIR}/configs/secrets.env.example" <<'SECRETS_EOF'
# Orchestrator Secrets Template
# Copy to secrets.env and fill in values before deployment
# PostgreSQL password (required)
POSTGRES_PASSWORD=
# Authority JWT signing key (if using local Authority)
AUTHORITY_SIGNING_KEY=
# OpenTelemetry endpoint (optional)
OTEL_EXPORTER_OTLP_ENDPOINT=
# Tenant encryption key for multi-tenant isolation (optional)
TENANT_ENCRYPTION_KEY=
SECRETS_EOF
# ------------------------------------------------------------------------------
# Stage 3: Generate bundle manifest
# ------------------------------------------------------------------------------
echo "[orchestrator-airgap] Generating bundle manifest..."
# Calculate checksums for all bundle files
MANIFEST_FILE="${BUNDLE_DIR}/manifests/bundle-manifest.json"
# Build file list with checksums
FILES_JSON="[]"
while IFS= read -r -d '' file; do
rel_path="${file#$BUNDLE_DIR/}"
if [[ "$rel_path" != "manifests/bundle-manifest.json" ]]; then
sha=$(sha256sum "$file" | cut -d' ' -f1)
size=$(stat -f%z "$file" 2>/dev/null || stat -c%s "$file" 2>/dev/null || echo "0")
FILES_JSON=$(echo "$FILES_JSON" | jq --arg name "$rel_path" --arg sha "$sha" --arg size "$size" \
'. + [{"name": $name, "sha256": $sha, "size": ($size | tonumber)}]')
fi
done < <(find "$BUNDLE_DIR" -type f -print0 | sort -z)
cat > "$MANIFEST_FILE" <<EOF
{
"bundle": {
"name": "stellaops-orchestrator",
"version": "${VERSION}",
"channel": "${CHANNEL}",
"createdAt": "${TIMESTAMP}",
"components": [
{
"name": "orchestrator-web",
"type": "container",
"image": "registry.stella-ops.org/stellaops/orchestrator-web:${VERSION}"
},
{
"name": "orchestrator-worker",
"type": "container",
"image": "registry.stella-ops.org/stellaops/orchestrator-worker:${VERSION}"
},
{
"name": "orchestrator-postgres",
"type": "infrastructure",
"image": "docker.io/library/postgres:16-alpine"
}
]
},
"files": ${FILES_JSON}
}
EOF
# Checksum the manifest itself
sha256sum "$MANIFEST_FILE" | cut -d' ' -f1 > "${MANIFEST_FILE}.sha256"
# ------------------------------------------------------------------------------
# Stage 4: Copy documentation
# ------------------------------------------------------------------------------
echo "[orchestrator-airgap] Copying documentation..."
# Module architecture
if [[ -f "$REPO_ROOT/docs/modules/orchestrator/architecture.md" ]]; then
cp "$REPO_ROOT/docs/modules/orchestrator/architecture.md" \
"${BUNDLE_DIR}/docs/architecture.md"
fi
# GA checklist
if [[ -f "$REPO_ROOT/ops/orchestrator/GA_CHECKLIST.md" ]]; then
cp "$REPO_ROOT/ops/orchestrator/GA_CHECKLIST.md" \
"${BUNDLE_DIR}/docs/GA_CHECKLIST.md"
fi
# Quick deployment guide
cat > "${BUNDLE_DIR}/docs/DEPLOY.md" <<'DEPLOY_EOF'
# Orchestrator Air-Gap Deployment Guide
## Prerequisites
- Docker or containerd runtime
- Kubernetes 1.28+ (for Helm deployment) or Docker Compose
- PostgreSQL 16+ (included as container or external)
## Quick Start (Docker)
1. Load images:
```bash
for img in images/*.oci.tar.gz; do
gunzip -c "$img" | docker load
done
```
2. Configure secrets:
```bash
cp configs/secrets.env.example secrets.env
# Edit secrets.env with your values
```
3. Start services:
```bash
docker compose -f docker-compose.orchestrator.yaml up -d
```
## Helm Deployment
1. Import images to registry:
```bash
for img in images/*.oci.tar.gz; do
crane push "$img" your-registry.local/stellaops/$(basename "$img" .oci.tar.gz)
done
```
2. Install chart:
```bash
helm upgrade --install stellaops ./stellaops \
-f configs/values-orchestrator.yaml \
--set global.imageRegistry=your-registry.local
```
## Verification
Check health endpoints:
```bash
curl http://localhost:8080/healthz
curl http://localhost:8080/readyz
```
DEPLOY_EOF
# ------------------------------------------------------------------------------
# Stage 5: Create final tarball
# ------------------------------------------------------------------------------
echo "[orchestrator-airgap] Creating final tarball..."
TARBALL="${BUNDLE_DIR}.tar.gz"
tar -C "$(dirname "$BUNDLE_DIR")" -czf "$TARBALL" "$(basename "$BUNDLE_DIR")"
# Checksum the tarball
sha256sum "$TARBALL" | cut -d' ' -f1 > "${TARBALL}.sha256"
echo "[orchestrator-airgap] Bundle created successfully:"
echo " Tarball: ${TARBALL}"
echo " SHA256: $(cat "${TARBALL}.sha256")"
echo " Size: $(du -h "$TARBALL" | cut -f1)"

View File

@@ -0,0 +1,106 @@
{
"_type": "https://in-toto.io/Statement/v1",
"subject": [
{
"name": "registry.stella-ops.org/stellaops/orchestrator-web",
"digest": {
"sha256": "<IMAGE_DIGEST_WEB>"
}
},
{
"name": "registry.stella-ops.org/stellaops/orchestrator-worker",
"digest": {
"sha256": "<IMAGE_DIGEST_WORKER>"
}
}
],
"predicateType": "https://slsa.dev/provenance/v1",
"predicate": {
"buildDefinition": {
"buildType": "https://stella-ops.org/OrchestratorBuild/v1",
"externalParameters": {
"source": {
"uri": "git+https://git.stella-ops.org/stella-ops/stellaops.git",
"digest": {
"gitCommit": "<GIT_SHA>"
}
},
"builderImage": {
"uri": "mcr.microsoft.com/dotnet/nightly/sdk:10.0",
"digest": {
"sha256": "<SDK_DIGEST>"
}
}
},
"internalParameters": {
"dockerfile": "ops/orchestrator/Dockerfile",
"targetStages": ["orchestrator-web", "orchestrator-worker"],
"buildArgs": {
"VERSION": "<VERSION>",
"CHANNEL": "<CHANNEL>",
"GIT_SHA": "<GIT_SHA>",
"SOURCE_DATE_EPOCH": "<SOURCE_DATE_EPOCH>"
}
},
"resolvedDependencies": [
{
"uri": "pkg:nuget/Microsoft.Extensions.Hosting@10.0.0",
"digest": {
"sha256": "<NUGET_HOSTING_DIGEST>"
}
},
{
"uri": "pkg:nuget/Npgsql.EntityFrameworkCore.PostgreSQL@10.0.0",
"digest": {
"sha256": "<NUGET_NPGSQL_DIGEST>"
}
},
{
"uri": "pkg:nuget/Cronos@0.10.0",
"digest": {
"sha256": "<NUGET_CRONOS_DIGEST>"
}
}
]
},
"runDetails": {
"builder": {
"id": "https://git.stella-ops.org/stella-ops/stellaops/-/runners/1",
"builderDependencies": [
{
"uri": "docker.io/moby/buildkit:latest",
"digest": {
"sha256": "<BUILDKIT_DIGEST>"
}
}
],
"version": {
"buildkit": "0.14.0"
}
},
"metadata": {
"invocationId": "<INVOCATION_ID>",
"startedOn": "<BUILD_START_TIME>",
"finishedOn": "<BUILD_END_TIME>"
},
"byproducts": [
{
"name": "sbom-web",
"uri": "registry.stella-ops.org/stellaops/orchestrator-web:sbom",
"mediaType": "application/spdx+json",
"digest": {
"sha256": "<SBOM_WEB_DIGEST>"
}
},
{
"name": "sbom-worker",
"uri": "registry.stella-ops.org/stellaops/orchestrator-worker:sbom",
"mediaType": "application/spdx+json",
"digest": {
"sha256": "<SBOM_WORKER_DIGEST>"
}
}
]
}
}
}