work
This commit is contained in:
StellaOps Bot
@@ -24,3 +24,25 @@ openssl pkey -in "$KEYFILE" -pubout -out "$KEYDIR/ci-ed25519.pub" >/dev/null 2>&
|
||||
STAGE=${STAGE:-$ROOT/out/mirror/thin/stage-v1}
|
||||
CREATED=${CREATED:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}
|
||||
SIGN_KEY="$KEYFILE" STAGE="$STAGE" CREATED="$CREATED" "$ROOT/src/Mirror/StellaOps.Mirror.Creator/make-thin-v1.sh"
|
||||
|
||||
# Emit milestone summary with hashes for downstream consumers
|
||||
MANIFEST_PATH="$ROOT/out/mirror/thin/mirror-thin-v1.manifest.json"
|
||||
TAR_PATH="$ROOT/out/mirror/thin/mirror-thin-v1.tar.gz"
|
||||
DSSE_PATH="$ROOT/out/mirror/thin/mirror-thin-v1.manifest.dsse.json"
|
||||
SUMMARY_PATH="$ROOT/out/mirror/thin/milestone.json"
|
||||
|
||||
sha256() {
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
}
|
||||
|
||||
cat > "$SUMMARY_PATH" <<JSON
|
||||
{
|
||||
"created": "$CREATED",
|
||||
"manifest": {"path": "$(basename "$MANIFEST_PATH")", "sha256": "$(sha256 "$MANIFEST_PATH")"},
|
||||
"tarball": {"path": "$(basename "$TAR_PATH")", "sha256": "$(sha256 "$TAR_PATH")"},
|
||||
"dsse": $( [[ -f "$DSSE_PATH" ]] && echo "{\"path\": \"$(basename "$DSSE_PATH")\", \"sha256\": \"$(sha256 "$DSSE_PATH")\"}" || echo "null" ),
|
||||
"time_anchor": $( [[ -n "${TIME_ANCHOR_FILE:-}" && -f "$TIME_ANCHOR_FILE" ]] && echo "{\"path\": \"$(basename "$TIME_ANCHOR_FILE")\", \"sha256\": \"$(sha256 "$TIME_ANCHOR_FILE")\"}" || echo "null" )
|
||||
}
|
||||
JSON
|
||||
|
||||
echo "Milestone summary written to $SUMMARY_PATH"
|
||||
|
||||
134
scripts/observability/incident-mode.sh
Normal file
134
scripts/observability/incident-mode.sh
Normal file
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Incident mode automation
|
||||
# - Enables a feature-flag JSON when burn rate crosses threshold
|
||||
# - Writes retention override parameters for downstream storage/ingest systems
|
||||
# - Resets automatically after a cooldown period once burn subsides
|
||||
# All inputs are provided via CLI flags or env vars to remain offline-friendly.
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: incident-mode.sh --burn-rate <float> [--threshold 2.0] [--reset-threshold 0.5] \
|
||||
[--state-dir out/incident-mode] [--retention-hours 24] \
|
||||
[--cooldown-mins 30] [--note "text"]
|
||||
|
||||
Environment overrides:
|
||||
INCIDENT_STATE_DIR default: out/incident-mode
|
||||
INCIDENT_THRESHOLD default: 2.0 (fast burn multiple)
|
||||
INCIDENT_RESET_TH default: 0.5 (burn multiple to exit)
|
||||
INCIDENT_COOLDOWN default: 30 (minutes below reset threshold)
|
||||
INCIDENT_RETENTION_H default: 24 (hours)
|
||||
|
||||
Outputs (in state dir):
|
||||
flag.json feature flag payload (enabled/disabled + metadata)
|
||||
retention.json retention override (hours, applied_at)
|
||||
last_burn.txt last burn rate observed
|
||||
cooldown.txt consecutive minutes below reset threshold
|
||||
|
||||
Examples:
|
||||
incident-mode.sh --burn-rate 3.1 --note "fast burn" # enter incident mode
|
||||
incident-mode.sh --burn-rate 0.2 # progress cooldown / exit
|
||||
USAGE
|
||||
}
|
||||
|
||||
if [[ $# -eq 0 ]]; then usage; exit 1; fi
|
||||
|
||||
BURN_RATE=""
|
||||
NOTE=""
|
||||
STATE_DIR=${INCIDENT_STATE_DIR:-out/incident-mode}
|
||||
THRESHOLD=${INCIDENT_THRESHOLD:-2.0}
|
||||
RESET_TH=${INCIDENT_RESET_TH:-0.5}
|
||||
COOLDOWN_MINS=${INCIDENT_COOLDOWN:-30}
|
||||
RETENTION_H=${INCIDENT_RETENTION_H:-24}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--burn-rate) BURN_RATE="$2"; shift 2;;
|
||||
--threshold) THRESHOLD="$2"; shift 2;;
|
||||
--reset-threshold) RESET_TH="$2"; shift 2;;
|
||||
--state-dir) STATE_DIR="$2"; shift 2;;
|
||||
--retention-hours) RETENTION_H="$2"; shift 2;;
|
||||
--cooldown-mins) COOLDOWN_MINS="$2"; shift 2;;
|
||||
--note) NOTE="$2"; shift 2;;
|
||||
-h|--help) usage; exit 0;;
|
||||
*) echo "Unknown arg: $1" >&2; usage; exit 1;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$BURN_RATE" ]]; then echo "--burn-rate is required" >&2; exit 1; fi
|
||||
mkdir -p "$STATE_DIR"
|
||||
FLAG_FILE="$STATE_DIR/flag.json"
|
||||
RET_FILE="$STATE_DIR/retention.json"
|
||||
LAST_FILE="$STATE_DIR/last_burn.txt"
|
||||
COOLDOWN_FILE="$STATE_DIR/cooldown.txt"
|
||||
|
||||
jq_escape() { python - <<PY "$1"
|
||||
import json,sys
|
||||
print(json.dumps(sys.argv[1]))
|
||||
PY
|
||||
}
|
||||
|
||||
now_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
burn_float=$(python - <<PY "$BURN_RATE"
|
||||
import sys
|
||||
print(float(sys.argv[1]))
|
||||
PY)
|
||||
|
||||
cooldown_current=0
|
||||
if [[ -f "$COOLDOWN_FILE" ]]; then
|
||||
cooldown_current=$(cat "$COOLDOWN_FILE")
|
||||
fi
|
||||
|
||||
enter_incident=false
|
||||
exit_incident=false
|
||||
|
||||
if (( $(echo "$burn_float >= $THRESHOLD" | bc -l) )); then
|
||||
enter_incident=true
|
||||
cooldown_current=0
|
||||
elif (( $(echo "$burn_float <= $RESET_TH" | bc -l) )); then
|
||||
cooldown_current=$((cooldown_current + 1))
|
||||
if (( cooldown_current >= COOLDOWN_MINS )); then
|
||||
exit_incident=true
|
||||
fi
|
||||
else
|
||||
cooldown_current=0
|
||||
fi
|
||||
|
||||
echo "$burn_float" > "$LAST_FILE"
|
||||
echo "$cooldown_current" > "$COOLDOWN_FILE"
|
||||
|
||||
write_flag() {
|
||||
local enabled="$1"
|
||||
cat > "$FLAG_FILE" <<JSON
|
||||
{
|
||||
"enabled": $enabled,
|
||||
"updated_at": "$now_utc",
|
||||
"reason": "incident-mode",
|
||||
"note": $(jq_escape "$NOTE"),
|
||||
"burn_rate": $burn_float
|
||||
}
|
||||
JSON
|
||||
}
|
||||
|
||||
if $enter_incident; then
|
||||
write_flag true
|
||||
cat > "$RET_FILE" <<JSON
|
||||
{
|
||||
"retention_hours": $RETENTION_H,
|
||||
"applied_at": "$now_utc"
|
||||
}
|
||||
JSON
|
||||
echo "incident-mode: activated (burn_rate=$burn_float)" >&2
|
||||
elif $exit_incident; then
|
||||
write_flag false
|
||||
echo "incident-mode: cleared after cooldown (burn_rate=$burn_float)" >&2
|
||||
else
|
||||
# no change; preserve prior flag if exists
|
||||
if [[ ! -f "$FLAG_FILE" ]]; then
|
||||
write_flag false
|
||||
fi
|
||||
echo "incident-mode: steady (burn_rate=$burn_float, cooldown=$cooldown_current/$COOLDOWN_MINS)" >&2
|
||||
fi
|
||||
|
||||
exit 0
|
||||
59
scripts/orchestrator/smoke.sh
Normal file
59
scripts/orchestrator/smoke.sh
Normal file
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT=$(cd "$(dirname "$0")/.." && pwd)
|
||||
COMPOSE_FILE="${COMPOSE_FILE:-$ROOT/devops/orchestrator/docker-compose.orchestrator.yml}"
|
||||
STATE_DIR="${STATE_DIR:-$ROOT/out/orchestrator-smoke}"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Orchestrator infra smoke test
|
||||
- Starts postgres + mongo + nats via docker-compose
|
||||
- Verifies basic connectivity and prints ready endpoints
|
||||
|
||||
Env/flags:
|
||||
COMPOSE_FILE path to compose file (default: ops/devops/orchestrator/docker-compose.orchestrator.yml)
|
||||
STATE_DIR path for logs (default: out/orchestrator-smoke)
|
||||
SKIP_UP set to 1 to skip compose up (assumes already running)
|
||||
USAGE
|
||||
}
|
||||
|
||||
if [[ ${1:-} == "-h" || ${1:-} == "--help" ]]; then usage; exit 0; fi
|
||||
mkdir -p "$STATE_DIR"
|
||||
|
||||
if [[ "${SKIP_UP:-0}" != "1" ]]; then
|
||||
docker compose -f "$COMPOSE_FILE" up -d
|
||||
fi
|
||||
|
||||
log() { echo "[smoke] $*"; }
|
||||
|
||||
log "waiting for postgres..."
|
||||
for i in {1..12}; do
|
||||
if docker compose -f "$COMPOSE_FILE" exec -T orchestrator-postgres pg_isready -U orch >/dev/null 2>&1; then break; fi
|
||||
sleep 5;
|
||||
done
|
||||
|
||||
log "waiting for mongo..."
|
||||
for i in {1..12}; do
|
||||
if docker compose -f "$COMPOSE_FILE" exec -T orchestrator-mongo mongosh --quiet --eval "db.adminCommand('ping')" >/dev/null 2>&1; then break; fi
|
||||
sleep 5;
|
||||
done
|
||||
|
||||
log "waiting for nats..."
|
||||
for i in {1..12}; do
|
||||
if docker compose -f "$COMPOSE_FILE" exec -T orchestrator-nats nats --server localhost:4222 ping >/dev/null 2>&1; then break; fi
|
||||
sleep 5;
|
||||
done
|
||||
|
||||
log "postgres DSN: postgres://orch:orchpass@localhost:55432/orchestrator"
|
||||
log "mongo uri: mongodb://localhost:57017"
|
||||
log "nats uri: nats://localhost:4222"
|
||||
|
||||
# Write readiness summary
|
||||
cat > "$STATE_DIR/readiness.txt" <<EOF
|
||||
postgres=postgres://orch:orchpass@localhost:55432/orchestrator
|
||||
mongo=mongodb://localhost:57017
|
||||
nats=nats://localhost:4222
|
||||
ready_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
EOF
|
||||
|
||||
log "smoke completed; summary at $STATE_DIR/readiness.txt"
|
||||
Reference in New Issue
Block a user