save progress
This commit is contained in:
StellaOps Bot
299
devops/offline/scripts/rotate-secrets-bundle.sh
Normal file
299
devops/offline/scripts/rotate-secrets-bundle.sh
Normal file
@@ -0,0 +1,299 @@
|
||||
#!/usr/bin/env bash
|
||||
# -----------------------------------------------------------------------------
|
||||
# rotate-secrets-bundle.sh
|
||||
# Sprint: SPRINT_20260104_005_AIRGAP (Secret Offline Kit Integration)
|
||||
# Task: OKS-006 - Add bundle rotation/upgrade workflow
|
||||
# Description: Safely rotate/upgrade secrets rule bundle with backup and rollback
|
||||
# -----------------------------------------------------------------------------
|
||||
# Usage: ./rotate-secrets-bundle.sh <new-bundle-path> [install-path]
|
||||
# Example: ./rotate-secrets-bundle.sh /mnt/offline-kit/rules/secrets/2026.02
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
NEW_BUNDLE_PATH="${1:?New bundle path required (e.g., /mnt/offline-kit/rules/secrets/2026.02)}"
|
||||
INSTALL_PATH="${2:-/opt/stellaops/plugins/scanner/analyzers/secrets}"
|
||||
BACKUP_BASE="${BACKUP_BASE:-/opt/stellaops/backups/secrets-bundles}"
|
||||
BUNDLE_ID="${BUNDLE_ID:-secrets.ruleset}"
|
||||
ATTESTOR_MIRROR="${ATTESTOR_MIRROR:-}"
|
||||
RESTART_WORKERS="${RESTART_WORKERS:-true}"
|
||||
KUBERNETES_NAMESPACE="${KUBERNETES_NAMESPACE:-stellaops}"
|
||||
KUBERNETES_DEPLOYMENT="${KUBERNETES_DEPLOYMENT:-scanner-worker}"
|
||||
MAX_BACKUPS="${MAX_BACKUPS:-5}"
|
||||
|
||||
# Script directory for calling install script
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Color output helpers
|
||||
if [[ -t 1 ]]; then
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[0;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
else
|
||||
RED=''
|
||||
GREEN=''
|
||||
YELLOW=''
|
||||
BLUE=''
|
||||
NC=''
|
||||
fi
|
||||
|
||||
log_info() { echo -e "${GREEN}==>${NC} $*"; }
|
||||
log_warn() { echo -e "${YELLOW}WARN:${NC} $*" >&2; }
|
||||
log_error() { echo -e "${RED}ERROR:${NC} $*" >&2; }
|
||||
log_step() { echo -e "${BLUE}--->${NC} $*"; }
|
||||
|
||||
# Error handler
|
||||
cleanup_on_error() {
|
||||
log_error "Rotation failed! Attempting rollback..."
|
||||
if [[ -n "${BACKUP_DIR:-}" && -d "${BACKUP_DIR}" ]]; then
|
||||
perform_rollback "${BACKUP_DIR}"
|
||||
fi
|
||||
}
|
||||
|
||||
perform_rollback() {
|
||||
local backup_dir="$1"
|
||||
log_info "Rolling back to backup: ${backup_dir}"
|
||||
|
||||
if [[ ! -d "${backup_dir}" ]]; then
|
||||
log_error "Backup directory not found: ${backup_dir}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Restore files
|
||||
cp -a "${backup_dir}"/* "${INSTALL_PATH}/" 2>/dev/null || {
|
||||
log_error "Failed to restore files from backup"
|
||||
return 1
|
||||
}
|
||||
|
||||
log_info "Rollback completed"
|
||||
|
||||
# Restart workers after rollback
|
||||
if [[ "${RESTART_WORKERS}" == "true" ]]; then
|
||||
restart_workers "rollback"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
restart_workers() {
|
||||
local reason="${1:-upgrade}"
|
||||
log_info "Restarting scanner workers (${reason})..."
|
||||
|
||||
# Try Kubernetes first
|
||||
if command -v kubectl &>/dev/null; then
|
||||
if kubectl get deployment "${KUBERNETES_DEPLOYMENT}" -n "${KUBERNETES_NAMESPACE}" &>/dev/null; then
|
||||
log_step "Performing Kubernetes rolling restart..."
|
||||
kubectl rollout restart deployment/"${KUBERNETES_DEPLOYMENT}" -n "${KUBERNETES_NAMESPACE}"
|
||||
log_step "Waiting for rollout to complete..."
|
||||
kubectl rollout status deployment/"${KUBERNETES_DEPLOYMENT}" -n "${KUBERNETES_NAMESPACE}" --timeout=300s || {
|
||||
log_warn "Rollout status check timed out (workers may still be restarting)"
|
||||
}
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Try systemd
|
||||
if command -v systemctl &>/dev/null; then
|
||||
if systemctl is-active stellaops-scanner-worker &>/dev/null 2>&1; then
|
||||
log_step "Restarting systemd service..."
|
||||
systemctl restart stellaops-scanner-worker
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
log_warn "Could not auto-restart workers (no Kubernetes or systemd found)"
|
||||
log_warn "Please restart scanner workers manually"
|
||||
}
|
||||
|
||||
cleanup_old_backups() {
|
||||
log_info "Cleaning up old backups (keeping last ${MAX_BACKUPS})..."
|
||||
|
||||
if [[ ! -d "${BACKUP_BASE}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# List backups sorted by name (which includes timestamp)
|
||||
local backups
|
||||
backups=$(find "${BACKUP_BASE}" -maxdepth 1 -type d -name "20*" | sort -r)
|
||||
local count=0
|
||||
|
||||
for backup in ${backups}; do
|
||||
count=$((count + 1))
|
||||
if [[ ${count} -gt ${MAX_BACKUPS} ]]; then
|
||||
log_step "Removing old backup: ${backup}"
|
||||
rm -rf "${backup}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Main rotation logic
|
||||
main() {
|
||||
echo ""
|
||||
log_info "Secrets Bundle Rotation"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Validate new bundle
|
||||
log_info "Step 1/6: Validating new bundle..."
|
||||
if [[ ! -d "${NEW_BUNDLE_PATH}" ]]; then
|
||||
log_error "New bundle directory not found: ${NEW_BUNDLE_PATH}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
NEW_MANIFEST="${NEW_BUNDLE_PATH}/${BUNDLE_ID}.manifest.json"
|
||||
if [[ ! -f "${NEW_MANIFEST}" ]]; then
|
||||
log_error "New bundle manifest not found: ${NEW_MANIFEST}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
NEW_VERSION=$(jq -r '.version // "unknown"' "${NEW_MANIFEST}" 2>/dev/null || echo "unknown")
|
||||
NEW_RULE_COUNT=$(jq -r '.ruleCount // 0' "${NEW_MANIFEST}" 2>/dev/null || echo "0")
|
||||
log_step "New version: ${NEW_VERSION} (${NEW_RULE_COUNT} rules)"
|
||||
|
||||
# Check current installation
|
||||
log_info "Step 2/6: Checking current installation..."
|
||||
CURRENT_VERSION="(none)"
|
||||
CURRENT_RULE_COUNT="0"
|
||||
|
||||
if [[ -f "${INSTALL_PATH}/${BUNDLE_ID}.manifest.json" ]]; then
|
||||
CURRENT_VERSION=$(jq -r '.version // "unknown"' "${INSTALL_PATH}/${BUNDLE_ID}.manifest.json" 2>/dev/null || echo "unknown")
|
||||
CURRENT_RULE_COUNT=$(jq -r '.ruleCount // 0' "${INSTALL_PATH}/${BUNDLE_ID}.manifest.json" 2>/dev/null || echo "0")
|
||||
log_step "Current version: ${CURRENT_VERSION} (${CURRENT_RULE_COUNT} rules)"
|
||||
else
|
||||
log_step "No current installation found"
|
||||
fi
|
||||
|
||||
# Version comparison
|
||||
if [[ "${CURRENT_VERSION}" != "(none)" ]]; then
|
||||
if [[ "${CURRENT_VERSION}" == "${NEW_VERSION}" ]]; then
|
||||
log_warn "New version (${NEW_VERSION}) is the same as current"
|
||||
if [[ "${FORCE_ROTATION:-false}" != "true" ]]; then
|
||||
log_warn "Use FORCE_ROTATION=true to reinstall"
|
||||
exit 0
|
||||
fi
|
||||
elif [[ "${CURRENT_VERSION}" > "${NEW_VERSION}" ]]; then
|
||||
log_warn "New version (${NEW_VERSION}) is older than current (${CURRENT_VERSION})"
|
||||
if [[ "${FORCE_ROTATION:-false}" != "true" ]]; then
|
||||
log_warn "Use FORCE_ROTATION=true to downgrade"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
log_info "Upgrade: ${CURRENT_VERSION} -> ${NEW_VERSION}"
|
||||
echo ""
|
||||
|
||||
# Backup current installation
|
||||
log_info "Step 3/6: Creating backup..."
|
||||
BACKUP_DIR="${BACKUP_BASE}/$(date +%Y%m%d_%H%M%S)_${CURRENT_VERSION}"
|
||||
|
||||
if [[ -d "${INSTALL_PATH}" && -f "${INSTALL_PATH}/${BUNDLE_ID}.manifest.json" ]]; then
|
||||
mkdir -p "${BACKUP_DIR}"
|
||||
cp -a "${INSTALL_PATH}"/* "${BACKUP_DIR}/" 2>/dev/null || {
|
||||
log_error "Failed to create backup"
|
||||
exit 1
|
||||
}
|
||||
log_step "Backup created: ${BACKUP_DIR}"
|
||||
|
||||
# Create backup metadata
|
||||
cat > "${BACKUP_DIR}/.backup-metadata.json" <<EOF
|
||||
{
|
||||
"version": "${CURRENT_VERSION}",
|
||||
"backupAt": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||
"reason": "rotation-to-${NEW_VERSION}",
|
||||
"hostname": "$(hostname -f 2>/dev/null || hostname)"
|
||||
}
|
||||
EOF
|
||||
else
|
||||
log_step "No existing installation to backup"
|
||||
BACKUP_DIR=""
|
||||
fi
|
||||
|
||||
# Set up error handler for rollback
|
||||
trap cleanup_on_error ERR
|
||||
|
||||
# Install new bundle
|
||||
log_info "Step 4/6: Installing new bundle..."
|
||||
export FORCE_INSTALL=true
|
||||
export REQUIRE_SIGNATURE="${REQUIRE_SIGNATURE:-true}"
|
||||
|
||||
if [[ -n "${ATTESTOR_MIRROR}" ]]; then
|
||||
"${SCRIPT_DIR}/install-secrets-bundle.sh" "${NEW_BUNDLE_PATH}" "${INSTALL_PATH}" "${ATTESTOR_MIRROR}"
|
||||
else
|
||||
"${SCRIPT_DIR}/install-secrets-bundle.sh" "${NEW_BUNDLE_PATH}" "${INSTALL_PATH}"
|
||||
fi
|
||||
|
||||
# Verify installation
|
||||
log_info "Step 5/6: Verifying installation..."
|
||||
INSTALLED_VERSION=$(jq -r '.version' "${INSTALL_PATH}/${BUNDLE_ID}.manifest.json" 2>/dev/null || echo "unknown")
|
||||
|
||||
if [[ "${INSTALLED_VERSION}" != "${NEW_VERSION}" ]]; then
|
||||
log_error "Installation verification failed"
|
||||
log_error "Expected version: ${NEW_VERSION}"
|
||||
log_error "Installed version: ${INSTALLED_VERSION}"
|
||||
exit 1
|
||||
fi
|
||||
log_step "Installation verified: ${INSTALLED_VERSION}"
|
||||
|
||||
# Remove error trap since installation succeeded
|
||||
trap - ERR
|
||||
|
||||
# Restart workers
|
||||
log_info "Step 6/6: Restarting workers..."
|
||||
if [[ "${RESTART_WORKERS}" == "true" ]]; then
|
||||
restart_workers "upgrade"
|
||||
else
|
||||
log_step "Worker restart skipped (RESTART_WORKERS=false)"
|
||||
fi
|
||||
|
||||
# Cleanup old backups
|
||||
cleanup_old_backups
|
||||
|
||||
# Generate rotation report
|
||||
REPORT_FILE="${INSTALL_PATH}/.rotation-report.json"
|
||||
cat > "${REPORT_FILE}" <<EOF
|
||||
{
|
||||
"previousVersion": "${CURRENT_VERSION}",
|
||||
"newVersion": "${NEW_VERSION}",
|
||||
"previousRuleCount": ${CURRENT_RULE_COUNT},
|
||||
"newRuleCount": ${NEW_RULE_COUNT},
|
||||
"rotatedAt": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
||||
"backupPath": "${BACKUP_DIR:-null}",
|
||||
"hostname": "$(hostname -f 2>/dev/null || hostname)"
|
||||
}
|
||||
EOF
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
log_info "Rotation completed successfully!"
|
||||
echo ""
|
||||
echo "Summary:"
|
||||
echo " Previous version: ${CURRENT_VERSION} (${CURRENT_RULE_COUNT} rules)"
|
||||
echo " New version: ${NEW_VERSION} (${NEW_RULE_COUNT} rules)"
|
||||
if [[ -n "${BACKUP_DIR}" ]]; then
|
||||
echo " Backup path: ${BACKUP_DIR}"
|
||||
fi
|
||||
echo " Report: ${REPORT_FILE}"
|
||||
echo ""
|
||||
echo "To verify the upgrade:"
|
||||
echo " kubectl logs -l app=scanner-worker --tail=100 | grep SecretsAnalyzerHost"
|
||||
echo ""
|
||||
echo "To rollback if needed:"
|
||||
echo " $0 --rollback ${BACKUP_DIR:-/path/to/backup}"
|
||||
}
|
||||
|
||||
# Handle rollback command
|
||||
if [[ "${1:-}" == "--rollback" ]]; then
|
||||
ROLLBACK_BACKUP="${2:?Backup directory required for rollback}"
|
||||
perform_rollback "${ROLLBACK_BACKUP}"
|
||||
if [[ "${RESTART_WORKERS}" == "true" ]]; then
|
||||
restart_workers "rollback"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Run main
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user