219 lines
18 KiB
Bash
219 lines
18 KiB
Bash
#!/usr/bin/env bash
|
|
# CLI Tier 2b test runner - captures output from all CLI help commands
|
|
# Each test outputs a JSON line to be assembled into evidence
|
|
|
|
CLI_PROJECT="C:/dev/New folder/git.stella-ops.org/src/Cli/StellaOps.Cli/StellaOps.Cli.csproj"
|
|
OUTDIR="C:/dev/New folder/git.stella-ops.org/docs/qa/feature-checks/runs/cli/run-20260213-deep-e2e"
|
|
RESULTS_FILE="$OUTDIR/raw-results.jsonl"
|
|
|
|
> "$RESULTS_FILE"
|
|
|
|
run_test() {
|
|
local feature="$1"
|
|
local args="$2"
|
|
local expect_pattern="$3"
|
|
local notes="$4"
|
|
|
|
local ts
|
|
ts=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
# Run the command, capture stdout+stderr, and exit code
|
|
local output
|
|
output=$(dotnet run --project "$CLI_PROJECT" --no-build -- $args 2>&1)
|
|
local ec=$?
|
|
|
|
# Filter noise from output
|
|
local clean_output
|
|
clean_output=$(echo "$output" | grep -v "^ln:\|^/usr/bin\|^rm:\|info:\|warn:\|^'" | head -40)
|
|
|
|
# Check for expected pattern
|
|
local verdict="fail"
|
|
if [ $ec -eq 0 ] && echo "$clean_output" | grep -qiE "$expect_pattern"; then
|
|
verdict="pass"
|
|
fi
|
|
|
|
# Take first 300 chars for snippet
|
|
local snippet
|
|
snippet=$(echo "$clean_output" | head -20 | cut -c1-300 | tr '\n' '|' | sed 's/|/\\n/g' | sed 's/"/\\"/g')
|
|
|
|
# stderr snippet (only actual errors)
|
|
local stderr_snip=""
|
|
if echo "$output" | grep -q "Unhandled exception\|Error:\|FATAL"; then
|
|
stderr_snip=$(echo "$output" | grep "Unhandled exception\|Error:\|FATAL" | head -3 | tr '\n' '|' | sed 's/|/\\n/g' | sed 's/"/\\"/g')
|
|
fi
|
|
|
|
echo "{\"feature\":\"$feature\",\"tier\":\"2b\",\"timestamp\":\"$ts\",\"command\":\"stella $args\",\"exitCode\":$ec,\"stdoutSnippet\":\"$snippet\",\"stderrSnippet\":\"$stderr_snip\",\"verdict\":\"$verdict\",\"notes\":\"$notes\"}" >> "$RESULTS_FILE"
|
|
|
|
echo "[$verdict] $feature ($args) -> exit=$ec"
|
|
}
|
|
|
|
echo "=== Category 1: Core Infrastructure ==="
|
|
|
|
run_test "cli-command-router-infrastructure.md" "--help" "Commands:" "Verify all command groups listed in help"
|
|
run_test "cli-help-text-and-discoverability.md" "scan --help" "Commands:" "Verify scan help text shows subcommands"
|
|
run_test "cli-help-text-and-discoverability-policy.md" "policy --help" "Commands:" "Verify policy help text shows subcommands"
|
|
run_test "resource-oriented-cli-hierarchy.md" "--help" "scanner|scan|policy|config|auth" "Verify resource-oriented hierarchy in root help"
|
|
run_test "cli-config-command-hub.md" "config --help" "show|list|notify|integrations" "Verify config command hub subcommands"
|
|
run_test "settings-consolidation-under-stella-config.md" "config show --help" "Display resolved" "Verify config show command exists"
|
|
run_test "setup-wizard-cli.md" "setup --help" "run|resume|status|reset|validate" "Verify setup wizard subcommands"
|
|
run_test "backward-compatible-command-aliases.md" "function-map --help" "function.map|fmap" "Test function-map alias"
|
|
run_test "cli-deprecation-warning-system.md" "fmap --help" "function.map|fmap|Runtime" "Test fmap alias"
|
|
run_test "cli-plugin-module-loading-architecture.md" "-v --help" "plug-in|plugin|Commands:" "Verify plugin loading in verbose mode"
|
|
run_test "cli-with-plugin-based-command-modules.md" "--help" "Commands:" "Verify plugin-based command loading"
|
|
run_test "tenant-context-management-cli.md" "tenants --help" "tenant" "Verify tenant management commands"
|
|
run_test "token-minting-and-delegation-cli.md" "auth --help" "token|login|logout" "Verify auth commands"
|
|
run_test "auth-revocation-bundle-export-verify-cli.md" "auth --help" "revok|bundle|token" "Verify auth revocation commands"
|
|
run_test "cli-parity.md" "--help" "Commands:" "Verify comprehensive command set"
|
|
|
|
echo ""
|
|
echo "=== Category 2: Scan & Policy ==="
|
|
|
|
run_test "baseline-selection-logic.md" "scan --help" "replay|delta|gate" "Verify baseline-related scan subcommands"
|
|
run_test "cli-scan-command-consolidation.md" "scan --help" "run|upload|download|diff|delta" "Verify consolidated scan subcommands"
|
|
run_test "scan-reproducibility-verification-flag.md" "scan replay --help" "replay|hash|deterministic" "Verify scan replay/reproducibility"
|
|
run_test "scan-snapshot-compare-cli.md" "scan diff --help" "base|target|compare" "Verify scan diff/compare"
|
|
run_test "scan-entry-trace-analysis-cli.md" "scan entrytrace --help" "entry.trace|trace|scan" "Verify scan entrytrace"
|
|
run_test "delta-scan-cli-command.md" "scan delta --help" "old|new|delta" "Verify scan delta"
|
|
run_test "cli-policy-lifecycle-commands.md" "policy --help" "activate|submit|review|publish|rollback" "Verify policy lifecycle subcommands"
|
|
run_test "policy-dsl-compiler-cli.md" "policy compile --help" "compile|file|IR" "Verify policy compile"
|
|
run_test "policy-dsl-testing-cli.md" "policy test --help" "test|fixture|coverage" "Verify policy test"
|
|
run_test "policy-history-cli.md" "policy history --help" "history|policy" "Verify policy history"
|
|
run_test "policy-publish-and-sign-cli.md" "policy publish --help" "publish|policy" "Verify policy publish"
|
|
run_test "policy-review-workflow-cli.md" "policy review --help" "review" "Verify policy review"
|
|
run_test "policy-rollback-cli.md" "policy rollback --help" "rollback|version" "Verify policy rollback"
|
|
run_test "policy-scaffolding-cli.md" "policy new --help" "new|name|template" "Verify policy new/scaffold"
|
|
run_test "policy-simulation-batch-mode-with-sbom-selectors.md" "policy simulate --help" "simulate|sbom|policy" "Verify policy simulate"
|
|
run_test "policy-simulation-reachability-overrides.md" "policy simulate --help" "simulate|reachability|override" "Verify reachability overrides in simulate"
|
|
run_test "policy-version-bump-cli.md" "policy version --help" "version|bump|semver" "Verify policy version"
|
|
run_test "policy-workspace-initialization-cli.md" "policy init --help" "init|path|workspace" "Verify policy init"
|
|
run_test "vex-gated-policy-decisions.md" "scan gate-policy --help" "gate|vex|policy" "Verify VEX gated policy"
|
|
|
|
echo ""
|
|
echo "=== Category 3: Evidence, VEX & SBOM ==="
|
|
|
|
run_test "evidence-pack-download-and-verification.md" "evidence --help" "evidence|pack|download|verify" "Verify evidence commands"
|
|
run_test "cli-verify-command-for-attestation-chain-validation.md" "verify --help" "attest|chain|verify" "Verify unified verify command"
|
|
run_test "vex-generation-with-evidence-links.md" "vex --help" "vex|consensus|evidence" "Verify VEX commands"
|
|
run_test "sbom-format-conversion-cli.md" "sbom --help" "sbom|format|convert" "Verify SBOM commands"
|
|
run_test "proof-of-exposure-export-verify-cli.md" "proof --help" "proof|chain|verify" "Verify proof commands"
|
|
run_test "rekor-cli-commands.md" "verify --help" "rekor|verify" "Verify Rekor-related commands"
|
|
run_test "witness-cli-commands.md" "witness --help" "witness|micro|binary" "Verify witness commands"
|
|
run_test "cli-offline-offline-poe-verification.md" "offline --help" "offline|air.gap|kit" "Verify offline commands"
|
|
run_test "cli-forensic-snapshot-commands.md" "forensic --help" "forensic|snapshot|locker" "Verify forensic commands"
|
|
run_test "evidence-legal-holds-cli.md" "forensic --help" "legal.hold|forensic|hold" "Verify evidence legal holds"
|
|
run_test "evidence-card-and-remediation-pr-cli-commands.md" "evidence --help" "card|evidence" "Verify evidence card commands"
|
|
run_test "cli-and-web-ui-for-proof-inspection.md" "proof --help" "inspect|proof|chain" "Verify proof inspection CLI"
|
|
run_test "audit-bundle-generation-and-verification-cli.md" "bundle --help" "bundle|offline|evidence" "Verify audit bundle commands"
|
|
run_test "offline-sbom-verification-cli.md" "verify --help" "sbom|verify|offline" "Verify offline SBOM verification"
|
|
run_test "offline-verdict-verification-cli-plugin.md" "verify --help" "verdict|verify|offline" "Verify offline verdict verification"
|
|
run_test "verification-receipt-cli.md" "verify --help" "receipt|verify" "Verify receipt verification"
|
|
run_test "verification-command-consolidation.md" "verify --help" "attest|sbom|vex|patch|evidence" "Verify unified verify subcommands"
|
|
run_test "vex-observation-and-webhooks-cli.md" "vex --help" "observe|webhook|vex" "Verify VEX observation commands"
|
|
run_test "cli-vex-consensus-commands.md" "vex --help" "consensus|vex" "Verify VEX consensus commands"
|
|
|
|
echo ""
|
|
echo "=== Category 4: Remaining Commands ==="
|
|
|
|
run_test "advisory-database-status-and-connector-cli-commands.md" "config sources --help" "source|advisory|status" "Verify advisory source commands"
|
|
run_test "advisory-source-management-cli.md" "advisory --help" "advisory|observation|linkset" "Verify advisory management CLI"
|
|
run_test "advisoryai-chat-cli.md" "advise --help" "advise|chat|ai" "Verify AdvisoryAI chat CLI"
|
|
run_test "ai-code-guard-cli.md" "advise --help" "advise|guard|code" "Verify AI code guard"
|
|
run_test "ci-template-generator-cli-command.md" "ci --help" "template|ci|generate" "Verify CI template generator"
|
|
run_test "cli-api-spec-download-command.md" "api --help" "api|spec|download" "Verify API spec download"
|
|
run_test "cli-commands-for-ground-truth-and-golden-set-management.md" "golden --help" "golden|ground.truth|set" "Verify golden set commands"
|
|
run_test "cli-determinism-score-report-generator.md" "detscore --help" "determinism|score|report" "Verify determinism score commands"
|
|
run_test "cli-export-profile-and-run-management.md" "export --help" "export|profile|run" "Verify export commands"
|
|
run_test "cli-ir-commands.md" "policy compile --help" "compile|IR|file" "Verify IR compilation"
|
|
run_test "cli-notification-simulation-and-acknowledgment.md" "notify --help" "notify|channel|rule" "Verify notification commands"
|
|
run_test "cli-observability-dashboard-commands.md" "obs --help" "observability|health|SLO|metric" "Verify observability commands"
|
|
run_test "cli-reachability-trace-export.md" "reachability --help" "reachability|trace|export" "Verify reachability commands"
|
|
run_test "cli-reachability-upload-and-explain-commands.md" "reachability --help" "upload|explain|reachability" "Verify reachability upload/explain"
|
|
run_test "cli-slice-management-commands.md" "sbom --help" "sbom|slice|manage" "Verify SBOM slice management"
|
|
run_test "cli-tools.md" "tools --help" "tools|policy|local" "Verify tools commands"
|
|
run_test "cli-vulnerability-workflow-commands.md" "vuln --help" "vuln|observation|overlay" "Verify vuln workflow commands"
|
|
run_test "concelier-database-operations-cli.md" "db --help" "db|Concelier|database" "Verify DB operations CLI"
|
|
run_test "deltasig-cli-module.md" "deltasig --help" "delta|sig|binary|backport" "Verify deltasig commands"
|
|
run_test "doctor-cli-command-group.md" "doctor --help" "doctor|diagnostic|check" "Verify doctor commands"
|
|
run_test "evidence-legal-holds-cli-2.md" "forensic --help" "forensic|locker" "Verify forensic/evidence"
|
|
run_test "excititor-vex-ingest-management-cli.md" "vex --help" "vex|ingest|excititor" "Verify VEX ingest management"
|
|
run_test "explain-block-cli-command.md" "explain --help" "explain|policy|decision|trace" "Verify explain command"
|
|
run_test "feed-snapshotting-for-deterministic-replay.md" "config feeds --help" "feed|snapshot|source" "Verify feed snapshotting config"
|
|
run_test "function-map-cli.md" "fmap --help" "function.map|fmap|runtime" "Verify function map CLI"
|
|
run_test "github-code-scanning-endpoints.md" "github --help" "github|code.scan|integration" "Verify GitHub integration"
|
|
run_test "gitops-controller.md" "pack --help" "pack|plan|run|push|pull" "Verify pack/GitOps commands"
|
|
run_test "hlc-status-and-timeline-query-cli-commands.md" "orch --help" "orch|job|status" "Verify orchestrator commands"
|
|
run_test "image-inspect-cli-command.md" "image --help" "image|inspect|OCI" "Verify image inspect command"
|
|
run_test "incident-response-cli.md" "forensic --help" "forensic|incident|snapshot" "Verify incident response CLI"
|
|
run_test "key-rotation-cli.md" "key --help" "key|rotation|manage" "Verify key rotation"
|
|
run_test "kms-key-export-import-cli.md" "kms --help" "kms|key|sign" "Verify KMS commands"
|
|
run_test "local-validator-for-offline-config-checking.md" "config show --help" "config|show|display" "Verify local config validation"
|
|
run_test "notification-channel-management-cli-commands.md" "notify --help" "notify|channel|rule|delivery" "Verify notification channel management"
|
|
run_test "oci-referrer-based-artifact-association.md" "image --help" "image|oci|referrer" "Verify OCI referrer commands"
|
|
run_test "policy-dsl-compiler-cli-2.md" "policy lint --help" "lint|validate|file" "Verify policy lint"
|
|
run_test "policy-review-workflow-cli-2.md" "policy submit --help" "submit|policy" "Verify policy submit"
|
|
run_test "policy-simulation-batch-mode-with-sbom-selectors-2.md" "policy evaluate --help" "evaluate|evidence|policy" "Verify policy evaluate"
|
|
run_test "proof-chain-cli-commands-with-structured-exit-codes.md" "chain --help" "chain|attestation|verification" "Verify chain commands"
|
|
run_test "python-workspace-analyzer-cli.md" "python --help" "python|PHP|analyzer" "Verify Python analyzer CLI"
|
|
run_test "reachability-aware-security-as-gate.md" "gate --help" "gate|release|CI" "Verify gate commands"
|
|
run_test "reachability-query-api-and-triage-flow.md" "reachability --help" "reachability|query|triage" "Verify reachability query"
|
|
run_test "rekor-cli-commands-2.md" "verify --help" "rekor|verify|attest" "Verify Rekor in verify"
|
|
run_test "replay-button-determinism-as-ux.md" "replay --help" "replay|scan|manifest|verdict" "Verify replay commands"
|
|
run_test "replay-command-generator-service.md" "prove --help" "prove|replay|proof|image" "Verify prove command"
|
|
run_test "runtime-observations-query-cli.md" "observations --help" "observation|runtime" "Verify observations CLI"
|
|
run_test "sbom-analytics-cli-commands.md" "analytics --help" "analytics|insight|reporting" "Verify analytics commands"
|
|
run_test "sbom-deterministic-generation-cli.md" "sbomer --help" "sbom|layer|fragment|merge|merkle" "Verify sbomer commands"
|
|
run_test "sbom-format-conversion-cli-2.md" "sbom --help" "sbom|format" "Verify SBOM format commands"
|
|
run_test "scan-reproducibility-verification-flag-2.md" "detscore --help" "detscore|determinism|reproducibility" "Verify detscore reproducibility"
|
|
run_test "scan-snapshot-compare-cli-2.md" "change-trace --help" "change.trace|build|export" "Verify change-trace commands"
|
|
run_test "stella-admin-cli-command-group.md" "admin --help" "admin|Administrative|platform" "Verify admin commands"
|
|
run_test "symbol-ingestion-cli.md" "symbols --help" "symbol|bundle|air.gap" "Verify symbols commands"
|
|
run_test "system-database-migrations-cli.md" "system --help" "system|migration" "Verify system commands"
|
|
run_test "trust-anchor-management-cli.md" "verify --help" "trust|verify|anchor" "Verify trust anchor in verify"
|
|
run_test "unknowns-export-artifacts.md" "unknowns --help" "unknown|registry|unmatched" "Verify unknowns commands"
|
|
run_test "verdict-ladder-ui.md" "score --help" "score|computation|replay" "Verify score commands"
|
|
run_test "zastava-cli-commands.md" "risk-profile --help" "risk.profile|schema|validation" "Verify risk-profile/zastava commands"
|
|
run_test "sdk-management.md" "sdk --help" "sdk|SDK|management" "Verify SDK commands"
|
|
run_test "mirror-commands.md" "mirror --help" "mirror|air.gap|offline" "Verify mirror commands"
|
|
run_test "airgap-commands.md" "airgap --help" "airgap|air.gap|offline" "Verify airgap commands"
|
|
run_test "trust-profile-commands.md" "trust-profile --help" "trust.profile|offline|verification" "Verify trust-profile commands"
|
|
run_test "devportal-commands.md" "devportal --help" "devportal|offline" "Verify devportal commands"
|
|
run_test "delta-verdict-commands.md" "delta --help" "delta|verdict" "Verify delta commands"
|
|
run_test "budget-commands.md" "budget --help" "budget|risk|release" "Verify budget commands"
|
|
run_test "watchlist-commands.md" "watchlist --help" "watchlist|identity|transparency" "Verify watchlist commands"
|
|
run_test "exception-commands.md" "exception --help" "exception|approval|workflow" "Verify exception commands"
|
|
run_test "exceptions-commands.md" "exceptions --help" "exception|governance|list" "Verify exceptions commands"
|
|
run_test "feedser-commands.md" "feedser --help" "federation|bundle|sync" "Verify feedser commands"
|
|
run_test "cvss-commands.md" "cvss --help" "cvss|v4|receipt|score" "Verify CVSS commands"
|
|
run_test "risk-commands.md" "risk --help" "risk|profile|scoring|bundle" "Verify risk commands"
|
|
run_test "graph-commands.md" "graph --help" "graph|call|evidence" "Verify graph commands"
|
|
run_test "binary-commands.md" "binary --help" "binary|reachability|analysis" "Verify binary commands"
|
|
run_test "attest-commands.md" "attest --help" "attest|verify|inspect|DSSE" "Verify attest commands"
|
|
run_test "promotion-commands.md" "promotion --help" "promotion|attestation" "Verify promotion commands"
|
|
run_test "seal-commands.md" "seal --help" "seal|facet|image|cryptographic" "Verify seal commands"
|
|
run_test "drift-commands.md" "drift --help" "drift|facet|baseline|image" "Verify drift commands"
|
|
run_test "verify-fix-command.md" "verify-fix --help" "verify.fix|patch|vulnerability" "Verify verify-fix command"
|
|
run_test "ts-commands.md" "ts --help" "RFC.3161|timestamp" "Verify timestamp commands"
|
|
run_test "license-commands.md" "license --help" "license|detection|compliance" "Verify license commands"
|
|
run_test "findings-commands.md" "findings --help" "findings|inspect|policy" "Verify findings commands"
|
|
run_test "scanner-commands.md" "scanner --help" "scanner|artifact|lifecycle" "Verify scanner commands"
|
|
run_test "ruby-commands.md" "ruby --help" "ruby|analyzer" "Verify ruby commands"
|
|
run_test "php-commands.md" "php --help" "php|analyzer" "Verify PHP commands"
|
|
run_test "bun-commands.md" "bun --help" "bun|analyzer" "Verify bun commands"
|
|
run_test "sources-commands.md" "sources --help" "source|ingestion" "Verify sources commands"
|
|
run_test "aoc-commands.md" "aoc --help" "aoc|Aggregation|Contract" "Verify AOC commands"
|
|
run_test "task-runner-commands.md" "task-runner --help" "task.runner|Task Runner" "Verify task-runner commands"
|
|
run_test "issuer-commands.md" "issuer --help" "issuer|key" "Verify issuer commands"
|
|
run_test "decision-commands.md" "decision --help" "decision|VEX|DSSE|Rekor" "Verify decision commands"
|
|
run_test "crypto-commands.md" "crypto --help" "crypto|sign|verify|profile" "Verify crypto commands"
|
|
|
|
echo ""
|
|
echo "=== Additional deep tests ==="
|
|
|
|
run_test "doctor-run.md" "doctor run --help" "diagnostic|check|run" "Verify doctor run subcommand"
|
|
run_test "config-show.md" "config show" "config|setting|value" "Actually run config show"
|
|
run_test "version-check.md" "--version" "version|StellaOps|[0-9]" "Verify version output"
|
|
|
|
echo ""
|
|
echo "=== DONE ==="
|
|
wc -l "$RESULTS_FILE"
|