Add property-based tests for SBOM/VEX document ordering and Unicode normalization determinism
- Implement `SbomVexOrderingDeterminismProperties` for testing component list and vulnerability metadata hash consistency. - Create `UnicodeNormalizationDeterminismProperties` to validate NFC normalization and Unicode string handling. - Add project file for `StellaOps.Testing.Determinism.Properties` with necessary dependencies. - Introduce CI/CD template validation tests including YAML syntax checks and documentation content verification. - Create validation script for CI/CD templates ensuring all required files and structures are present.
This commit is contained in:
StellaOps Bot
424
tests/cicd-templates/validate-templates.sh
Normal file
424
tests/cicd-templates/validate-templates.sh
Normal file
@@ -0,0 +1,424 @@
|
||||
#!/bin/bash
|
||||
# CI/CD Template Validation Tests
|
||||
# Sprint: SPRINT_20251226_004_BE_cicd_signing_templates
|
||||
# Tasks: 0020-0024
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PASS_COUNT=0
|
||||
FAIL_COUNT=0
|
||||
|
||||
log_pass() {
|
||||
echo -e "${GREEN}✓ PASS${NC}: $1"
|
||||
((PASS_COUNT++))
|
||||
}
|
||||
|
||||
log_fail() {
|
||||
echo -e "${RED}✗ FAIL${NC}: $1"
|
||||
((FAIL_COUNT++))
|
||||
}
|
||||
|
||||
log_skip() {
|
||||
echo -e "${YELLOW}○ SKIP${NC}: $1"
|
||||
}
|
||||
|
||||
log_section() {
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "$1"
|
||||
echo "========================================"
|
||||
}
|
||||
|
||||
# Check for required tools
|
||||
check_tools() {
|
||||
log_section "Checking required tools"
|
||||
|
||||
if command -v yq &> /dev/null; then
|
||||
log_pass "yq is installed"
|
||||
else
|
||||
log_skip "yq not installed - YAML structure tests will be skipped"
|
||||
fi
|
||||
|
||||
if command -v actionlint &> /dev/null; then
|
||||
log_pass "actionlint is installed"
|
||||
else
|
||||
log_skip "actionlint not installed - GitHub Actions lint tests will be skipped"
|
||||
fi
|
||||
}
|
||||
|
||||
# Test: GitHub Actions templates exist
|
||||
test_github_templates_exist() {
|
||||
log_section "Testing GitHub Actions templates exist"
|
||||
|
||||
local templates=(
|
||||
".github/workflows/examples/stellaops-sign.yml"
|
||||
".github/workflows/examples/stellaops-verify.yml"
|
||||
".github/workflows/examples/example-container-sign.yml"
|
||||
".github/workflows/examples/example-sbom-sign.yml"
|
||||
".github/workflows/examples/example-verdict-sign.yml"
|
||||
".github/workflows/examples/example-verification-gate.yml"
|
||||
)
|
||||
|
||||
for template in "${templates[@]}"; do
|
||||
if [[ -f "$ROOT_DIR/$template" ]]; then
|
||||
log_pass "$template exists"
|
||||
else
|
||||
log_fail "$template not found"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Test: GitLab CI templates exist
|
||||
test_gitlab_templates_exist() {
|
||||
log_section "Testing GitLab CI templates exist"
|
||||
|
||||
local templates=(
|
||||
"deploy/gitlab/examples/.gitlab-ci-stellaops.yml"
|
||||
"deploy/gitlab/examples/example-pipeline.gitlab-ci.yml"
|
||||
"deploy/gitlab/README.md"
|
||||
)
|
||||
|
||||
for template in "${templates[@]}"; do
|
||||
if [[ -f "$ROOT_DIR/$template" ]]; then
|
||||
log_pass "$template exists"
|
||||
else
|
||||
log_fail "$template not found"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Test: Gitea workflows exist
|
||||
test_gitea_workflows_exist() {
|
||||
log_section "Testing Gitea workflows exist"
|
||||
|
||||
local workflows=(
|
||||
".gitea/workflows/release-keyless-sign.yml"
|
||||
".gitea/workflows/deploy-keyless-verify.yml"
|
||||
)
|
||||
|
||||
for workflow in "${workflows[@]}"; do
|
||||
if [[ -f "$ROOT_DIR/$workflow" ]]; then
|
||||
log_pass "$workflow exists"
|
||||
else
|
||||
log_fail "$workflow not found"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Test: Documentation exists
|
||||
test_documentation_exists() {
|
||||
log_section "Testing documentation exists"
|
||||
|
||||
local docs=(
|
||||
"docs/guides/identity-constraints.md"
|
||||
"docs/guides/keyless-signing-troubleshooting.md"
|
||||
"docs/guides/keyless-signing-quickstart.md"
|
||||
)
|
||||
|
||||
for doc in "${docs[@]}"; do
|
||||
if [[ -f "$ROOT_DIR/$doc" ]]; then
|
||||
log_pass "$doc exists"
|
||||
else
|
||||
log_fail "$doc not found"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Test: YAML syntax validation
|
||||
test_yaml_syntax() {
|
||||
log_section "Testing YAML syntax"
|
||||
|
||||
if ! command -v yq &> /dev/null; then
|
||||
log_skip "yq not available - skipping YAML syntax tests"
|
||||
return
|
||||
fi
|
||||
|
||||
local yaml_files=(
|
||||
".github/workflows/examples/stellaops-sign.yml"
|
||||
".github/workflows/examples/stellaops-verify.yml"
|
||||
".github/workflows/examples/example-container-sign.yml"
|
||||
"deploy/gitlab/examples/.gitlab-ci-stellaops.yml"
|
||||
".gitea/workflows/release-keyless-sign.yml"
|
||||
)
|
||||
|
||||
for yaml_file in "${yaml_files[@]}"; do
|
||||
local full_path="$ROOT_DIR/$yaml_file"
|
||||
if [[ -f "$full_path" ]]; then
|
||||
if yq eval '.' "$full_path" > /dev/null 2>&1; then
|
||||
log_pass "$yaml_file has valid YAML syntax"
|
||||
else
|
||||
log_fail "$yaml_file has invalid YAML syntax"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Test: GitHub Actions workflow structure
|
||||
test_github_workflow_structure() {
|
||||
log_section "Testing GitHub Actions workflow structure"
|
||||
|
||||
if ! command -v yq &> /dev/null; then
|
||||
log_skip "yq not available - skipping structure tests"
|
||||
return
|
||||
fi
|
||||
|
||||
local sign_workflow="$ROOT_DIR/.github/workflows/examples/stellaops-sign.yml"
|
||||
|
||||
if [[ -f "$sign_workflow" ]]; then
|
||||
# Check for required fields
|
||||
if yq eval '.on.workflow_call' "$sign_workflow" | grep -q "inputs"; then
|
||||
log_pass "stellaops-sign.yml has workflow_call inputs"
|
||||
else
|
||||
log_fail "stellaops-sign.yml missing workflow_call inputs"
|
||||
fi
|
||||
|
||||
if yq eval '.jobs.sign.permissions' "$sign_workflow" | grep -q "id-token"; then
|
||||
log_pass "stellaops-sign.yml has id-token permission"
|
||||
else
|
||||
log_fail "stellaops-sign.yml missing id-token permission"
|
||||
fi
|
||||
|
||||
if yq eval '.jobs.sign.outputs' "$sign_workflow" > /dev/null 2>&1; then
|
||||
log_pass "stellaops-sign.yml has job outputs"
|
||||
else
|
||||
log_fail "stellaops-sign.yml missing job outputs"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Test: GitLab CI template structure
|
||||
test_gitlab_template_structure() {
|
||||
log_section "Testing GitLab CI template structure"
|
||||
|
||||
if ! command -v yq &> /dev/null; then
|
||||
log_skip "yq not available - skipping structure tests"
|
||||
return
|
||||
fi
|
||||
|
||||
local gitlab_template="$ROOT_DIR/deploy/gitlab/examples/.gitlab-ci-stellaops.yml"
|
||||
|
||||
if [[ -f "$gitlab_template" ]]; then
|
||||
# Check for hidden job templates
|
||||
if grep -q "\.stellaops-sign:" "$gitlab_template"; then
|
||||
log_pass "GitLab template has .stellaops-sign hidden job"
|
||||
else
|
||||
log_fail "GitLab template missing .stellaops-sign hidden job"
|
||||
fi
|
||||
|
||||
if grep -q "\.stellaops-verify:" "$gitlab_template"; then
|
||||
log_pass "GitLab template has .stellaops-verify hidden job"
|
||||
else
|
||||
log_fail "GitLab template missing .stellaops-verify hidden job"
|
||||
fi
|
||||
|
||||
if grep -q "id_tokens:" "$gitlab_template"; then
|
||||
log_pass "GitLab template has id_tokens configuration"
|
||||
else
|
||||
log_fail "GitLab template missing id_tokens configuration"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Test: Identity constraint documentation content
|
||||
test_identity_docs_content() {
|
||||
log_section "Testing identity constraint documentation content"
|
||||
|
||||
local doc="$ROOT_DIR/docs/guides/identity-constraints.md"
|
||||
|
||||
if [[ -f "$doc" ]]; then
|
||||
if grep -q "GitHub Actions" "$doc"; then
|
||||
log_pass "Identity docs cover GitHub Actions"
|
||||
else
|
||||
log_fail "Identity docs missing GitHub Actions coverage"
|
||||
fi
|
||||
|
||||
if grep -q "GitLab" "$doc"; then
|
||||
log_pass "Identity docs cover GitLab CI"
|
||||
else
|
||||
log_fail "Identity docs missing GitLab CI coverage"
|
||||
fi
|
||||
|
||||
if grep -q "certificate-identity" "$doc"; then
|
||||
log_pass "Identity docs explain certificate-identity"
|
||||
else
|
||||
log_fail "Identity docs missing certificate-identity explanation"
|
||||
fi
|
||||
|
||||
if grep -q "certificate-oidc-issuer" "$doc"; then
|
||||
log_pass "Identity docs explain certificate-oidc-issuer"
|
||||
else
|
||||
log_fail "Identity docs missing certificate-oidc-issuer explanation"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Test: Troubleshooting guide content
|
||||
test_troubleshooting_content() {
|
||||
log_section "Testing troubleshooting guide content"
|
||||
|
||||
local doc="$ROOT_DIR/docs/guides/keyless-signing-troubleshooting.md"
|
||||
|
||||
if [[ -f "$doc" ]]; then
|
||||
if grep -q "OIDC" "$doc"; then
|
||||
log_pass "Troubleshooting covers OIDC issues"
|
||||
else
|
||||
log_fail "Troubleshooting missing OIDC coverage"
|
||||
fi
|
||||
|
||||
if grep -q "Fulcio" "$doc"; then
|
||||
log_pass "Troubleshooting covers Fulcio issues"
|
||||
else
|
||||
log_fail "Troubleshooting missing Fulcio coverage"
|
||||
fi
|
||||
|
||||
if grep -q "Rekor" "$doc"; then
|
||||
log_pass "Troubleshooting covers Rekor issues"
|
||||
else
|
||||
log_fail "Troubleshooting missing Rekor coverage"
|
||||
fi
|
||||
|
||||
if grep -q "verification" "$doc"; then
|
||||
log_pass "Troubleshooting covers verification issues"
|
||||
else
|
||||
log_fail "Troubleshooting missing verification coverage"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Test: Quick-start guide content
|
||||
test_quickstart_content() {
|
||||
log_section "Testing quick-start guide content"
|
||||
|
||||
local doc="$ROOT_DIR/docs/guides/keyless-signing-quickstart.md"
|
||||
|
||||
if [[ -f "$doc" ]]; then
|
||||
if grep -q "GitHub Actions" "$doc"; then
|
||||
log_pass "Quick-start covers GitHub Actions"
|
||||
else
|
||||
log_fail "Quick-start missing GitHub Actions"
|
||||
fi
|
||||
|
||||
if grep -q "GitLab" "$doc"; then
|
||||
log_pass "Quick-start covers GitLab CI"
|
||||
else
|
||||
log_fail "Quick-start missing GitLab CI"
|
||||
fi
|
||||
|
||||
if grep -q "id-token: write" "$doc"; then
|
||||
log_pass "Quick-start shows id-token permission"
|
||||
else
|
||||
log_fail "Quick-start missing id-token permission example"
|
||||
fi
|
||||
|
||||
if grep -q "stella attest" "$doc"; then
|
||||
log_pass "Quick-start shows stella CLI usage"
|
||||
else
|
||||
log_fail "Quick-start missing stella CLI examples"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Test: GitHub Actions linting (if actionlint available)
|
||||
test_actionlint() {
|
||||
log_section "Testing GitHub Actions with actionlint"
|
||||
|
||||
if ! command -v actionlint &> /dev/null; then
|
||||
log_skip "actionlint not available - skipping lint tests"
|
||||
return
|
||||
fi
|
||||
|
||||
local workflows=(
|
||||
".github/workflows/examples/stellaops-sign.yml"
|
||||
".github/workflows/examples/stellaops-verify.yml"
|
||||
".github/workflows/examples/example-container-sign.yml"
|
||||
)
|
||||
|
||||
for workflow in "${workflows[@]}"; do
|
||||
local full_path="$ROOT_DIR/$workflow"
|
||||
if [[ -f "$full_path" ]]; then
|
||||
if actionlint "$full_path" 2>&1 | grep -q "error"; then
|
||||
log_fail "$workflow has actionlint errors"
|
||||
else
|
||||
log_pass "$workflow passes actionlint"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Test: Cross-platform verification pattern
|
||||
test_cross_platform_pattern() {
|
||||
log_section "Testing cross-platform verification patterns"
|
||||
|
||||
local github_verify="$ROOT_DIR/.github/workflows/examples/stellaops-verify.yml"
|
||||
local gitlab_template="$ROOT_DIR/deploy/gitlab/examples/.gitlab-ci-stellaops.yml"
|
||||
|
||||
# Check that both platforms use the same verification parameters
|
||||
if [[ -f "$github_verify" ]] && [[ -f "$gitlab_template" ]]; then
|
||||
if grep -q "certificate-identity" "$github_verify" && grep -q "CERTIFICATE_IDENTITY" "$gitlab_template"; then
|
||||
log_pass "Cross-platform: Both use certificate-identity pattern"
|
||||
else
|
||||
log_fail "Cross-platform: Missing consistent certificate-identity pattern"
|
||||
fi
|
||||
|
||||
if grep -q "certificate-oidc-issuer" "$github_verify" && grep -q "CERTIFICATE_OIDC_ISSUER" "$gitlab_template"; then
|
||||
log_pass "Cross-platform: Both use certificate-oidc-issuer pattern"
|
||||
else
|
||||
log_fail "Cross-platform: Missing consistent certificate-oidc-issuer pattern"
|
||||
fi
|
||||
|
||||
if grep -q "require-rekor" "$github_verify" && grep -q "REQUIRE_REKOR" "$gitlab_template"; then
|
||||
log_pass "Cross-platform: Both support Rekor requirement"
|
||||
else
|
||||
log_fail "Cross-platform: Missing consistent Rekor requirement support"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Run all tests
|
||||
main() {
|
||||
echo "============================================"
|
||||
echo "CI/CD Template Validation Tests"
|
||||
echo "Sprint: SPRINT_20251226_004_BE"
|
||||
echo "============================================"
|
||||
|
||||
check_tools
|
||||
test_github_templates_exist
|
||||
test_gitlab_templates_exist
|
||||
test_gitea_workflows_exist
|
||||
test_documentation_exists
|
||||
test_yaml_syntax
|
||||
test_github_workflow_structure
|
||||
test_gitlab_template_structure
|
||||
test_identity_docs_content
|
||||
test_troubleshooting_content
|
||||
test_quickstart_content
|
||||
test_actionlint
|
||||
test_cross_platform_pattern
|
||||
|
||||
echo ""
|
||||
echo "============================================"
|
||||
echo "Test Summary"
|
||||
echo "============================================"
|
||||
echo -e "${GREEN}Passed: $PASS_COUNT${NC}"
|
||||
echo -e "${RED}Failed: $FAIL_COUNT${NC}"
|
||||
echo ""
|
||||
|
||||
if [[ $FAIL_COUNT -gt 0 ]]; then
|
||||
echo -e "${RED}Some tests failed!${NC}"
|
||||
exit 1
|
||||
else
|
||||
echo -e "${GREEN}All tests passed!${NC}"
|
||||
exit 0
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user