save progress

devops/docker/repro-builders/rhel/scripts/build.sh

@@ -0,0 +1,213 @@
+#!/bin/bash
+# RHEL Reproducible Build Script
+# Sprint: SPRINT_1227_0002_0001 (Reproducible Builders)
+#
+# Usage: build.sh --srpm <url_or_path> [--patch <patch_file>] [--output <dir>]
+
+set -euo pipefail
+
+# Default values
+OUTPUT_DIR="/build/output"
+WORK_DIR="/build/work"
+SRPM=""
+PATCH_FILE=""
+SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-}"
+
+usage() {
+    cat <<EOF
+RHEL Reproducible Build Script
+
+Usage: $0 [OPTIONS]
+
+Options:
+    --srpm <path>       Path or URL to SRPM file (required)
+    --patch <path>      Path to security patch file (optional)
+    --output <dir>      Output directory (default: /build/output)
+    --epoch <timestamp> SOURCE_DATE_EPOCH value (default: from changelog)
+    --help              Show this help message
+
+Examples:
+    $0 --srpm openssl-3.0.7-1.el9.src.rpm --patch CVE-2023-0286.patch
+    $0 --srpm https://mirror/srpms/curl-8.0.1-1.el9.src.rpm
+
+EOF
+    exit 0
+}
+
+log() {
+    echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] $*"
+}
+
+error() {
+    log "ERROR: $*" >&2
+    exit 1
+}
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --srpm)
+            SRPM="$2"
+            shift 2
+            ;;
+        --patch)
+            PATCH_FILE="$2"
+            shift 2
+            ;;
+        --output)
+            OUTPUT_DIR="$2"
+            shift 2
+            ;;
+        --epoch)
+            SOURCE_DATE_EPOCH="$2"
+            shift 2
+            ;;
+        --help)
+            usage
+            ;;
+        *)
+            error "Unknown option: $1"
+            ;;
+    esac
+done
+
+[[ -z "${SRPM}" ]] && error "SRPM path required. Use --srpm <path>"
+
+# Create directories
+mkdir -p "${OUTPUT_DIR}" "${WORK_DIR}"
+cd "${WORK_DIR}"
+
+log "Starting RHEL reproducible build"
+log "SRPM: ${SRPM}"
+
+# Download or copy SRPM
+if [[ "${SRPM}" =~ ^https?:// ]]; then
+    log "Downloading SRPM..."
+    curl -fsSL -o source.src.rpm "${SRPM}"
+    SRPM="source.src.rpm"
+elif [[ ! -f "${SRPM}" ]]; then
+    error "SRPM file not found: ${SRPM}"
+fi
+
+# Install SRPM
+log "Installing SRPM..."
+rpm2cpio "${SRPM}" | cpio -idmv
+
+# Extract SOURCE_DATE_EPOCH from changelog if not provided
+if [[ -z "${SOURCE_DATE_EPOCH}" ]]; then
+    SPEC_FILE=$(find . -name "*.spec" | head -1)
+    if [[ -n "${SPEC_FILE}" ]]; then
+        # Extract date from first changelog entry
+        CHANGELOG_DATE=$(grep -m1 '^\*' "${SPEC_FILE}" | sed 's/^\* //' | cut -d' ' -f1-3)
+        if [[ -n "${CHANGELOG_DATE}" ]]; then
+            SOURCE_DATE_EPOCH=$(date -d "${CHANGELOG_DATE}" +%s 2>/dev/null || echo "")
+        fi
+    fi
+    
+    if [[ -z "${SOURCE_DATE_EPOCH}" ]]; then
+        SOURCE_DATE_EPOCH=$(date +%s)
+        log "Warning: Using current time for SOURCE_DATE_EPOCH"
+    fi
+fi
+
+export SOURCE_DATE_EPOCH
+log "SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH}"
+
+# Apply security patch if provided
+if [[ -n "${PATCH_FILE}" ]]; then
+    if [[ ! -f "${PATCH_FILE}" ]]; then
+        error "Patch file not found: ${PATCH_FILE}"
+    fi
+    
+    log "Applying security patch: ${PATCH_FILE}"
+    
+    # Copy patch to SOURCES
+    PATCH_NAME=$(basename "${PATCH_FILE}")
+    cp "${PATCH_FILE}" SOURCES/
+    
+    # Add patch to spec file
+    SPEC_FILE=$(find . -name "*.spec" | head -1)
+    if [[ -n "${SPEC_FILE}" ]]; then
+        # Find last Patch line or Source line
+        LAST_PATCH=$(grep -n '^Patch[0-9]*:' "${SPEC_FILE}" | tail -1 | cut -d: -f1)
+        if [[ -z "${LAST_PATCH}" ]]; then
+            LAST_PATCH=$(grep -n '^Source[0-9]*:' "${SPEC_FILE}" | tail -1 | cut -d: -f1)
+        fi
+        
+        # Calculate next patch number
+        PATCH_NUM=$(grep -c '^Patch[0-9]*:' "${SPEC_FILE}" || echo 0)
+        PATCH_NUM=$((PATCH_NUM + 100))  # Use 100+ for security patches
+        
+        # Insert patch declaration
+        sed -i "${LAST_PATCH}a Patch${PATCH_NUM}: ${PATCH_NAME}" "${SPEC_FILE}"
+        
+        # Add %patch to %prep if not using autosetup
+        if ! grep -q '%autosetup' "${SPEC_FILE}"; then
+            PREP_LINE=$(grep -n '^%prep' "${SPEC_FILE}" | head -1 | cut -d: -f1)
+            if [[ -n "${PREP_LINE}" ]]; then
+                # Find last %patch line in %prep
+                LAST_PATCH_LINE=$(sed -n "${PREP_LINE},\$p" "${SPEC_FILE}" | grep -n '^%patch' | tail -1 | cut -d: -f1)
+                if [[ -n "${LAST_PATCH_LINE}" ]]; then
+                    INSERT_LINE=$((PREP_LINE + LAST_PATCH_LINE))
+                else
+                    INSERT_LINE=$((PREP_LINE + 1))
+                fi
+                sed -i "${INSERT_LINE}a %patch${PATCH_NUM} -p1" "${SPEC_FILE}"
+            fi
+        fi
+    fi
+fi
+
+# Set up rpmbuild tree
+log "Setting up rpmbuild tree..."
+rpmdev-setuptree || true
+
+# Copy sources and spec
+cp -r SOURCES/* ~/rpmbuild/SOURCES/ 2>/dev/null || true
+cp *.spec ~/rpmbuild/SPECS/ 2>/dev/null || true
+
+# Build using mock for isolation and reproducibility
+log "Building with mock (stellaops-repro config)..."
+SPEC_FILE=$(find ~/rpmbuild/SPECS -name "*.spec" | head -1)
+
+if [[ -n "${SPEC_FILE}" ]]; then
+    # Build SRPM first
+    rpmbuild -bs "${SPEC_FILE}"
+    
+    BUILT_SRPM=$(find ~/rpmbuild/SRPMS -name "*.src.rpm" | head -1)
+    
+    if [[ -n "${BUILT_SRPM}" ]]; then
+        # Build with mock
+        mock -r stellaops-repro --rebuild "${BUILT_SRPM}" --resultdir="${OUTPUT_DIR}/rpms"
+    else
+        error "SRPM build failed"
+    fi
+else
+    error "No spec file found"
+fi
+
+# Extract function fingerprints from built RPMs
+log "Extracting function fingerprints..."
+for rpm in "${OUTPUT_DIR}/rpms"/*.rpm; do
+    if [[ -f "${rpm}" ]] && [[ ! "${rpm}" =~ \.src\.rpm$ ]]; then
+        /usr/local/bin/extract-functions.sh "${rpm}" "${OUTPUT_DIR}/fingerprints"
+    fi
+done
+
+# Generate build manifest
+log "Generating build manifest..."
+cat > "${OUTPUT_DIR}/manifest.json" <<EOF
+{
+  "builder": "rhel",
+  "base_image": "${BASE_IMAGE:-almalinux:9}",
+  "source_date_epoch": ${SOURCE_DATE_EPOCH},
+  "build_timestamp": "$(date -u '+%Y-%m-%dT%H:%M:%SZ')",
+  "srpm": "${SRPM}",
+  "patch_applied": $(if [[ -n "${PATCH_FILE}" ]]; then echo "\"${PATCH_FILE}\""; else echo "null"; fi),
+  "rpm_outputs": $(find "${OUTPUT_DIR}/rpms" -name "*.rpm" ! -name "*.src.rpm" -printf '"%f",' 2>/dev/null | sed 's/,$//' | sed 's/^/[/' | sed 's/$/]/'),
+  "fingerprint_files": $(find "${OUTPUT_DIR}/fingerprints" -name "*.json" -printf '"%f",' 2>/dev/null | sed 's/,$//' | sed 's/^/[/' | sed 's/$/]/')
+}
+EOF
+
+log "Build complete. Output in: ${OUTPUT_DIR}"
+log "Manifest: ${OUTPUT_DIR}/manifest.json"

devops/docker/repro-builders/rhel/scripts/extract-functions.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+# RHEL Function Extraction Script
+# Sprint: SPRINT_1227_0002_0001 (Reproducible Builders)
+#
+# Extracts function-level fingerprints from RPM packages
+
+set -euo pipefail
+
+RPM_PATH="${1:-}"
+OUTPUT_DIR="${2:-/build/fingerprints}"
+
+[[ -z "${RPM_PATH}" ]] && { echo "Usage: $0 <rpm_path> [output_dir]"; exit 1; }
+[[ ! -f "${RPM_PATH}" ]] && { echo "RPM not found: ${RPM_PATH}"; exit 1; }
+
+mkdir -p "${OUTPUT_DIR}"
+
+RPM_NAME=$(rpm -qp --qf '%{NAME}' "${RPM_PATH}" 2>/dev/null)
+RPM_VERSION=$(rpm -qp --qf '%{VERSION}-%{RELEASE}' "${RPM_PATH}" 2>/dev/null)
+
+WORK_DIR=$(mktemp -d)
+trap "rm -rf ${WORK_DIR}" EXIT
+
+cd "${WORK_DIR}"
+
+# Extract RPM contents
+rpm2cpio "${RPM_PATH}" | cpio -idmv 2>/dev/null
+
+# Find ELF binaries
+find . -type f -exec file {} \; | grep -E 'ELF.*(executable|shared object)' | cut -d: -f1 | while read -r binary; do
+    BINARY_NAME=$(basename "${binary}")
+    BINARY_PATH="${binary#./}"
+    
+    # Get build-id if present
+    BUILD_ID=$(readelf -n "${binary}" 2>/dev/null | grep 'Build ID:' | awk '{print $3}' || echo "")
+    
+    # Extract function symbols
+    OUTPUT_FILE="${OUTPUT_DIR}/${RPM_NAME}_${BINARY_NAME}.json"
+    
+    {
+        echo "{"
+        echo "  \"package\": \"${RPM_NAME}\","
+        echo "  \"version\": \"${RPM_VERSION}\","
+        echo "  \"binary\": \"${BINARY_PATH}\","
+        echo "  \"build_id\": \"${BUILD_ID}\","
+        echo "  \"extracted_at\": \"$(date -u '+%Y-%m-%dT%H:%M:%SZ')\","
+        echo "  \"functions\": ["
+        
+        # Extract function addresses and sizes using nm and objdump
+        FIRST=true
+        nm -S --defined-only "${binary}" 2>/dev/null | grep -E '^[0-9a-f]+ [0-9a-f]+ [Tt]' | while read -r addr size type name; do
+            if [[ "${FIRST}" == "true" ]]; then
+                FIRST=false
+            else
+                echo ","
+            fi
+            
+            # Calculate function hash from disassembly
+            FUNC_HASH=$(objdump -d --start-address=0x${addr} --stop-address=$((0x${addr} + 0x${size})) "${binary}" 2>/dev/null | \
+                        grep -E '^\s+[0-9a-f]+:' | awk '{$1=""; print}' | sha256sum | cut -d' ' -f1)
+            
+            printf '    {"name": "%s", "address": "0x%s", "size": %d, "hash": "%s"}' \
+                "${name}" "${addr}" "$((0x${size}))" "${FUNC_HASH}"
+        done || true
+        
+        echo ""
+        echo "  ]"
+        echo "}"
+    } > "${OUTPUT_FILE}"
+    
+    echo "Extracted: ${OUTPUT_FILE}"
+done
+
+echo "Function extraction complete for: ${RPM_NAME}"

devops/docker/repro-builders/rhel/scripts/mock-build.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# RHEL Mock Build Script
+# Sprint: SPRINT_1227_0002_0001 (Reproducible Builders)
+#
+# Builds SRPMs using mock for isolation and reproducibility
+
+set -euo pipefail
+
+SRPM="${1:-}"
+RESULT_DIR="${2:-/build/output}"
+CONFIG="${3:-stellaops-repro}"
+
+[[ -z "${SRPM}" ]] && { echo "Usage: $0 <srpm> [result_dir] [mock_config]"; exit 1; }
+[[ ! -f "${SRPM}" ]] && { echo "SRPM not found: ${SRPM}"; exit 1; }
+
+mkdir -p "${RESULT_DIR}"
+
+echo "Building SRPM with mock: ${SRPM}"
+echo "Config: ${CONFIG}"
+echo "Output: ${RESULT_DIR}"
+
+# Initialize mock if needed
+mock -r "${CONFIG}" --init
+
+# Build with reproducibility settings
+mock -r "${CONFIG}" \
+    --rebuild "${SRPM}" \
+    --resultdir="${RESULT_DIR}" \
+    --define "SOURCE_DATE_EPOCH ${SOURCE_DATE_EPOCH:-$(date +%s)}" \
+    --define "_buildhost stellaops.build" \
+    --define "debug_package %{nil}"
+
+echo "Build complete. Results in: ${RESULT_DIR}"
+ls -la "${RESULT_DIR}"

devops/docker/repro-builders/rhel/scripts/normalize.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+# RHEL Build Normalization Script
+# Sprint: SPRINT_1227_0002_0001 (Reproducible Builders)
+#
+# Normalizes RPM build environment for reproducibility
+
+set -euo pipefail
+
+# Normalize environment
+export TZ=UTC
+export LC_ALL=C.UTF-8
+export LANG=C.UTF-8
+
+# Deterministic compiler flags
+export CFLAGS="${CFLAGS:--fno-record-gcc-switches -fdebug-prefix-map=$(pwd)=/buildroot -O2 -g}"
+export CXXFLAGS="${CXXFLAGS:-${CFLAGS}}"
+
+# Disable debug info that varies
+export DEB_BUILD_OPTIONS="nostrip noopt"
+
+# RPM-specific reproducibility
+export RPM_BUILD_NCPUS=1
+
+# Normalize timestamps in archives
+normalize_ar() {
+    local archive="$1"
+    if command -v llvm-ar &>/dev/null; then
+        llvm-ar --format=gnu --enable-deterministic-archives rcs "${archive}.new" "${archive}"
+        mv "${archive}.new" "${archive}"
+    fi
+}
+
+# Normalize timestamps in tar archives
+normalize_tar() {
+    local archive="$1"
+    local mtime="${SOURCE_DATE_EPOCH:-0}"
+    
+    # Repack with deterministic settings
+    local tmp_dir=$(mktemp -d)
+    tar -xf "${archive}" -C "${tmp_dir}"
+    tar --sort=name \
+        --mtime="@${mtime}" \
+        --owner=0 --group=0 \
+        --numeric-owner \
+        -cf "${archive}.new" -C "${tmp_dir}" .
+    mv "${archive}.new" "${archive}"
+    rm -rf "${tmp_dir}"
+}
+
+# Normalize __pycache__ timestamps
+normalize_python() {
+    find . -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null || true
+    find . -name '*.pyc' -delete 2>/dev/null || true
+}
+
+# Strip build paths from binaries
+strip_build_paths() {
+    local binary="$1"
+    if command -v objcopy &>/dev/null; then
+        # Remove .note.gnu.build-id if it contains build path
+        objcopy --remove-section=.note.gnu.build-id "${binary}" 2>/dev/null || true
+    fi
+}
+
+# Main normalization
+normalize_build() {
+    echo "Normalizing build environment..."
+    
+    # Normalize Python bytecode
+    normalize_python
+    
+    # Find and normalize archives
+    find . -name '*.a' -type f | while read -r ar; do
+        normalize_ar "${ar}"
+    done
+    
+    echo "Normalization complete"
+}
+
+# If sourced, export functions; if executed, run normalization
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    normalize_build
+fi