Fix build and code structure improvements. New but essential UI functionality. CI improvements. Documentation improvements. AI module improvements.
This commit is contained in:
StellaOps Bot
59
devops/docker/repro-builders/debian/Dockerfile
Normal file
59
devops/docker/repro-builders/debian/Dockerfile
Normal file
@@ -0,0 +1,59 @@
|
||||
# Debian Reproducible Builder
|
||||
# Creates deterministic builds of Debian packages for fingerprint diffing
|
||||
#
|
||||
# Usage:
|
||||
# docker build -t repro-builder-debian:bookworm --build-arg RELEASE=bookworm .
|
||||
# docker run -v ./output:/output repro-builder-debian:bookworm build openssl 3.0.7-1
|
||||
|
||||
ARG RELEASE=bookworm
|
||||
FROM debian:${RELEASE}
|
||||
|
||||
ARG RELEASE
|
||||
ENV DEBIAN_RELEASE=${RELEASE}
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Install build tools
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
devscripts \
|
||||
dpkg-dev \
|
||||
equivs \
|
||||
fakeroot \
|
||||
git \
|
||||
curl \
|
||||
ca-certificates \
|
||||
binutils \
|
||||
elfutils \
|
||||
coreutils \
|
||||
patch \
|
||||
diffutils \
|
||||
file \
|
||||
jq \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Create build user
|
||||
RUN useradd -m -s /bin/bash builder \
|
||||
&& echo "builder ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
|
||||
|
||||
USER builder
|
||||
WORKDIR /home/builder
|
||||
|
||||
# Copy scripts
|
||||
COPY --chown=builder:builder scripts/build.sh /usr/local/bin/build.sh
|
||||
COPY --chown=builder:builder scripts/extract-functions.sh /usr/local/bin/extract-functions.sh
|
||||
COPY --chown=builder:builder scripts/normalize.sh /usr/local/bin/normalize.sh
|
||||
|
||||
USER root
|
||||
RUN chmod +x /usr/local/bin/*.sh
|
||||
USER builder
|
||||
|
||||
# Environment for reproducibility
|
||||
ENV TZ=UTC
|
||||
ENV LC_ALL=C.UTF-8
|
||||
ENV LANG=C.UTF-8
|
||||
|
||||
VOLUME /output
|
||||
WORKDIR /build
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/build.sh"]
|
||||
CMD ["--help"]
|
||||
233
devops/docker/repro-builders/debian/scripts/build.sh
Normal file
233
devops/docker/repro-builders/debian/scripts/build.sh
Normal file
@@ -0,0 +1,233 @@
|
||||
#!/bin/bash
|
||||
# Debian Reproducible Build Script
|
||||
# Builds packages with deterministic settings for fingerprint generation
|
||||
#
|
||||
# Usage: build.sh [build|diff] <package> <version> [patch_url...]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
COMMAND="${1:-help}"
|
||||
PACKAGE="${2:-}"
|
||||
VERSION="${3:-}"
|
||||
OUTPUT_DIR="${OUTPUT_DIR:-/output}"
|
||||
|
||||
log() {
|
||||
echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" >&2
|
||||
}
|
||||
|
||||
show_help() {
|
||||
cat <<EOF
|
||||
Debian Reproducible Builder
|
||||
|
||||
Usage:
|
||||
build.sh build <package> <version> [patch_urls...]
|
||||
Build a package with reproducible settings
|
||||
|
||||
build.sh diff <package> <vuln_version> <patched_version>
|
||||
Build two versions and compute fingerprint diff
|
||||
|
||||
build.sh --help
|
||||
Show this help message
|
||||
|
||||
Environment:
|
||||
SOURCE_DATE_EPOCH Override timestamp (extracted from changelog if not set)
|
||||
OUTPUT_DIR Output directory (default: /output)
|
||||
DEB_BUILD_OPTIONS Additional build options
|
||||
|
||||
Examples:
|
||||
build.sh build openssl 3.0.7-1
|
||||
build.sh diff curl 8.1.0-1 8.1.0-2
|
||||
EOF
|
||||
}
|
||||
|
||||
setup_reproducible_env() {
|
||||
local pkg="$1"
|
||||
|
||||
# Reproducible build flags
|
||||
export DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS:-} reproducible=+all"
|
||||
export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(date +%s)}"
|
||||
|
||||
# Compiler flags for reproducibility
|
||||
export CFLAGS="${CFLAGS:-} -fno-record-gcc-switches -fdebug-prefix-map=$(pwd)=/build"
|
||||
export CXXFLAGS="${CXXFLAGS:-} ${CFLAGS}"
|
||||
|
||||
export LC_ALL=C.UTF-8
|
||||
export TZ=UTC
|
||||
|
||||
log "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
|
||||
}
|
||||
|
||||
fetch_source() {
|
||||
local pkg="$1"
|
||||
local ver="$2"
|
||||
|
||||
log "Fetching source for $pkg=$ver"
|
||||
|
||||
mkdir -p /build/src
|
||||
cd /build/src
|
||||
|
||||
# Enable source repositories
|
||||
sudo sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list.d/*.sources 2>/dev/null || \
|
||||
sudo sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list 2>/dev/null || true
|
||||
sudo apt-get update
|
||||
|
||||
# Fetch source
|
||||
if [ -n "$ver" ]; then
|
||||
apt-get source "${pkg}=${ver}" || apt-get source "$pkg"
|
||||
else
|
||||
apt-get source "$pkg"
|
||||
fi
|
||||
|
||||
# Find extracted directory
|
||||
local src_dir=$(ls -d "${pkg}"*/ 2>/dev/null | head -1)
|
||||
if [ -z "$src_dir" ]; then
|
||||
log "ERROR: Could not find source directory for $pkg"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Extract SOURCE_DATE_EPOCH from changelog
|
||||
if [ -z "${SOURCE_DATE_EPOCH:-}" ]; then
|
||||
if [ -f "$src_dir/debian/changelog" ]; then
|
||||
SOURCE_DATE_EPOCH=$(dpkg-parsechangelog -l "$src_dir/debian/changelog" -S Timestamp 2>/dev/null || date +%s)
|
||||
export SOURCE_DATE_EPOCH
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "$src_dir"
|
||||
}
|
||||
|
||||
install_build_deps() {
|
||||
local src_dir="$1"
|
||||
|
||||
log "Installing build dependencies"
|
||||
cd "$src_dir"
|
||||
sudo apt-get build-dep -y . || true
|
||||
}
|
||||
|
||||
apply_patches() {
|
||||
local src_dir="$1"
|
||||
shift
|
||||
|
||||
cd "$src_dir"
|
||||
for patch_url in "$@"; do
|
||||
log "Applying patch: $patch_url"
|
||||
curl -sSL "$patch_url" | patch -p1
|
||||
done
|
||||
}
|
||||
|
||||
build_package() {
|
||||
local pkg="$1"
|
||||
local ver="$2"
|
||||
shift 2
|
||||
local patches="${@:-}"
|
||||
|
||||
log "Building $pkg version $ver"
|
||||
|
||||
setup_reproducible_env "$pkg"
|
||||
|
||||
cd /build
|
||||
local src_dir=$(fetch_source "$pkg" "$ver")
|
||||
|
||||
install_build_deps "$src_dir"
|
||||
|
||||
if [ -n "$patches" ]; then
|
||||
apply_patches "$src_dir" $patches
|
||||
fi
|
||||
|
||||
cd "$src_dir"
|
||||
|
||||
# Build with reproducible settings
|
||||
dpkg-buildpackage -b -us -uc
|
||||
|
||||
# Copy output
|
||||
local out_dir="$OUTPUT_DIR/$pkg-$ver"
|
||||
mkdir -p "$out_dir"
|
||||
cp -r /build/src/*.deb "$out_dir/" 2>/dev/null || true
|
||||
|
||||
# Extract and fingerprint
|
||||
for deb in "$out_dir"/*.deb; do
|
||||
[ -f "$deb" ] || continue
|
||||
local deb_name=$(basename "$deb" .deb)
|
||||
mkdir -p "$out_dir/extracted/$deb_name"
|
||||
dpkg-deb -x "$deb" "$out_dir/extracted/$deb_name"
|
||||
|
||||
# Extract function fingerprints
|
||||
/usr/local/bin/extract-functions.sh "$out_dir/extracted/$deb_name" > "$out_dir/$deb_name.functions.json"
|
||||
done
|
||||
|
||||
log "Build complete: $out_dir"
|
||||
}
|
||||
|
||||
diff_versions() {
|
||||
local pkg="$1"
|
||||
local vuln_ver="$2"
|
||||
local patched_ver="$3"
|
||||
|
||||
log "Building and diffing $pkg: $vuln_ver vs $patched_ver"
|
||||
|
||||
# Build vulnerable version
|
||||
build_package "$pkg" "$vuln_ver"
|
||||
|
||||
# Clean build environment
|
||||
rm -rf /build/src/*
|
||||
|
||||
# Build patched version
|
||||
build_package "$pkg" "$patched_ver"
|
||||
|
||||
# Compute diff
|
||||
local diff_out="$OUTPUT_DIR/$pkg-diff-$vuln_ver-vs-$patched_ver.json"
|
||||
|
||||
jq -s '
|
||||
.[0] as $vuln |
|
||||
.[1] as $patched |
|
||||
{
|
||||
package: "'"$pkg"'",
|
||||
vulnerable_version: "'"$vuln_ver"'",
|
||||
patched_version: "'"$patched_ver"'",
|
||||
vulnerable_functions: ($vuln | length),
|
||||
patched_functions: ($patched | length),
|
||||
added: [($patched[] | select(.name as $n | ($vuln | map(.name) | index($n)) == null))],
|
||||
removed: [($vuln[] | select(.name as $n | ($patched | map(.name) | index($n)) == null))],
|
||||
modified: [
|
||||
$vuln[] | .name as $n | .hash as $h |
|
||||
($patched[] | select(.name == $n and .hash != $h)) |
|
||||
{name: $n, vuln_hash: $h, patched_hash: .hash}
|
||||
]
|
||||
}
|
||||
' \
|
||||
"$OUTPUT_DIR/$pkg-$vuln_ver"/*.functions.json \
|
||||
"$OUTPUT_DIR/$pkg-$patched_ver"/*.functions.json \
|
||||
> "$diff_out" 2>/dev/null || log "Warning: Could not compute diff"
|
||||
|
||||
log "Diff complete: $diff_out"
|
||||
}
|
||||
|
||||
case "$COMMAND" in
|
||||
build)
|
||||
if [ -z "$PACKAGE" ]; then
|
||||
log "ERROR: Package required"
|
||||
show_help
|
||||
exit 1
|
||||
fi
|
||||
shift 2 # Remove command, package
|
||||
[ -n "${VERSION:-}" ] && shift # Remove version if present
|
||||
build_package "$PACKAGE" "${VERSION:-}" "$@"
|
||||
;;
|
||||
diff)
|
||||
PATCHED_VERSION="${4:-}"
|
||||
if [ -z "$PACKAGE" ] || [ -z "$VERSION" ] || [ -z "$PATCHED_VERSION" ]; then
|
||||
log "ERROR: Package, vulnerable version, and patched version required"
|
||||
show_help
|
||||
exit 1
|
||||
fi
|
||||
diff_versions "$PACKAGE" "$VERSION" "$PATCHED_VERSION"
|
||||
;;
|
||||
--help|help)
|
||||
show_help
|
||||
;;
|
||||
*)
|
||||
log "ERROR: Unknown command: $COMMAND"
|
||||
show_help
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/bin/bash
|
||||
# Extract function fingerprints from ELF binaries
|
||||
# Outputs JSON array with function name, offset, size, and hashes
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DIR="${1:-.}"
|
||||
|
||||
extract_functions_from_binary() {
|
||||
local binary="$1"
|
||||
|
||||
# Skip non-ELF files
|
||||
file "$binary" 2>/dev/null | grep -q "ELF" || return 0
|
||||
|
||||
# Get function symbols with objdump
|
||||
objdump -t "$binary" 2>/dev/null | \
|
||||
awk '/\.text.*[0-9a-f]+.*F/ {
|
||||
gsub(/\*.*\*/, "", $1)
|
||||
if ($5 != "" && length($4) > 0) {
|
||||
size = strtonum("0x" $4)
|
||||
if (size >= 16) {
|
||||
print $1, $4, $NF
|
||||
}
|
||||
}
|
||||
}' | while read -r offset size name; do
|
||||
# Skip compiler-generated symbols
|
||||
case "$name" in
|
||||
__*|_GLOBAL_*|.plt*|.text*|frame_dummy|register_tm_clones|deregister_tm_clones|_start|_init|_fini)
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
# Convert hex size
|
||||
dec_size=$((16#$size))
|
||||
|
||||
# Compute hash of function bytes
|
||||
local hash=$(objdump -d --start-address="0x$offset" --stop-address="$((16#$offset + dec_size))" "$binary" 2>/dev/null | \
|
||||
grep -E "^[[:space:]]*[0-9a-f]+:" | \
|
||||
awk '{for(i=2;i<=NF;i++){if($i~/^[0-9a-f]{2}$/){printf "%s", $i}}}' | \
|
||||
sha256sum | cut -d' ' -f1)
|
||||
|
||||
[ -n "$hash" ] || hash="unknown"
|
||||
|
||||
printf '{"name":"%s","offset":"0x%s","size":%d,"hash":"%s"}\n' \
|
||||
"$name" "$offset" "$dec_size" "$hash"
|
||||
done
|
||||
}
|
||||
|
||||
# Output JSON array
|
||||
echo "["
|
||||
first=true
|
||||
|
||||
find "$DIR" -type f \( -executable -o -name "*.so" -o -name "*.so.*" \) 2>/dev/null | while read -r binary; do
|
||||
file "$binary" 2>/dev/null | grep -q "ELF" || continue
|
||||
|
||||
extract_functions_from_binary "$binary" | while read -r json; do
|
||||
[ -z "$json" ] && continue
|
||||
if [ "$first" = "true" ]; then
|
||||
first=false
|
||||
echo "$json"
|
||||
else
|
||||
echo ",$json"
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
echo "]"
|
||||
29
devops/docker/repro-builders/debian/scripts/normalize.sh
Normal file
29
devops/docker/repro-builders/debian/scripts/normalize.sh
Normal file
@@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
# Normalization scripts for Debian reproducible builds
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DIR="${1:-.}"
|
||||
|
||||
log() {
|
||||
echo "[normalize] $*" >&2
|
||||
}
|
||||
|
||||
normalize_archives() {
|
||||
log "Normalizing ar archives..."
|
||||
find "$DIR" -name "*.a" -type f | while read -r archive; do
|
||||
if ar --version 2>&1 | grep -q "GNU ar"; then
|
||||
ar -rcsD "$archive.tmp" "$archive" 2>/dev/null && mv "$archive.tmp" "$archive" || true
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
strip_debug_timestamps() {
|
||||
log "Stripping debug timestamps..."
|
||||
# Handled by SOURCE_DATE_EPOCH and DEB_BUILD_OPTIONS
|
||||
}
|
||||
|
||||
normalize_archives
|
||||
strip_debug_timestamps
|
||||
|
||||
log "Normalization complete"
|
||||
Reference in New Issue
Block a user