containers/scripts/build-base.sh

0001 #!/bin/bash
0002 # Build the EIC base container image (debian_stable_base, cuda_devel, or cuda_runtime).
0003 #
0004 # This script is used in GitLab CI, GitHub Actions, and for local builds.
0005 # CI mode is detected via CI_REGISTRY (GitLab) or GITHUB_ACTIONS=true (GitHub Actions).
0006 #
0007 # Run `bash scripts/build-base.sh --help` for usage, options, and CI-specific details.
0008
0009 set -e
0010
0011 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
0012 REPO_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
0013 cd "${REPO_DIR}"
0014
0015 print_help() {
0016   cat <<EOF
0017 Build the EIC base container image (debian_stable_base, cuda_devel, or cuda_runtime).
0018
0019 Usage (local):
0020   bash scripts/build-base.sh [options]
0021
0022 Usage (CI, called from .gitlab-ci.yml or build-push.yml with matrix variables in env):
0023   bash scripts/build-base.sh
0024
0025 Options:
0026   --image IMAGE       Image to build: debian_stable_base, cuda_devel, cuda_runtime
0027                       (default: \$BUILD_IMAGE or debian_stable_base)
0028   --base-image IMAGE  Upstream base image (default: derived from --image)
0029   --platform PLATFORM Build platform, e.g. linux/amd64, linux/arm64
0030                       (default: \$PLATFORM or linux/amd64)
0031   --jobs N            Number of parallel Spack build jobs
0032                       (default: \$JOBS or \$(getconf _NPROCESSORS_ONLN))
0033   --tag TAG           Local tag for the image (default: local; ignored in CI)
0034   -h, --help          Show this help and exit
0035
0036 GitHub Actions mode (GITHUB_ACTIONS=true):
0037   Set GH_REGISTRY, GH_REGISTRY_USER, JOBS. The script derives cache-key slugs
0038   from GITHUB_HEAD_REF (PR source branch, when set) or GITHUB_REF_NAME
0039   (push/schedule branch), and from GITHUB_BASE_REF (PR target branch, empty on
0040   push events) or DEFAULT_BRANCH (repo default branch fallback). Writes the
0041   image digest to METADATA_FILE (default: /tmp/build-metadata.json).
0042 EOF
0043 }
0044
0045 ## Defaults (may be overridden by env vars set from CI matrix or command-line flags)
0046 BUILD_IMAGE="${BUILD_IMAGE:-debian_stable_base}"
0047 BASE_IMAGE="${BASE_IMAGE:-}"
0048 PLATFORM="${PLATFORM:-linux/amd64}"
0049 JOBS="${JOBS:-$(getconf _NPROCESSORS_ONLN)}"
0050 LOCAL_TAG="${LOCAL_TAG:-local}"
0051 METADATA_FILE="${METADATA_FILE:-/tmp/build-metadata.json}"
0052 ## CUDA defaults (used when building cuda_devel or cuda_runtime)
0053 CUDA_VERSION="${CUDA_VERSION:-12.5.1}"
0054 CUDA_OS="${CUDA_OS:-ubuntu24.04}"
0055
0056 while [[ $# -gt 0 ]]; do
0057   case "$1" in
0058     -h|--help)    print_help; exit 0 ;;
0059     --image)     BUILD_IMAGE="$2";   shift 2 ;;
0060     --base-image) BASE_IMAGE="$2";   shift 2 ;;
0061     --platform)  PLATFORM="$2";     shift 2 ;;
0062     --jobs)      JOBS="$2";         shift 2 ;;
0063     --tag)       LOCAL_TAG="$2";    shift 2 ;;
0064     *) echo "Unknown argument: $1" >&2; echo "Try 'bash scripts/build-base.sh --help' for usage." >&2; exit 1 ;;
0065   esac
0066 done
0067
0068 ## Source version files
0069 source "${REPO_DIR}/spack.sh"
0070 source "${REPO_DIR}/spack-packages.sh"
0071 source "${REPO_DIR}/key4hep-spack.sh"
0072 source "${REPO_DIR}/eic-spack.sh"
0073
0074 ## Convert an arbitrary git ref/branch name to a valid OCI tag component.
0075 ## Mirrors GitLab's CI_COMMIT_REF_SLUG: lowercase, non-alnum runs → '-',
0076 ## strip leading/trailing '-', truncate to 63 chars.
0077 slugify() {
0078   echo "$1" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-+//; s/-+$//' | cut -c1-63
0079 }
0080
0081 ## Detect CI mode and normalise environment variables
0082 if [ -n "${CI_REGISTRY}" ]; then
0083   ## GitLab CI — all CI_* variables are already set by the runner
0084   CI_MODE="gitlab"
0085 elif [ "${GITHUB_ACTIONS}" = "true" ]; then
0086   ## GitHub Actions — map GitHub variables to the names used below.
0087   ## Use GITHUB_HEAD_REF for PR source branches (better cache reuse across PR
0088   ## updates), then fall back to GITHUB_REF_NAME for push/schedule events.
0089   ## GITHUB_BASE_REF is the PR target branch (empty on push events). DEFAULT_BRANCH
0090   ## should be supplied by the workflow so cache keys remain stable when
0091   ## GITHUB_BASE_REF is empty.
0092   CI_MODE="github"
0093   CI_REGISTRY="${GH_REGISTRY}"
0094   CI_PROJECT_PATH="${GH_REGISTRY_USER}"
0095   CI_COMMIT_REF_SLUG="$(slugify "${GITHUB_HEAD_REF:-${GITHUB_REF_NAME:-master}}")"
0096   CI_DEFAULT_BRANCH_SLUG="$(slugify "${GITHUB_BASE_REF:-${DEFAULT_BRANCH:-master}}")"
0097   INTERNAL_TAG="${INTERNAL_TAG:-pipeline-${GITHUB_RUN_ID}}"
0098 else
0099   CI_MODE="local"
0100 fi
0101
0102 ## Derive BASE_IMAGE from BUILD_IMAGE if not provided
0103 if [ -z "${BASE_IMAGE}" ]; then
0104   case "${BUILD_IMAGE}" in
0105     debian_stable_base) BASE_IMAGE="debian:trixie-slim" ;;
0106     cuda_devel)         BASE_IMAGE="nvidia/cuda:${CUDA_VERSION}-devel-${CUDA_OS}" ;;
0107     cuda_runtime)       BASE_IMAGE="nvidia/cuda:${CUDA_VERSION}-runtime-${CUDA_OS}" ;;
0108     *) echo "Unknown BUILD_IMAGE '${BUILD_IMAGE}'; please specify --base-image" >&2; exit 1 ;;
0109   esac
0110 fi
0111
0112 ## Resolve SHAs (network calls — skipped if version is already a SHA)
0113 echo "Resolving git SHAs..."
0114 SPACK_SHA=$(sh "${REPO_DIR}/scripts/resolve_git_ref" "${SPACK_ORGREPO}" "${SPACK_VERSION}")
0115 SPACKPACKAGES_SHA=$(sh "${REPO_DIR}/scripts/resolve_git_ref" "${SPACKPACKAGES_ORGREPO}" "${SPACKPACKAGES_VERSION}")
0116 KEY4HEPSPACK_SHA=$(sh "${REPO_DIR}/scripts/resolve_git_ref" "${KEY4HEPSPACK_ORGREPO}" "${KEY4HEPSPACK_VERSION}")
0117 EICSPACK_SHA=$(sh "${REPO_DIR}/scripts/resolve_git_ref" "${EICSPACK_ORGREPO}" "${EICSPACK_VERSION}")
0118
0119 ## Normalize arch string for cache tag names while preserving platform variants
0120 ## Examples: linux/amd64 -> amd64, linux/amd64/v3 -> amd64_v3, linux/arm/v7 -> arm_v7
0121 ARCH=$(echo "${PLATFORM}" | sed 's|^linux/||; s|/|_|g')
0122
0123 ## Build the docker buildx command as an array for safe quoting
0124 build_cmd=(docker buildx build)
0125 # shellcheck disable=SC2206  # word splitting is intentional: BUILD_OPTIONS is a space-separated list
0126 build_cmd+=(${BUILD_OPTIONS})
0127
0128 ## Derive shared registry prefix (used for image push, caching, and DOCKER_REGISTRY build-arg)
0129 CI_REGISTRY_PREFIX="${CI_REGISTRY}/${CI_PROJECT_PATH}"
0130 IMAGE_REPO="${CI_REGISTRY_PREFIX}/${BUILD_IMAGE}"
0131
0132 ## Output mode: push-by-digest in all CI modes; load locally
0133 if [ "${CI_MODE}" != "local" ]; then
0134   ## Push by digest; CI wrapper creates final tags via imagetools create
0135   build_cmd+=(--output "type=image,name=${IMAGE_REPO},push-by-digest=true,name-canonical=true,push=true")
0136   build_cmd+=(--metadata-file "${METADATA_FILE}")
0137 else
0138   build_cmd+=(--load)
0139 fi
0140
0141 ## Cache sources: CI registry (if in CI) plus public ghcr.io/eic (GitLab and local modes)
0142 BUILDCACHE_REPOS=()
0143 [ "${CI_MODE}" != "local" ] && BUILDCACHE_REPOS+=("${CI_REGISTRY_PREFIX}")
0144 [ "${CI_MODE}" != "github" ] && BUILDCACHE_REPOS+=("ghcr.io/eic")
0145 for REPO in "${BUILDCACHE_REPOS[@]}"; do
0146   build_cmd+=(--cache-from "type=registry,ref=${REPO}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG:-master}-${ARCH}")
0147   build_cmd+=(--cache-from "type=registry,ref=${REPO}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG:-master}-${ARCH}")
0148 done
0149
0150 ## Cache destination (CI only)
0151 if [ "${CI_MODE}" != "local" ]; then
0152   build_cmd+=(--cache-to "type=registry,ref=${CI_REGISTRY_PREFIX}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG:-master}-${ARCH},mode=max")
0153 fi
0154
0155 ## Image tag (local only; CI creates tags via imagetools create after build)
0156 if [ "${CI_MODE}" = "local" ]; then
0157   build_cmd+=(--tag "${BUILD_IMAGE}:${LOCAL_TAG}")
0158 fi
0159
0160 ## Dockerfile and platform
0161 build_cmd+=(--file containers/debian/Dockerfile)
0162 build_cmd+=(--platform "${PLATFORM}")
0163
0164 ## Build arguments
0165 build_cmd+=(--build-arg "BASE_IMAGE=${BASE_IMAGE}")
0166 build_cmd+=(--build-arg "BUILD_IMAGE=${BUILD_IMAGE}")
0167 case "${BUILD_IMAGE}" in
0168   cuda*)
0169     build_cmd+=(--build-arg "NVIDIA_VISIBLE_DEVICES=all")
0170     build_cmd+=(--build-arg "NVIDIA_DRIVER_CAPABILITIES=all")
0171     ;;
0172 esac
0173 build_cmd+=(--build-arg "SPACK_ORGREPO=${SPACK_ORGREPO}")
0174 build_cmd+=(--build-arg "SPACK_VERSION=${SPACK_VERSION}")
0175 build_cmd+=(--build-arg "SPACK_SHA=${SPACK_SHA}")
0176 build_cmd+=(--build-arg "SPACK_CHERRYPICKS=${SPACK_CHERRYPICKS}")
0177 build_cmd+=(--build-arg "SPACK_CHERRYPICKS_FILES=${SPACK_CHERRYPICKS_FILES}")
0178 build_cmd+=(--build-arg "SPACKPACKAGES_ORGREPO=${SPACKPACKAGES_ORGREPO}")
0179 build_cmd+=(--build-arg "SPACKPACKAGES_VERSION=${SPACKPACKAGES_VERSION}")
0180 build_cmd+=(--build-arg "SPACKPACKAGES_SHA=${SPACKPACKAGES_SHA}")
0181 build_cmd+=(--build-arg "SPACKPACKAGES_CHERRYPICKS=${SPACKPACKAGES_CHERRYPICKS}")
0182 build_cmd+=(--build-arg "SPACKPACKAGES_CHERRYPICKS_FILES=${SPACKPACKAGES_CHERRYPICKS_FILES}")
0183 build_cmd+=(--build-arg "KEY4HEPSPACK_ORGREPO=${KEY4HEPSPACK_ORGREPO}")
0184 build_cmd+=(--build-arg "KEY4HEPSPACK_VERSION=${KEY4HEPSPACK_VERSION}")
0185 build_cmd+=(--build-arg "KEY4HEPSPACK_SHA=${KEY4HEPSPACK_SHA}")
0186 build_cmd+=(--build-arg "EICSPACK_ORGREPO=${EICSPACK_ORGREPO}")
0187 build_cmd+=(--build-arg "EICSPACK_VERSION=${EICSPACK_VERSION}")
0188 build_cmd+=(--build-arg "EICSPACK_SHA=${EICSPACK_SHA}")
0189 build_cmd+=(--build-arg "jobs=${JOBS}")
0190 build_cmd+=(--build-arg "BUILDWEEK=${BUILDWEEK:-0}")
0191
0192 ## Suppress provenance attestation (matches CI behaviour)
0193 build_cmd+=(--provenance false)
0194
0195 ## Build context
0196 build_cmd+=(containers/debian)
0197
0198 ## Execute
0199 set -o xtrace -o pipefail
0200 "${build_cmd[@]}" 2>&1 | tee build.log