containers/scripts/build-eic.sh

0001 #!/bin/bash
0002 # Build an EIC container image (eic_ci, eic_xl, eic_cuda, etc.).
0003 #
0004 # This script is used in GitLab CI, GitHub Actions, and for local builds.
0005 # CI mode is detected via CI_REGISTRY (GitLab) or GITHUB_ACTIONS=true (GitHub Actions).
0006 #
0007 # Run `bash scripts/build-eic.sh --help` for usage, options, and CI-specific details.
0008
0009 set -e
0010
0011 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
0012 REPO_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
0013 cd "${REPO_DIR}"
0014
0015 print_help() {
0016   cat <<EOF
0017 Build an EIC container image (eic_ci, eic_xl, eic_cuda, etc.).
0018
0019 Usage (local):
0020   bash scripts/build-eic.sh [options]
0021
0022 Usage (CI, called from .gitlab-ci.yml or build-push.yml with matrix variables in env):
0023   bash scripts/build-eic.sh
0024
0025 Options:
0026   --env ENV           Environment: ci, xl, cuda, dbg, jl, prod, cvmfs, tf, ...
0027                       (default: \$ENV or xl)
0028   --build-type TYPE   Comma-separated list of build types: default, nightly, or both
0029                       (default: \$BUILD_TYPE or default,nightly)
0030   --builder-image IMG Builder base image name (default: \$BUILDER_IMAGE or debian_stable_base)
0031   --runtime-image IMG Runtime base image name (default: \$RUNTIME_IMAGE or debian_stable_base)
0032   --target STAGE      Docker build target stage (default: \$BUILD_TARGET or final)
0033   --platform PLATFORM Build platform, e.g. linux/amd64 (default: \$PLATFORM or linux/amd64)
0034   --jobs N            Number of parallel Spack build jobs
0035                       (default: \$JOBS or \$(getconf _NPROCESSORS_ONLN))
0036   --base-tag TAG      Tag of the locally built base image to use (default: local); if the image
0037                       is not found in the local Docker daemon, ghcr.io/eic/ is used with tag
0038                       'latest' as fallback (ignored in CI)
0039   --tag TAG           Local tag for the output image (default: local; ignored in CI)
0040   -h, --help          Show this help and exit
0041
0042 When multiple build types are given (e.g. "default,nightly"), both are built sequentially
0043 in the same Docker session so that the shared base stages (default environment
0044 concretization and installation) are only built once and reused from the local BuildKit
0045 layer cache.
0046
0047 GitHub Actions mode (GITHUB_ACTIONS=true):
0048   Set GH_REGISTRY, GH_REGISTRY_USER, JOBS. The script derives cache-key slugs
0049   from GITHUB_HEAD_REF (PR source branch, when set) or GITHUB_REF_NAME
0050   (push/schedule branch), and from GITHUB_BASE_REF (PR target branch, empty on
0051   push events) or DEFAULT_BRANCH (repo default branch fallback). Writes each image digest to
0052   \${METADATA_FILE%.json}-<build_type>.json (default base: /tmp/build-metadata.json).
0053 EOF
0054 }
0055
0056 ## Defaults (may be overridden by env vars set from CI matrix or command-line flags)
0057 BUILD_IMAGE="${BUILD_IMAGE:-eic_}"
0058 ENV="${ENV:-xl}"
0059 BUILD_TYPE="${BUILD_TYPE:-default,nightly}"
0060 BUILDER_IMAGE="${BUILDER_IMAGE:-debian_stable_base}"
0061 RUNTIME_IMAGE="${RUNTIME_IMAGE:-debian_stable_base}"
0062 BUILD_TARGET="${BUILD_TARGET:-final}"
0063 PLATFORM="${PLATFORM:-linux/amd64}"
0064 JOBS="${JOBS:-$(getconf _NPROCESSORS_ONLN)}"
0065 LOCAL_TAG="${LOCAL_TAG:-local}"
0066 LOCAL_BASE_TAG="${LOCAL_BASE_TAG:-local}"
0067 METADATA_FILE="${METADATA_FILE:-/tmp/build-metadata.json}"
0068
0069 while [[ $# -gt 0 ]]; do
0070   case "$1" in
0071     -h|--help)      print_help; exit 0 ;;
0072     --env)           ENV="$2";           shift 2 ;;
0073     --build-type)    BUILD_TYPE="$2";    shift 2 ;;
0074     --builder-image) BUILDER_IMAGE="$2"; shift 2 ;;
0075     --runtime-image) RUNTIME_IMAGE="$2"; shift 2 ;;
0076     --target)        BUILD_TARGET="$2";  shift 2 ;;
0077     --platform)      PLATFORM="$2";     shift 2 ;;
0078     --jobs)          JOBS="$2";         shift 2 ;;
0079     --base-tag)      LOCAL_BASE_TAG="$2"; shift 2 ;;
0080     --tag)           LOCAL_TAG="$2";    shift 2 ;;
0081     *) echo "Unknown argument: $1" >&2; echo "Try 'bash scripts/build-eic.sh --help' for usage." >&2; exit 1 ;;
0082   esac
0083 done
0084
0085 ## Source version files (only spack-packages version is needed for mirrors.yaml)
0086 source "${REPO_DIR}/spack-packages.sh"
0087
0088 ## Convert an arbitrary git ref/branch name to a valid OCI tag component.
0089 ## Mirrors GitLab's CI_COMMIT_REF_SLUG: lowercase, non-alnum runs → '-',
0090 ## strip leading/trailing '-', truncate to 63 chars.
0091 slugify() {
0092   echo "$1" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-+//; s/-+$//' | cut -c1-63
0093 }
0094
0095 ## Escape a string for safe use as a sed replacement (handles \, &, and | delimiter).
0096 sed_escape() {
0097   printf '%s' "$1" | sed 's/[\\&|]/\\&/g'
0098 }
0099
0100 ## Detect CI mode and normalise environment variables
0101 if [ -n "${CI_REGISTRY}" ]; then
0102   ## GitLab CI — all CI_* variables are already set by the runner
0103   CI_MODE="gitlab"
0104 elif [ "${GITHUB_ACTIONS}" = "true" ]; then
0105   ## GitHub Actions — map GitHub variables to the names used below.
0106   ## Use GITHUB_HEAD_REF for PR source branches (better cache reuse across PR
0107   ## updates), then fall back to GITHUB_REF_NAME for push/schedule events.
0108   ## GITHUB_BASE_REF is the PR target branch (empty on push events). DEFAULT_BRANCH
0109   ## should be supplied by the workflow so cache keys remain stable when
0110   ## GITHUB_BASE_REF is empty.
0111   CI_MODE="github"
0112   CI_REGISTRY="${GH_REGISTRY}"
0113   CI_PROJECT_PATH="${GH_REGISTRY_USER}"
0114   CI_COMMIT_REF_SLUG="$(slugify "${GITHUB_HEAD_REF:-${GITHUB_REF_NAME:-master}}")"
0115   CI_DEFAULT_BRANCH_SLUG="$(slugify "${GITHUB_BASE_REF:-${DEFAULT_BRANCH:-master}}")"
0116   CI_COMMIT_SHA="${GITHUB_SHA:-}"
0117   INTERNAL_TAG="${INTERNAL_TAG:-pipeline-${GITHUB_RUN_ID}}"
0118 else
0119   CI_MODE="local"
0120 fi
0121
0122 ## Generate mirrors.yaml in a temp file; the trap ensures cleanup on exit.
0123 ## sed replacement values are escaped to handle special chars (\, &, |).
0124 MIRRORS_YAML=$(mktemp "${TMPDIR:-/tmp}/mirrors-XXXXXX.yaml")
0125 trap 'rm -f "${MIRRORS_YAML}"' EXIT INT TERM
0126 if [ "${CI_MODE}" != "local" ]; then
0127   ## CI mode: expand CI_REGISTRY/CI_PROJECT_PATH variables in the template
0128   sed -e "s|\${CI_REGISTRY}|$(sed_escape "${CI_REGISTRY}")|g" \
0129       -e "s|\${CI_PROJECT_PATH}|$(sed_escape "${CI_PROJECT_PATH}")|g" \
0130       -e "s|\${SPACKPACKAGES_VERSION}|$(sed_escape "${SPACKPACKAGES_VERSION}")|g" \
0131       "${REPO_DIR}/mirrors.yaml.in" > "${MIRRORS_YAML}"
0132 else
0133   ## Local mode: public-only mirrors (no credentials required)
0134   cat > "${MIRRORS_YAML}" <<EOF
0135 mirrors:
0136   ghcr:
0137     url: oci://ghcr.io/eic/spack-${SPACKPACKAGES_VERSION}
0138     signed: false
0139   spack:
0140     url: https://binaries.spack.io/v1.0
0141     signed: false
0142 EOF
0143 fi
0144
0145 ## Resolve SHAs (network calls — skipped if version is already a SHA)
0146 echo "Resolving git SHAs..."
0147 BENCHMARK_COM_SHA=$(sh "${REPO_DIR}/scripts/resolve_git_ref" eic/common_bench master)
0148 BENCHMARK_DET_SHA=$(sh "${REPO_DIR}/scripts/resolve_git_ref" eic/detector_benchmarks master)
0149 BENCHMARK_PHY_SHA=$(sh "${REPO_DIR}/scripts/resolve_git_ref" eic/physics_benchmarks master)
0150 CAMPAIGNS_HEPMC3_SHA=$(sh "${REPO_DIR}/scripts/resolve_git_ref" eic/simulation_campaign_hepmc3 main)
0151 CAMPAIGNS_CONDOR_SHA=$(sh "${REPO_DIR}/scripts/resolve_git_ref" eic/job_submission_condor main)
0152 CAMPAIGNS_SLURM_SHA=$(sh "${REPO_DIR}/scripts/resolve_git_ref" eic/job_submission_slurm main)
0153
0154 ## Compute per-ENV duplicate allowlist (independent of build type)
0155 case "${ENV}" in
0156   xl|tf)
0157     SPACK_DUPLICATE_ALLOWLIST="epic|llvm|py-setuptools|py-urllib3|py-dask|py-dask-awkward|py-dask-histogram|py-distributed|py-requests" ;;
0158   *)
0159     SPACK_DUPLICATE_ALLOWLIST="epic|llvm|py-setuptools|py-urllib3" ;;
0160 esac
0161
0162 ## Normalize arch string for cache tag names while preserving platform variants
0163 ## Examples: linux/amd64 -> amd64, linux/amd64/v3 -> amd64_v3, linux/arm/v7 -> arm_v7
0164 ARCH=$(echo "${PLATFORM}" | sed 's|linux/||; s|/|_|g')
0165
0166 ## Derive shared registry prefix (used for image push, caching, and DOCKER_REGISTRY build-arg)
0167 CI_REGISTRY_PREFIX="${CI_REGISTRY}/${CI_PROJECT_PATH}"
0168 IMAGE_REPO="${CI_REGISTRY_PREFIX}/${BUILD_IMAGE}${ENV}"
0169
0170 ## Validate and split the build-type list
0171 IFS=',' read -ra BUILD_TYPES <<< "${BUILD_TYPE}"
0172 for _bt in "${BUILD_TYPES[@]}"; do
0173   _bt="${_bt#"${_bt%%[![:space:]]*}"}"; _bt="${_bt%"${_bt##*[![:space:]]}"}"  # trim whitespace
0174   case "${_bt}" in
0175     default|nightly) ;;
0176     *) echo "Unknown build type '${_bt}'; must be 'default' or 'nightly'." >&2; exit 1 ;;
0177   esac
0178 done
0179
0180 ## Enable xtrace and pipefail for the build loop
0181 set -o xtrace -o pipefail
0182
0183 ## Build each type sequentially; the shared base Docker stages (default env concretization
0184 ## and installation) are reused from BuildKit's layer cache after the first build.
0185 for build_type in "${BUILD_TYPES[@]}"; do
0186   ## Trim whitespace from the build type
0187   build_type="${build_type#"${build_type%%[![:space:]]*}"}"; build_type="${build_type%"${build_type##*[![:space:]]}"}"
0188
0189   ## Resolve optional version overrides (nightly always resolves; default only if version set)
0190   unset EDM4EIC_SHA EICRECON_SHA EPIC_SHA
0191   if [ "${build_type}" = "nightly" ]; then
0192     EDM4EIC_SHA=$(sh "${REPO_DIR}/scripts/resolve_git_ref" eic/EDM4eic "${EDM4EIC_VERSION:-main}")
0193     EICRECON_SHA=$(sh "${REPO_DIR}/scripts/resolve_git_ref" eic/EICrecon "${EICRECON_VERSION:-main}")
0194     EPIC_SHA=$(sh "${REPO_DIR}/scripts/resolve_git_ref" eic/epic "${EPIC_VERSION:-main}")
0195   else
0196     ## default build: only resolve if version is explicitly provided
0197     [ -n "${EDM4EIC_VERSION}" ]  && EDM4EIC_SHA=$(sh "${REPO_DIR}/scripts/resolve_git_ref" eic/EDM4eic  "${EDM4EIC_VERSION}")
0198     [ -n "${EICRECON_VERSION}" ] && EICRECON_SHA=$(sh "${REPO_DIR}/scripts/resolve_git_ref" eic/EICrecon "${EICRECON_VERSION}")
0199     [ -n "${EPIC_VERSION}" ]     && EPIC_SHA=$(sh "${REPO_DIR}/scripts/resolve_git_ref"     eic/epic     "${EPIC_VERSION}")
0200   fi
0201
0202   ## Build the docker buildx command as an array for safe quoting
0203   build_cmd=(docker buildx build)
0204   # shellcheck disable=SC2206  # word splitting is intentional: BUILD_OPTIONS is a space-separated list
0205   build_cmd+=(${BUILD_OPTIONS})
0206
0207   ## Output mode: push-by-digest in all CI modes; load locally
0208   if [ "${CI_MODE}" != "local" ]; then
0209     ## Push by digest; CI wrapper creates final tags via imagetools create.
0210     ## Always write a per-build-type metadata file so manifest jobs can identify it.
0211     build_cmd+=(--output "type=image,name=${IMAGE_REPO},push-by-digest=true,name-canonical=true,push=true")
0212     build_cmd+=(--metadata-file "${METADATA_FILE%.json}-${build_type}.json")
0213   else
0214     build_cmd+=(--load)
0215   fi
0216
0217   ## Cache sources: CI registry (if in CI) plus public ghcr.io/eic (GitLab and local modes)
0218   CACHE_KEY="${BUILD_IMAGE}${ENV}-${build_type}"
0219   BUILDCACHE_REPOS=()
0220   [ "${CI_MODE}" != "local" ] && BUILDCACHE_REPOS+=("${CI_REGISTRY_PREFIX}")
0221   [ "${CI_MODE}" != "github" ] && BUILDCACHE_REPOS+=("ghcr.io/eic")
0222   for REPO in "${BUILDCACHE_REPOS[@]}"; do
0223     build_cmd+=(--cache-from "type=registry,ref=${REPO}/buildcache:${CACHE_KEY}-${CI_COMMIT_REF_SLUG:-master}-${ARCH}")
0224     build_cmd+=(--cache-from "type=registry,ref=${REPO}/buildcache:${CACHE_KEY}-${CI_DEFAULT_BRANCH_SLUG:-master}-${ARCH}")
0225   done
0226
0227   ## Cache destination (CI only)
0228   if [ "${CI_MODE}" != "local" ]; then
0229     build_cmd+=(--cache-to "type=registry,ref=${CI_REGISTRY_PREFIX}/buildcache:${CACHE_KEY}-${CI_COMMIT_REF_SLUG:-master}-${ARCH},mode=max")
0230   fi
0231
0232   ## Image tag (local only; CI creates tags via imagetools create after build)
0233   if [ "${CI_MODE}" = "local" ]; then
0234     build_cmd+=(--tag "${BUILD_IMAGE}${ENV}:${LOCAL_TAG}-${build_type}")
0235   fi
0236
0237   ## Dockerfile, target stage, and platform
0238   build_cmd+=(--file containers/eic/Dockerfile)
0239   [ -n "${BUILD_TARGET}" ] && build_cmd+=(--target "${BUILD_TARGET}")
0240   build_cmd+=(--platform "${PLATFORM}")
0241
0242   ## Build arguments
0243   build_cmd+=(--build-arg "BENCHMARK_COM_SHA=${BENCHMARK_COM_SHA}")
0244   build_cmd+=(--build-arg "BENCHMARK_DET_SHA=${BENCHMARK_DET_SHA}")
0245   build_cmd+=(--build-arg "BENCHMARK_PHY_SHA=${BENCHMARK_PHY_SHA}")
0246   build_cmd+=(--build-arg "CAMPAIGNS_HEPMC3_SHA=${CAMPAIGNS_HEPMC3_SHA}")
0247   build_cmd+=(--build-arg "CAMPAIGNS_CONDOR_SHA=${CAMPAIGNS_CONDOR_SHA}")
0248   build_cmd+=(--build-arg "CAMPAIGNS_SLURM_SHA=${CAMPAIGNS_SLURM_SHA}")
0249
0250   if [ "${CI_MODE}" != "local" ]; then
0251     build_cmd+=(--build-arg "DOCKER_REGISTRY=${CI_REGISTRY_PREFIX}/")
0252     build_cmd+=(--build-arg "INTERNAL_TAG=${INTERNAL_TAG}")
0253     build_cmd+=(--build-arg "CI_COMMIT_SHA=${CI_COMMIT_SHA}")
0254   else
0255     ## Auto-detect: use locally built base images if available, otherwise pull from ghcr.io/eic/.
0256     ## Both BUILDER_IMAGE and RUNTIME_IMAGE must exist locally to avoid a mixed local/remote build.
0257     if docker image inspect "${BUILDER_IMAGE}:${LOCAL_BASE_TAG}" >/dev/null 2>&1 \
0258        && docker image inspect "${RUNTIME_IMAGE}:${LOCAL_BASE_TAG}" >/dev/null 2>&1; then
0259       echo "Using local base images: ${BUILDER_IMAGE}:${LOCAL_BASE_TAG}, ${RUNTIME_IMAGE}:${LOCAL_BASE_TAG}"
0260       build_cmd+=(--build-arg "DOCKER_REGISTRY=")
0261       build_cmd+=(--build-arg "INTERNAL_TAG=${LOCAL_BASE_TAG}")
0262     else
0263       echo "Local base images not found (${BUILDER_IMAGE}:${LOCAL_BASE_TAG} and/or ${RUNTIME_IMAGE}:${LOCAL_BASE_TAG}); pulling from ghcr.io/eic/:latest"
0264       build_cmd+=(--build-arg "DOCKER_REGISTRY=ghcr.io/eic/")
0265       build_cmd+=(--build-arg "INTERNAL_TAG=latest")
0266     fi
0267   fi
0268   ## EIC_CONTAINER_VERSION format is intentionally different per CI system
0269   if [ "${CI_MODE}" = "gitlab" ]; then
0270     build_cmd+=(--build-arg "EIC_CONTAINER_VERSION=${EXPORT_TAG}-${build_type}-$(git rev-parse HEAD)")
0271   elif [ "${CI_MODE}" = "github" ]; then
0272     build_cmd+=(--build-arg "EIC_CONTAINER_VERSION=github-${build_type}-${CI_COMMIT_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}")
0273   else
0274     build_cmd+=(--build-arg "EIC_CONTAINER_VERSION=local-${build_type}-$(git rev-parse HEAD 2>/dev/null || echo unknown)")
0275   fi
0276
0277   build_cmd+=(--build-arg "BUILDER_IMAGE=${BUILDER_IMAGE}")
0278   build_cmd+=(--build-arg "RUNTIME_IMAGE=${RUNTIME_IMAGE}")
0279   build_cmd+=(--build-arg "ENV=${ENV}")
0280   build_cmd+=(--build-arg "SPACK_DUPLICATE_ALLOWLIST=${SPACK_DUPLICATE_ALLOWLIST}")
0281   build_cmd+=(--build-arg "jobs=${JOBS}")
0282
0283   ## Optional version overrides
0284   [ -n "${EDM4EIC_SHA}" ]  && build_cmd+=(--build-arg "EDM4EIC_SHA=${EDM4EIC_SHA}")
0285   [ -n "${EICRECON_SHA}" ] && build_cmd+=(--build-arg "EICRECON_SHA=${EICRECON_SHA}")
0286   [ -n "${EPIC_SHA}" ]     && build_cmd+=(--build-arg "EPIC_SHA=${EPIC_SHA}")
0287
0288   ## Additional build contexts
0289   build_cmd+=(--build-context "spack-environment=spack-environment")
0290
0291   ## Secrets
0292   build_cmd+=(--secret "id=mirrors,src=${MIRRORS_YAML}")
0293   if [ "${CI_MODE}" != "local" ]; then
0294     if [ -n "${CI_REGISTRY_USER}" ] && [ -n "${CI_REGISTRY_PASSWORD}" ]; then
0295       build_cmd+=(--secret "type=env,id=CI_REGISTRY_USER,env=CI_REGISTRY_USER")
0296       build_cmd+=(--secret "type=env,id=CI_REGISTRY_PASSWORD,env=CI_REGISTRY_PASSWORD")
0297     fi
0298     if [ -n "${GITHUB_REGISTRY_USER}" ] && [ -n "${GITHUB_REGISTRY_TOKEN}" ]; then
0299       build_cmd+=(--secret "type=env,id=GITHUB_REGISTRY_USER,env=GITHUB_REGISTRY_USER")
0300       build_cmd+=(--secret "type=env,id=GITHUB_REGISTRY_TOKEN,env=GITHUB_REGISTRY_TOKEN")
0301     fi
0302   fi
0303
0304   ## Suppress provenance attestation (matches CI behaviour)
0305   build_cmd+=(--provenance false)
0306
0307   ## Build context
0308   build_cmd+=(containers/eic)
0309
0310   ## Execute
0311   "${build_cmd[@]}" 2>&1 | tee "build-${build_type}.log"
0312 done