CI/dependencies/setup.sh

0001 #!/bin/bash
0002 set -u
0003 set -e
0004 set -o pipefail
0005
0006 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
0007
0008 export SPACK_COLOR=always
0009
0010 function start_section() {
0011     local section_name="$1"
0012     if [ -n "${GITHUB_ACTIONS:-}" ]; then
0013         echo "::group::${section_name}"
0014     else
0015         echo "${section_name}"
0016     fi
0017 }
0018
0019 function end_section() {
0020     if [ -n "${GITHUB_ACTIONS:-}" ]; then
0021         echo "::endgroup::"
0022     fi
0023 }
0024
0025 # Parse command line arguments
0026 while getopts "c:t:d:e:fh" opt; do
0027   case ${opt} in
0028     c )
0029       compiler=$OPTARG
0030       ;;
0031     t )
0032       tag=$OPTARG
0033       ;;
0034     d )
0035       destination=$OPTARG
0036       ;;
0037     e )
0038       env_file=$OPTARG
0039       ;;
0040     f )
0041       full_install=true
0042       ;;
0043     h )
0044       echo "Usage: $0 [-c compiler] [-t tag] [-d destination] [-e env_file] [-h]"
0045       echo "Options:"
0046       echo "  -c <compiler>    Specify compiler (defaults to CXX env var)"
0047       echo "  -t <tag>         Specify dependency tag (defaults to DEPENDENCY_TAG env var)"
0048       echo "  -d <destination> Specify install destination (defaults based on CI environment)"
0049       echo "  -e <env_file>    Specify environment file to output environments to"
0050       echo "  -f               Full dependency installation. Includes Geant4 datasets and Python packages."
0051       echo "  -h               Show this help message"
0052       exit 0
0053       ;;
0054     \? )
0055       echo "Invalid option: -$OPTARG" 1>&2
0056       exit 1
0057       ;;
0058     : )
0059       echo "Option -$OPTARG requires an argument" 1>&2
0060       exit 1
0061       ;;
0062   esac
0063 done
0064
0065 script_start=$(date +%s.%N)
0066
0067 # Helper to print elapsed time since previous checkpoint
0068 checkpoint() {
0069     local label=$1
0070     local now
0071     now=$(date +%s.%N)
0072     local elapsed
0073     elapsed=$(echo "$now - ${last_time:-$script_start}" | bc)
0074     printf "[%s] %.3f s\n" "$label" "$elapsed"
0075     last_time=$now
0076 }
0077
0078 # Set defaults if not specified
0079 if [ -z "${compiler:-}" ]; then
0080   compiler="${CXX:-default}"
0081 fi
0082
0083 if [ -z "${tag:-}" ]; then
0084   tag="${DEPENDENCY_TAG:-}"
0085   if [ -z "${tag:-}" ]; then
0086     echo "No tag specified via -t or DEPENDENCY_TAG environment variable"
0087     exit 1
0088   fi
0089 fi
0090
0091 if [ -z "${destination:-}" ]; then
0092   if [ -n "${GITHUB_ACTIONS:-}" ]; then
0093     destination="${GITHUB_WORKSPACE}/dependencies"
0094   elif [ -n "${GITLAB_CI:-}" ]; then
0095     destination="${CI_PROJECT_DIR}/dependencies"
0096   else
0097     echo "No destination specified via -d and not running in CI"
0098     exit 1
0099   fi
0100 fi
0101
0102 if [ -z "${env_file:-}" ]; then
0103   if [ -n "${GITHUB_ACTIONS:-}" ]; then
0104     env_file="${GITHUB_ENV}"
0105   else
0106     echo "No environment file specified via -e and not running in GitHub Actions"
0107     exit 1
0108   fi
0109 fi
0110
0111 export env_file
0112
0113 function set_env {
0114   key="$1"
0115   value="$2"
0116
0117   echo "=> ${key}=${value}"
0118
0119   if [ -n "${GITHUB_ACTIONS:-}" ]; then
0120     echo "${key}=${value}" >> "$env_file"
0121   else
0122     echo "export ${key}=${value}" >> "$env_file"
0123   fi
0124 }
0125
0126
0127
0128 checkpoint "Starting setup script"
0129
0130 echo "Install tag: $tag"
0131 echo "Install destination: $destination"
0132
0133 mkdir -p "${destination}"
0134
0135 if [ -n "${GITLAB_CI:-}" ]; then
0136     _spack_folder=${CI_PROJECT_DIR}/spack
0137 else
0138     _spack_folder=${PWD}/spack
0139 fi
0140
0141 start_section "Install spack if not already installed"
0142 if ! command -v spack &> /dev/null; then
0143   "${SCRIPT_DIR}/setup_spack.sh" "${_spack_folder}"
0144   source "${_spack_folder}/share/spack/setup-env.sh"
0145 fi
0146 checkpoint "Spack install complete"
0147
0148 _spack_repo_version=${SPACK_REPO_VERSION:-develop}
0149 _spack_repo_directory="$(realpath "$(spack location --repo builtin)/../../../")"
0150
0151 echo "Ensure repo is synced with version ${_spack_repo_version}"
0152
0153 git config --global --add safe.directory "${_spack_repo_directory}"
0154 spack repo update builtin --tag "${_spack_repo_version}"
0155 checkpoint "Spack repository updated"
0156
0157 end_section
0158
0159 if [ -n "${GITLAB_CI:-}" ]; then
0160   # Use the project spack config for GitLab CI so we can cache it
0161   mkdir -p ${CI_PROJECT_DIR}/.spack
0162   ln -s ${CI_PROJECT_DIR}/.spack ${HOME}/.spack
0163 fi
0164
0165
0166
0167 if [ -n "${CI:-}" ]; then
0168   start_section "Add buildcache mirror"
0169   mirror_name="acts-spack-buildcache"
0170   mirror_url="oci://ghcr.io/acts-project/spack-buildcache"
0171   if [ -n "${GITLAB_CI:-}" ]; then
0172   # Use CERN mirror for non-Github Actions
0173     mirror_url="oci://registry.cern.ch/ghcr.io/acts-project/spack-buildcache"
0174   fi
0175
0176   # Check if this buildcache is already configured
0177   if ! spack mirror list | grep -q ${mirror_name}; then
0178     echo "Adding buildcache ${mirror_name}"
0179     spack mirror add ${mirror_name} ${mirror_url} --unsigned
0180   fi
0181   end_section
0182
0183   start_section "Locate OpenGL"
0184   "${SCRIPT_DIR}/opengl.sh"
0185   checkpoint "OpenGL location complete"
0186   end_section
0187 fi
0188
0189 start_section "Get spack lock file"
0190 arch=$(spack arch --family)
0191
0192 env_dir="${destination}/env"
0193 view_dir="${destination}/view"
0194 venv_dir="${destination}/venv"
0195 mkdir -p ${env_dir}
0196
0197 lock_file_path="${destination}/spack.lock"
0198 cmd=(
0199     "${SCRIPT_DIR}/select_lockfile.py"
0200     "--tag" "${tag}"
0201     "--arch" "${arch}"
0202     "--output" "${lock_file_path}"
0203 )
0204
0205 if [ "${compiler}" != "default" ]; then
0206     cmd+=("--compiler-binary" "${compiler}")
0207 fi
0208
0209 "${cmd[@]}"
0210
0211 checkpoint "Lock file prepared"
0212
0213 end_section
0214
0215
0216
0217 start_section "Create spack environment"
0218 spack env create -d "${env_dir}" "${lock_file_path}" --with-view "$view_dir"
0219 checkpoint "Spack environment created"
0220 spack -e "${env_dir}" spec -l
0221 checkpoint "Spack spec complete"
0222 spack -e "${env_dir}" find
0223 checkpoint "Spack find complete"
0224 end_section
0225
0226 start_section "Install spack packages"
0227 spack -e "${env_dir}" install --fail-fast --use-buildcache only --concurrent-packages 10
0228 checkpoint "Spack install complete"
0229 end_section
0230
0231 start_section "Patch up Geant4 data directory"
0232 if [ "${full_install:-false}" == "true" ]; then
0233   if ! which uv &> /dev/null ; then
0234     echo "uv not found, installing uv"
0235     curl -LsSf https://astral.sh/uv/install.sh | sh
0236     UV_EXE="/root/.local/bin/uv"
0237     checkpoint "uv installation complete"
0238   else
0239     UV_EXE=$(which uv)
0240   fi
0241   $UV_EXE run "$SCRIPT_DIR/download_geant4_datasets.py" -j8 --config "${view_dir}/bin/geant4-config"
0242   checkpoint "Geant4 datasets download complete"
0243 fi
0244 geant4_dir=$(spack -e "${env_dir}" location -i geant4)
0245 # Prepare the folder for G4 data, and symlink it to where G4 will look for it
0246 mkdir -p "${geant4_dir}/share/Geant4"
0247 ln -s "${geant4_dir}/share/Geant4/data" "${view_dir}/share/Geant4/data"
0248 end_section
0249
0250 start_section "Prepare python environment"
0251 "${view_dir}/bin/python3" -m venv --system-site-packages "$venv_dir"
0252 "${venv_dir}/bin/python3" -m pip install pyyaml jinja2
0253 if [ "${full_install:-false}" == "true" ]; then
0254   "${venv_dir}/bin/python3" -m pip install -r "${SCRIPT_DIR}/../../Examples/Python/tests/requirements.txt"
0255   "${venv_dir}/bin/python3" -m pip install histcmp==0.8.1 matplotlib
0256   "${venv_dir}/bin/python3" -m pip install pytest-md-report
0257 fi
0258 checkpoint "Python environment prepared"
0259 end_section
0260
0261 start_section "Set environment variables"
0262 if [ -n "${GITHUB_ACTIONS:-}" ]; then
0263   echo "${view_dir}/bin" >> "$GITHUB_PATH"
0264   echo "${venv_dir}/bin" >> "$GITHUB_PATH"
0265 fi
0266 set_env PATH "${venv_dir}/bin:${view_dir}/bin/:${PATH}"
0267 set_env LD_LIBRARY_PATH "${venv_dir}/lib:${view_dir}/lib"
0268 set_env CMAKE_PREFIX_PATH "${venv_dir}:${view_dir}"
0269 set_env ROOT_SETUP_SCRIPT "${view_dir}/bin/thisroot.sh"
0270 set_env ROOT_INCLUDE_PATH "${view_dir}/include"
0271 # cleanup setup-python mess
0272 set_env PKG_CONFIG_PATH ""
0273 set_env pythonLocation ""
0274 set_env Python_ROOT_DIR ""
0275 set_env Python2_ROOT_DIR ""
0276 set_env Python3_ROOT_DIR ""
0277 end_section
0278
0279 checkpoint "Setup script complete"