job_submission_condor/scripts/run_osg_test.sh

0001 #!/bin/bash
0002 set -Euo pipefail
0003 trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
0004 IFS=$'\n\t'
0005
0006 # Load job environment (includes secrets, so delete when read)
0007 if [ -f environment.sh ] ; then
0008   grep -v SECRET environment.sh
0009   source environment.sh
0010   rm environment.sh
0011 fi
0012
0013 # Startup
0014 echo "date sys: $(date)"
0015 echo "date web: $(date -d "$(curl -Is --max-redirs 0 google.com 2>&1 | grep Date: | cut -d' ' -f2-7)")"
0016 echo "hostname: $(hostname -f)"
0017 echo "uname:    $(uname -a)"
0018 echo "whoami:   $(whoami)"
0019 echo "pwd:      $(pwd)"
0020 echo "site:     ${GLIDEIN_Site:-}"
0021 echo "resource: ${GLIDEIN_ResourceName:-}"
0022 echo "http_proxy: ${http_proxy:-}"
0023 df -h --exclude-type=fuse --exclude-type=tmpfs
0024 ls -al
0025 test -f .job.ad && cat .job.ad
0026 test -f .machine.ad && cat .machine.ad
0027 eic-info
0028
0029 INPUT_FILE=EVGEN/CI/pythia8NCDIS_5x41_minQ2=1_beamEffects_xAngle=-0.025_hiDiv_1_20ev.hepmc.gz
0030
0031 # Retry function
0032 function retry {
0033   local n=0
0034   local max=5
0035   local delay=20
0036   while [[ $n -lt $max ]] ; do
0037     n=$((n+1))
0038     s=0
0039     "$@" || s=$?
0040     [ $s -eq 0 ] && {
0041       return $s
0042     }
0043     [ $n -ge $max ] && {
0044       echo "Failed after $n retries, exiting with $s"
0045       return $s
0046     }
0047     echo "Retrying in $delay seconds..."
0048     sleep $delay
0049   done
0050 }
0051
0052 # S3 locations
0053 MC="/usr/local/bin/mc"
0054 S3URL="https://eics3.sdcc.bnl.gov:9000"
0055 S3RO="S3"
0056 S3RODIR="${S3RO}/eictest/EPIC"
0057
0058 # Local temp dir
0059 echo "SLURM_TMPDIR=${SLURM_TMPDIR:-}"
0060 echo "SLURM_JOB_ID=${SLURM_JOB_ID:-}"
0061 echo "SLURM_ARRAY_JOB_ID=${SLURM_ARRAY_JOB_ID:-}"
0062 echo "SLURM_ARRAY_TASK_ID=${SLURM_ARRAY_TASK_ID:-}"
0063 echo "_CONDOR_SCRATCH_DIR=${_CONDOR_SCRATCH_DIR:-}"
0064 echo "OSG_WN_TMP=${OSG_WN_TMP:-}"
0065 if [ -n "${SLURM_TMPDIR:-}" ] ; then
0066   TMPDIR=${SLURM_TMPDIR}
0067 elif [ -n "${_CONDOR_SCRATCH_DIR:-}" ] ; then
0068   TMPDIR=${_CONDOR_SCRATCH_DIR}
0069 else
0070   if [ -d "/scratch/slurm/${SLURM_JOB_ID:-}" ] ; then
0071     TMPDIR="/scratch/slurm/${SLURM_JOB_ID:-}"
0072   else
0073     TMPDIR=${TMPDIR:-/tmp}/${$}
0074   fi
0075 fi
0076 echo "TMPDIR=${TMPDIR}"
0077 mkdir -p ${TMPDIR}
0078 ls -al ${TMPDIR}
0079
0080 # Internet connectivity check
0081 if curl --connect-timeout 30 --retry 5 --silent --show-error ${S3URL} > /dev/null ; then
0082   echo "$(hostname) is online."
0083   export ONLINE=true
0084 else
0085   echo "$(hostname) is NOT online."
0086   if which tracepath ; then
0087     echo "tracepath -b -p 9000 eics3.sdcc.bnl.gov"
0088     tracepath -b -p 9000 eics3.sdcc.bnl.gov
0089     echo "tracepath -b www.bnl.gov"
0090     tracepath -b www.bnl.gov
0091     echo "tracepath -b google.com"
0092     tracepath -b google.com
0093   fi
0094   export ONLINE=
0095 fi
0096
0097 # Retrieve test file if S3_ACCESS_KEY and S3_SECRET_KEY in environment
0098 if [ -x ${MC} ] ; then
0099   if [ -n "${ONLINE:-}" ] ; then
0100     if [ -n "${S3_ACCESS_KEY:-}" -a -n "${S3_SECRET_KEY:-}" ] ; then
0101       MC_CONFIG=$(mktemp -d $PWD/mc_config.XXXX)
0102       retry ${MC} -C ${MC_CONFIG} config host add ${S3RO} ${S3URL} ${S3_ACCESS_KEY} ${S3_SECRET_KEY}
0103       retry ${MC} -C ${MC_CONFIG} config host list ${S3RO} | grep -v SecretKey
0104       retry ${MC} -C ${MC_CONFIG} cp --disable-multipart --insecure ${S3RODIR}/${INPUT_FILE} .
0105       retry ${MC} -C ${MC_CONFIG} config host remove ${S3RO}
0106       ls -al
0107     else
0108       echo "No S3 credentials. Provide (readonly) S3 credentials."
0109       exit -1
0110     fi
0111   else
0112     echo "No internet connection. Pre-cache input file."
0113     exit -1
0114   fi
0115 fi
0116
0117 # closeout
0118 date
0119