simulation_campaign_single/scripts/run.sh

0001 #!/bin/bash
0002 set -Euo pipefail
0003 trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
0004 IFS=$'\n\t'
0005
0006 # Load job environment (mask secrets)
0007 if ls environment*.sh ; then
0008   grep -v BEARER environment*.sh
0009   source environment*.sh
0010 fi
0011
0012 # Check arguments
0013 if [ $# -lt 1 ] ; then
0014   echo "Usage: "
0015   echo "  $0 <input> [n_chunk=10000] [i_chunk=]"
0016   echo
0017   echo "A typical npsim run requires from 0.5 to 5 core-seconds per event,"
0018   echo "and uses under 3 GB of memory. The output ROOT file for"
0019   echo "10k events take up about 2 GB in disk space."
0020   exit
0021 fi
0022
0023 # Startup
0024 echo "date sys: $(date)"
0025 echo "date web: $(date -d "$(curl --insecure --head --silent --max-redirs 0 google.com 2>&1 | grep Date: | cut -d' ' -f2-7)")"
0026 echo "hostname: $(hostname -f)"
0027 echo "uname:    $(uname -a)"
0028 echo "whoami:   $(whoami)"
0029 echo "pwd:      $(pwd)"
0030 echo "site:     ${GLIDEIN_Site:-}"
0031 echo "resource: ${GLIDEIN_ResourceName:-}"
0032 echo "http_proxy: ${http_proxy:-}"
0033 df -h --exclude-type=fuse --exclude-type=tmpfs
0034 ls -al
0035 test -f .job.ad && cat .job.ad
0036 test -f .machine.ad && cat .machine.ad
0037
0038 # Load container environment (include ${DETECTOR_VERSION})
0039 export DETECTOR_CONFIG_REQUESTED=${DETECTOR_CONFIG:-}
0040 export DETECTOR_VERSION_REQUESTED=${DETECTOR_VERSION:-main}
0041 source /opt/detector/epic-${DETECTOR_VERSION_REQUESTED}/bin/thisepic.sh
0042 export DETECTOR_VERSION=${DETECTOR_VERSION_REQUESTED}
0043 export DETECTOR_CONFIG=${DETECTOR_CONFIG_REQUESTED:-${DETECTOR_CONFIG:-$DETECTOR}}
0044
0045 # Argument parsing
0046 # - input file
0047 INPUT_FILE=${1}
0048 # - number of events
0049 EVENTS_PER_TASK=${2:-10000}
0050 # - current chunk
0051 if [ ${#} -lt 3 ] ; then
0052   TASK=""
0053   SEED=1
0054   SKIP_N_EVENTS=0
0055 else
0056   # 10-base input task number to 4-zero-padded task number
0057   TASK=".${3}"
0058   SEED=$((10#${3}+1))
0059   # assumes zero-based task number, can be zero-padded
0060   SKIP_N_EVENTS=0
0061 fi
0062
0063 # Output location
0064 BASEDIR=${DATADIR:-${PWD}}
0065
0066 # XRD Write locations
0067 XRDWURL="xroots://dtn2201.jlab.org/"
0068 XRDWBASE=${XRDWBASE:-"/eic/eic2/EPIC"}
0069
0070 # XRD Read locations
0071 XRDRURL="root://dtn-eic.jlab.org/"
0072 XRDRBASE="/work/eic2/EPIC"
0073
0074 # Local temp dir
0075 echo "SLURM_TMPDIR=${SLURM_TMPDIR:-}"
0076 echo "SLURM_JOB_ID=${SLURM_JOB_ID:-}"
0077 echo "SLURM_ARRAY_JOB_ID=${SLURM_ARRAY_JOB_ID:-}"
0078 echo "SLURM_ARRAY_TASK_ID=${SLURM_ARRAY_TASK_ID:-}"
0079 echo "_CONDOR_SCRATCH_DIR=${_CONDOR_SCRATCH_DIR:-}"
0080 echo "OSG_WN_TMP=${OSG_WN_TMP:-}"
0081 if [ -n "${_CONDOR_SCRATCH_DIR:-}" ] ; then
0082   TMPDIR=${_CONDOR_SCRATCH_DIR}
0083 elif [ -n "${SLURM_TMPDIR:-}" ] ; then
0084   TMPDIR=${SLURM_TMPDIR}
0085 else
0086   if [ -d "/scratch/slurm/${SLURM_JOB_ID:-}" ] ; then
0087     TMPDIR="/scratch/slurm/${SLURM_JOB_ID:-}"
0088   else
0089     TMPDIR=${TMPDIR:-/tmp}/${$}
0090   fi
0091 fi
0092 echo "TMPDIR=${TMPDIR}"
0093 mkdir -p ${TMPDIR}
0094 ls -al ${TMPDIR}
0095
0096 # Input file parsing
0097 BASENAME=$(basename ${INPUT_FILE} .steer)
0098 TASKNAME=${BASENAME}${TASK}
0099 INPUT_DIR=$(dirname $(realpath --canonicalize-missing --relative-to=${BASEDIR} ${INPUT_FILE}))
0100 # - file.hepmc              -> TAG=""
0101 # - EVGEN/file.hepmc        -> TAG=""
0102 # - EVGEN/DIS/file.hepmc    -> TAG="DIS"
0103 # - EVGEN/DIS/NC/file.hepmc -> TAG="DIS/NC"
0104 # - ../file.hepmc           -> error
0105 if [ ! "${INPUT_DIR/\.\.\//}" = "${INPUT_DIR}" ] ; then
0106   echo "Error: Input file must be below current directory."
0107   exit
0108 fi
0109 INPUT_PREFIX=${INPUT_DIR/\/*/}
0110 TAG=${INPUT_DIR/${INPUT_PREFIX}\//}
0111 INPUT_DIR=${BASEDIR}/EVGEN/${TAG}
0112 mkdir -p ${INPUT_DIR}
0113 TAG=${DETECTOR_VERSION:-main}/${DETECTOR_CONFIG}/${TAG}
0114
0115 # Copy input file from xrootd
0116 xrdcp -f ${XRDRURL}/${XRDRBASE}/${INPUT_FILE} ${INPUT_DIR}
0117
0118 # Output file names
0119 LOG_DIR=LOG/${TAG}
0120 LOG_TEMP=${TMPDIR}/${LOG_DIR}
0121 mkdir -p ${LOG_TEMP} ${BASEDIR}/${LOG_DIR}
0122 #
0123 FULL_DIR=FULL/${TAG}
0124 FULL_TEMP=${TMPDIR}/${FULL_DIR}
0125 mkdir -p ${FULL_TEMP} ${BASEDIR}/${FULL_DIR}
0126 #
0127 RECO_DIR=RECO/${TAG}
0128 RECO_TEMP=${TMPDIR}/${RECO_DIR}
0129 mkdir -p ${RECO_TEMP} ${BASEDIR}/${RECO_DIR}
0130
0131 # Run simulation
0132 {
0133   date
0134   eic-info
0135   prmon \
0136     --filename ${LOG_TEMP}/${TASKNAME}.npsim.prmon.txt \
0137     --json-summary ${LOG_TEMP}/${TASKNAME}.npsim.prmon.json \
0138     -- \
0139   npsim \
0140     --runType run \
0141     --random.seed ${SEED:-1} \
0142     --random.enableEventSeed \
0143     --printLevel WARNING \
0144     --enableGun \
0145     --steeringFile ${INPUT_FILE} \
0146     --numberOfEvents ${EVENTS_PER_TASK} \
0147     --filter.tracker 'edep0' \
0148     --compactFile ${DETECTOR_PATH}/${DETECTOR_CONFIG}.xml \
0149     --outputFile ${FULL_TEMP}/${TASKNAME}.edm4hep.root
0150   ls -al ${FULL_TEMP}/${TASKNAME}.edm4hep.root
0151 } 2>&1 | tee ${LOG_TEMP}/${TASKNAME}.npsim.log | tail -n1000
0152
0153 # Data egress to directory
0154 if [ "${COPYFULL:-false}" == "true" ] ; then
0155   # Token for write authentication
0156   export BEARER_TOKEN=$(cat ${_CONDOR_CREDS}/eic.use)
0157   xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${FULL_DIR} || echo "Cannot write simulation outputs to xrootd server"
0158   xrdcp --force --recursive ${FULL_TEMP}/${TASKNAME}.edm4hep.root ${XRDWURL}/${XRDWBASE}/${FULL_DIR}
0159 fi
0160
0161 # Run eicrecon reconstruction
0162 {
0163   date
0164   eic-info
0165   prmon \
0166     --filename ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.txt \
0167     --json-summary ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.json \
0168     -- \
0169   eicrecon \
0170     -Ppodio:output_file="${RECO_TEMP}/${TASKNAME}.eicrecon.tree.edm4eic.root" \
0171     -Pjana:warmup_timeout=0 -Pjana:timeout=0 \
0172     -Pplugins=janadot \
0173     "${FULL_TEMP}/${TASKNAME}.edm4hep.root"
0174   if [ -f jana.dot ] ; then mv jana.dot ${LOG_TEMP}/${TASKNAME}.eicrecon.dot ; fi
0175   ls -al ${RECO_TEMP}/${TASKNAME}*.eicrecon.tree.edm4eic.root
0176 } 2>&1 | tee ${LOG_TEMP}/${TASKNAME}.eicrecon.log | tail -n1000
0177
0178 # List log files
0179 ls -al ${LOG_TEMP}/${TASKNAME}.*
0180
0181 # Data egress to directory
0182 if [ "${COPYRECO:-false}" == "true" ] ; then
0183   # Token for write authentication
0184   export BEARER_TOKEN=$(cat ${_CONDOR_CREDS}/eic.use)
0185   xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${RECO_DIR} || echo "Cannot write reconstructed outputs to xrootd server"
0186   xrdcp --force --recursive ${RECO_TEMP}/${TASKNAME}*.edm4eic.root ${XRDWURL}/${XRDWBASE}/${RECO_DIR}
0187 fi
0188 if [ "${COPYLOG:-false}" == "true" ] ; then
0189   # Token for write authentication
0190   export BEARER_TOKEN=$(cat ${_CONDOR_CREDS}/eic.use)
0191   xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${LOG_DIR} || echo "Cannot write log outputs to xrootd server"
0192   xrdcp --force --recursive ${LOG_TEMP}/${TASKNAME}.* ${XRDWURL}/${XRDWBASE}/${LOG_DIR}
0193 fi
0194
0195 # closeout
0196 date
0197 find ${TMPDIR}
0198 du -sh ${TMPDIR}