job_submission_slurm/scripts/submit_csv.sh

0001 #!/bin/bash
0002 set -Euo pipefail
0003 trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
0004 IFS=$'\n\t'
0005
0006 # check arguments
0007 if [ $# -lt 2 ] ; then
0008   echo "Usage: "
0009   echo "  $0 <template> <type> <file or url> [target hours = 2]"
0010   exit
0011 fi
0012
0013 # Project configuration
0014 BASEURL="https://eicweb.phy.anl.gov/api/v4/projects/491/jobs/artifacts/main/raw/results/datasets/timings/"
0015 BASEJOB="?job=collect"
0016
0017 # Parse arguments
0018 # - type of simulation
0019 TYPE=${1}
0020 shift
0021 # - input file
0022 FILE=${1}
0023 shift
0024 # - target hours
0025 TARGET=${1:-2}
0026 shift
0027
0028 # Parse input URI
0029 if [ -f "${FILE}" ] ; then
0030   echo "Using local file as input"
0031   INPUT=(cat ${FILE})
0032 else
0033   echo "Using ${BASEURL}${FILE}${BASEJOB} as input"
0034   INPUT=(curl ${BASEURL}${FILE}${BASEJOB})
0035 fi
0036
0037 # Loop over input
0038 SBATCH=(sbatch --mail-user=${EMAIL:-${USER}@jlab.org} --parsable --requeue --time=$((2*TARGET)):00:00)
0039 "${INPUT[@]}" | grep -v ^curl | while IFS="," read file ntotal dt0 dt1 ; do
0040   if [ -z "${file}" ] ; then
0041     continue
0042   fi
0043   nevents=$(echo "n=(3600*$TARGET-$dt0)/$dt1; if (n>$ntotal) print($ntotal) else print(n)" | bc)
0044   nchunks=$(echo "n=$ntotal/$nevents; if (n==0) print(1) else print(n)" | bc)
0045   echo "${file}: ${dt0}+N*${dt1} s, for ${nchunks} chunks for ${TARGET} hours. Submit? [Y,n] "
0046   "${SBATCH[@]}" --array=1-${nchunks} $(dirname $0)/submit.sh ${TYPE} ${file} ${nevents}
0047 done | tee $(basename ${FILE} .csv)-$(date --iso-8601=minutes).log