job_submission_condor/scripts/submit_csv.sh

0001 #!/bin/bash
0002 set -Euo pipefail
0003 trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
0004 IFS=$'\n\t'
0005
0006 # check arguments
0007 if [ $# -lt 2 ] ; then
0008   echo "Usage: "
0009   echo "  $0 <template> <type> <file or url> [target hours = 2]"
0010   exit
0011 fi
0012
0013 # Project configuration
0014 BASEURL="https://eicweb.phy.anl.gov/api/v4/projects/491/jobs/artifacts/${DETECTOR_VERSION:-main}/raw/results/datasets/timings/"
0015 BASEJOB="?job=collect"
0016
0017 # Parse arguments
0018 # - condor template
0019 TEMPLATE=${1}
0020 shift
0021 # - type of simulation
0022 TYPE=${1}
0023 shift
0024 # - input file
0025 FILE=${1}
0026 shift
0027 # - target hours
0028 TARGET=${1:-2}
0029 shift
0030
0031 # process csv file into jobs
0032 if [ -n "${CSV_FILE:-}" ]; then
0033   # allow to set custom csv file for job instead of fetching from web archive
0034   CSV_FILE=$(realpath -e ${CSV_FILE})
0035 else
0036   CSV_FILE=$($(dirname $0)/csv_to_chunks.sh ${FILE} ${TARGET})
0037 fi
0038
0039 # create command line
0040 EXECUTABLE="$(dirname $0)/run.sh"
0041 ARGUMENTS="EVGEN/\$(file) \$(ext) \$(nevents) \$(ichunk)"
0042
0043 # construct environment file
0044 ENVIRONMENT=environment-$(date --iso-8601=minutes).sh
0045
0046 # extract certificate name
0047 X509_USER_PROXY_BASE=$(basename ${X509_USER_PROXY:-})
0048
0049 sed "
0050   s|%COPYRECO%|${COPYRECO:-}|g;
0051   s|%COPYFULL%|${COPYFULL:-}|g;
0052   s|%COPYLOG%|${COPYLOG:-}|g;
0053   s|%USERUCIO%|${USERUCIO:-}|g;
0054   s|%X509_USER_PROXY%|${X509_USER_PROXY_BASE:-}|g;
0055   s|%TAG_PREFIX%|${TAG_PREFIX:-}|g;
0056   s|%TAG_SUFFIX%|${TAG_SUFFIX:-}|g;
0057   s|%DETECTOR_VERSION%|${DETECTOR_VERSION}|g;
0058   s|%DETECTOR_CONFIG%|${DETECTOR_CONFIG}|g;
0059   s|%EBEAM%|${EBEAM}|g;
0060   s|%PBEAM%|${PBEAM}|g;
0061 " templates/${TEMPLATE}.sh.in > ${ENVIRONMENT}
0062
0063 # construct requirements
0064 REQUIREMENTS=""
0065
0066 # construct input files
0067 INPUT_FILES=${ENVIRONMENT},${X509_USER_PROXY}
0068
0069 # construct submission file
0070 SUBMIT_FILE=$(basename ${CSV_FILE} .csv).submit
0071 sed "
0072   s|%EXECUTABLE%|${EXECUTABLE}|g;
0073   s|%ARGUMENTS%|${ARGUMENTS}|g;
0074   s|%JUG_XL_TAG%|${JUG_XL_TAG:-nightly}|g;
0075   s|%DETECTOR_VERSION%|${DETECTOR_VERSION}|g;
0076   s|%DETECTOR_CONFIG%|${DETECTOR_CONFIG}|g;
0077   s|%INPUT_FILES%|${INPUT_FILES}|g;
0078   s|%REQUIREMENTS%|${REQUIREMENTS}|g;
0079   s|%CSV_FILE%|${CSV_FILE}|g;
0080 " templates/${TEMPLATE}.submit.in > ${SUBMIT_FILE}
0081
0082 # submit job
0083 condor_submit -verbose -file ${SUBMIT_FILE}
0084
0085 # create log dir
0086 if [ $? -eq 0 ] ; then
0087   for i in `condor_q --batch | grep ^${USER} | tail -n1 | awk '{print($NF)}' | cut -d. -f1` ; do
0088     mkdir -p LOG/CONDOR/osg_$i/
0089   done
0090 fi