job_submission_condor/scripts/csv_to_chunks.sh

0001 #!/bin/bash
0002 set -Euo pipefail
0003 trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
0004 IFS=$'\n\t'
0005
0006 # check arguments
0007 if [ $# -lt 2 ] ; then
0008   echo "Usage: "
0009   echo "  $0 <file or url> [target hours = 2]"
0010   exit
0011 fi
0012
0013 # Project configuration
0014 BASEURL="https://eicweb.phy.anl.gov/api/v4/projects/491/jobs/artifacts/main/raw/results/datasets/timings/"
0015 BASEJOB="?job=collect"
0016
0017 # Parse arguments
0018 # - input file
0019 FILE=${1}
0020 shift
0021 # - target hours
0022 TARGET=${1:-2}
0023 shift
0024
0025 # Parse input URI
0026 if [ -f "${FILE}" ] ; then
0027   echo "Using local file as input" 1>&2
0028   INPUT=(cat ${FILE})
0029 else
0030   echo "Using ${BASEURL}${FILE}${BASEJOB} as input" 1>&2
0031   INPUT=(curl ${BASEURL}${FILE}${BASEJOB})
0032 fi
0033
0034 # Output file
0035 output=$(basename ${FILE} .csv)-$(date --iso-8601=minutes).csv
0036
0037 # Loop over input
0038 "${INPUT[@]}" | grep -v ^curl | while IFS="," read file ext ntotal dt0 dt1 duf0 duf1 dur0 dur1 ; do
0039   if [[ "${file}" =~ csv$ ]] ; then
0040     ${0} ${TEMPLATE} ${TYPE} ${file} ${TARGET}
0041   else
0042     nevents=$(echo "scale=0; n=(3600*$TARGET-$dt0)/$dt1; if (n>$ntotal) print($ntotal) else print(n)" | bc -l)
0043     nchunks=$(echo "scale=0; n=$ntotal/$nevents+1; if (n==0) print(1) else print(n)" | bc -l)
0044     nevents=$(echo "scale=0; $ntotal/$nchunks" | bc -l)
0045     actualt=$(echo "scale=2; ($dt0+$nevents*$dt1)/3600" | bc -l)
0046     for ichunk in `seq 0 $((nchunks-1))` ; do
0047       echo "${file},${ext},${nevents},$(printf '%04d' $ichunk)"
0048     done
0049   fi
0050 done > ${output}
0051 echo "${output}"