eic-opticks/sysrap/smonitor.sh

0001 #!/bin/bash
0002 usage(){ cat << EOU
0003 smonitor.sh : NVML based GPU memory monitor
0004 =============================================
0005
0006 Usage:
0007
0008 1. start the monitor process in one tab::
0009
0010     ~/o/sysrap/smonitor.sh   # builds and starts runloop
0011
0012 2. start the GPU memory using job in another tab
0013
0014 3. once the job has completed, ctrl-C the monitor process
0015    which catches the SIGINT signal and saves the memory
0016    profile into smonitor.npy array
0017
0018 4. make a plot::
0019
0020     ~/o/sysrap/smonitor.sh grab
0021     ~/o/sysrap/smonitor.sh ana
0022
0023
0024 5. screen capture the plot with annotation
0025
0026     ~/o/sysrap/smonitor.sh mpcap
0027     PUB=cxs_min_igs_with_rg_dummy ~/o/sysrap/smonitor.sh mppub
0028
0029
0030 EOU
0031 }
0032
0033 SDIR=$(dirname $(realpath $BASH_SOURCE))
0034
0035 name=smonitor
0036
0037 vars="BASH_SOURCE SDIR PWD name TMP FOLD CUDA_PREFIX CUDA_TARGET bin"
0038
0039 cuda_prefix=/usr/local/cuda-11.7
0040 CUDA_PREFIX=${CUDA_PREFIX:-$cuda_prefix}
0041 CUDA_TARGET=$CUDA_PREFIX/targets/x86_64-linux
0042
0043 tmp=/tmp/$USER/opticksGG
0044 TMP=${TMP:-$tmp}
0045 export FOLD=$TMP/$name
0046 mkdir -p $FOLD
0047
0048 bin=$FOLD/$name
0049 script=$SDIR/$name.py
0050
0051 cd $FOLD
0052 LOGDIR=$PWD
0053
0054 #stem=smonitor_okjob
0055 stem=smonitor_cxs_min_igs
0056 pub=no-PUB
0057 export STEM=${STEM:-$stem}
0058 export PUB=${PUB:-$pub}
0059
0060 export smonitor__SLEEP_US=100000  # 0.1s
0061
0062
0063 defarg="info_build_run"
0064 arg=${1:-$defarg}
0065
0066 if [ "${arg/info}" != "$arg" ]; then
0067     for var in $vars ; do printf "%25s : %s \n" "$var" "${!var}" ; done
0068 fi
0069
0070 if [ "${arg/build}" != "$arg" ]; then
0071     gcc $SDIR/$name.cc -std=c++11 -lstdc++ \
0072          -I$SDIR \
0073          -I$CUDA_TARGET/include \
0074          -L$CUDA_TARGET/lib \
0075          -lnvidia-ml  \
0076          -o $bin
0077     [ $? -ne 0 ] && echo $BASH_SOURCE build error && exit 1
0078 fi
0079
0080 if [ "${arg/run}" != "$arg" ]; then
0081     $bin
0082     [ $? -ne 0 ] && echo $BASH_SOURCE run error && exit 2
0083 fi
0084
0085 if [ "${arg/grab}" != "$arg" ]; then
0086     source $OPTICKS_HOME/bin/rsync.sh $LOGDIR
0087     [ $? -ne 0 ] && echo $BASH_SOURCE grab error && exit 3
0088 fi
0089
0090 if [ "${arg/ana}" != "$arg" ]; then
0091     ${IPYTHON:-ipython} --pdb -i $script
0092     [ $? -ne 0 ] && echo $BASH_SOURCE ana error && exit 4
0093 fi
0094
0095 if [ "$arg" == "mpcap" -o "$arg" == "mppub" ]; then
0096     export CAP_BASE=$FOLD/figs
0097     export CAP_REL=smonitor
0098     export CAP_STEM=${STEM}_${PUB}
0099     case $arg in
0100        mpcap) source mpcap.sh cap  ;;
0101        mppub) source mpcap.sh env  ;;
0102     esac
0103     if [ "$arg" == "mppub" ]; then
0104         source epub.sh
0105     fi
0106 fi
0107
0108 exit 0