eic-opticks/ana/profilesmry.py

0001 #!/usr/bin/env python
0002 #
0003 # Copyright (c) 2019 Opticks Team. All Rights Reserved.
0004 #
0005 # This file is part of Opticks
0006 # (see https://bitbucket.org/simoncblyth/opticks).
0007 #
0008 # Licensed under the Apache License, Version 2.0 (the "License");
0009 # you may not use this file except in compliance with the License.
0010 # You may obtain a copy of the License at
0011 #
0012 #   http://www.apache.org/licenses/LICENSE-2.0
0013 #
0014 # Unless required by applicable law or agreed to in writing, software
0015 # distributed under the License is distributed on an "AS IS" BASIS,
0016 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0017 # See the License for the specific language governing permissions and
0018 # limitations under the License.
0019 #
0020
0021 """
0022 profilesmry.py
0023 ===============
0024
0025 ::
0026
0027     an ; ip profilesmry.py
0028       ## loads times from scans, after manual adjustment of pfx and cat startswith in __main__
0029
0030 """
0031
0032 from __future__ import print_function
0033 import os, sys, re, logging, numpy as np, argparse, textwrap
0034 from collections import OrderedDict as odict
0035 log = logging.getLogger(__name__)
0036
0037 from opticks.ana.num import Num
0038 from opticks.ana.base import findfile
0039 from opticks.ana.profile_ import Profile
0040 from opticks.ana.bashnotes import BashNotes
0041
0042 class ProfileSmry(object):
0043     """
0044     A ProfileSmry is an ordered dict of Profiles, keyed by cat
0045     the Profiles are read from OpticksProfile.npy files written
0046     by Opticks executables.
0047     """
0048
0049     BASE = "$OPTICKS_EVENT_BASE"
0050
0051     @classmethod
0052     def LoadHit_(cls, htpath):
0053         return np.load(htpath) if os.path.exists(htpath) else None
0054
0055
0056     @classmethod
0057     def Load_(cls, pfx, base, startswith=None):
0058         """
0059         :param pfx: prefix, see scan-vi, eg scan-ph-0
0060         :param base: directory
0061         :param startswith: used to select the *cat* category of runs
0062                            the cat is the path element after evt,
0063                            an example of *cat* for scan-ph is cvd_0_rtx_1_100M
0064
0065         :return s: odict keyed by cat with Profile instances
0066
0067         Finds all persisted profiles with selected prefix that meet the startswith selection,
0068         collecting them into an odict which is returned.
0069         """
0070         assert not base is None
0071         if startswith is None:
0072             select_ = lambda n:True
0073         else:
0074             select_ = lambda n:n.find(startswith) == 0
0075         pass
0076
0077         relp = findfile(base, Profile.NAME )   # list of relative paths beneath base
0078         #log.info("base %s relp %d : NAME %s startswith %s " % (base, len(relp), Profile.NAME, startswith ))
0079
0080         s = odict()
0081         for rp in relp:
0082             path = os.path.join(base, rp)
0083             elem = path.split("/")
0084             cat = elem[elem.index("evt")+1]
0085             sel = select_(cat)
0086
0087             #log.info("path %s cat %s sel %s " % (path, cat, sel) )
0088
0089             if not sel: continue
0090             name = cat
0091             ecat = cat.split("_")
0092             npho = Num.Int( ecat[-1] )
0093
0094             dir_ = os.path.dirname(path)
0095
0096             prof = Profile(dir_, name)
0097             prof.npho = npho
0098
0099             htpath1 = os.path.join(dir_, "1", "ht.npy")
0100             ## with multi-event running, currently get all the same hits for tags 1,2,3,...
0101             ## the differences are between cats
0102             ht = cls.LoadHit_(htpath1)
0103             nhit = ht.shape[0] if not ht is None else -1
0104             prof.ht = ht
0105             prof.nhit = nhit
0106
0107             ihit = prof.npho/prof.nhit
0108
0109             #print("car %20s npho %9d nhit %9d ihit %5d     path %s " % (cat, prof.npho, prof.nhit, ihit,  path))
0110             s[cat] = prof
0111         pass
0112         return s
0113
0114     @classmethod
0115     def Base(cls, pfx, base=None):
0116         if base is None:
0117             base = cls.BASE
0118         pass
0119         base = os.path.expandvars(os.path.join(base,pfx))
0120         return base
0121
0122
0123     CATPTN = re.compile("^cvd_(?P<cvd>\d)_rtx_(?P<rtx>\d)_(?P<M>\d*)M$")
0124
0125     @classmethod
0126     def ExamineCats(cls, pfx, base=None):
0127         base = cls.Base(pfx, base)
0128         evtdir = os.path.join(base, "evt")
0129         cats = os.listdir(evtdir)
0130         c = {}
0131         for cat in cats:
0132             m = cls.CATPTN.match(cat)
0133             if not m:
0134                 log.error("failed to match %s " % cat )
0135                 continue
0136             pass
0137             c[cat] = m.groupdict()
0138         pass
0139         return c
0140
0141     @classmethod
0142     def UCVD(cls, c ):
0143         ucvd = list(set(map(lambda d:d["cvd"], c.values())))
0144         return ucvd
0145
0146     @classmethod
0147     def Load(cls, pfx, base=None, startswith=None, gpufallback=None):
0148         base = cls.Base(pfx, base)
0149         s = cls.Load_(pfx, base, startswith)
0150         ps = cls.FromDict(s, pfx, startswith)
0151         ps.base = base
0152         ps.gpufallback = gpufallback
0153         return ps
0154
0155     @classmethod
0156     def FromDict(cls, s, pfx, startswith):
0157         """
0158         :param s: raw odict keyed with cat with Profile instance values
0159
0160         Creates ProfileSmry instance comprising arrays populated
0161         from the Profile instances
0162         """
0163         launch = np.zeros( [len(s), 10], dtype=np.float32 )
0164         alaunch = np.zeros( [len(s) ], dtype=np.float32 )
0165         interval = np.zeros( [len(s), 9], dtype=np.float32 )
0166         ainterval = np.zeros( [len(s)], dtype=np.float32 )
0167         npho = np.zeros( len(s), dtype=np.int32 )
0168         nhit = np.zeros( len(s), dtype=np.int32 )
0169         meta = np.zeros( len(s), dtype=np.object )
0170
0171         for i, kv in enumerate(sorted(s.items(), key=lambda kv:kv[1].npho )):
0172             prof = kv[1]
0173
0174             npho[i] = prof.npho
0175             nhit[i] = prof.nhit
0176             launch[i] = prof.launch
0177             meta[i] = prof.meta
0178
0179             alaunch[i] = np.average( launch[i][1:] )
0180             interval[i] = prof.start_interval
0181             ainterval[i] = np.average( interval[i][1:] )
0182         pass
0183
0184         ps = cls(s)
0185         ps.launch = launch
0186         ps.alaunch = alaunch
0187         ps.interval = interval
0188         ps.ainterval = ainterval
0189         ps.npho = npho
0190         ps.nhit = nhit
0191         ps.creator = "FromDict:%s:%s" % (pfx,startswith )
0192         ps.meta = meta
0193         ps.postinit()
0194
0195         return ps
0196
0197
0198     @classmethod
0199     def FromExtrapolation(cls, npho, seconds_1M=0. ):
0200         """
0201         See notes/issues/geant4-beamOn-profiling.rst
0202
0203         100 s : for tboolean-box scan-ph
0204         239 s : for full JUNO  scan-pf before alignment shakedown
0205         """
0206         assert seconds_1M > 0, seconds_1M
0207
0208         s = odict()
0209         ps = cls(s)
0210         ps.npho = npho
0211         xtim =  (npho/1e6)*seconds_1M
0212         ps.alaunch = xtim
0213         ps.ainterval = xtim
0214         ps.creator = "FromExtrapolation"
0215         return ps
0216
0217     @classmethod
0218     def FromAB(cls, a, b, att="ainterval"):
0219         s = odict()
0220         ps = cls(s)
0221         assert np.all( a.npho == b.npho )
0222         ps.npho = a.npho
0223         ps.ratio = getattr(b,att)/getattr(a, att)
0224         ps.creator = "FromAB"
0225         return ps
0226
0227     @classmethod
0228     def FromAtt(cls, a, num_att="ainterval", den_att="alaunch" ):
0229         s = odict()
0230         ps = cls(s)
0231         ps.npho = a.npho
0232         ps.ratio = getattr(a,num_att)/getattr(a, den_att)
0233         ps.creator = "FromAtt"
0234         return ps
0235
0236
0237     def __init__(self, s):
0238         self.s = s
0239         self.d = odict()
0240         self.gpufallback = "?"
0241
0242
0243     COMMON = r"""
0244     CDeviceBriefAll
0245     CDeviceBriefVis
0246     RTXMode
0247     NVIDIA_DRIVER_VERSION
0248     """
0249     def commonk(self):
0250         return filter(None,textwrap.dedent(self.COMMON).split("\n"))
0251
0252     def postinit(self):
0253         for k in self.commonk():
0254             self.d[k] = self.metacommon(k)
0255         pass
0256
0257     def descmeta(self):
0258         return "\n".join(["%25s : %s " % (k, v) for k,v in self.d.items()])
0259
0260     def _get_gpu(self):
0261         return self.d.get('CDeviceBriefVis',self.gpufallback)
0262     gpu = property(_get_gpu)
0263
0264     def _get_rtx(self):
0265         RTXMode = self.d.get('RTXMode', None)
0266         assert RTXMode in [None,0,1,2]
0267         e = { None:"?", 0:"OFF", 1:"ON", 2:"ON.Tri" }
0268         return "RTX %s" % e[RTXMode]
0269     rtx = property(_get_rtx)
0270
0271     def _get_autolabel(self):
0272         return "%s, %s" % (self.gpu, self.rtx)
0273     autolabel = property(_get_autolabel)
0274
0275     def metacommon(self, k):
0276         vv = list(set(map( lambda m:m.get(k, None), self.meta )))
0277         assert len(vv) in [0,1], vv
0278         return vv[0] if len(vv) == 1 else None
0279
0280     def desc(self):
0281         return "ProfileSmry %s %s %s " % (self.creator, getattr(self, 'base',""), self.d.get('CDeviceBriefVis','-') )
0282
0283     def __repr__(self):
0284         return "\n".join([self.desc(), self.autolabel, self.descmeta(), Profile.Labels()]+map(lambda kv:repr(kv[1]), sorted(self.s.items(), key=lambda kv:kv[1].npho )  ))
0285
0286
0287
0288 class ProfileMain(object):
0289     @classmethod
0290     def ParseArgs(cls, doc):
0291         parser = argparse.ArgumentParser(__doc__)
0292         default_cvd = os.environ.get("OPTICKS_DEFAULT_INTEROP_CVD", "0")  ## hmm this is broken by scan-rsync when looking as scans from another machine
0293         parser.add_argument( "--pfx", default="scan-pf", help="Start of prefix to be appended with a hyphen and integer, beneath which to search for OpticksProfile.npy" )
0294         parser.add_argument( "--g4_seconds_1M", default=239.0, help="Number of seconds for G4 obtained by 1M run of OKG4Test" )
0295         parser.add_argument( "vers", nargs="*", default=[10], type=int, help="Prefix beneath which to search for OpticksProfile.npy" )
0296         parser.add_argument( "--cvd", default=default_cvd, help="CUDA_VISIBLE_DEVICE for the named GPU" )
0297         parser.add_argument( "--gpufallback", default="Quadro_RTX_8000", help="Fallback GPU Name for older scans without this metadata, eg TITAN_RTX" )
0298         args = parser.parse_args()
0299         return cls(args)
0300
0301     def get_pfx(self, v):
0302         return "%s-%s" % ( self.args.pfx, v)
0303
0304     def get_cvd(self, pfx):
0305         """
0306         When only one cvd in the cats return it,
0307         otherwise return the argument
0308         """
0309         c = ProfileSmry.ExamineCats(pfx)
0310         ucvd = ProfileSmry.UCVD(c)
0311         if len(ucvd) == 1:
0312             cvd = ucvd[0]
0313         else:
0314             log.info("mixed cvd using argument %s " % pm.cvd )
0315             cvd = pm.cvd
0316         pass
0317         return cvd
0318
0319     def _get_bashcmd(self):
0320         pfx = self.args.pfx  # without version tail -0 -1
0321         elem = pfx.split("-")
0322         assert len(elem) == 2
0323         return "%s-;%s-notes" % (elem[0], pfx)
0324     bashcmd = property(_get_bashcmd)
0325
0326     def __init__(self, args):
0327         self.args = args
0328         self.vers = args.vers
0329         self.pfx0 = self.get_pfx(self.vers[0])
0330         self.cvd = args.cvd
0331         self.gpufallback = args.gpufallback
0332         self.g4_seconds_1M = args.g4_seconds_1M
0333
0334         bashcmd = self.bashcmd
0335         log.info("lookup BashNotes from %s " % bashcmd )
0336         self.bnote = BashNotes(bashcmd)
0337
0338
0339 if __name__ == '__main__':
0340     logging.basicConfig(level=logging.INFO)
0341     np.set_printoptions(precision=4, linewidth=200)
0342
0343     pm = ProfileMain.ParseArgs(__doc__)
0344
0345     ps = {}
0346     for v in pm.vers:
0347         pfx = pm.get_pfx(v)
0348         cvd = pm.get_cvd(pfx)
0349
0350         print(" v %d  pfx %s " % (v, pfx))
0351         print(" %s " % (pm.bnote(v)))
0352
0353         ps[0] = ProfileSmry.Load(pfx, startswith="cvd_%s_rtx_0" % cvd)
0354         ps[1] = ProfileSmry.Load(pfx, startswith="cvd_%s_rtx_1" % cvd)
0355
0356         #ps[9] = ProfileSmry.FromExtrapolation( ps[0].npho,  time_for_1M=239. )
0357
0358         print("\n")
0359         print(ps[0])
0360         print("\n")
0361         print(ps[1])
0362         #print(ps[9])
0363     pass
0364
0365
0366