eic-opticks/ana/bench.py

0001 #!/usr/bin/env python
0002 #
0003 # Copyright (c) 2019 Opticks Team. All Rights Reserved.
0004 #
0005 # This file is part of Opticks
0006 # (see https://bitbucket.org/simoncblyth/opticks).
0007 #
0008 # Licensed under the Apache License, Version 2.0 (the "License");
0009 # you may not use this file except in compliance with the License.
0010 # You may obtain a copy of the License at
0011 #
0012 #   http://www.apache.org/licenses/LICENSE-2.0
0013 #
0014 # Unless required by applicable law or agreed to in writing, software
0015 # distributed under the License is distributed on an "AS IS" BASIS,
0016 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0017 # See the License for the specific language governing permissions and
0018 # limitations under the License.
0019 #
0020
0021 """
0022 bench.py
0023 ============
0024
0025 Presents launchAVG times and prelaunch times for groups of Opticks runs
0026 with filtering based on commandline arguments of the runs and the digest
0027 of the geocache used.
0028
0029 ::
0030
0031     bench.py --include xanalytic --digest f6cc352e44243f8fa536ab483ad390ce
0032     bench.py --include xanalytic --digest f6
0033         selecting analytic results for a particular geometry
0034
0035     bench.py --include xanalytic --digest 52e --since May22_1030
0036         selecting analytic results for a particular geometry after some time
0037
0038     bench.py --digest 52 --since 6pm
0039
0040     bench.py --name geocache-bench360
0041          fullname of the results dir
0042
0043     bench.py --name 360
0044          also works with just a tail string, so long as it selects
0045          one of the results dirs
0046
0047
0048     bench.py --name 360 --runlabel R1
0049           select runs with runlabel starting R1
0050
0051
0052 ::
0053
0054     ipython -i $(which bench.py) -- --name geocache-bench360 --include xanalytic --include 10240,5760,1
0055
0056
0057 """
0058 import os, re, logging, sys, argparse
0059 from collections import OrderedDict as odict
0060 import numpy as np
0061 log = logging.getLogger(__name__)
0062
0063 from dateutil.parser import parse
0064 from datetime import datetime
0065 from opticks.ana.datedfolder import DatedFolder, dateparser
0066 from opticks.ana.meta import Meta
0067 from opticks.ana.key import Key
0068
0069
0070 def lol2l(lol):
0071     l = []
0072     if lol is not None:
0073         for ii in lol:
0074             assert len(ii) == 1, "expecting a list of lists of length 1"
0075             i = ii[0]
0076             l.append(i)
0077         pass
0078     pass
0079     return l
0080
0081
0082 class Bench(object):
0083
0084     @classmethod
0085     def Args(cls):
0086         argline = "bench.py %s" % " ".join(sys.argv[1:])
0087         print(argline)
0088
0089         parser = argparse.ArgumentParser(__doc__)
0090
0091         resultsprefix = "$OPTICKS_RESULTS_PREFIX" if os.environ.has_key("OPTICKS_RESULTS_PREFIX") else "$TMP"  ## equivalent to BOpticksResource::ResolveResultsPrefix
0092         parser.add_argument( "--resultsdir", default=os.path.join(resultsprefix, "results"), help="Directory path to results" )
0093         parser.add_argument( "--name", default="bench", help="String at the end of the directory names beneath resultsdir in which to look for results. Default %(default)s.")
0094         parser.add_argument( "--digest", default=None, help="Select result groups using geocaches with digests that start with the option string")
0095         parser.add_argument( "--since", default=None, help="Select results from dated folders following the date string provided, eg May22_1030 or 20190522_173746")
0096         parser.add_argument( "--include", default=None, action='append', nargs='*', help="Select result groups with commandline containing the string provided. ALL the strings when repeated" )
0097         parser.add_argument( "--exclude", default=None, action='append', nargs='*', help="Select result groupd with commandline NOT containing the string. NOT containing ANY of the strings when repeated" )
0098         parser.add_argument( "--runlabel", default=None, help="Select result groups with runlabel starting with the string provided." )
0099         parser.add_argument( "--xrunlabel", default=None, help="Exclude result groups with runlabel starting with the string provided." )
0100         parser.add_argument( "--metric", default="launchAVG", help="Quantity key to present in comparison tables. Default %(default)s." );
0101         parser.add_argument( "--other", default="prelaunch000", help="Another quantity key to list in tables. Default %(default)s." );
0102         parser.add_argument( "--nodirs", dest="dirs", action="store_false", default=True, help="Skip the listing of results dirs, for more compact output." );
0103         parser.add_argument( "--splay",  action="store_true", default=False, help="Display the example commandline in a more readable but space consuming form." );
0104         parser.add_argument( "--nosort",  action="store_true", default=False, help="Dont display time sorted." );
0105         args = parser.parse_args()
0106         print(args)
0107         args.argline = argline
0108         return args
0109
0110
0111     def _get_since(self):
0112         """
0113         Parse since strings such as 0930, May22_1030 or 20190522_173746 into a datetime
0114         """
0115         args = self.args
0116         if args.since is not None:
0117             now = datetime.now()
0118             default = datetime(now.year, now.month, now.day)
0119             since = parse(args.since.replace("_"," "), default=default)
0120             print("since : %s " % since )
0121         else:
0122             since = None
0123         pass
0124     since = property(_get_since)
0125
0126
0127     def _get_base(self):
0128         args = self.args
0129         if self._base is None:
0130             rnames = filter( lambda rname:rname.endswith(args.name),  os.listdir(os.path.expandvars(args.resultsdir)) )
0131             #print(rnames)
0132             assert len(rnames) == 1, rnames
0133             rname = rnames[0]
0134
0135             base = os.path.join( args.resultsdir, rname )
0136             base = os.path.expandvars(base)
0137             self._base = base
0138             print("base %s" % base)
0139         pass
0140         return self._base
0141     base = property(_get_base)
0142
0143
0144     def get_udirs(self, df):
0145         args = self.args
0146         udirs = filter(lambda _:_.endswith(df),self.dirs)
0147         if args.runlabel is not None:
0148             udirs = filter(lambda udir:os.path.dirname(udir).startswith(args.runlabel),  udirs)
0149         pass
0150         if args.xrunlabel is not None:
0151             udirs = filter(lambda udir:not os.path.dirname(udir).startswith(args.xrunlabel),  udirs)
0152         pass
0153         return udirs
0154
0155     def find(self, df):
0156         rgs = filter(lambda rg:rg.df == df,  self.rgs)
0157         assert len(rgs) == 1
0158         return rgs[0]
0159
0160     def __init__(self, args):
0161         self.args = args
0162
0163         self.metric = args.metric
0164         self.metric_ = lambda m:float(m.d["OTracerTimes"][args.metric])
0165         self.other = args.other
0166         self.other_ = lambda m:float(m.d["OTracerTimes"][args.other])
0167
0168         self.argline = args.argline
0169         self._base = None
0170         self.findRunGroups()
0171
0172     def findRunGroups(self):
0173         """
0174         Arrange into groups of runs with the same runstamp/datedfolder
0175         """
0176         dirs, dfolds, dtimes = DatedFolder.find(self.base)
0177         assert len(dfolds) == len(dtimes)
0178
0179         self.dirs = dirs
0180         self.dfolds = dfolds
0181         self.dtimes = dtimes
0182
0183         order = sorted(range(len(dfolds)), key=lambda i:dtimes[i])   ## sorted by run datetimes
0184         self.order = order
0185
0186         rgs = []
0187         for i in order:
0188             rg = RunGroup.Make(self, i)
0189             if rg is None: continue
0190             rgs.append(rg)
0191         pass
0192         self.rgs = rgs
0193
0194     def head(self):
0195         return []
0196
0197     def body(self):
0198         return map(repr, self.rgs)
0199
0200     def tail(self):
0201         return ["", self.argline]
0202
0203     def __repr__(self):
0204         return "\n".join(self.head() + self.body() + self.tail())
0205
0206     def __len__(self):
0207         return len(self.rgs)
0208
0209     def __getitem__(self, i):
0210         return self.rgs[i]
0211
0212
0213
0214 labfmt_ = lambda lab:" %30s %10s %10s %10s      %10s " %  lab
0215 rowfmt_ = lambda row:" %30s %10.3f %10.3f %10.3f      %10.3f  %s " % ( row.label, row.metric, row.rfast, row.rslow, row.other, row.absdir )
0216
0217 class RunGroup(object):
0218
0219     dtype = [
0220           ("index", np.int32),
0221           ("label", "|S30"),
0222           ("metric", np.float32),
0223           ("rfast", np.float32),
0224           ("rslow", np.float32),
0225           ("other", np.float32),
0226           ("absdir", "|S64"),
0227             ]
0228
0229     @classmethod
0230     def Make(cls, b, i):
0231
0232         df = b.dfolds[i]
0233         dt = b.dtimes[i]
0234         udirs = b.get_udirs(df)
0235
0236         if len(udirs) == 0: return None
0237         mm,smm,mcmd,geof,key,cmdline = cls.LoadMeta(b, udirs, dt)
0238         if key is None: return None
0239
0240         rg = cls(b, i, df, dt, mm,smm,mcmd,geof,key,cmdline)
0241         return rg
0242
0243     @classmethod
0244     def LoadMeta(cls, b, udirs, dt ):
0245
0246         mm = [Meta(p, b.base) for p in udirs]
0247         smm = sorted(mm, key=b.metric_)
0248         cmdline = smm[0].d["parameters"]["CMDLINE"]
0249
0250         selected = cls.Selected( b, dt, cmdline )
0251         if selected:
0252             def key_(m):
0253                 d = m.d["parameters"]
0254                 k0 = d.get("OPTICKS_KEY",None)
0255                 k1 = d.get("KEY",None)
0256                 kk = list(set(filter(None, [k0,k1])))
0257                 assert len(kk) == 1, d
0258                 return kk[0]
0259             pass
0260             groupcommand_ = lambda m:m.d["parameters"].get("GROUPCOMMAND","-")
0261             geofunc_ = lambda m:m.d["parameters"].get("GEOFUNC","-")
0262
0263             keys = map(key_, smm)
0264             mcmds = map(groupcommand_, smm)
0265             geofs = map(geofunc_, smm)
0266
0267             assert len(set(mcmds)) == 1, "all OPTICKS_GROUPCOMMAND for a group of runs with same dated folder should be identical"
0268             assert len(set(geofs)) == 1, "all OPTICKS_GEOFUNC for a group of runs with same dated folder should be identical"
0269             assert len(set(keys)) == 1, "all OPTICKS_KEY for a group of runs with same dated folder should be identical "
0270
0271             mcmd = mcmds[0]
0272             geof = geofs[0]
0273             key = keys[0]
0274         else:
0275             mcmd = None
0276             geof = None
0277             key = None
0278         pass
0279         return mm,smm,mcmd,geof,key,cmdline
0280
0281     @classmethod
0282     def Selected(cls, b, dt, cmdline):
0283         args = b.args
0284         select = False
0285
0286         includes = lol2l(args.include)
0287         excludes = lol2l(args.exclude)
0288
0289         if len(includes)>0:
0290             found = list(set(map(lambda include:cmdline.find(include) > -1, includes)))
0291             if len(found) == 1 and found[0] == True:
0292                 select=True
0293             else:
0294                 select=False
0295             pass
0296         elif len(excludes)>0:
0297             notfound = list(set(map(lambda exclude:cmdline.find(exclude) == -1, excludes)))
0298             if len(notfound) == 1 and notfound[0] == True:
0299                 select=True
0300             else:
0301                 select=False
0302             pass
0303         elif args.digest is not None and not digest.startswith(args.digest):
0304             select=False
0305         elif b.since is not None and not dt > b.since:
0306             select=False
0307         else:
0308             select=True
0309         pass
0310         return select
0311
0312     def __repr__(self):
0313         return "\n".join(self.head() + self.body() + self.tail())
0314
0315     def _get_path(self):
0316         return os.path.expandvars(os.path.join("$TMP/ana", "%s_%s.png" % ("bench",self.df) ))
0317     path = property(_get_path)
0318
0319     def head(self):
0320         lines = ["---  GROUPCOMMAND : %s  GEOFUNC : %s " % (self.mcmd, self.geof)]
0321         if self.b.args.splay:
0322             lines.append("\\\n    --".join(self.cmdline.split("--")))
0323         else:
0324             lines.append(self.cmdline)
0325         pass
0326         k = Key(key=self.key)
0327         digest = k.digest
0328         idpath = k.keydir
0329         lines.extend([self.name, self.key, idpath, self.labels])
0330         return lines
0331
0332     def body(self):
0333         return map(rowfmt_, self.a )
0334
0335     def tail(self):
0336         return map(lambda kv:"  %30s %10.3f " % (kv[0], kv[1]), self.r.items() )
0337
0338     def make_a(self):
0339         b = self.b
0340         args = b.args
0341         mm = self.mm
0342         smm = self.smm
0343         umm = mm if args.nosort else smm
0344         ffast = b.metric_(smm[0])
0345         fslow = b.metric_(smm[-1])
0346         a = np.recarray((len(self.mm),), dtype=self.dtype )
0347         d = odict()
0348         for i, m in enumerate(umm):
0349             f = b.metric_(m)
0350             rfast = f/ffast
0351             rslow = f/fslow
0352             o = b.other_(m)
0353             a[i] = (i, m.parentfold, f, rfast, rslow, o, m.absdir )
0354             d[m.parentfold] = f
0355         pass
0356         r = odict()
0357         for k,v in args.ratios.items():
0358             assert len(v) == 2, v
0359             num,den = v
0360             if num in d and den in d:
0361                 r[k] = d[num]/d[den]
0362             pass
0363         pass
0364         return a, d, r
0365
0366     def __init__(self, b, i, df, dt, mm, smm, mcmd, geof, key, cmdline):
0367         self.b = b
0368         self.args = b.args
0369         self.base = b.base
0370         self.metric = b.metric
0371
0372         self.i = i
0373         self.df = df
0374         self.dt = dt
0375         self.mm = mm
0376         self.smm = smm
0377         self.mcmd = mcmd
0378         self.geof = geof
0379         self.key = key
0380         self.cmdline = cmdline
0381         self.name = "bench%d" % i
0382
0383         lab = ( df, b.metric,"rfast", "rslow", b.other)
0384         self.labels = labfmt_(lab)
0385
0386         a, d, r = self.make_a()
0387
0388         self.a = a
0389         self.d = d
0390         self.r = r
0391
0392
0393
0394
0395 if __name__ == '__main__':
0396     logging.basicConfig(level=logging.INFO)
0397
0398     ratios = odict()
0399     ratios["R0/1_TITAN_V"] = "R0_TITAN_V R1_TITAN_V".split()
0400     ratios["R0/1_TITAN_RTX"] = "R0_TITAN_RTX R1_TITAN_RTX".split()
0401     ratios["R1/0_TITAN_V"] = "R1_TITAN_V R0_TITAN_V".split()
0402     ratios["R1/0_TITAN_RTX"] = "R1_TITAN_RTX R0_TITAN_RTX".split()
0403
0404     args = Bench.Args()
0405     args.ratios = ratios
0406
0407     b = Bench(args)
0408     print(b)
0409
0410
0411