eic-opticks/ana/metadata.py

0001 #!/usr/bin/env python
0002 #
0003 # Copyright (c) 2019 Opticks Team. All Rights Reserved.
0004 #
0005 # This file is part of Opticks
0006 # (see https://bitbucket.org/simoncblyth/opticks).
0007 #
0008 # Licensed under the Apache License, Version 2.0 (the "License");
0009 # you may not use this file except in compliance with the License.
0010 # You may obtain a copy of the License at
0011 #
0012 #   http://www.apache.org/licenses/LICENSE-2.0
0013 #
0014 # Unless required by applicable law or agreed to in writing, software
0015 # distributed under the License is distributed on an "AS IS" BASIS,
0016 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0017 # See the License for the specific language governing permissions and
0018 # limitations under the License.
0019 #
0020
0021 """
0022 metadata.py
0023 =============
0024
0025 Access the metadata json files written by Opticks runs,
0026 allowing evt digests and run times to be compared.
0027
0028 See also meta.py a more generalized version of this, but
0029 not so fleshed out.
0030
0031 TODO: extract the good stuff from here as migrate from metadata.py to meta.py
0032
0033 ::
0034
0035     [blyth@localhost tmp]$ OPTICKS_EVENT_BASE=tboolean-box metadata.py --det tboolean-box --src torch --pfx .
0036
0037 """
0038
0039 from datetime import datetime
0040 import os, re, logging
0041 import numpy as np
0042 from collections import OrderedDict as odict
0043 log = logging.getLogger(__name__)
0044
0045 from opticks.ana.main import opticks_main
0046 from opticks.ana.base import ini_, json_, splitlines_
0047 from opticks.ana.datedfolder import DatedFolder, dateparser
0048
0049
0050 class DeltaTime(object):
0051     """
0052     hmm reading from delta is brittle, as will change
0053     on updates of profiling points.  Better to read two times
0054     and calculate the delta : as that does not depend in
0055     interposing profile points.
0056     """
0057     NAME = "DeltaTime.ini"
0058     PROPAGATE_G4 = "CG4::propagate_0"
0059     PROPAGATE_OK = "OPropagator::launch_0"
0060
0061     @classmethod
0062     def Path(cls, base, tag):
0063         path = os.path.join(base, cls.NAME)
0064         log.debug("path %s " % path)
0065         return path
0066
0067     def __init__(self, base, tag):
0068         path = self.Path(base, tag)
0069         ini = ini_(path) if os.path.exists(path) else {}
0070         itag = int(tag)
0071         key = self.PROPAGATE_OK if itag > 0 else self.PROPAGATE_G4
0072         propagate = ini.get(key,-1)
0073         self.propagate = propagate
0074         self.ini = ini
0075     pass
0076 pass
0077
0078 class LaunchTimes(object):
0079     NAME = "OpticksEvent_launch.ini"
0080     PROPAGATE_OK = "launch001"
0081     def __init__(self, base, tag):
0082         path = os.path.join(base, self.NAME)
0083         if not os.path.exists(path):
0084             ini = None
0085             propagate = -99.
0086             log.debug("path %s does not exist " % path )
0087         else:
0088             ini = ini_(path)
0089             itag = int(tag)
0090             propagate = ini.get(self.PROPAGATE_OK,-1) if itag > 0 else -1
0091         pass
0092         self.propagate = propagate
0093         self.ini = ini
0094     pass
0095 pass
0096
0097
0098 class CommandLine(object):
0099     def __init__(self, cmdline):
0100         self.cmdline = cmdline
0101     def has(self, opt):
0102         return 1 if self.cmdline.find(opt) > -1 else 0
0103     def __repr__(self):
0104         return "\n".join(self.cmdline.split())
0105
0106
0107 ratio_ = lambda num,den:float(num)/float(den) if den != 0 else -1
0108
0109 class CompareMetadata(object):
0110
0111     permit_mismatch = ["cmdline", "mode"]
0112
0113     def __init__(self, am, bm):
0114         self.am = am
0115         self.bm = bm
0116
0117         self.GEOCACHE = self.expected_common("GEOCACHE", parameter=False)
0118         self.numPhotons = self.expected_common("numPhotons", parameter=False)
0119         self.mode =  self.expected_common("mode", parameter=False)
0120
0121         self.TestCSGPath =  self.expected_common("TestCSGPath", parameter=False)
0122         self.csgmeta0 = self.expected_common("csgmeta0", parameter=False)  # container metadata, usually an emitter
0123
0124         cmdline = self.expected_common( "cmdline", parameter=True)
0125         self.cmdline = CommandLine(cmdline)
0126         self.Switches = self.expected_common( "Switches", parameter=True)
0127         self.align = self.cmdline.has("--align ")
0128         self.utaildebug = self.cmdline.has("--utaildebug ")
0129         self.reflectcheat = self.cmdline.has("--reflectcheat ")
0130         self.factor = ratio_(bm.propagate0, am.propagate0)
0131
0132     def _get_crucial(self):
0133         """
0134         Amplify some of the crucial options for comparisons
0135         """
0136         return " ".join([
0137             "ALIGN" if self.align==1 else "non-align" ,
0138             "REFLECTCHEAT" if self.reflectcheat==1 else "non-reflectcheat" ,
0139             "UTAILDEBUG" if self.utaildebug==1 else "non-utaildebug"
0140             ])
0141     crucial = property(_get_crucial)
0142
0143     def expected_common(self, key, parameter=True):
0144         if parameter:
0145             av = self.am.parameters.get(key,"")
0146             bv = self.bm.parameters.get(key,"")
0147         else:
0148             av = getattr( self.am, key )
0149             bv = getattr( self.bm, key )
0150         pass
0151
0152         match = av == bv
0153
0154         if not match:
0155             if key in self.permit_mismatch:
0156                 log.warning("note permitted expected_common mismatch for key %s " % key )
0157                 log.info(av)
0158                 log.info(bv)
0159             else:
0160                 log.fatal("expected_common mismatch for key %s " % key )
0161                 log.fatal("av:%s"%av)
0162                 log.fatal("bv:%s"%bv)
0163                 assert match, key
0164             pass
0165         return av
0166
0167     def __repr__(self):
0168
0169         dfmt_ = lambda d:"-" if d is None else " ".join(["%s:%s" % (kv[0],kv[1]) for kv in sorted(d.items(),key=lambda kv:kv[0])])
0170
0171         return "\n".join([
0172               "ab.cfm",
0173               self.brief(),
0174               "ab.a.metadata:%s" % self.am,
0175               "ab.b.metadata:%s" % self.bm,
0176               self.Switches,
0177               ".",
0178               dfmt_(self.csgmeta0),
0179               #repr(self.cmdline),
0180               "."
0181                         ])
0182
0183     def brief(self):
0184         return "nph:%8d A:%10.4f B:%10.4f B/A:%10.1f %s %s " % (self.numPhotons, self.am.propagate0, self.bm.propagate0, self.factor, self.mode, self.crucial )
0185
0186
0187 class Metadata(object):
0188     """
0189     v2 layout::
0190
0191         simon:ana blyth$ l $TMP/evt/PmtInBox/torch/10/
0192         total 55600
0193         drwxr-xr-x  6 blyth  wheel       204 Aug 19 15:32 20160819_153245
0194         -rw-r--r--  1 blyth  wheel       100 Aug 19 15:32 Boundary_IndexLocal.json
0195         -rw-r--r--  1 blyth  wheel       111 Aug 19 15:32 Boundary_IndexSource.json
0196         ...
0197         -rw-r--r--  1 blyth  wheel   6400080 Aug 19 15:32 ox.npy
0198         -rw-r--r--  1 blyth  wheel      1069 Aug 19 15:32 parameters.json
0199         -rw-r--r--  1 blyth  wheel   1600080 Aug 19 15:32 ph.npy
0200         -rw-r--r--  1 blyth  wheel    400080 Aug 19 15:32 ps.npy
0201         -rw-r--r--  1 blyth  wheel      2219 Aug 19 15:32 report.txt
0202         -rw-r--r--  1 blyth  wheel   4000096 Aug 19 15:32 rs.npy
0203         -rw-r--r--  1 blyth  wheel  16000096 Aug 19 15:32 rx.npy
0204         -rw-r--r--  1 blyth  wheel       763 Aug 19 15:32 t_absolute.ini
0205         -rw-r--r--  1 blyth  wheel       817 Aug 19 15:32 t_delta.ini
0206         drwxr-xr-x  6 blyth  wheel       204 Aug 18 20:53 20160818_205342
0207
0208     timestamp folders contain just metadata for prior runs not full evt::
0209
0210         simon:ana blyth$ l /tmp/blyth/opticks/evt/PmtInBox/torch/10/20160819_153245/
0211         total 32
0212         -rw-r--r--  1 blyth  wheel  1069 Aug 19 15:32 parameters.json
0213         -rw-r--r--  1 blyth  wheel  2219 Aug 19 15:32 report.txt
0214         -rw-r--r--  1 blyth  wheel   763 Aug 19 15:32 t_absolute.ini
0215         -rw-r--r--  1 blyth  wheel   817 Aug 19 15:32 t_delta.ini
0216
0217
0218     """
0219
0220     COMPUTE = 0x1 << 1
0221     INTEROP = 0x1 << 2
0222     CFG4    = 0x1 << 3
0223
0224     date_ptn = re.compile("\d{8}_\d{6}")  # eg 20160817_141731
0225
0226
0227
0228
0229     def __init__(self, path, base=None):
0230         """
0231         Path assumed to be a directory with one of two forms::
0232
0233               $TMP/evt/PmtInBox/torch/10/                ## ending with tag
0234               $TMP/evt/PmtInBox/torch/10/20160817_141731 ## ending with datefold
0235
0236         In both cases the directory must contain::
0237
0238               parameters.json
0239               DeltaTime.ini
0240
0241         """
0242         if base is not None:
0243             path = os.path.join(base, path)
0244         pass
0245
0246         jsonpath = os.path.join(path, "parameters.json")
0247         parameters = json_(jsonpath) if os.path.exists(jsonpath) else {}
0248
0249         self.path = path
0250         self.parameters = parameters
0251
0252         basename = os.path.basename(path)
0253         if self.date_ptn.match(basename):
0254              datefold = basename
0255              timestamp = dateparser(datefold)
0256              tag = os.path.basename(os.path.dirname(path))
0257         else:
0258              datefold = None
0259              timestamp = None
0260              tag = basename
0261         pass
0262         self.datefold = datefold
0263         self.tag = tag
0264         self.timestamp = timestamp
0265
0266         self.delta_times = DeltaTime(self.path, tag)
0267         self.launch_times = LaunchTimes(self.path, tag)
0268         self.propagate0 = float(self.delta_times.propagate)
0269         self.propagate = float(self.launch_times.propagate)
0270
0271         self._solids = None
0272
0273
0274     # parameter accessors (from the json)
0275     mode = property(lambda self:self.parameters.get('mode',"no-mode") )  # eg COMPUTE_MODE
0276     photonData = property(lambda self:self.parameters.get('photonData',"no-photonData") ) # digest
0277     recordData = property(lambda self:self.parameters.get('recordData',"no-recordData") )  # digest
0278     sequenceData = property(lambda self:self.parameters.get('sequenceData',"no-sequenceData") )  # digest
0279     numPhotons = property(lambda self:int(self.parameters.get('NumPhotons',"-1")) )
0280     TestCSGPath = property(lambda self:self.parameters.get('TestCSGPath',None) )  # eg tboolean-box
0281     GEOCACHE = property(lambda self:self.parameters.get('GEOCACHE',None) )
0282     Note = property(lambda self:self.parameters.get('Note',"") )
0283
0284     def _flags(self):
0285         flgs = 0
0286         if self.mode.lower().startswith("compute"):
0287             flgs |= self.COMPUTE
0288         elif self.mode.lower().startswith("interop"):
0289             flgs |= self.INTEROP
0290         elif self.mode.lower().startswith("cfg4"):
0291             flgs |= self.CFG4
0292         pass
0293         return flgs
0294     flags = property(_flags)
0295
0296     def __repr__(self):
0297         return "%-60s ox:%32s rx:%32s np:%7d pr:%10.4f %s" % (self.path, self.photonData, self.recordData, self.numPhotons, self.propagate0, self.mode )
0298
0299
0300     def _get_csgbnd(self):
0301         """
0302         csg.txt contains a list of boundaries, one per line::
0303
0304             [blyth@localhost tboolean-box]$ cat csg.txt
0305             Rock//perfectAbsorbSurface/Vacuum
0306             Vacuum///GlassSchottF2
0307
0308         * observed no newline at end
0309         """
0310         if self.TestCSGPath is None:
0311             return []
0312         pass
0313         csgtxt = os.path.join(self.TestCSGPath, "csg.txt")
0314         csgbnd = splitlines_(csgtxt) if os.path.exists(csgtxt) else []
0315         return csgbnd
0316     csgbnd = property(_get_csgbnd)
0317
0318
0319     def _get_csgmeta0(self):
0320         """
0321         Metadata from the outer CSG solid, typically the container::
0322
0323             [blyth@localhost 0]$ js.py meta.json
0324             {u'container': 1,
0325              u'containerscale': 3.0,
0326              u'ctrl': 0,
0327              u'emit': -1,
0328              u'emitconfig': u'photons:100000,wavelength:380,time:0.2,posdelta:0.1,sheetmask:0x1,umin:0.45,umax:0.55,vmin:0.45,vmax:0.55',
0329              u'poly': u'IM',
0330              u'resolution': u'20',
0331              u'verbosity': u'0'}
0332
0333         """
0334         if self.TestCSGPath is None:
0335             return None
0336         pass
0337         csgmeta0_ = os.path.join(self.TestCSGPath, "0", "meta.json")
0338         csgmeta0 = json_(csgmeta0_) if os.path.exists(csgmeta0_) else []
0339         return csgmeta0
0340     csgmeta0 = property(_get_csgmeta0)
0341
0342
0343     def _get_lv(self):
0344         path = self.TestCSGPath
0345         if path is None or len(path) == 0:
0346             lv = None
0347         else:
0348             name = os.path.basename(path)
0349             lv = name.split("-")[-1]
0350         return lv
0351     lv = property(_get_lv)
0352
0353     def _get_solid(self):
0354         lv = self.lv
0355         if lv is None: return None
0356
0357         try:
0358             ilv = int(lv)
0359             solid = self.solids[ilv]
0360         except ValueError:
0361             solid = lv
0362         pass
0363         return solid
0364     solid = property(_get_solid)
0365
0366     def _get_solids(self):
0367         """
0368         List of solids (aka meshes) obtained from "GItemList/GMeshLib.txt" within the basis geocache
0369         """
0370         if self._solids is None:
0371             path = os.path.join(self.GEOCACHE, "GItemList/GMeshLib.txt")
0372             self._solids = splitlines_(path)
0373         return self._solids
0374     solids = property(_get_solids)
0375
0376
0377     def dump(self):
0378         for k,v in self.parameters.items():
0379             print("%20s : %s " % (k, v))
0380         for k,v in self.delta_times.ini.items():
0381             print("%20s : %s " % (k, v))
0382         for k,v in self.launch_times.ini.items():
0383             print("%20s : %s " % (k, v))
0384
0385
0386
0387
0388 def test_metadata():
0389     td = tagdir_("PmtInBox", "torch", "10")
0390     md = Metadata(td)
0391
0392
0393 def test_tagdir():
0394     td = os.path.expandvars("/tmp/$USER/opticks/evt/boolean/torch/1")
0395     md = Metadata(td)
0396     print(md)
0397
0398
0399
0400 if __name__ == '__main__':
0401     ok = opticks_main()
0402     print("ok.brief : %s " % ok.brief)
0403     print("ok.tagdir : %s " % ok.tagdir)
0404
0405     md = Metadata(ok.tagdir)
0406     print("md : %s " % md)
0407
0408 if 0:
0409     md.dump()
0410     csgpath = md.TestCSGPath
0411     print("csgpath", csgpath)
0412     print("csgbnd", md.csgbnd)
0413
0414
0415
0416
0417
0418
0419
0420
0421
0422
0423
0424