eic-opticks/ana/base.py

0001 #!/usr/bin/env python
0002 #
0003 # Copyright (c) 2019 Opticks Team. All Rights Reserved.
0004 #
0005 # This file is part of Opticks
0006 # (see https://bitbucket.org/simoncblyth/opticks).
0007 #
0008 # Licensed under the Apache License, Version 2.0 (the "License");
0009 # you may not use this file except in compliance with the License.
0010 # You may obtain a copy of the License at
0011 #
0012 #   http://www.apache.org/licenses/LICENSE-2.0
0013 #
0014 # Unless required by applicable law or agreed to in writing, software
0015 # distributed under the License is distributed on an "AS IS" BASIS,
0016 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0017 # See the License for the specific language governing permissions and
0018 # limitations under the License.
0019 #
0020
0021 """
0022 Strictly Non-numpy basics
0023
0024
0025 """
0026
0027 import os, logging, json, ctypes, subprocess, datetime, re
0028 log = logging.getLogger(__name__)
0029
0030 from collections import OrderedDict as odict
0031 import numpy as np
0032 from opticks.ana.enum_ import Enum
0033
0034 from opticks.ana.key import keydir
0035 KEYDIR = keydir()
0036
0037 log = logging.getLogger(__name__)
0038
0039
0040 try:
0041     cpp = ctypes.cdll.LoadLibrary('libc++.1.dylib')
0042     ffs_ = lambda _:cpp.ffs(_)
0043 except OSError:
0044     pass
0045
0046
0047 #idp_ = lambda _:"%s/%s" % (os.environ["IDPATH"],_)
0048 #uidp_ = lambda _:_.replace(os.environ["IDPATH"],"$IDPATH")
0049 #
0050 #idp_ = lambda _:"%s/%s" % (KEYDIR,_)
0051 #uidp_ = lambda _:_.replace(KEYDIR,"$KEYDIR")
0052
0053 gcp_ = lambda _:"%s/%s" % (os.environ["GEOCACHE"],_)
0054
0055
0056 import sys, codecs
0057 if sys.version_info.major > 2:
0058     u_ = lambda _:_                            # py3 strings are unicode already
0059     b_ = lambda _:codecs.latin_1_encode(_)[0]  # from py3 unicode string to bytes
0060     d_ = lambda _:codecs.latin_1_decode(_)[0]  # from bytes to py3 unicode string
0061 else:
0062     u_ = lambda _:unicode(_, "utf-8")          # py2 strings are bytes
0063     b_ = lambda _:_
0064     d_ = lambda _:_
0065 pass
0066
0067
0068 def findfile(base, name, relative=True):
0069     paths = []
0070     for root, dirs, files in os.walk(base):
0071         if name in files:
0072             path = os.path.join(root,name)
0073             paths.append(path[len(base)+1:] if relative else path)
0074         pass
0075     pass
0076     return paths
0077
0078
0079 def translate_xml_identifier_(name):
0080     return name.replace("__","/").replace("--","#").replace("..",":")
0081
0082
0083 splitlines_ = lambda txtpath:open(txtpath).read().splitlines()
0084
0085 def now_(fmt="%Y%m%d-%H%M"):
0086     return datetime.datetime.now().strftime(fmt)
0087
0088 def stamp_(path, fmt="%Y%m%d-%H%M"):
0089    if path is None:
0090        return None
0091    elif not os.path.exists(path):
0092        return None
0093    else:
0094        return datetime.datetime.fromtimestamp(os.stat(path).st_ctime).strftime(fmt)
0095    pass
0096
0097
0098 def is_integer_string(s):
0099     try:
0100         int(s)
0101         iis = True
0102     except ValueError:
0103         iis = False
0104     pass
0105     return iis
0106
0107 def list_integer_subdirs(base):
0108     """
0109     return list of subdirs beneath base with names that are lexically integers, sorted as integers
0110     """
0111     return list(sorted(map(int,filter(is_integer_string,os.listdir(os.path.expandvars(base))))))
0112
0113 def dump_extras_meta(base, name="meta.json", fmt=" %(idx)5s : %(height)6s : %(lvname)-40s : %(soname)-40s : %(err)s "):
0114     """
0115     Tabulate content of meta.json files from subdirs with integer names
0116     """
0117     idxs = list_integer_subdirs(base)
0118     assert idxs == range(len(idxs))
0119
0120     log.info("dump_extras_meta base:%s xbase:%s " % (base,expand_(base)))
0121
0122     keys = re.compile("\((\w*)\)").findall(fmt)
0123     print(fmt % dict(zip(keys,keys)))
0124
0125     for idx in idxs:
0126         meta = json_load_(os.path.join(base,str(idx),name))
0127         meta['idx'] = idx
0128         meta['err'] = meta.get('err',"-")
0129         print(fmt % meta)
0130     pass
0131
0132
0133 def makedirs_(path):
0134     pdir = os.path.dirname(path)
0135     if not os.path.exists(pdir):
0136         os.makedirs(pdir)
0137     pass
0138     return path
0139
0140 expand_ = lambda path:os.path.expandvars(os.path.expanduser(path))
0141 json_load_ = lambda path:json.load(open(expand_(path)))
0142 json_save_ = lambda path, d:json.dump(d, open(makedirs_(expand_(path)),"w"), cls=NPEncoder)
0143 json_save_pretty_ = lambda path, d:json.dump(d, open(makedirs_(expand_(path)),"w"),cls=NPEncoder, sort_keys=True, indent=4, separators=(',', ': '))
0144
0145 class NPEncoder(json.JSONEncoder):
0146     """
0147     Custom encoder for numpy data types
0148     https://stackoverflow.com/questions/50916422/python-typeerror-object-of-type-int64-is-not-json-serializable/50916741
0149     """
0150     def default(self, obj):
0151         if isinstance(obj, (np.int_, np.intc, np.intp, np.int8,
0152                             np.int16, np.int32, np.int64, np.uint8,
0153                             np.uint16, np.uint32, np.uint64)):
0154
0155             return int(obj)
0156
0157         elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)):
0158             return float(obj)
0159
0160         elif isinstance(obj, (np.complex_, np.complex64, np.complex128)):
0161             return {'real': obj.real, 'imag': obj.imag}
0162
0163         elif isinstance(obj, (np.ndarray,)):
0164             return obj.tolist()
0165
0166         elif isinstance(obj, (np.bool_)):
0167             return bool(obj)
0168
0169         elif isinstance(obj, (np.void)):
0170             return None
0171
0172         return json.JSONEncoder.default(self, obj)
0173
0174
0175
0176
0177 def manual_mixin( dst, src ):
0178     """
0179     Add all methods from the src class to the destination class
0180
0181     :param dst: destination class
0182     :param src: source class
0183     """
0184     for k,fn in src.__dict__.items():
0185         if k.startswith("_"): continue
0186         setattr(dst, k, fn )
0187     pass
0188
0189 def _subprocess_output(args):
0190     p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
0191     return p.communicate()
0192
0193 def _opticks_default_idpath_from_exe(exe="OpticksIDPATH"):
0194     """
0195     This provides a way to discern the IDPATH by running an
0196     Opticks executable.  This works fine for the default geometry with
0197     no arguments picking alternate geometries.
0198
0199     To make this work for non default geometries would have to
0200     somehow save the opticks geometry selection arguments, given
0201     this might as way stay simple and require the IDPATH envvar
0202     as an input to analysis.
0203     """
0204     stdout, stderr = _subprocess_output([exe])
0205     idpath = stderr.strip()
0206     return idpath
0207
0208 def _opticks_env(st="OPTICKS_ IDPATH"):
0209     return filter(lambda _:_[0].startswith(st.split()), os.environ.items())
0210
0211
0212 def ihex_(i):
0213     """
0214     Trim the 0x and any L from a hex string::
0215
0216         assert ihex_(0xccd) == 'ccd'
0217         assert ihex_(0xccdL) == 'ccd'    # assumed
0218
0219     """
0220     xs = hex(i)[2:]
0221     xs = xs[:-1] if xs[-1] == 'L' else xs
0222     return xs
0223
0224
0225 _ini = {}
0226 def ini_(path):
0227     global _ini
0228     if _ini.get(path,None):
0229         log.debug("return cached ini for key %s" % path)
0230         return _ini[path]
0231     try:
0232         log.debug("parsing ini for key %s" % path)
0233         xpath = os.path.expandvars(os.path.expanduser(path))
0234         txt = open(xpath,"r").read()
0235         lines = list(filter(None, txt.split("\n")))
0236         d = dict(map(lambda _:_.split("="), lines))
0237         _ini[path] = d
0238     except IOError:
0239         log.fatal("failed to load ini from %s" % path)
0240         assert 0, ( path )
0241         _ini[path] = {}
0242     pass
0243     return _ini[path]
0244
0245
0246 _json = {}
0247 def json_(path):
0248     global _json
0249     if _json.get(path,None):
0250         log.debug("return cached json for key %s" % path)
0251         return _json[path]
0252         return js
0253     try:
0254         log.debug("parsing json for key %s" % path)
0255         xpath = os.path.expandvars(os.path.expanduser(path))
0256         #log.info("xpath:%s"%xpath)
0257         js = json.load(open(xpath,"r"))
0258         #js[u"jsonLoadPath"] = unicode(xpath)
0259         _json[path] = js
0260     except IOError:
0261         log.warning("failed to load json from %s : %s " % (path,xpath))
0262         assert 0
0263         _json[path] = {}
0264     pass
0265     return _json[path]
0266
0267
0268 _enum = {}
0269 def enum_(path):
0270     global _enum
0271     if _enum.get(path, None):
0272         log.debug("return cached enum for key %s" % path)
0273         return _enum[path]
0274     try:
0275         log.debug("parsing enum for key %s" % path)
0276         d = Enum(path)
0277         _enum[path] = d
0278     except IOError:
0279         log.fatal("failed to load enum from %s" % path)
0280         _enum[path] = {}
0281     pass
0282     return _enum[path]
0283
0284
0285 class Abbrev(object):
0286     """
0287     simon:opticksdata blyth$ find . -name abbrev.json
0288     ./export/DayaBay/GMaterialLib/abbrev.json
0289     ./resource/GFlags/abbrev.json
0290     simon:opticksdata blyth$
0291
0292     $OPTICKS_DATA_DIR/resource/GFlags/abbrev.json
0293     """
0294     def __init__(self, path):
0295         js = json_(path)
0296
0297         if "abbrev" in js:
0298             js = js["abbrev"]
0299         pass
0300         #log.info("js:%s" % js.keys() )
0301
0302         names = list(map(str,js.keys()))    # CAUTION ORDER DOES NOT MATCH PhotonCodeFlags
0303         abbrs = list(map(str,js.values()))
0304
0305         self.names = names
0306         self.abbrs = abbrs
0307         self.name2abbr = dict(zip(names, abbrs))
0308         self.abbr2name = dict(zip(abbrs, names))
0309
0310
0311     def __repr__(self):
0312         lines = []
0313         lines.append(".names")
0314         lines.extend(self.names)
0315         lines.append(".abbrs")
0316         lines.extend(self.abbrs)
0317         return "\n".join(lines)
0318
0319
0320 class ItemList(object): # formerly ListFlags
0321     @classmethod
0322     def Path(cls, txt, reldir=None):
0323         """
0324         :param txt: eg GMaterialLib
0325         :param reldir:  normally relative to IDPATH, for test geometry provide an absolute path
0326         """
0327         if reldir is not None and reldir.startswith("/"):
0328             npath = os.path.join(reldir, txt+".txt" )
0329         else:
0330             if reldir is None:
0331                 reldir = "GItemList"
0332             pass
0333             npath=idp_("%(reldir)s/%(txt)s.txt" % locals())
0334         pass
0335         return npath
0336
0337     def __init__(self, txt="GMaterialLib", offset=1, translate_=None, reldir=None):
0338         """
0339         :param reldir: when starts with "/" an absolute path is assumed
0340         """
0341         log.debug("txt %s reldir  %s " % (txt, reldir))
0342         npath=self.Path(txt, reldir)
0343         names = list(map(lambda _:_[:-1],open(npath,"r").readlines()))
0344         if translate_ is not None:
0345             log.info("translating")
0346             names = list(map(translate_, names))
0347         pass
0348         codes = list(map(lambda _:_ + offset, range(len(names))))
0349
0350         self.npath = npath
0351         self.offset = offset
0352         self.names = names
0353         self.codes = codes
0354         self.name2code = dict(zip(names, codes))
0355         self.code2name = dict(zip(codes, names))
0356
0357     def find_index(self, name):
0358         return self.names.index(name)
0359
0360     def __str__(self):
0361         return "ItemLists names %6d name2code %6d code2name %6d offset %5d npath %s " % (len(self.names), len(self.name2code), len(self.code2name), self.offset, uidp_(self.npath))
0362
0363     __repr__ = __str__
0364
0365
0366
0367 class IniFlags(object):
0368     """
0369     Formerly from $OPTICKS_INSTALL_CACHE/OKC/GFlagIndexLocal.ini
0370     """
0371     def __init__(self, path="$OPTICKS_PREFIX/include/SysRap/OpticksPhoton_Enum.ini"):
0372         ini = ini_(path)
0373         assert len(ini) > 0, "IniFlags bad path/flags %s " % path
0374
0375         #assert 0, "who uses this ?"
0376
0377         ini = dict(zip(ini.keys(),map(int,ini.values())))  # convert values to int
0378         names = map(str,ini.keys())
0379         codes = map(int,ini.values())
0380
0381         self.names = names
0382         self.codes = codes
0383         self.name2code = dict(zip(names, codes))
0384         self.code2name = dict(zip(codes, names))
0385
0386 class EnumFlags(object):
0387     """
0388     With the default of mask2int False the values are::
0389
0390         1 << 0, 1 << 1, 1 << 2, ...
0391
0392     Otherwise with mask2int True they are::
0393
0394         1,2,3,...
0395
0396     """
0397     def __init__(self, path, mask2int=False):
0398         d = enum_(path)
0399         ini = dict(zip(d.keys(),list(map(int,d.values()))))
0400
0401         names = list(map(str,ini.keys()))
0402         codes = list(map(int,ini.values()))
0403
0404         if mask2int:
0405             mask2int = {}
0406             for i in range(32):
0407                 mask2int[1 << i] = i + 1
0408             pass
0409             codes = list(map(lambda _:mask2int.get(_,-1), codes))
0410         pass
0411
0412         self.names = names
0413         self.codes = codes
0414         self.name2code = dict(zip(names, codes))
0415         self.code2name = dict(zip(codes, names))
0416
0417     def __repr__(self):
0418         return "\n".join([self.__class__.__name__, "names", str(self.names), "codes", str(self.codes), "name2code", str(self.name2code), "code2name", str(self.code2name) ])
0419
0420 class PhotonMaskFlags(EnumFlags):
0421     """
0422     This is used by hismask.py for pflags_ana
0423
0424     Note this is partially duplicating optickscore/OpticksFlags.cc
0425     Former positions of Abbrev : $OPTICKS_INSTALL_CACHE/OKC/OpticksFlagsAbbrevMeta.json
0426     """
0427     def __init__(self):
0428         EnumFlags.__init__(self, path="$OPTICKS_PREFIX/include/SysRap/OpticksPhoton.h", mask2int=False)
0429         self.abbrev = Abbrev("$OPTICKS_PREFIX/include/SysRap/OpticksPhoton_Abbrev.json")
0430         ## looks like SysRap suffers case inconsistency
0431
0432
0433
0434 class PhotonCodeFlags(EnumFlags):
0435     """
0436     This is used by histype.py for seqhis_ana
0437     """
0438     def __init__(self):
0439         EnumFlags.__init__(self, path="$OPTICKS_PREFIX/include/SysRap/OpticksPhoton.h", mask2int=True)
0440
0441         abbrev = Abbrev("$OPTICKS_PREFIX/include/SysRap/OpticksPhoton_Abbrev.json")
0442         name2abbr = abbrev.name2abbr
0443         names = self.names
0444
0445         abbr2code = {}
0446         abbr = []
0447         for code, name in enumerate(names):
0448             abr = name2abbr.get(name, "??")
0449             abbr.append(abr)
0450             abbr2code[abr] = code + 1
0451         pass
0452
0453         fln = np.array(["~~"]+names)
0454         fla = np.array(["~~"]+abbr)
0455         ftab = np.c_[np.arange(len(fla)),fla,fln]
0456
0457         self.abbrev = abbrev
0458         self.abbr = abbr      # these abbr follow same order as names unlike abbrev.names
0459         self.fln = fln
0460         self.fla = fla
0461         self.ftab = ftab
0462         self.abbr2code = abbr2code
0463
0464     def __getattr__(self, arg):
0465         code = self.abbr2code.get(arg, -1)
0466         return code
0467
0468
0469
0470 def test_basics():
0471     from opticks.ana.main import opticks_main
0472     ok = opticks_main()
0473
0474     lf = ItemList("GMaterialLib")
0475     print("lf:ItemList(GMaterialLib).name2code")
0476     print("\n".join([" %30s : %s " % (k,v) for k,v in sorted(lf.name2code.items(),key=lambda kv:kv[1])]))
0477
0478     inif = IniFlags()
0479     print("inif:IniFlags(photon flags)")
0480     print("\n".join([" %30s : %s " % (k,v) for k,v in sorted(inif.name2code.items(),key=lambda kv:kv[1])]))
0481
0482     phmf = PhotonMaskFlags()
0483     print("phmf:PhotonMaskFlags()")
0484     print("\n".join([" %30s : %s " % (k,v) for k,v in sorted(phmf.name2code.items(),key=lambda kv:kv[1])]))
0485
0486     phcf = PhotonCodeFlags()
0487     print("phcf:PhotonCodeFlags()")
0488     print("\n".join([" %30s : %s " % (k,v) for k,v in sorted(phcf.name2code.items(),key=lambda kv:kv[1])]))
0489
0490
0491
0492 if __name__ == '__main__':
0493
0494     pcf = PhotonCodeFlags()
0495     print(pcf.ftab)
0496
0497
0498
0499
0500