eic-opticks/ana/nload.py

0001 #!/usr/bin/env python
0002 #
0003 # Copyright (c) 2019 Opticks Team. All Rights Reserved.
0004 #
0005 # This file is part of Opticks
0006 # (see https://bitbucket.org/simoncblyth/opticks).
0007 #
0008 # Licensed under the Apache License, Version 2.0 (the "License");
0009 # you may not use this file except in compliance with the License.
0010 # You may obtain a copy of the License at
0011 #
0012 #   http://www.apache.org/licenses/LICENSE-2.0
0013 #
0014 # Unless required by applicable law or agreed to in writing, software
0015 # distributed under the License is distributed on an "AS IS" BASIS,
0016 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0017 # See the License for the specific language governing permissions and
0018 # limitations under the License.
0019 #
0020
0021 """
0022 nload.py
0023 ==========
0024
0025 With current defaults in environment::
0026
0027     export OPTICKS_ANA_DEFAULTS="det=tboolean-box,src=torch,tag=1,pfx=tboolean-box"
0028     export OPTICKS_EVENT_BASE=/tmp
0029
0030 ::
0031
0032    cd /tmp
0033    OPTICKS_EVENT_BASE=tboolean-box nload.py
0034         ## no longer works
0035
0036    OPTICKS_EVENT_BASE=. nload.py
0037    nload.py
0038         ## these work
0039
0040    OPTICKS_EVENT_BASE=/timbucku nload.py
0041         ## looks in /timbucku/tboolean-box/evt/tboolean-box/torch/1
0042
0043
0044 """
0045 import os, sys, logging, datetime
0046 log = logging.getLogger(__name__)
0047
0048 try:
0049     from StringIO import StringIO
0050 except ImportError:
0051     from io import StringIO
0052 pass
0053
0054 import numpy as np
0055 from opticks.ana.base import ini_
0056 from opticks.ana.num  import _slice, slice_, Num
0057
0058
0059 def time_(path):
0060     st = os.stat(path) if os.path.exists(path) else None
0061     t = None if st is None else datetime.datetime.fromtimestamp(st.st_ctime)
0062     return t
0063
0064 def tfmt_(dt, fmt="%Y%m%d-%H%M"):
0065     return dt.strftime(fmt) if not dt is None else "-"
0066
0067 def stmp_(st, fmt="%Y%m%d-%H%M"):
0068     return datetime.datetime.fromtimestamp(st.st_ctime).strftime(fmt)
0069
0070 def stamp_(path, fmt="%Y%m%d-%H%M"):
0071     try:
0072         st = os.stat(path)
0073     except OSError:
0074         return "FILE-DOES-NOT-EXIST"
0075     return stmp_(st, fmt=fmt)
0076
0077 def x_(_):
0078     p = os.path.expandvars(_)
0079     st = stamp_(p)
0080     log.info( " %s -> %s (%s) " % (_, p, st))
0081     return p
0082
0083 txt_ = lambda _:np.loadtxt(StringIO(_))
0084
0085
0086
0087
0088
0089 def np_paths(fold):
0090     """
0091     :param fold: directory containing .npy arrays
0092     :return paths: list of file paths of .npy arrays in *fold*
0093     """
0094     paths = []
0095     for name in os.listdir(fold):
0096         if not name.endswith(".npy"): continue
0097         path = os.path.join(fold, name)
0098         paths.append(path)
0099     pass
0100     return sorted(paths)
0101
0102 def np_concatenate(paths):
0103     """
0104     :param paths: list of paths of .npy arrays
0105     :return c: concatenated array
0106     """
0107     aa = []
0108     print("\n".join(paths))
0109     for path in paths:
0110         a = np.load(path)
0111         aa.append(a)
0112     pass
0113     c = np.concatenate(aa)
0114     return c
0115
0116
0117 def np_load(base,sub=None,rel=None):
0118     """
0119     Loads single np array or loads and concats a directory of them,
0120     returning None if non-existing
0121
0122     :param arg: path to .npy array or directory containing .npy to be concatenated
0123     :return c: array
0124     """
0125     path_ = os.path.join(*filter(None,[base,sub,rel]))
0126     path = os.path.expandvars(path_)
0127
0128     if os.path.exists(path):
0129         if path.endswith(".npy"):
0130             a = np.load(path)
0131             paths = [path]
0132         elif os.path.isdir(path):
0133             paths = np_paths(path)
0134             a = np_concatenate(paths)
0135         else:
0136             a = None
0137             paths = []
0138         pass
0139     else:
0140         log.debug("np_load path_:%s path:%s DOES NOT EXIST " % ( path_, path ))
0141         a = None
0142         paths = []
0143     pass
0144     return a, paths
0145
0146
0147
0148
0149
0150
0151
0152
0153
0154
0155
0156 DEFAULT_BASE = "$OPTICKS_EVENT_BASE/$0/evt"
0157 DEFAULT_DIR_TEMPLATE = DEFAULT_BASE + "/$1/$2/$3"  ## cf C++  brap- BOpticksEvent
0158
0159 def tagdir_(det, typ, tag, pfx=".", layout=2):
0160     """
0161     layout version 1 (which is still used for source gensteps) does
0162     not have the tag in the directory
0163
0164     OPTICKS_EVENT_BASE
0165         in direct running this is the geocache directory, with "$0" pfx
0166         being the name of the executable or "source" for the primary (geocache creating)
0167         executable
0168
0169         for test running from legacy basis geometry this is for example /tmp
0170         with pfx being the name of the test eg "tboolean-box"
0171
0172     """
0173     log.debug("tagdir_ det %s typ %s tag %s layout %s DEFAULT_DIR_TEMPLATE %s " % (det,typ,tag,layout, DEFAULT_DIR_TEMPLATE))
0174     log.debug("tagdir_ type(tag) %s " % type(tag))
0175
0176     if layout == 1:
0177         utag = "."
0178         tmpl = DEFAULT_DIR_TEMPLATE
0179         tmpl = tmpl.replace("$0", pfx)
0180         tmpl = tmpl.replace("$1", det)
0181         tmpl = tmpl.replace("$2", typ)
0182         tmpl = tmpl.replace("$3",utag)
0183     elif layout == 2:
0184         tmpl = DEFAULT_DIR_TEMPLATE
0185         tmpl = tmpl.replace("$0", pfx)
0186         tmpl = tmpl.replace("$1", det)
0187         tmpl = tmpl.replace("$2", typ)
0188
0189         if tag is None or tag == "0" or tag == 0:
0190             tmpl = tmpl.replace("$3", "/")
0191         else:
0192             tmpl = tmpl.replace("$3", str(tag))
0193         pass
0194     else:
0195         assert 0, "bad layout"
0196     pass
0197
0198     xdir = os.path.expandvars(tmpl)
0199     while xdir.endswith("/"):xdir = xdir[:-1]
0200     log.debug("tagdir_ tmpl %s xdir %s " % (tmpl, xdir) )
0201
0202     if not os.path.exists(xdir):
0203         log.error("NON EXISTING tagdir : %s  expanded from %s " % (xdir, DEFAULT_DIR_TEMPLATE))
0204         #assert 0, (xdir, tmpl, DEFAULT_DIR_TEMPLATE)
0205     pass
0206     return xdir
0207
0208
0209 def typdirs_(evtdir=None):
0210     """
0211     :return typdirs: list of absolute paths to type dirs
0212     """
0213     if evtdir is None:
0214         evtdir = os.path.expandvars("/tmp/$USER/opticks/evt")
0215     pass
0216     dets = os.listdir(evtdir)
0217     typdirs = []
0218     for det in filter(lambda det:os.path.isdir(os.path.join(evtdir,det)),os.listdir(evtdir)):
0219         detdir = os.path.join(evtdir, det)
0220         for typ in os.listdir(detdir):
0221             typdir = os.path.join(detdir, typ)
0222             print("typdir : ", typdir)
0223             typdirs.append(typdir)
0224         pass
0225     pass
0226     return typdirs
0227
0228 def path_(typ, tag, det="dayabay", name=None, pfx="source" ):
0229     """
0230     :param typ:
0231     :param tag:
0232     :param det:
0233     :param name:
0234     :param layout:
0235
0236     *layout 1*
0237          typ is of form oxtorch with the
0238          constituent and source combined and the tag is in the
0239          filename stem.
0240
0241     *layout 2*
0242          tag is in the directory and the filename stem
0243          is the constituent eg ox
0244
0245
0246     Signal use of layout 2 by specifying the "ox" "rx" "idom" etc
0247     in the name rather than on the "typ"
0248     """
0249     if name is None:
0250         layout = 1
0251     else:
0252         layout = 2
0253     pass
0254     tagdir = tagdir_(det, typ, tag, layout=layout, pfx=pfx )
0255     if layout == 1:
0256         tmpl = os.path.join(tagdir, "%s.npy" % tag)
0257     elif layout == 2:
0258         tmpl = os.path.join(tagdir, name)
0259     else:
0260         assert 0, "bad layout"
0261     pass
0262     return tmpl
0263
0264
0265 def tpaths_(typ, tag, det="dayabay", name=None, pfx="source"):
0266     """
0267     :return tnams: paths of files named *name* that are contained in directories within the tagdir
0268                    typically these are time stamped dirs
0269     """
0270     assert name is not None
0271
0272     tagdir = tagdir_(det, typ, tag, pfx=pfx)
0273
0274     if os.path.isdir(tagdir):
0275         names = os.listdir(tagdir)
0276     else:
0277         log.warning("tpaths_ tagdir %s does not exist" % tagdir)
0278         names = []
0279     pass
0280
0281     tdirs = filter( os.path.isdir, map(lambda name:os.path.join(tagdir, name), names))
0282     tdirs = sorted(tdirs, reverse=True)  # paths of dirs within tagdir
0283     tnams = filter( os.path.exists, map(lambda tdir:os.path.join(tdir, name), tdirs))
0284     # paths of named files
0285     return tnams
0286
0287
0288 def gspath_(typ, tag, det, gsbase=None):
0289     if gsbase is None:
0290         gsbase= os.path.expandvars("$LOCAL_BASE/opticks/opticksdata/gensteps")
0291     gspath = os.path.join(gsbase, det, typ, "%s.npy" % tag)
0292     return gspath
0293
0294 class A(np.ndarray):
0295
0296     @classmethod
0297     def path_(cls, stem, typ, tag, det="dayabay", pfx="source"):
0298         """
0299         :param stem: gs,ox,ht,rs,so,ph,fdom,idom
0300         :param typ: natural ok.src
0301         :param tag: 1,-1
0302         :param det: g4live
0303         :param pfx: source for 1st executable, the name of the executable for subsequent ones eg OKG4Test
0304         :return path:
0305         """
0306         if stem == "gensteps":
0307             path = gspath_(typ, tag, det)
0308             assert 0, "suspect these paths are now wrong"
0309         else:
0310             path = path_(typ,tag, det, name="%s.npy" % stem, pfx=pfx)
0311         pass
0312         return path
0313
0314     @classmethod
0315     def load_(cls, stem, typ, tag, det="dayabay", pfx="source", dbg=False, optional=False, msli=None, g4only=False, okonly=False):
0316         """
0317         :param stem: gs,ox,ht,rs,so,ph,fdom,idom
0318         :param typ: natural
0319         :param tag: 1,-1
0320         :param det: g4live
0321         :param pfx: source for 1st executable, the name of the executable for subsequent ones eg OKG4Test
0322         :param dbg:
0323         :param optional:
0324         :param msli: np.load mmap_mode slice, OR None for ordinary np.load
0325
0326         When msli is specified the np.load is used in memory mapped readonly mode, mmap_mode="r",
0327         meaning that the full array is not loaded into memory until later when slicing
0328         specifies how much of the array should be loaded.
0329         """
0330         if dbg:
0331             print("stem:%s typ:%s tag:%s det:%s pfx:%s dbg:%s optional:%s" % (stem, typ, tag, det, pfx, dbg, optional))
0332         pass
0333         path = cls.path_(stem, typ, tag, det, pfx)
0334         a = None
0335         missing = False
0336         oshape = None
0337         itag = int(tag)
0338
0339         exists = os.path.exists(path)
0340
0341         fake_missing = exists and ( ( g4only and itag > 0 and stem == "dx" ) or ( okonly and itag < 0 and stem == "bn" ))
0342         if fake_missing:
0343             log.debug("fake_missing %s " % path)
0344         pass
0345         if exists and not fake_missing:
0346
0347             st = os.stat(path)
0348             sz = st.st_size
0349
0350             log.debug(" path %s size %s " % (path, sz) )
0351
0352             log.debug("loading %s " % path )
0353             if dbg:
0354                 os.system("ls -l %s " % path)
0355             pass
0356             if msli is None:
0357                 arr = np.load(path)
0358             else:
0359                 arr = np.load(path, mmap_mode="r")
0360                 oshape = arr.shape        #
0361                 arr = arr[msli]
0362             pass
0363         else:
0364             if not optional:
0365                 raise IOError("cannot load %s " % path)
0366             pass
0367             arr = np.zeros(())
0368             missing = True
0369         pass
0370
0371         a = arr.view(cls)
0372         a.path = path
0373         a.typ = typ
0374         a.tag = tag
0375         a.det = det
0376         a.pfx = pfx
0377         a.stamp = stamp_(path)
0378         a.missing = missing
0379         a.oshape = oshape
0380         a.msli = msli
0381
0382         return a
0383
0384     def origshape(self):
0385         return Num.String(self.oshape) if hasattr(self, 'oshape') else "-"
0386
0387     def __repr__(self):
0388         if hasattr(self, 'typ'):
0389             return "A(%s,%s,%s)%s\n%s" % (self.typ, self.tag, self.det, getattr(self,'desc','-'),np.ndarray.__repr__(self))
0390         pass
0391         return "%s" % (np.ndarray.__repr__(self))
0392
0393     def derivative_path(self, postfix="track"):
0394         tag = "%s_%s" % (self.tag, postfix)
0395         return path_(self.typ, tag, self.det )
0396
0397     def derivative_save(self, drv, postfix="track"):
0398         path = self.derivative_path(postfix)
0399         if os.path.exists(path):
0400             log.warning("derivative of %s at path %s exists already, delete and rerun to update" % (repr(self),path) )
0401         else:
0402             log.info("saving derivative of %s to %s " % (repr(self), path ))
0403             np.save( path, drv )
0404         pass
0405
0406
0407
0408 class I(dict):
0409     @classmethod
0410     def loadpath_(cls, path, dbg=False):
0411         if os.path.exists(path):
0412             log.debug("loading %s " % path )
0413             if dbg:
0414                 os.system("ls -l %s " % path)
0415             pass
0416             d = ini_(path)
0417         else:
0418             log.warning("cannot load %s " % path)
0419             #raise IOError("cannot load %s " % path)
0420             d = {}
0421         return d
0422
0423     @classmethod
0424     def load_(cls, typ, tag, det="dayabay", pfx="source", name="DeltaTime.ini", dbg=False):
0425         path = path_(typ, tag, det, name=name, pfx=pfx )
0426         i = cls(path, typ=typ, tag=tag, det=det, name=name)
0427         return i
0428
0429     def __init__(self, path, typ=None, tag=None, det=None, name=None, dbg=False):
0430         d = self.loadpath_(path,dbg=dbg)
0431         dict.__init__(self, d)
0432         self.path = path
0433         self.stamp = stamp_(path)
0434         self.fold = os.path.basename(os.path.dirname(path))
0435         self.typ = typ
0436         self.tag = tag
0437         self.det = det
0438         self.name = name
0439
0440     def __repr__(self):
0441         return "I %5s %10s %10s %10s %10s " % (self.tag, self.typ, self.det, self.name, self.fold )
0442
0443
0444 class II(list):
0445     """
0446     List of I instances holding ini file dicts, providing a history of
0447     event metadata
0448     """
0449     @classmethod
0450     def load_(cls, typ, tag, det="dayabay", pfx="source", name="DeltaTime.ini", dbg=False):
0451         tpaths = tpaths_(typ, tag, det, pfx=pfx, name=name)
0452         ii = map(lambda path:I(path, typ=typ, tag=tag, det=det, name=name, dbg=dbg), tpaths)
0453         return cls(ii)
0454
0455     def __init__(self, ii):
0456         list.__init__(self, ii)
0457
0458     def __getitem__(self, k):
0459         return map(lambda d:d.get(k, None), self)
0460
0461     def folds(self):
0462         return map(lambda i:i.fold, self)
0463
0464     def __repr__(self):
0465         return "\n".join(map(repr, self))
0466
0467
0468
0469 if __name__ == '__main__':
0470     from opticks.ana.main import opticks_main
0471     #ok = opticks_main(src="torch", tag="10", det="PmtInBox")
0472     ok = opticks_main()
0473
0474     try:
0475         i = I.load_(typ=ok.src, tag=ok.tag, det=ok.det, pfx=ok.pfx, name="DeltaTime.ini")
0476     except IOError as err:
0477         log.fatal(err)
0478         sys.exit(ok.mrc)
0479
0480     log.info(" loaded i %s %s " % (i.path, i.stamp))
0481     print("\n".join([" %20s : %s " % (k,v) for k,v in i.items()]))
0482
0483
0484     #a = A.load_("ph", "torch","5", "rainbow")
0485     #i = I.load_("torch","5", "rainbow", name="t_delta.ini")
0486
0487     #ii = II.load_("torch","5", "rainbow", name="t_delta.ini")
0488     #iprp = map(float, filter(None,ii['propagate']) )
0489
0490     #jj = II.load_("torch","-5", "rainbow", name="t_delta.ini")
0491     #jprp = map(float, filter(None,jj['propagate']) )
0492
0493
0494