eic-opticks/ana/fold.py

0001 #!/usr/bin/env python
0002
0003 import os, sys, logging, textwrap, numpy as np, datetime, builtins
0004 from opticks.ana.npmeta import NPMeta
0005 from opticks.sysrap.sframe import sframe
0006
0007 CMDLINE = " ".join(sys.argv)
0008
0009 log = logging.getLogger(__name__)
0010 np.set_printoptions(suppress=True, edgeitems=5, linewidth=200,precision=3)
0011
0012 def EXPR_(rtxt):
0013     return list(map(str.strip,textwrap.dedent(rtxt).split("\n")))
0014
0015 class STR(str):
0016     """STR inherits from str and changes the repr to provide the str : useful for interactive ipython"""
0017     def __repr__(self):
0018         return str(self)
0019
0020
0021 def IsEmptyNPY(path):
0022     """
0023     Heuristic to detect empty .npy before trying to np.load them
0024     BUT: why do that : as NumPy should do that justfine ?
0025
0026     Observe some pho0.npy with header only and no content ?
0027     Also the stree _names.npy sidecars are all empty.
0028     """
0029     empty = False
0030     with open(path, mode="rb") as fp:
0031         contents = fp.read()
0032         if contents[0:6] == b'\x93NUMPY' and contents[-1:] == b'\n' and len(contents) == 128:
0033             empty = True
0034         pass
0035     pass
0036     return empty
0037
0038
0039 def IsRemoteSession():
0040     """
0041     Heuristic to detect remote SSH session
0042     """
0043     has_SSH_CLIENT = not os.environ.get("SSH_CLIENT", None) is None
0044     has_SSH_TTY = not os.environ.get("SSH_TTY", None) is None
0045     return has_SSH_CLIENT or has_SSH_TTY
0046
0047 class AttrBase(object):
0048     def __init__(self, symbol="t", prefix="", publish=False):
0049         self.__dict__["_symbol"] = symbol
0050         self.__dict__["_prefix"] = prefix
0051         self.__dict__["_publish"] = publish
0052
0053     def __setattr__(self, name, value):
0054         self.__dict__[name] = value
0055         key = "%s%s" % (self._prefix, name)
0056         if self._publish and not name.startswith("_"):
0057             print("publish key:%s " % key)
0058             setattr(builtins, key, value)
0059         pass
0060     pass
0061
0062     def __repr__(self):
0063         lines = []
0064         for k,v in self.__dict__.items():
0065             line = k
0066             lines.append(line)
0067         pass
0068         return "\n".join(lines)
0069
0070
0071 class Fold(object):
0072
0073     @classmethod
0074     def MultiLoad(cls, symbols=None):
0075         """
0076         Used by::
0077
0078             CSG/tests/CSGSimtraceTest.py
0079             extg4/tests/X4SimtraceTest.py
0080
0081         Values are read from the environment and added to the invoking global context
0082         using the builtins module. The prefixes are obtained from the SYMBOLS envvar
0083         which defaults to "S,T".
0084
0085         Crucially in addition directory paths provided by S_FOLD and T_FOLD
0086         are loaded as folders and variables are added to global context.
0087
0088         The default SYMBOLS would transmogrify:
0089
0090         * S_LABEL -> s_label
0091         * T_LABEL -> t_label
0092         * S_FOLD -> s
0093         * T_FOLD -> t
0094
0095         In addition an array [s,t] is returned
0096         """
0097         if symbols is None:
0098             symbols = os.environ.get("SYMBOLS", "S,T")
0099             if "," in symbols:
0100                 symbols = symbols.split(",")
0101             else:
0102                 symbols = list(symbols)
0103             pass
0104         pass
0105         print("symbols:%s " % str(symbols))
0106         ff = []
0107         for symbol in symbols:
0108             ekey = "$%s_FOLD" % symbol.upper()
0109             label = os.environ.get("%s_LABEL" % symbol.upper(), None)
0110             setattr(builtins, "%s_label" % symbol.lower(), label)
0111             log.info("symbol %s ekey %s label %s " % (symbol,ekey, label))
0112             if not label is None:
0113                 f = Fold.Load(ekey, symbol=symbol.lower() )
0114                 ff.append(f)
0115             else:
0116                 f = None
0117             pass
0118             setattr(builtins, symbol.lower(), f)
0119             pass
0120         pass
0121         for f in ff:
0122             log.info(repr(f))
0123         pass
0124         return ff
0125
0126
0127     @classmethod
0128     def IsQuiet(cls, **kwa):
0129         quiet_default = True
0130         quiet = kwa.get("quiet", quiet_default)
0131         return quiet
0132
0133     @classmethod
0134     def LoadConcat(cls, *args, **kwa0):
0135         kwa = kwa0.copy()
0136         NEVT = kwa.pop("NEVT", 0)
0137         log.info("LoadConcat NEVT %s " % NEVT)
0138         ff = {}
0139         for evt in range(NEVT):
0140             kwa["evt"] = evt
0141             ff[evt] = cls.Load(*args, **kwa)
0142         pass
0143         fc = Fold.Concatenate(ff, **kwa0)
0144         fc.ff = ff
0145         return fc
0146
0147     @classmethod
0148     def Load(cls, *args, **kwa):
0149         """
0150         :param kwa: "parent=True" loads the parent folder
0151         """
0152         reldir = kwa.pop("reldir", None)
0153         parent = kwa.pop("parent", False)
0154         evt = kwa.pop("evt", None)
0155
0156         if len(args) == 0:
0157             args = list(filter(None, [os.environ["FOLD"], reldir]))
0158         pass
0159
0160
0161         relbase = os.path.join(*args[1:]) if len(args) > 1 else args[0]
0162         kwa["relbase"] = relbase   # relbase is the dir path excluding the first element
0163
0164         base = os.path.join(*args)
0165         base = os.path.expandvars(base)
0166         quiet = cls.IsQuiet(**kwa)
0167
0168         log.info("Fold.Load args %s quiet:%d" % (str(args), quiet))
0169
0170         if not evt is None:
0171             base = base % evt
0172         pass
0173         ubase = os.path.dirname(base) if parent else base
0174         fold = cls(ubase, **kwa) if os.path.isdir(ubase) else None
0175         if fold is None:
0176             log.error("Fold.Load FAILED for base [%s]" % base )
0177         pass
0178         if quiet == False:
0179             print(repr(fold))
0180             print(str(fold))
0181         pass
0182         return fold
0183
0184     # TRY USING DEFAULT : FOR ORDINARY hasattr
0185     #def __getattr__(self, name):
0186     #    """Only called when there is no *name* attr"""
0187     #    return None
0188     #
0189     #def __getattr__(self, item):
0190     #    try:
0191     #        return self.__dict__[item]
0192     #    except KeyError:
0193     #        classname = type(self).__name__
0194     #        msg = f'{classname!r} object has no attribute {item!r}'
0195     #        raise AttributeError(msg)
0196     #
0197
0198
0199     SFRAME = "sframe.npy"
0200     INDEX = "NPFold_index.txt"
0201
0202     def brief(self):
0203         return "Fold : symbol %30s base %s " % (self.symbol, self.base)
0204
0205     @classmethod
0206     def IsFold(cls, base, name):
0207         """
0208         :param base:
0209         :param name:
0210         :return bool: True when indexpath:base/name/NPFold_index.txt exists
0211         """
0212         path = os.path.join(base, name)
0213         indexpath = os.path.join(path, cls.INDEX )
0214         return os.path.isdir(path) and os.path.exists(indexpath)
0215
0216     def get_names(self, base):
0217         """
0218         :param base: directory path
0219         :return names: list of names that are either .txt .npy or subfold
0220         """
0221         nn = os.listdir(base)
0222         if not self.order is None:
0223             sizes = list(map(lambda name:os.stat(os.path.join(base, name)).st_size, nn))
0224             order = np.argsort(np.array(sizes))
0225             if self.order == "descend":
0226                 order = order[::-1]
0227             pass
0228             nn = np.array(nn)[order]
0229         pass
0230
0231         names = []
0232         for n in nn:
0233             if n.endswith(".npy") or n.endswith(".txt"):
0234                 names.append(n)
0235             elif self.IsFold(base,n):
0236                 names.append(n)
0237             else:
0238                 pass
0239             pass
0240         pass
0241         return names
0242
0243
0244     def __init__(self, base, **kwa):
0245         """
0246         Fold ctor
0247         """
0248         self.base = base
0249         self.kwa = kwa
0250         self.quiet = self.IsQuiet(**kwa)
0251         self.symbol = kwa.get("symbol", "t")
0252         self.relbase = kwa.get("relbase")
0253         self.is_concat = kwa.get("is_concat", False)
0254         self.globals = kwa.get("globals", False) == True
0255         self.globals_prefix = kwa.get("globals_prefix", "")
0256         self.order = kwa.get("order", None)
0257         assert self.order in ["ascend", "descend", None ]
0258
0259         if self.quiet == False:
0260             print(self.brief())
0261             print("Fold : setting globals %s globals_prefix %s " % (self.globals, self.globals_prefix))
0262         pass
0263
0264         self.paths = []
0265         self.stems = []
0266         self.abbrev = []
0267         self.txts = {}
0268         self.ff = []
0269
0270         if base is None or self.is_concat:
0271             log.info("not loading as base None or is_concat")
0272         else:
0273             self.load()
0274         pass
0275
0276
0277     def load(self):
0278         """
0279         Similar impl over in np is no longer limited to 26*2 symbols
0280         """
0281         base = self.base
0282         symbols = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
0283         self.names = self.get_names(base)
0284
0285         enough_symbols = len(self.names) <= len(symbols)
0286
0287         for i, name in enumerate(self.names):
0288             path = os.path.join(base, name)
0289             symbol = symbols[i] if enough_symbols else "sym%d" % i
0290             stem = name[:-4] if name.endswith(".npy") or name.endswith(".txt") else name
0291
0292             self.paths.append(path)
0293             self.stems.append(stem)
0294             self.abbrev.append(symbol)
0295
0296             if name == self.SFRAME:
0297                 a = sframe.Load(path)
0298             elif name.endswith(".npy"):
0299                 is_empty_npy = IsEmptyNPY(path)
0300                 a = None
0301                 a = np.load(path)
0302                 #if is_empty_npy:
0303                 #    print("fold.load detected empty .npy %s " % path )
0304                 #else:
0305                 #    try:
0306                 #        a = np.load(path)
0307                 #    except ValueError:
0308                 #        print("fold.load error with np.load of %s " % path )
0309                 #    pass
0310                 #pass
0311             elif name.endswith("_meta.txt"):
0312                 a = NPMeta.Load(path)
0313                 self.txts[name] = a
0314             elif name.endswith(".txt"):
0315                 a = NPMeta.LoadAsArray(path)
0316                 self.txts[name] = a
0317             elif self.IsFold(base, name):
0318                 a = Fold(path, symbol=name)
0319                 self.ff.append(name)
0320             pass
0321             setattr(self, stem, a )
0322
0323             if self.globals:
0324                 gstem = self.globals_prefix + stem
0325                 setattr( builtins, gstem, a )
0326                 setattr( builtins, symbol, a )
0327                 print("setting builtins symbol:%s gstem:%s" % (symbol, gstem) )
0328             pass
0329         pass
0330
0331     @classmethod
0332     def CommonNames(cls, ff):
0333         """
0334         Currently asserts that all Fold instances contain the same names and stems
0335
0336         :param ff: dictionary holding Fold instances keyed by integers specifying order
0337         :return names, stems: two lists of common names and stems present in all Fold instances
0338         """
0339         names = []
0340         stems = []
0341         for k,v in ff.items():
0342             if len(names) == 0:
0343                 names = v.names
0344             else:
0345                 assert names == v.names
0346             pass
0347             if len(stems) == 0:
0348                 stems = v.stems
0349             else:
0350                 assert stems == v.stems
0351             pass
0352         pass
0353         return names, stems
0354
0355
0356     @classmethod
0357     def BaseLabel(cls, base, shorten=False):
0358         """
0359         Example baselabel and shortened baselabel::
0360
0361              /tmp/blyth/opticks/GEOM/R0J008/ntds2/ALL0/00[0,1,2,3,4,5,6,7,8,9]
0362              R0J008/ntds2/ALL0
0363
0364         """
0365         label = "BaseLabel:unexpected-base-type"
0366         if type(base) is str:
0367             label = base
0368         elif type(base) is list:
0369             pfx = os.path.commonprefix(base)
0370             tail = []
0371             for _ in base:
0372                 tail.append(_[len(pfx):])
0373             pass
0374             label = "%s[%s]" % ( pfx, ",".join(tail))
0375         pass
0376
0377         if shorten and "GEOM" in label:
0378             label = os.path.dirname(label.split("GEOM")[1][1:])
0379         pass
0380         return label
0381
0382     @classmethod
0383     def Concatenate(cls, ff, **kwa0):
0384         """
0385         :param ff: dictionary holding Fold instances keyed by integers specifying order
0386         :return f: Fold instance created by concatenating the ff Fold arrays
0387         """
0388         kwa = kwa0.copy()
0389         kwa["is_concat"] = True
0390
0391         names, stems = cls.CommonNames(ff)
0392         assert len(names) == len(stems)
0393         kk = list(ff.keys())
0394         log.info("Concatenating %d Fold into one " % len(kk))
0395
0396         bases = []
0397         for k in kk:
0398             f = ff[k]
0399             bases.append(f.base)
0400         pass
0401         base = bases
0402         baselabel = cls.BaseLabel(bases)
0403
0404         fc = Fold(base, **kwa )
0405         fc.names = names
0406         fc.stems = stems
0407         fc.paths = []
0408         for i in range(len(stems)):
0409             name = names[i]
0410             stem = stems[i]
0411             if name.endswith(".npy") and not stem == "sframe":
0412                 aa = []
0413                 for k in kk:
0414                     f = ff[k]
0415                     a = getattr(f, stem, None)
0416                     assert not a is None
0417                     aa.append(a)
0418                 pass
0419                 setattr(fc, stem, np.concatenate(tuple(aa)) )
0420             pass
0421         pass
0422         return fc
0423
0424     def _get_baselabel(self):
0425         return self.BaseLabel(self.base, shorten=True)
0426
0427     baselabel = property(_get_baselabel)
0428
0429     def desc(self):
0430         """
0431         for is_concat:True cannot assume there are .npy files
0432
0433         """
0434         now_stamp = datetime.datetime.now()
0435         l = []
0436         l.append(self.symbol)
0437         l.append("")
0438         l.append("CMDLINE:%s" % CMDLINE )
0439         l.append("%s.base:%s" % (self.symbol,self.base) )
0440         l.append("")
0441         stamps = []
0442
0443         # stems include both files and subfold
0444         for i in range(len(self.stems)):
0445             stem = self.stems[i]
0446             path = self.paths[i] if i < len(self.paths) else None
0447             abbrev = self.abbrev[i] if i < len(self.abbrev) else None
0448             abbrev_ = abbrev if self.globals else " "
0449             aname = "%s.%s" % (self.symbol,stem)
0450             line = "%1s : %-50s :" % ( abbrev_, aname )
0451
0452             is_subfold = os.path.isdir(path)
0453
0454             if is_subfold:
0455                 line += " SUBFOLD "
0456             else:
0457                 a = getattr(self, stem)
0458
0459                 if a is None:
0460                     line += " NO ATTR "
0461                 else:
0462                     kls = a.__class__.__name__
0463                     ext = ".txt" if kls == 'NPMeta' else ".npy"
0464                     name = "%s%s" % (stem,ext)
0465
0466                     sh = str(len(a)) if ext == ".txt" else str(a.shape)
0467                     line += " %20s :" % ( sh )
0468                 pass
0469
0470
0471                 if not path is None and os.path.exists(path):
0472                     st = os.stat(path)
0473                     stamp = datetime.datetime.fromtimestamp(st.st_ctime)
0474                     age_stamp = now_stamp - stamp
0475                     stamps.append(stamp)
0476                     line += " %s " % age_stamp
0477                 else:
0478                     line += " NO path "
0479                 pass
0480             pass
0481             l.append(line)
0482         pass
0483         l.append("")
0484
0485         if len(stamps) > 0:
0486             min_stamp = min(stamps)
0487             max_stamp = max(stamps)
0488             dif_stamp = max_stamp - min_stamp
0489             age_stamp = now_stamp - max_stamp
0490             l.append(" min_stamp : %s " % str(min_stamp))
0491             l.append(" max_stamp : %s " % str(max_stamp))
0492             l.append(" dif_stamp : %s " % str(dif_stamp))
0493             l.append(" age_stamp : %s " % str(age_stamp))
0494             assert dif_stamp.microseconds < 1e6, "stamp divergence detected microseconds %d : so are seeing mixed up results from multiple runs " % dif_stamp.microseconds
0495         else:
0496             l.append("WARNING THERE ARE NO TIME STAMPS")
0497         pass
0498         return l
0499
0500     def __repr__(self):
0501         return "\n".join(self.desc())
0502
0503     def __str__(self):
0504         l = []
0505         for k in self.txts:
0506             l.append("")
0507             l.append(k)
0508             l.append("")
0509             l.append(str(self.txts[k]))
0510         pass
0511         return "\n".join(l)
0512
0513
0514 if __name__  == '__main__':
0515     pass
0516     logging.basicConfig(level=logging.INFO)
0517     NEVT = int(os.environ.get("NEVT",3))
0518     fc = Fold.LoadConcat(NEVT=NEVT, symbol="fc")
0519     print(repr(fc))
0520
0521
0522
0523