sphenixprod/devonly/bugfixer.py

0001 #!/usr/bin/env python
0002
0003 from pathlib import Path
0004 from datetime import datetime
0005 import subprocess
0006 import sys
0007 import shutil
0008 import math
0009 from typing import Tuple,List
0010
0011 from sphenixdbutils import cnxn_string_map, dbQuery
0012 #from sphenixdbutils import filedb_info, upsert_filecatalog, update_proddb
0013
0014
0015 # ============================================================================================
0016 def shell_command(command: str) -> List[str]:
0017     """Minimal wrapper to hide away subbprocess tedium"""
0018     # CHATTY(f"[shell_command] Command: {command}")
0019     ret=[]
0020     try:
0021         ret = subprocess.run(command, shell=True, check=True, capture_output=True).stdout.decode('utf-8').split()
0022     except subprocess.CalledProcessError as e:
0023         print("[shell_command] Command failed with exit code:", e.returncode)
0024     finally:
0025         pass
0026
0027     return ret
0028
0029 # ============================================================================
0030 def my_parse_spiderstuff(filename: str) -> Tuple[str,...] :
0031     try:
0032         # lfn,_,nevents,_,first,_,last,_,md5,_,dbid = filename.split(':')
0033         lfn = filename
0034         lfn=Path(lfn).name
0035     except Exception as e:
0036         print(f"Error: {e}")
0037         print(filename)
0038         print(filename.split(':'))
0039         exit(-1)
0040
0041     return lfn,-1,-1,-1,-1,-1
0042
0043 # ============================================================================
0044 def my_parse_lfn(lfn: str, outtriplet:str ):
0045     try:
0046         name=lfn.split(':')[0]
0047         name=Path(name).name # could throw an error instead if we're handed a full path.
0048          #  split at, and remove, run3auau_new_nocbdtag_v001, remainder is 'DST_...', '-00066582-00000.root' (or .finished)
0049         dsttype,runsegend=name.split(f'_run3auau_{outtriplet}')
0050         _,run,segend=runsegend.split('-')
0051         seg,end=segend.split('.')
0052     except ValueError as e:
0053         print(f"[parse_lfn] Caught error {e}")
0054         print(f"lfn = {lfn}")
0055         exit(-1)
0056     return dsttype,int(run),int(seg),end
0057
0058 # try:
0059 #         dsttype,runsegend=lfn.split(dataset) # 'DST_..._run3auau', '-00066582-00000.root' (or .finished)
0060 #         _,run,segend=runsegend.split('-')
0061 #         seg,end=segend.split('.')
0062 #     except ValueError as e:
0063 #         print(f"[parse_lfn] Caught error {e}")
0064 #         print(f"lfn = {lfn}")
0065 #         print(f"lfn.split(':') = {lfn.split(':')}")
0066 #         print(f"name = {lfn.split(':')[0]}")
0067 #         name=lfn.split(':')[0]
0068 #         print(f"dsttype,runsegend = name.split(rule.dataset) = {name.split(rule.dataset)}")
0069 #         dsttype,runsegend=name.split(rule.dataset) # 'DST_..._run3auau', '-00066582-00000.root' (or .finished)
0070 #         print(f"_,run,segend = runsegend.split('-') = {runsegend.split('-')}")
0071 #         _,run,segend=runsegend.split('-')
0072 #         print(f"seg,end = segend.split('.') = {segend.split('.')})")
0073 #         seg,end=segend.split('.')
0074 #         exit(-1)
0075
0076
0077 #     # "dsttype" as currently used in the datasets table is e.g. DST_STREAMING_EVENT_ebdc01_1_run3auau
0078 #     # We almost have that but need to strip off a trailing "_"
0079 #     if dsttype[-1] == '_':
0080 #         dsttype=dsttype[0:-1]
0081
0082 #    return dsttype,int(run),int(seg),end
0083
0084 # ============================================================================================
0085
0086 update_files_tmpl="""
0087 update files
0088 set full_file_path='{full_file_path}'
0089 where lfn='{lfn}'
0090 ;
0091 """
0092
0093 # ============================================================================================
0094
0095 def main():
0096     rungroup_tmpl = "run_{a:08d}_{b:08d}"
0097
0098     prod='physics'
0099     if prod=='physics':
0100         prodname='run3auau'
0101     elif prod=='cosmics':
0102         prodname='run3cosmics'
0103     elif prod=='line_laser':
0104         prodname='run3line_laser'
0105     else:
0106         print("don't know that prod")
0107         exit(-1)
0108
0109     # type='DST_TRIGGERED'
0110     type='DST_STREAMING'
0111     outtriplet='new_nocdbtag_v001'
0112     if 'TRKR' in type:
0113         outtriplet='new_newcdbtag_v001'
0114     wrongdir=f'/sphenix/lustre01/sphnxpro/production/run3auau/{prod}/{outtriplet}'
0115     print(wrongdir)
0116     exit()
0117
0118     dpattern=f'{type}_\*'
0119     # tmpfound = shell_command(f"find {wrongdir} -type f -name {pattern}")
0120     cmd=f'find {wrongdir} -maxdepth 1 -type d -name {dpattern}'
0121     tdirs=shell_command(cmd)
0122     for tdir in tdirs:
0123         cmd=f'find {tdir} -maxdepth 1 -type d -name run_\*'
0124         rdirs=shell_command(cmd)
0125         for rdir in rdirs:
0126             print('--- Working on {rdir}')
0127             chunksize=10000
0128             chunkcmd = f'/usr/bin/ls -1 {rdir} | grep -v :: | head -n {chunksize}'
0129             while True:
0130                 chunk=shell_command(chunkcmd)
0131                 if len(chunk)==0:
0132                     break
0133                 for lfn in chunk:
0134                     dsttype,run,seg,_=my_parse_lfn(lfn,outtriplet=outtriplet)
0135                     leaf=dsttype.split(f"_{prodname}")[0]
0136                     rungroup=rungroup_tmpl.format(a=100*math.floor(run/100), b=100*math.ceil((run+1)/100))
0137                     #rightdir=
0138
0139     exit()
0140
0141     foundhists = [ file for file in tmpfound ]
0142     print(f"Found {len(foundhists)} histograms to register and move.")
0143         # dsttype,run,seg,_=my_parse_lfn(lfn,dataset='new_nocdbtag_v001')
0144         # leaf=dsttype.split(f"_{prodname}")[0]
0145         # leaf=leaf.split("HIST_")[1]
0146         # rungroup=rungroup_tmpl.format(a=100*math.floor(run/100), b=100*math.ceil((run+1)/100))
0147         # rightdir=f'/sphenix/data/data02/sphnxpro/production/run3auau/{prod}/new_nocdbtag_v001/{leaf}/{rungroup}/hist'
0148         # fullpath=rightdir+'/'+lfn
0149
0150         # ### For additional db info. Note: stat is costly. Could be omitted with filestat=None
0151         # # Do it before the mv.
0152         # filestat=Path(file).stat()
0153
0154         # ### Extract what else we need for file databases
0155         # full_file_path = fullpath
0156
0157         # dryrun=False
0158         # dbstring = 'fcw'
0159         # ### Move
0160         # if dryrun:
0161         #     if f%when2blurb == 0:
0162         #         print( f"Dryrun: Pretending to do:\n mv {file} {full_file_path}" )
0163         # else:
0164         #     # Move (rename) the file
0165         #     try:
0166         #         shutil.move( file, full_file_path )
0167         #     except Exception as e:
0168         #         WARN(e)
0169
0170
0171         # ### ... and update files catalog table
0172         # update_files=update_files_tmpl.format(
0173         #     full_file_path=full_file_path,
0174         #     lfn=lfn
0175         #     )
0176         # dbstring
0177         # if not dryrun:
0178         #     files_curs = dbQuery( cnxn_string_map[ dbstring ], update_files )
0179         #     files_curs.commit()
0180         # else:
0181         #     print(update_files)
0182
0183 # ============================================================================================
0184
0185 if __name__ == '__main__':
0186     main()
0187     exit(0)
0188
0189
0190
0191