pandaharvester/harvesterfilesyncer/git_file_syncer.py

0001 import pathlib
0002 import subprocess
0003
0004 from pandaharvester.harvestercore import core_utils
0005
0006 from .base_file_syncer import BaseFileSyncer
0007
0008 # logger
0009 _logger = core_utils.setup_logger("git_file_syncer")
0010
0011
0012 # run command
0013 def run_command(command_str, cwd=None):
0014     p = subprocess.Popen(command_str.split(), shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
0015     std_out, std_err = p.communicate()
0016     ret_code = p.returncode
0017     return ret_code, std_out, std_err
0018
0019
0020 # file syncer with git tools
0021 class GitFileSyncer(BaseFileSyncer):
0022     # constructor
0023     def __init__(self, **kwarg):
0024         BaseFileSyncer.__init__(self, **kwarg)
0025         # make logger
0026         main_log = self.make_logger(_logger, method_name="__init__")
0027         # set up with direct attributes
0028         self.setupMap = dict(vars(self))
0029         # setupMap
0030         # self.checkPeriod = self.setupMap.get('checkPeriod', 1)
0031         # self.lifetime = self.setupMap.get('lifetime', 96)
0032         self.targetDir = self.setupMap.get("targetDir")
0033         self.sourceURL = self.setupMap.get("sourceURL")
0034         self.sourceBranch = self.setupMap.get("sourceBranch", "master")
0035         self.sourceRemoteName = self.setupMap.get("sourceRemoteName", "origin")
0036         self.sourceSubdir = self.setupMap.get("sourceSubdir", "")
0037
0038     # update
0039     def update(self):
0040         # make logger
0041         main_log = self.make_logger(_logger, method_name="update")
0042         main_log.info("start")
0043         # initialize
0044         err_msg_list = []
0045         ret_val = False
0046         # execute command and store result in result list
0047
0048         def execute_command(command, **kwargs):
0049             ret_code, std_out, std_err = run_command(command, **kwargs)
0050             if ret_code != 0:
0051                 main_log.error(f"command: {command} ; kwargs: {str(kwargs)} ; ret_code={ret_code} ; stdout: {std_out} ; stderr: {std_err}")
0052                 err_msg_list.append(std_err)
0053             else:
0054                 main_log.debug(f"command: {command} ; kwargs: {str(kwargs)} ; ret_code={ret_code} ; stdout: {std_out} ; stderr: {std_err}")
0055             return ret_code, std_out, std_err
0056
0057         # run
0058         try:
0059             # assure the local target directory
0060             target_dir_path = pathlib.Path(self.targetDir)
0061             target_dir_path.mkdir(mode=0o755, parents=True, exist_ok=True)
0062             main_log.debug(f"assure local target directory {str(target_dir_path)}")
0063             # git init
0064             execute_command("git init -q", cwd=target_dir_path)
0065             # git remote
0066             ret_code, std_out, std_err = execute_command(
0067                 f"git remote set-url {self.sourceRemoteName} {self.sourceURL}",
0068                 cwd=target_dir_path,
0069             )
0070             if ret_code != 0:
0071                 execute_command(
0072                     f"git remote add -f -t {self.sourceBranch} {self.sourceRemoteName} {self.sourceURL}",
0073                     cwd=target_dir_path,
0074                 )
0075                 execute_command(
0076                     f"git remote set-branches {self.sourceRemoteName} {self.sourceBranch}",
0077                     cwd=target_dir_path,
0078                 )
0079             # git config
0080             execute_command("git config core.sparseCheckout true", cwd=target_dir_path)
0081             # modify sparse checkout list file
0082             sparse_checkout_config_path = target_dir_path / ".git/info/sparse-checkout"
0083             with sparse_checkout_config_path.open("w") as f:
0084                 f.write(self.sourceSubdir)
0085             main_log.debug(f"wrote {self.sourceSubdir} in git sparse-checkout file")
0086             # git fetch (without refspec so remote can be updated)
0087             execute_command(
0088                 f"git fetch {self.sourceRemoteName}",
0089                 cwd=target_dir_path,
0090             )
0091             # git reset to the branch
0092             execute_command(
0093                 f"git reset --hard {self.sourceRemoteName}/{self.sourceBranch}",
0094                 cwd=target_dir_path,
0095             )
0096             # git clean
0097             execute_command("git clean -d -x -f", cwd=target_dir_path)
0098             # return val
0099             ret_val = True
0100             main_log.info("done")
0101         except Exception:
0102             err_msg_list.append(core_utils.dump_error_message(main_log))
0103         return ret_val, str(err_msg_list)