acts/CI/check_unused_files.py

0001 #!/usr/bin/env python3
0002
0003 from pathlib import Path
0004 import os
0005 import sys
0006 import subprocess
0007
0008
0009 def file_can_be_removed(searchstring, scope):
0010     cmd = "grep -IR '" + searchstring + "' " + " ".join(scope)
0011
0012     p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
0013     output, _ = p.communicate()
0014     return output == b""
0015
0016
0017 def count_files(path=".", exclude_dirs=(), exclude_files=()):
0018     count = 0
0019     for root, dirs, files in os.walk(path):
0020         dirs[:] = [d for d in dirs if d not in exclude_dirs]
0021         files[:] = [f for f in files if f not in exclude_files]
0022         count += len(files)
0023
0024     return count
0025
0026
0027 def main():
0028     print("\033[32mINFO\033[0m Start check_unused_files.py ...")
0029     exclude_dirs = (
0030         "Scripts",
0031         "thirdparty",
0032         "CI",
0033         "git",
0034         "cmake",
0035         ".git",
0036         ".github",
0037         ".idea",
0038         ".devcontainer",
0039     )
0040     exclude_files = (
0041         "acts_logo_colored.svg",
0042         ".gitignore",
0043         "README.md",
0044         "CMakeLists.txt",
0045         # Filename not completed in source
0046         "vertexing_event_mu20_beamspot.csv",
0047         "vertexing_event_mu20_tracks.csv",
0048         "vertexing_event_mu20_vertices_AMVF.csv",
0049         "event000000001-MuonDriftCircle.csv",
0050         "event000000001-MuonSimHit.csv",
0051         # TODO Move the following files to a better place?
0052         "Magfield.ipynb",
0053         "SolenoidField.ipynb",
0054         # TODO Add README next to the following files?
0055         "generic-input-config.json",
0056         "generic-alignment-geo.json",
0057         "odd-digi-smearing-config-notime.json",
0058         # TODO Mention these files somewhere?
0059         "generate_particle_data_table.py",
0060         "lazy_autodoc.py",
0061         "codegen/src/codegen/sympy_common.py",
0062         "CompressedIO.h",
0063         # Files for python binding generation
0064         "tgeo_aux.py.in",
0065     )
0066
0067     suffix_header = (
0068         ".hpp",
0069         ".cuh",
0070     )
0071     suffix_source = (
0072         ".ipp",
0073         ".cpp",
0074         ".cu",
0075     )
0076     suffix_image = (
0077         ".png",
0078         ".svg",
0079         ".jpg",
0080         ".gif",
0081     )
0082     suffix_python = (".py",)
0083     suffix_doc = (
0084         ".md",
0085         ".rst",
0086     )
0087     suffix_other = (
0088         "",
0089         ".csv",
0090         ".css",
0091         ".gdml",
0092         ".hepmc3",
0093         ".in",
0094         ".ipynb",
0095         ".json",
0096         ".j2",
0097         ".onnx",
0098         ".root",
0099         ".toml",
0100         ".txt",
0101         ".yml",
0102     )
0103     suffix_allowed = (
0104         suffix_header
0105         + suffix_source
0106         + suffix_image
0107         + suffix_python
0108         + suffix_doc
0109         + suffix_other
0110     )
0111
0112     exit = 0
0113
0114     dirs_base = next(os.walk("."))[1]
0115     dirs_base.append(".")
0116     dirs_base[:] = [d for d in dirs_base if d not in exclude_dirs]
0117     dirs_base_docs = ("docs",)
0118     dirs_base_code = [d for d in dirs_base if d not in dirs_base_docs]
0119
0120     # Collectors
0121     wrong_extension = ()
0122     unused_files = ()
0123
0124     # walk over all files
0125     for root, dirs, files in os.walk("."):
0126         dirs[:] = [d for d in dirs if d not in exclude_dirs]
0127         files[:] = [f for f in files if f not in exclude_files]
0128
0129         # Skip base-directory
0130         if str(Path(root)) == ".":
0131             continue
0132
0133         # Print progress
0134         if root[2:] in dirs_base:
0135             processed_files = 0
0136             current_base_dir = root
0137             number_files = count_files(root, exclude_dirs, exclude_files)
0138             # print empty to start a new line
0139             print("")
0140
0141         # Skip "white-paper-figures"
0142         # TODO Find a more elegant way
0143         if str(root).find("white_papers/figures") != -1:
0144             processed_files += count_files(root, exclude_dirs, exclude_files)
0145             continue
0146
0147         # Skip "DD4hep-tests" since their cmake looks a bit different
0148         # TODO Find a more elegant way
0149         if str(root).find("Tests/UnitTests/Plugins/DD4hep") != -1:
0150             processed_files += count_files(root, exclude_dirs, exclude_files)
0151             continue
0152
0153         root = Path(root)
0154         for filename in files:
0155             processed_files += 1
0156             # get the full path of the file
0157             filepath = root / filename
0158
0159             # Check for wrong extensions
0160             if filepath.suffix not in suffix_allowed:
0161                 wrong_extension += (str(filepath),)
0162
0163             # Check header files and remove
0164             elif filepath.suffix in suffix_header + suffix_source:
0165                 if file_can_be_removed(filepath.stem, dirs_base_code):
0166                     unused_files += (str(filepath),)
0167                     remove_cmd = "rm " + str(filepath)
0168                     os.system(remove_cmd)
0169
0170             elif filepath.suffix in suffix_python:
0171                 # Skip the python tests folder
0172                 if str(root).find("Python/Examples") != -1:
0173                     continue
0174
0175                 if not file_can_be_removed("import .*" + filepath.stem, dirs_base):
0176                     continue
0177
0178                 if not file_can_be_removed(
0179                     "from " + filepath.stem + " import", dirs_base
0180                 ):
0181                     continue
0182
0183                 if file_can_be_removed(filename, dirs_base):
0184                     unused_files += (str(filepath),)
0185                     remove_cmd = "rm " + str(filepath)
0186                     os.system(remove_cmd)
0187
0188             # Check documentation files (weak tests)
0189             # TODO find more reliable test for this
0190             elif filepath.suffix in suffix_doc:
0191                 if file_can_be_removed(filepath.stem, dirs_base_docs):
0192                     unused_files += (str(filepath),)
0193                     remove_cmd = "rm " + str(filepath)
0194                     os.system(remove_cmd)
0195
0196             # Check and print other files
0197             elif filepath.suffix in suffix_image + suffix_other:
0198                 if file_can_be_removed(filename, dirs_base):
0199                     unused_files += (str(filepath),)
0200                     remove_cmd = "rm " + str(filepath)
0201                     os.system(remove_cmd)
0202
0203         # Print the progress in place
0204         progress = int(20 * processed_files / number_files)
0205         sys.stdout.write("\r")
0206         sys.stdout.write(
0207             "Checked [%-20s] %d/%d files in %s"
0208             % ("=" * progress, processed_files, number_files, current_base_dir)
0209         )
0210         sys.stdout.flush()
0211
0212     if len(wrong_extension) != 0:
0213         print(
0214             "\n\n\033[31mERROR\033[0m "
0215             + f"The following {len(wrong_extension)} files have an unsupported extension:\n\n"
0216             + "\033[31m"
0217             + "\n".join(wrong_extension)
0218             + "\033[0m"
0219             + "\nCheck if you can change the format to one of the following:\n"
0220             + "\n".join(suffix_allowed)
0221             + "\nIf you really need that specific extension, add it to the list above.\n"
0222         )
0223
0224         exit += 1
0225
0226     if len(unused_files) != 0:
0227         print(
0228             "\n\n\033[31mERROR\033[0m "
0229             + f"The following {len(unused_files)} files seem to be unused:\n"
0230             + "\033[31m"
0231             + "\n".join(unused_files)
0232             + "\033[0m"
0233             + "\nYou have 3 options:"
0234             + "\n\t- Remove them"
0235             + "\n\t- Use them (check proper include)"
0236             + "\n\t- Modify the ignore list of this check\n"
0237         )
0238
0239         exit += 1
0240
0241     if exit == 0:
0242         print(
0243             "\n\n\033[32mINFO\033[0m Finished check_unused_files.py without any errors."
0244         )
0245
0246     return exit
0247
0248
0249 if "__main__" == __name__:
0250     sys.exit(main())