acts/CI/check_unused_files.py

0001 #!/usr/bin/env python3
0002
0003 from pathlib import Path
0004 import os
0005 import sys
0006 import subprocess
0007
0008
0009 def file_can_be_removed(searchstring, scope):
0010     cmd = "grep -IR '" + searchstring + "' " + " ".join(scope)
0011
0012     p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
0013     output, _ = p.communicate()
0014     return output == b""
0015
0016
0017 def count_files(path=".", exclude_dirs=(), exclude_files=()):
0018     count = 0
0019     for root, dirs, files in os.walk(path):
0020         dirs[:] = [d for d in dirs if d not in exclude_dirs]
0021         files[:] = [f for f in files if f not in exclude_files]
0022         count += len(files)
0023
0024     return count
0025
0026
0027 def main():
0028     print("\033[32mINFO\033[0m Start check_unused_files.py ...")
0029     exclude_dirs = (
0030         "Scripts",
0031         "thirdparty",
0032         "CI",
0033         "git",
0034         "cmake",
0035         ".git",
0036         ".github",
0037         ".idea",
0038         ".devcontainer",
0039     )
0040     exclude_files = (
0041         "acts_logo_colored.svg",
0042         ".gitignore",
0043         "README.md",
0044         "CMakeLists.txt",
0045         # Filename not completed in source
0046         "vertexing_event_mu20_beamspot.csv",
0047         "vertexing_event_mu20_tracks.csv",
0048         "vertexing_event_mu20_vertices_AMVF.csv",
0049         "event000000001-MuonDriftCircle.csv",
0050         "event000000001-MuonSimHit.csv",
0051         # TODO Move the following files to a better place?
0052         "Magfield.ipynb",
0053         "SolenoidField.ipynb",
0054         # TODO Add README next to the following files?
0055         "generic-input-config.json",
0056         "generic-alignment-geo.json",
0057         "odd-digi-smearing-config-notime.json",
0058         # TODO Mention these files somewhere?
0059         "generate_particle_data_table.py",
0060         "lazy_autodoc.py",
0061         "codegen/src/codegen/sympy_common.py",
0062         "CompressedIO.h",
0063     )
0064
0065     suffix_header = (
0066         ".hpp",
0067         ".cuh",
0068     )
0069     suffix_source = (
0070         ".ipp",
0071         ".cpp",
0072         ".cu",
0073     )
0074     suffix_image = (
0075         ".png",
0076         ".svg",
0077         ".jpg",
0078         ".gif",
0079     )
0080     suffix_python = (".py",)
0081     suffix_doc = (
0082         ".md",
0083         ".rst",
0084     )
0085     suffix_other = (
0086         "",
0087         ".csv",
0088         ".css",
0089         ".gdml",
0090         ".hepmc3",
0091         ".in",
0092         ".ipynb",
0093         ".json",
0094         ".j2",
0095         ".onnx",
0096         ".root",
0097         ".toml",
0098         ".txt",
0099         ".yml",
0100     )
0101     suffix_allowed = (
0102         suffix_header
0103         + suffix_source
0104         + suffix_image
0105         + suffix_python
0106         + suffix_doc
0107         + suffix_other
0108     )
0109
0110     exit = 0
0111
0112     dirs_base = next(os.walk("."))[1]
0113     dirs_base.append(".")
0114     dirs_base[:] = [d for d in dirs_base if d not in exclude_dirs]
0115     dirs_base_docs = ("docs",)
0116     dirs_base_code = [d for d in dirs_base if d not in dirs_base_docs]
0117
0118     # Collectors
0119     wrong_extension = ()
0120     unused_files = ()
0121
0122     # walk over all files
0123     for root, dirs, files in os.walk("."):
0124         dirs[:] = [d for d in dirs if d not in exclude_dirs]
0125         files[:] = [f for f in files if f not in exclude_files]
0126
0127         # Skip base-directory
0128         if str(Path(root)) == ".":
0129             continue
0130
0131         # Print progress
0132         if root[2:] in dirs_base:
0133             processed_files = 0
0134             current_base_dir = root
0135             number_files = count_files(root, exclude_dirs, exclude_files)
0136             # print empty to start a new line
0137             print("")
0138
0139         # Skip "white-paper-figures"
0140         # TODO Find a more elegant way
0141         if str(root).find("white_papers/figures") != -1:
0142             processed_files += count_files(root, exclude_dirs, exclude_files)
0143             continue
0144
0145         # Skip "DD4hep-tests" since their cmake looks a bit different
0146         # TODO Find a more elegant way
0147         if str(root).find("Tests/UnitTests/Plugins/DD4hep") != -1:
0148             processed_files += count_files(root, exclude_dirs, exclude_files)
0149             continue
0150
0151         root = Path(root)
0152         for filename in files:
0153             processed_files += 1
0154             # get the full path of the file
0155             filepath = root / filename
0156
0157             # Check for wrong extensions
0158             if filepath.suffix not in suffix_allowed:
0159                 wrong_extension += (str(filepath),)
0160
0161             # Check header files and remove
0162             elif filepath.suffix in suffix_header + suffix_source:
0163                 if file_can_be_removed(filepath.stem, dirs_base_code):
0164                     unused_files += (str(filepath),)
0165                     remove_cmd = "rm " + str(filepath)
0166                     os.system(remove_cmd)
0167
0168             elif filepath.suffix in suffix_python:
0169                 # Skip the python tests folder
0170                 if str(root).find("Examples/Python") != -1:
0171                     continue
0172
0173                 if not file_can_be_removed("import .*" + filepath.stem, dirs_base):
0174                     continue
0175
0176                 if not file_can_be_removed(
0177                     "from " + filepath.stem + " import", dirs_base
0178                 ):
0179                     continue
0180
0181                 if file_can_be_removed(filename, dirs_base):
0182                     unused_files += (str(filepath),)
0183                     remove_cmd = "rm " + str(filepath)
0184                     os.system(remove_cmd)
0185
0186             # Check documentation files (weak tests)
0187             # TODO find more reliable test for this
0188             elif filepath.suffix in suffix_doc:
0189                 if file_can_be_removed(filepath.stem, dirs_base_docs):
0190                     unused_files += (str(filepath),)
0191                     remove_cmd = "rm " + str(filepath)
0192                     os.system(remove_cmd)
0193
0194             # Check and print other files
0195             elif filepath.suffix in suffix_image + suffix_other:
0196                 if file_can_be_removed(filename, dirs_base):
0197                     unused_files += (str(filepath),)
0198                     remove_cmd = "rm " + str(filepath)
0199                     os.system(remove_cmd)
0200
0201         # Print the progress in place
0202         progress = int(20 * processed_files / number_files)
0203         sys.stdout.write("\r")
0204         sys.stdout.write(
0205             "Checked [%-20s] %d/%d files in %s"
0206             % ("=" * progress, processed_files, number_files, current_base_dir)
0207         )
0208         sys.stdout.flush()
0209
0210     if len(wrong_extension) != 0:
0211         print(
0212             "\n\n\033[31mERROR\033[0m "
0213             + f"The following {len(wrong_extension)} files have an unsupported extension:\n\n"
0214             + "\033[31m"
0215             + "\n".join(wrong_extension)
0216             + "\033[0m"
0217             + "\nCheck if you can change the format to one of the following:\n"
0218             + "\n".join(suffix_allowed)
0219             + "\nIf you really need that specific extension, add it to the list above.\n"
0220         )
0221
0222         exit += 1
0223
0224     if len(unused_files) != 0:
0225         print(
0226             "\n\n\033[31mERROR\033[0m "
0227             + f"The following {len(unused_files)} files seem to be unused:\n"
0228             + "\033[31m"
0229             + "\n".join(unused_files)
0230             + "\033[0m"
0231             + "\nYou have 3 options:"
0232             + "\n\t- Remove them"
0233             + "\n\t- Use them (check proper include)"
0234             + "\n\t- Modify the ignore list of this check\n"
0235         )
0236
0237         exit += 1
0238
0239     if exit == 0:
0240         print(
0241             "\n\n\033[32mINFO\033[0m Finished check_unused_files.py without any errors."
0242         )
0243
0244     return exit
0245
0246
0247 if "__main__" == __name__:
0248     sys.exit(main())