acts/CI/check_unused_files.py

0001 #!/usr/bin/env python3
0002
0003 from pathlib import Path
0004 import os
0005 import sys
0006 import subprocess
0007
0008
0009 def file_can_be_removed(searchstring, scope):
0010     cmd = "grep -IR '" + searchstring + "' " + " ".join(scope)
0011
0012     p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
0013     output, _ = p.communicate()
0014     return output == b""
0015
0016
0017 def count_files(path=".", exclude_dirs=(), exclude_files=()):
0018     count = 0
0019     for root, dirs, files in os.walk(path):
0020         dirs[:] = [d for d in dirs if d not in exclude_dirs]
0021         files[:] = [f for f in files if f not in exclude_files]
0022         count += len(files)
0023
0024     return count
0025
0026
0027 def main():
0028     print("\033[32mINFO\033[0m Start check_unused_files.py ...")
0029     exclude_dirs = (
0030         "Scripts",
0031         "thirdparty",
0032         "CI",
0033         "git",
0034         "cmake",
0035         ".git",
0036         ".github",
0037         ".idea",
0038     )
0039     exclude_files = (
0040         "acts_logo_colored.svg",
0041         ".gitignore",
0042         "README.md",
0043         "CMakeLists.txt",
0044         # Filename not completed in source
0045         "vertexing_event_mu20_beamspot.csv",
0046         "vertexing_event_mu20_tracks.csv",
0047         "vertexing_event_mu20_vertices_AMVF.csv",
0048         "event000000001-MuonDriftCircle.csv",
0049         "event000000001-MuonSimHit.csv",
0050         # TODO Move the following files to a better place?
0051         "Magfield.ipynb",
0052         "SolenoidField.ipynb",
0053         # TODO Add README next to the following files?
0054         "generic-input-config.json",
0055         "generic-alignment-geo.json",
0056         "odd-digi-smearing-config-notime.json",
0057         # TODO Mention these files somewhere?
0058         "generate_particle_data_table.py",
0059         "lazy_autodoc.py",
0060         "codegen/src/codegen/sympy_common.py",
0061         "CompressedIO.h",
0062     )
0063
0064     suffix_header = (
0065         ".hpp",
0066         ".cuh",
0067     )
0068     suffix_source = (
0069         ".ipp",
0070         ".cpp",
0071         ".cu",
0072     )
0073     suffix_image = (
0074         ".png",
0075         ".svg",
0076         ".jpg",
0077         ".gif",
0078     )
0079     suffix_python = (".py",)
0080     suffix_doc = (
0081         ".md",
0082         ".rst",
0083     )
0084     suffix_other = (
0085         "",
0086         ".csv",
0087         ".css",
0088         ".gdml",
0089         ".hepmc3",
0090         ".in",
0091         ".ipynb",
0092         ".json",
0093         ".j2",
0094         ".onnx",
0095         ".root",
0096         ".toml",
0097         ".txt",
0098         ".yml",
0099     )
0100     suffix_allowed = (
0101         suffix_header
0102         + suffix_source
0103         + suffix_image
0104         + suffix_python
0105         + suffix_doc
0106         + suffix_other
0107     )
0108
0109     exit = 0
0110
0111     dirs_base = next(os.walk("."))[1]
0112     dirs_base.append(".")
0113     dirs_base[:] = [d for d in dirs_base if d not in exclude_dirs]
0114     dirs_base_docs = ("docs",)
0115     dirs_base_code = [d for d in dirs_base if d not in dirs_base_docs]
0116
0117     # Collectors
0118     wrong_extension = ()
0119     unused_files = ()
0120
0121     # walk over all files
0122     for root, dirs, files in os.walk("."):
0123         dirs[:] = [d for d in dirs if d not in exclude_dirs]
0124         files[:] = [f for f in files if f not in exclude_files]
0125
0126         # Skip base-directory
0127         if str(Path(root)) == ".":
0128             continue
0129
0130         # Print progress
0131         if root[2:] in dirs_base:
0132             processed_files = 0
0133             current_base_dir = root
0134             number_files = count_files(root, exclude_dirs, exclude_files)
0135             # print empty to start a new line
0136             print("")
0137
0138         # Skip "white-paper-figures"
0139         # TODO Find a more elegant way
0140         if str(root).find("white_papers/figures") != -1:
0141             processed_files += count_files(root, exclude_dirs, exclude_files)
0142             continue
0143
0144         # Skip "DD4hep-tests" since their cmake looks a bit different
0145         # TODO Find a more elegant way
0146         if str(root).find("Tests/UnitTests/Plugins/DD4hep") != -1:
0147             processed_files += count_files(root, exclude_dirs, exclude_files)
0148             continue
0149
0150         root = Path(root)
0151         for filename in files:
0152             processed_files += 1
0153             # get the full path of the file
0154             filepath = root / filename
0155
0156             # Check for wrong extensions
0157             if filepath.suffix not in suffix_allowed:
0158                 wrong_extension += (str(filepath),)
0159
0160             # Check header files and remove
0161             elif filepath.suffix in suffix_header + suffix_source:
0162                 if file_can_be_removed(filepath.stem, dirs_base_code):
0163                     unused_files += (str(filepath),)
0164                     remove_cmd = "rm " + str(filepath)
0165                     os.system(remove_cmd)
0166
0167             elif filepath.suffix in suffix_python:
0168                 # Skip the python tests folder
0169                 if str(root).find("Examples/Python") != -1:
0170                     continue
0171
0172                 if not file_can_be_removed("import .*" + filepath.stem, dirs_base):
0173                     continue
0174
0175                 if not file_can_be_removed(
0176                     "from " + filepath.stem + " import", dirs_base
0177                 ):
0178                     continue
0179
0180                 if file_can_be_removed(filename, dirs_base):
0181                     unused_files += (str(filepath),)
0182                     remove_cmd = "rm " + str(filepath)
0183                     os.system(remove_cmd)
0184
0185             # Check documentation files (weak tests)
0186             # TODO find more reliable test for this
0187             elif filepath.suffix in suffix_doc:
0188                 if file_can_be_removed(filepath.stem, dirs_base_docs):
0189                     unused_files += (str(filepath),)
0190                     remove_cmd = "rm " + str(filepath)
0191                     os.system(remove_cmd)
0192
0193             # Check and print other files
0194             elif filepath.suffix in suffix_image + suffix_other:
0195                 if file_can_be_removed(filename, dirs_base):
0196                     unused_files += (str(filepath),)
0197                     remove_cmd = "rm " + str(filepath)
0198                     os.system(remove_cmd)
0199
0200         # Print the progress in place
0201         progress = int(20 * processed_files / number_files)
0202         sys.stdout.write("\r")
0203         sys.stdout.write(
0204             "Checked [%-20s] %d/%d files in %s"
0205             % ("=" * progress, processed_files, number_files, current_base_dir)
0206         )
0207         sys.stdout.flush()
0208
0209     if len(wrong_extension) != 0:
0210         print(
0211             "\n\n\033[31mERROR\033[0m "
0212             + f"The following {len(wrong_extension)} files have an unsupported extension:\n\n"
0213             + "\033[31m"
0214             + "\n".join(wrong_extension)
0215             + "\033[0m"
0216             + "\nCheck if you can change the format to one of the following:\n"
0217             + "\n".join(suffix_allowed)
0218             + "\nIf you really need that specific extension, add it to the list above.\n"
0219         )
0220
0221         exit += 1
0222
0223     if len(unused_files) != 0:
0224         print(
0225             "\n\n\033[31mERROR\033[0m "
0226             + f"The following {len(unused_files)} files seem to be unused:\n"
0227             + "\033[31m"
0228             + "\n".join(unused_files)
0229             + "\033[0m"
0230             + "\nYou have 3 options:"
0231             + "\n\t- Remove them"
0232             + "\n\t- Use them (check proper include)"
0233             + "\n\t- Modify the ignore list of this check\n"
0234         )
0235
0236         exit += 1
0237
0238     if exit == 0:
0239         print(
0240             "\n\n\033[32mINFO\033[0m Finished check_unused_files.py without any errors."
0241         )
0242
0243     return exit
0244
0245
0246 if "__main__" == __name__:
0247     sys.exit(main())