acts/CI/check_unused_files.py

0001 #!/usr/bin/env python3
0002
0003 from pathlib import Path
0004 import os
0005 import sys
0006 import subprocess
0007
0008
0009 def file_can_be_removed(searchstring, scope):
0010     cmd = "grep -IR '" + searchstring + "' " + " ".join(scope)
0011
0012     p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
0013     output, _ = p.communicate()
0014     return output == b""
0015
0016
0017 def count_files(path=".", exclude_dirs=(), exclude_files=()):
0018     count = 0
0019     for root, dirs, files in os.walk(path):
0020         dirs[:] = [d for d in dirs if d not in exclude_dirs]
0021         files[:] = [f for f in files if f not in exclude_files]
0022         count += len(files)
0023
0024     return count
0025
0026
0027 def main():
0028     print("\033[32mINFO\033[0m Start check_unused_files.py ...")
0029     exclude_dirs = (
0030         "Scripts",
0031         "thirdparty",
0032         "CI",
0033         "git",
0034         "cmake",
0035         ".git",
0036         ".github",
0037         ".idea",
0038     )
0039     exclude_files = (
0040         "acts_logo_colored.svg",
0041         ".gitignore",
0042         "README.md",
0043         "CMakeLists.txt",
0044         # Filename not completed in source
0045         "vertexing_event_mu20_beamspot.csv",
0046         "vertexing_event_mu20_tracks.csv",
0047         "vertexing_event_mu20_vertices_AMVF.csv",
0048         "event000000001-MuonDriftCircle.csv",
0049         "event000000001-MuonSimHit.csv",
0050         # TODO Move the following files to a better place?
0051         "Magfield.ipynb",
0052         "SolenoidField.ipynb",
0053         # TODO Add README next to the following files?
0054         "default-input-config-generic.json",
0055         "geoSelection-openDataDetector.json",
0056         "alignment-geo-contextualDetector.json",
0057         # TODO Mention these files somewhere?
0058         "generate_particle_data_table.py",
0059         "lazy_autodoc.py",
0060     )
0061
0062     suffix_header = (
0063         ".hpp",
0064         ".cuh",
0065     )
0066     suffix_source = (
0067         ".ipp",
0068         ".cpp",
0069         ".cu",
0070     )
0071     suffix_image = (
0072         ".png",
0073         ".svg",
0074         ".jpg",
0075         ".gif",
0076     )
0077     suffix_python = (".py",)
0078     suffix_doc = (
0079         ".md",
0080         ".rst",
0081     )
0082     suffix_other = (
0083         "",
0084         ".csv",
0085         ".css",
0086         ".gdml",
0087         ".hepmc3",
0088         ".in",
0089         ".ipynb",
0090         ".json",
0091         ".j2",
0092         ".onnx",
0093         ".root",
0094         ".toml",
0095         ".txt",
0096         ".yml",
0097     )
0098     suffix_allowed = (
0099         suffix_header
0100         + suffix_source
0101         + suffix_image
0102         + suffix_python
0103         + suffix_doc
0104         + suffix_other
0105     )
0106
0107     exit = 0
0108
0109     dirs_base = next(os.walk("."))[1]
0110     dirs_base.append(".")
0111     dirs_base[:] = [d for d in dirs_base if d not in exclude_dirs]
0112     dirs_base_docs = ("docs",)
0113     dirs_base_code = [d for d in dirs_base if d not in dirs_base_docs]
0114
0115     # Collectors
0116     wrong_extension = ()
0117     unused_files = ()
0118
0119     # walk over all files
0120     for root, dirs, files in os.walk("."):
0121         dirs[:] = [d for d in dirs if d not in exclude_dirs]
0122         files[:] = [f for f in files if f not in exclude_files]
0123
0124         # Skip base-directory
0125         if str(Path(root)) == ".":
0126             continue
0127
0128         # Print progress
0129         if root[2:] in dirs_base:
0130             processed_files = 0
0131             current_base_dir = root
0132             number_files = count_files(root, exclude_dirs, exclude_files)
0133             # print empty to start a new line
0134             print("")
0135
0136         # Skip "white-paper-figures"
0137         # TODO Find a more elegant way
0138         if str(root).find("white_papers/figures") != -1:
0139             processed_files += count_files(root, exclude_dirs, exclude_files)
0140             continue
0141
0142         # Skip "DD4hep-tests" since their cmake looks a bit different
0143         # TODO Find a more elegant way
0144         if str(root).find("Tests/UnitTests/Plugins/DD4hep") != -1:
0145             processed_files += count_files(root, exclude_dirs, exclude_files)
0146             continue
0147
0148         root = Path(root)
0149         for filename in files:
0150             processed_files += 1
0151             # get the full path of the file
0152             filepath = root / filename
0153
0154             # Check for wrong extensions
0155             if filepath.suffix not in suffix_allowed:
0156                 wrong_extension += (str(filepath),)
0157
0158             # Check header files and remove
0159             elif filepath.suffix in suffix_header + suffix_source:
0160                 if file_can_be_removed(filepath.stem, dirs_base_code):
0161                     unused_files += (str(filepath),)
0162                     remove_cmd = "rm " + str(filepath)
0163                     os.system(remove_cmd)
0164
0165             elif filepath.suffix in suffix_python:
0166                 # Skip the python tests folder
0167                 if str(root).find("Examples/Python") != -1:
0168                     continue
0169
0170                 if not file_can_be_removed("import .*" + filepath.stem, dirs_base):
0171                     continue
0172
0173                 if not file_can_be_removed(
0174                     "from " + filepath.stem + " import", dirs_base
0175                 ):
0176                     continue
0177
0178                 if file_can_be_removed(filename, dirs_base):
0179                     unused_files += (str(filepath),)
0180                     remove_cmd = "rm " + str(filepath)
0181                     os.system(remove_cmd)
0182
0183             # Check documentation files (weak tests)
0184             # TODO find more reliable test for this
0185             elif filepath.suffix in suffix_doc:
0186                 if file_can_be_removed(filepath.stem, dirs_base_docs):
0187                     unused_files += (str(filepath),)
0188                     remove_cmd = "rm " + str(filepath)
0189                     os.system(remove_cmd)
0190
0191             # Check and print other files
0192             elif filepath.suffix in suffix_image + suffix_other:
0193                 if file_can_be_removed(filename, dirs_base):
0194                     unused_files += (str(filepath),)
0195                     remove_cmd = "rm " + str(filepath)
0196                     os.system(remove_cmd)
0197
0198         # Print the progress in place
0199         progress = int(20 * processed_files / number_files)
0200         sys.stdout.write("\r")
0201         sys.stdout.write(
0202             "Checked [%-20s] %d/%d files in %s"
0203             % ("=" * progress, processed_files, number_files, current_base_dir)
0204         )
0205         sys.stdout.flush()
0206
0207     if len(wrong_extension) != 0:
0208         print(
0209             "\n\n\033[31mERROR\033[0m "
0210             + f"The following {len(wrong_extension)} files have an unsupported extension:\n\n"
0211             + "\033[31m"
0212             + "\n".join(wrong_extension)
0213             + "\033[0m"
0214             + "\nCheck if you can change the format to one of the following:\n"
0215             + "\n".join(suffix_allowed)
0216             + "\nIf you really need that specific extension, add it to the list above.\n"
0217         )
0218
0219         exit += 1
0220
0221     if len(unused_files) != 0:
0222         print(
0223             "\n\n\033[31mERROR\033[0m "
0224             + f"The following {len(unused_files)} files seem to be unused:\n"
0225             + "\033[31m"
0226             + "\n".join(unused_files)
0227             + "\033[0m"
0228             + "\nYou have 3 options:"
0229             + "\n\t- Remove them"
0230             + "\n\t- Use them (check proper include)"
0231             + "\n\t- Modify the ignore list of this check\n"
0232         )
0233
0234         exit += 1
0235
0236     if exit == 0:
0237         print(
0238             "\n\n\033[32mINFO\033[0m Finished check_unused_files.py without any errors."
0239         )
0240
0241     return exit
0242
0243
0244 if "__main__" == __name__:
0245     sys.exit(main())