acts/CI/check_unused_files.py

0001 #!/usr/bin/env python3
0002
0003 from pathlib import Path
0004 import os
0005 import sys
0006 import subprocess
0007
0008
0009 def file_can_be_removed(searchstring, scope):
0010     cmd = "grep -IR '" + searchstring + "' " + " ".join(scope)
0011
0012     p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
0013     output, _ = p.communicate()
0014     return output == b""
0015
0016
0017 def count_files(path=".", exclude_dirs=(), exclude_files=()):
0018     count = 0
0019     for root, dirs, files in os.walk(path):
0020         dirs[:] = [d for d in dirs if d not in exclude_dirs]
0021         files[:] = [f for f in files if f not in exclude_files]
0022         count += len(files)
0023
0024     return count
0025
0026
0027 def main():
0028     print("\033[32mINFO\033[0m Start check_unused_files.py ...")
0029     exclude_dirs = (
0030         "Scripts",
0031         "thirdparty",
0032         "CI",
0033         "Python",
0034         "git",
0035         "cmake",
0036         ".git",
0037         ".github",
0038         ".idea",
0039         ".devcontainer",
0040     )
0041     exclude_files = (
0042         "acts_logo_colored.svg",
0043         ".gitignore",
0044         "README.md",
0045         "CMakeLists.txt",
0046         # Filename not completed in source
0047         "vertexing_event_mu20_beamspot.csv",
0048         "vertexing_event_mu20_tracks.csv",
0049         "vertexing_event_mu20_vertices_AMVF.csv",
0050         "event000000001-MuonDriftCircle.csv",
0051         "event000000001-MuonSimHit.csv",
0052         # TODO Move the following files to a better place?
0053         "Magfield.ipynb",
0054         "SolenoidField.ipynb",
0055         # TODO Add README next to the following files?
0056         "generic-input-config.json",
0057         "generic-alignment-geo.json",
0058         "odd-digi-smearing-config-notime.json",
0059         # TODO Mention these files somewhere?
0060         "generate_particle_data_table.py",
0061         "lazy_autodoc.py",
0062         "codegen/src/codegen/sympy_common.py",
0063         "CompressedIO.h",
0064         # Files for python binding generation
0065         "tgeo_aux.py.in",
0066         "serve.py",
0067         "SNIPPETS.md",
0068         "todo.md",
0069         "bugs.md",
0070         "deprecated.md",
0071         "acts-version-manager.js",
0072         "tex-mml-chtml.js",
0073         "Python/conftest.py",
0074         # Temporarily excluded files. TODO remove in next major release.
0075         "Core/include/Acts/EventData/detail/ParameterTraits.hpp",
0076         "Core/include/Acts/Seeding/PathSeeder.hpp",
0077         "Tests/CommonHelpers/include/ActsTests/CommonHelpers/TestSpacePoint.hpp",
0078     )
0079
0080     suffix_header = (
0081         ".hpp",
0082         ".cuh",
0083     )
0084     suffix_source = (
0085         ".ipp",
0086         ".cpp",
0087         ".cu",
0088     )
0089     suffix_image = (
0090         ".png",
0091         ".svg",
0092         ".jpg",
0093         ".gif",
0094     )
0095     suffix_python = (".py",)
0096     suffix_doc = (
0097         ".md",
0098         ".rst",
0099         ".dox",
0100         ".html",
0101         ".bib",
0102     )
0103     suffix_other = (
0104         "",
0105         ".csv",
0106         ".css",
0107         ".gdml",
0108         ".hepmc3",
0109         ".in",
0110         ".ipynb",
0111         ".json",
0112         ".j2",
0113         ".onnx",
0114         ".root",
0115         ".toml",
0116         ".txt",
0117         ".yml",
0118         ".xml",
0119         ".sh",
0120     )
0121     suffix_allowed = (
0122         suffix_header
0123         + suffix_source
0124         + suffix_image
0125         + suffix_python
0126         + suffix_doc
0127         + suffix_other
0128     )
0129
0130     exit = 0
0131
0132     dirs_base = next(os.walk("."))[1]
0133     dirs_base.append(".")
0134     dirs_base[:] = [d for d in dirs_base if d not in exclude_dirs]
0135     dirs_base_docs = ("docs",)
0136     dirs_base_code = [d for d in dirs_base if d not in dirs_base_docs]
0137
0138     # Collectors
0139     wrong_extension = ()
0140     unused_files = ()
0141
0142     # walk over all files
0143     for root, dirs, files in os.walk("."):
0144         dirs[:] = [d for d in dirs if d not in exclude_dirs]
0145         files[:] = [f for f in files if f not in exclude_files]
0146
0147         # Skip base-directory
0148         if str(Path(root)) == ".":
0149             continue
0150
0151         # Print progress
0152         if root[2:] in dirs_base:
0153             processed_files = 0
0154             current_base_dir = root
0155             number_files = count_files(root, exclude_dirs, exclude_files)
0156             # print empty to start a new line
0157             print("")
0158
0159         # Skip "white-paper-figures"
0160         # TODO Find a more elegant way
0161         if str(root).find("white_papers/figures") != -1:
0162             processed_files += count_files(root, exclude_dirs, exclude_files)
0163             continue
0164
0165         # Skip "DD4hep-tests" since their cmake looks a bit different
0166         # TODO Find a more elegant way
0167         if str(root).find("Tests/UnitTests/Plugins/DD4hep") != -1:
0168             processed_files += count_files(root, exclude_dirs, exclude_files)
0169             continue
0170
0171         root = Path(root)
0172         for filename in files:
0173             processed_files += 1
0174             # get the full path of the file
0175             filepath = root / filename
0176
0177             # Check for wrong extensions
0178             if filepath.suffix not in suffix_allowed:
0179                 wrong_extension += (str(filepath),)
0180
0181             # Check header files and remove
0182             elif filepath.suffix in suffix_header + suffix_source:
0183                 if file_can_be_removed(filename, dirs_base_code):
0184                     unused_files += (str(filepath),)
0185                     remove_cmd = "rm " + str(filepath)
0186                     os.system(remove_cmd)
0187
0188             elif filepath.suffix in suffix_python:
0189                 if not file_can_be_removed("import .*" + filepath.stem, dirs_base):
0190                     continue
0191
0192                 if not file_can_be_removed(
0193                     "from " + filepath.stem + " import", dirs_base
0194                 ):
0195                     continue
0196
0197                 if file_can_be_removed(filename, dirs_base):
0198                     unused_files += (str(filepath),)
0199                     remove_cmd = "rm " + str(filepath)
0200                     os.system(remove_cmd)
0201
0202             # Check documentation files (weak tests)
0203             # TODO find more reliable test for this
0204             elif filepath.suffix in suffix_doc:
0205                 if file_can_be_removed(filepath.stem, dirs_base_docs):
0206                     unused_files += (str(filepath),)
0207                     remove_cmd = "rm " + str(filepath)
0208                     os.system(remove_cmd)
0209
0210             # Check and print other files
0211             elif filepath.suffix in suffix_image + suffix_other:
0212                 if file_can_be_removed(filename, dirs_base):
0213                     unused_files += (str(filepath),)
0214                     remove_cmd = "rm " + str(filepath)
0215                     os.system(remove_cmd)
0216
0217         # Print the progress in place
0218         progress = int(20 * processed_files / number_files)
0219         sys.stdout.write("\r")
0220         sys.stdout.write(
0221             "Checked [%-20s] %d/%d files in %s"
0222             % ("=" * progress, processed_files, number_files, current_base_dir)
0223         )
0224         sys.stdout.flush()
0225
0226     if len(wrong_extension) != 0:
0227         print(
0228             "\n\n\033[31mERROR\033[0m "
0229             + f"The following {len(wrong_extension)} files have an unsupported extension:\n\n"
0230             + "\033[31m"
0231             + "\n".join(wrong_extension)
0232             + "\033[0m"
0233             + "\nCheck if you can change the format to one of the following:\n"
0234             + "\n".join(suffix_allowed)
0235             + "\nIf you really need that specific extension, add it to the list above.\n"
0236         )
0237
0238         exit += 1
0239
0240     if len(unused_files) != 0:
0241         print(
0242             "\n\n\033[31mERROR\033[0m "
0243             + f"The following {len(unused_files)} files seem to be unused:\n"
0244             + "\033[31m"
0245             + "\n".join(unused_files)
0246             + "\033[0m"
0247             + "\nYou have 3 options:"
0248             + "\n\t- Remove them"
0249             + "\n\t- Use them (check proper include)"
0250             + "\n\t- Modify the ignore list of this check\n"
0251         )
0252
0253         exit += 1
0254
0255     if exit == 0:
0256         print(
0257             "\n\n\033[32mINFO\033[0m Finished check_unused_files.py without any errors."
0258         )
0259
0260     return exit
0261
0262
0263 if "__main__" == __name__:
0264     sys.exit(main())