acts/CI/check_unused_files.py

0001 #!/usr/bin/env python3
0002
0003 from pathlib import Path
0004 import os
0005 import sys
0006 import subprocess
0007
0008 EXCLUDE_PATHS = (
0009     ".devcontainer",
0010     ".git",
0011     ".github",
0012     ".idea",
0013     "CI",
0014     "cmake",
0015     # Used by traccc
0016     "Detray/detectors",
0017     # CLI tools
0018     "Detray/tests/tools",
0019     "Detray/python/Io/tests",
0020     "Detray/python/Tests/tests",
0021     "Detray/python/Plugins/tests",
0022     # TODO: Remove the traccc part.
0023     "Traccc",
0024     "git",
0025     "Python",
0026     "Scripts",
0027     # cmake for "DD4hep-tests" looks a bit different
0028     "Tests/UnitTests/Plugins/DD4hep",
0029     "thirdparty",
0030     "white_papers/figures",
0031 )
0032 EXCLUDE_FILES = (
0033     ".gersemirc",
0034     ".gitignore",
0035     ".kodiak.toml",
0036     ".merge-sentinel.yml",
0037     ".policy.yml",
0038     ".pre-commit-config.yaml",
0039     "acts_logo_colored.svg",
0040     "CITATION.cff",
0041     "CMakeLists.txt",
0042     "CMakePresets.json",
0043     "CODE_OF_CONDUCT.md",
0044     "CODEOWNERS",
0045     "codecov.yml",
0046     "pytest.ini",
0047     "README.md",
0048     "readthedocs.yml",
0049     "sonar-project.properties",
0050     # Filename not completed in source
0051     "vertexing_event_mu20_beamspot.csv",
0052     "vertexing_event_mu20_tracks.csv",
0053     "vertexing_event_mu20_vertices_AMVF.csv",
0054     "event000000001-MuonDriftCircle.csv",
0055     "event000000001-MuonSimHit.csv",
0056     # TODO Move the following files to a better place?
0057     "Magfield.ipynb",
0058     "SolenoidField.ipynb",
0059     # TODO Add README next to the following files?
0060     "generic-input-config.json",
0061     "generic-alignment-geo.json",
0062     # TODO Mention these files somewhere?
0063     "codegen/src/codegen/sympy_common.py",
0064     "codegen/src/codegen/detray_backend.py",
0065     "CompressedIO.h",
0066     "generate_particle_data_table.py",
0067     "GeometryModule.h",
0068     "lazy_autodoc.py",
0069     "runtime_geometry_modules.md",
0070     # Files for python binding generation
0071     "acts-version-manager.js",
0072     "bugs.md",
0073     "deprecated.md",
0074     "Python/conftest.py",
0075     "serve.py",
0076     "SNIPPETS.md",
0077     "tex-mml-chtml.js",
0078     "tgeo_aux.py.in",
0079     "todo.md",
0080     # Detray python tests for auto-generated code
0081     "Detray/codegen/detray-sympy/tests/test_assumptions_D.py",
0082     "Detray/codegen/detray-sympy/tests/test_matrices.py",
0083     # Used in traccc
0084     "Detray/tests/include/detray/test/utils/perigee_stopper.hpp",
0085     "Detray/tests/include/detray/test/validation/propagation_validation.hpp",
0086     # Build-time metadata generation
0087     "Detray/python/detray/detectors/impl/definitions.py",
0088     "Detray/python/detray/detectors/impl/type_helpers.py",
0089     # Python uv files
0090     "Detray/codegen/detray-sympy/uv.lock",
0091     "Detray/python/detray/uv.lock",
0092     # TODO: remove after file is gone
0093     "Core/include/Acts/Utilities/ProtoAxisHelpers.hpp",
0094 )
0095 SUFFIX_CPP = (
0096     ".hpp",
0097     ".cuh",
0098     ".sycl",
0099     ".hip",
0100     ".ipp",
0101     ".cpp",
0102     ".cu",
0103 )
0104 SUFFIX_IMAGE = (
0105     ".png",
0106     ".svg",
0107     ".jpg",
0108     ".gif",
0109 )
0110 SUFFIX_PYTHON = (".py",)
0111 SUFFIX_DOC = (
0112     ".md",
0113     ".rst",
0114     ".dox",
0115     ".html",
0116     ".bib",
0117 )
0118 SUFFIX_OTHER = (
0119     "",
0120     ".C",
0121     ".csv",
0122     ".css",
0123     ".gdml",
0124     ".hepmc3",
0125     ".lock",
0126     ".ico",
0127     ".in",
0128     ".ipynb",
0129     ".json",
0130     ".j2",
0131     ".onnx",
0132     ".root",
0133     ".toml",
0134     ".txt",
0135     ".yml",
0136     ".xml",
0137     ".sh",
0138 )
0139
0140
0141 def filter_paths(names, root, exclude_paths=(), exclude_files=()):
0142     """
0143     Filter names from os.walk() based on path substrings and file rules.
0144     Excludes entries if their full path matches exclude_paths or exclude_files.
0145     """
0146
0147     def keep(name):
0148         p = Path(root) / name
0149         p_str = p.as_posix()
0150         return not any(ep in p_str for ep in exclude_paths) and not any(
0151             ef in p_str if "/" in ef else p.name == ef for ef in exclude_files
0152         )
0153
0154     return [name for name in names if keep(name)]
0155
0156
0157 def file_can_be_removed(searchstring, scope):
0158     # Only existence matters: stop at the first match instead of scanning the
0159     # entire tree and collecting every matching line for each candidate.
0160     result = subprocess.run(["grep", "-IRq", "--", searchstring, *scope])
0161     if result.returncode not in (0, 1):
0162         result.check_returncode()
0163     return result.returncode == 1
0164
0165
0166 def count_files(path="."):
0167     count = 0
0168     for root, dirs, files in os.walk(path):
0169         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0170         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0171
0172         count += len(files)
0173
0174     return count
0175
0176
0177 def check_wrong_extensions(walk_root):
0178     """
0179     Collect files with disallowed suffixes. Returns the number of problematic files.
0180     """
0181
0182     suffix_allowed = (
0183         SUFFIX_CPP + SUFFIX_IMAGE + SUFFIX_PYTHON + SUFFIX_DOC + SUFFIX_OTHER
0184     )
0185
0186     wrong_extension = []
0187     for root, dirs, files in os.walk(walk_root):
0188         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0189         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0190
0191         for f in files:
0192             p = Path(root) / f
0193             if p.suffix not in suffix_allowed:
0194                 wrong_extension.append(str(p))
0195
0196     if len(wrong_extension) != 0:
0197         print(
0198             "\n\n\033[31mERROR\033[0m "
0199             + f"The following {len(wrong_extension)} files have an unsupported extension:\n\n"
0200             + "\033[31m"
0201             + "\n".join(wrong_extension)
0202             + "\033[0m"
0203             + "\nCheck if you can change the format to one of the following:\n"
0204             + "\n".join(suffix_allowed)
0205             + "\nIf you really need that specific extension, add it to the list above.\n"
0206         )
0207
0208     return len(wrong_extension)
0209
0210
0211 def find_unused_by_suffix(walk_root, suffixes, search_key, search_scope):
0212     unused = []
0213
0214     for root, dirs, files in os.walk(walk_root):
0215         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0216         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0217
0218         for f in files:
0219             p = Path(root) / f
0220             if p.suffix in suffixes and file_can_be_removed(
0221                 search_key(p), search_scope
0222             ):
0223                 unused.append(str(p))
0224
0225     return unused
0226
0227
0228 def find_unused_python_files(walk_root, dirs_base):
0229     unused = []
0230
0231     for root, dirs, files in os.walk(walk_root):
0232         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0233         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0234
0235         for f in files:
0236             p = Path(root) / f
0237             if p.suffix not in SUFFIX_PYTHON:
0238                 continue
0239
0240             if not file_can_be_removed(r"import .*" + p.stem, dirs_base):
0241                 continue
0242
0243             if not file_can_be_removed(r"from " + p.stem + r" import", dirs_base):
0244                 continue
0245
0246             if file_can_be_removed(p.name, dirs_base):
0247                 unused.append(str(p))
0248
0249     return unused
0250
0251
0252 def main():
0253     print("\033[32mINFO\033[0m Start check_unused_files.py ...")
0254
0255     exit = 0
0256
0257     dirs_base = next(os.walk("."))[1]
0258     dirs_base.append(".")
0259     dirs_base[:] = filter_paths(dirs_base, Path("."), EXCLUDE_PATHS)
0260     dirs_base_docs = ("docs",)
0261     dirs_base_code = filter_paths(dirs_base, Path("."), dirs_base_docs)
0262
0263     exit += check_wrong_extensions(".")
0264
0265     # Collector
0266     unused_files = []
0267
0268     unused_files += find_unused_by_suffix(
0269         ".", SUFFIX_CPP, lambda p: p.name, dirs_base_code
0270     )
0271
0272     unused_files += find_unused_python_files(".", dirs_base)
0273
0274     # TODO find more reliable test for this
0275     unused_files += find_unused_by_suffix(
0276         ".", SUFFIX_DOC, lambda p: p.stem, dirs_base_docs
0277     )
0278
0279     unused_files += find_unused_by_suffix(
0280         ".", SUFFIX_IMAGE + SUFFIX_OTHER, lambda p: p.name, dirs_base
0281     )
0282
0283     if len(unused_files) != 0:
0284         print(
0285             "\n\n\033[31mERROR\033[0m "
0286             + f"The following {len(unused_files)} files seem to be unused:\n"
0287             + "\033[31m"
0288             + "\n".join(unused_files)
0289             + "\033[0m"
0290             + "\nYou have 3 options:"
0291             + "\n\t- Remove them"
0292             + "\n\t- Use them (check proper include)"
0293             + "\n\t- Modify the ignore list of this check\n"
0294         )
0295
0296         exit += 1
0297
0298     if exit == 0:
0299         print(
0300             "\n\n\033[32mINFO\033[0m Finished check_unused_files.py without any errors."
0301         )
0302
0303     return exit
0304
0305
0306 if "__main__" == __name__:
0307     sys.exit(main())