acts/CI/check_unused_files.py

0001 #!/usr/bin/env python3
0002
0003 from pathlib import Path
0004 import os
0005 import sys
0006 import subprocess
0007
0008 EXCLUDE_PATHS = (
0009     ".devcontainer",
0010     ".git",
0011     ".github",
0012     ".idea",
0013     "CI",
0014     "cmake",
0015     # Used by traccc
0016     "Detray/detectors",
0017     # CLI tools
0018     "Detray/tests/tools",
0019     "Detray/python/Io/tests",
0020     "Detray/python/Tests/tests",
0021     "Detray/python/Plugins/tests",
0022     # TODO: Remove the traccc part.
0023     "Traccc",
0024     "git",
0025     "Python",
0026     "Scripts",
0027     # cmake for "DD4hep-tests" looks a bit different
0028     "Tests/UnitTests/Plugins/DD4hep",
0029     "thirdparty",
0030     "white_papers/figures",
0031 )
0032 EXCLUDE_FILES = (
0033     ".gersemirc",
0034     ".gitignore",
0035     ".kodiak.toml",
0036     ".merge-sentinel.yml",
0037     ".policy.yml",
0038     ".pre-commit-config.yaml",
0039     "acts_logo_colored.svg",
0040     "CITATION.cff",
0041     "CMakeLists.txt",
0042     "CMakePresets.json",
0043     "CODE_OF_CONDUCT.md",
0044     "CODEOWNERS",
0045     "codecov.yml",
0046     "pytest.ini",
0047     "README.md",
0048     "readthedocs.yml",
0049     "sonar-project.properties",
0050     # Filename not completed in source
0051     "vertexing_event_mu20_beamspot.csv",
0052     "vertexing_event_mu20_tracks.csv",
0053     "vertexing_event_mu20_vertices_AMVF.csv",
0054     "event000000001-MuonDriftCircle.csv",
0055     "event000000001-MuonSimHit.csv",
0056     # TODO Move the following files to a better place?
0057     "Magfield.ipynb",
0058     "SolenoidField.ipynb",
0059     # TODO Add README next to the following files?
0060     "generic-input-config.json",
0061     "generic-alignment-geo.json",
0062     # TODO Mention these files somewhere?
0063     "codegen/src/codegen/sympy_common.py",
0064     "codegen/src/codegen/detray_backend.py",
0065     "CompressedIO.h",
0066     "generate_particle_data_table.py",
0067     "GeometryModule.h",
0068     "lazy_autodoc.py",
0069     "runtime_geometry_modules.md",
0070     # Files for python binding generation
0071     "acts-version-manager.js",
0072     "bugs.md",
0073     "deprecated.md",
0074     "Python/conftest.py",
0075     "serve.py",
0076     "SNIPPETS.md",
0077     "tex-mml-chtml.js",
0078     "tgeo_aux.py.in",
0079     # Pytest collects these examples; Doxygen includes the group by ID.
0080     "docs/examples/test_generic.py",
0081     "docs/examples/test_performance_and_plotting.py",
0082     "docs/groups/python/performance_and_plotting.md",
0083     "todo.md",
0084     # Detray python tests for auto-generated code
0085     "Detray/codegen/detray-sympy/tests/test_assumptions_D.py",
0086     "Detray/codegen/detray-sympy/tests/test_matrices.py",
0087     # Used in traccc
0088     "Detray/tests/include/detray/test/utils/perigee_stopper.hpp",
0089     "Detray/tests/include/detray/test/validation/propagation_validation.hpp",
0090     # Build-time metadata generation
0091     "Detray/python/detray/detectors/impl/definitions.py",
0092     "Detray/python/detray/detectors/impl/type_helpers.py",
0093     # Python uv files
0094     "Detray/codegen/detray-sympy/uv.lock",
0095     "Detray/python/detray/uv.lock",
0096     # Generated PyPI readme, checked by Python/Examples/tests/test_pypi_readme.py
0097     "CI/pypi_readme.md",
0098     # TODO: remove after file is gone
0099     "Core/include/Acts/Utilities/ProtoAxisHelpers.hpp",
0100 )
0101 SUFFIX_CPP = (
0102     ".hpp",
0103     ".cuh",
0104     ".sycl",
0105     ".hip",
0106     ".ipp",
0107     ".cpp",
0108     ".cu",
0109 )
0110 SUFFIX_IMAGE = (
0111     ".png",
0112     ".svg",
0113     ".jpg",
0114     ".gif",
0115 )
0116 SUFFIX_PYTHON = (".py",)
0117 SUFFIX_DOC = (
0118     ".md",
0119     ".rst",
0120     ".dox",
0121     ".html",
0122     ".bib",
0123 )
0124 SUFFIX_OTHER = (
0125     "",
0126     ".C",
0127     ".csv",
0128     ".css",
0129     ".gdml",
0130     ".hepmc3",
0131     ".lock",
0132     ".ico",
0133     ".in",
0134     ".ipynb",
0135     ".json",
0136     ".j2",
0137     ".onnx",
0138     ".root",
0139     ".toml",
0140     ".txt",
0141     ".yml",
0142     ".xml",
0143     ".sh",
0144 )
0145
0146
0147 def filter_paths(names, root, exclude_paths=(), exclude_files=()):
0148     """
0149     Filter names from os.walk() based on path substrings and file rules.
0150     Excludes entries if their full path matches exclude_paths or exclude_files.
0151     """
0152
0153     def keep(name):
0154         p = Path(root) / name
0155         p_str = p.as_posix()
0156         return not any(ep in p_str for ep in exclude_paths) and not any(
0157             ef in p_str if "/" in ef else p.name == ef for ef in exclude_files
0158         )
0159
0160     return [name for name in names if keep(name)]
0161
0162
0163 def file_can_be_removed(searchstring, scope):
0164     # Only existence matters: stop at the first match instead of scanning the
0165     # entire tree and collecting every matching line for each candidate.
0166     result = subprocess.run(["grep", "-IRq", "--", searchstring, *scope])
0167     if result.returncode not in (0, 1):
0168         result.check_returncode()
0169     return result.returncode == 1
0170
0171
0172 def count_files(path="."):
0173     count = 0
0174     for root, dirs, files in os.walk(path):
0175         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0176         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0177
0178         count += len(files)
0179
0180     return count
0181
0182
0183 def check_wrong_extensions(walk_root):
0184     """
0185     Collect files with disallowed suffixes. Returns the number of problematic files.
0186     """
0187
0188     suffix_allowed = (
0189         SUFFIX_CPP + SUFFIX_IMAGE + SUFFIX_PYTHON + SUFFIX_DOC + SUFFIX_OTHER
0190     )
0191
0192     wrong_extension = []
0193     for root, dirs, files in os.walk(walk_root):
0194         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0195         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0196
0197         for f in files:
0198             p = Path(root) / f
0199             if p.suffix not in suffix_allowed:
0200                 wrong_extension.append(str(p))
0201
0202     if len(wrong_extension) != 0:
0203         print(
0204             "\n\n\033[31mERROR\033[0m "
0205             + f"The following {len(wrong_extension)} files have an unsupported extension:\n\n"
0206             + "\033[31m"
0207             + "\n".join(wrong_extension)
0208             + "\033[0m"
0209             + "\nCheck if you can change the format to one of the following:\n"
0210             + "\n".join(suffix_allowed)
0211             + "\nIf you really need that specific extension, add it to the list above.\n"
0212         )
0213
0214     return len(wrong_extension)
0215
0216
0217 def find_unused_by_suffix(walk_root, suffixes, search_key, search_scope):
0218     unused = []
0219
0220     for root, dirs, files in os.walk(walk_root):
0221         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0222         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0223
0224         for f in files:
0225             p = Path(root) / f
0226             if p.suffix in suffixes and file_can_be_removed(
0227                 search_key(p), search_scope
0228             ):
0229                 unused.append(str(p))
0230
0231     return unused
0232
0233
0234 def find_unused_python_files(walk_root, dirs_base):
0235     unused = []
0236
0237     for root, dirs, files in os.walk(walk_root):
0238         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0239         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0240
0241         for f in files:
0242             p = Path(root) / f
0243             if p.suffix not in SUFFIX_PYTHON:
0244                 continue
0245
0246             if not file_can_be_removed(r"import .*" + p.stem, dirs_base):
0247                 continue
0248
0249             if not file_can_be_removed(r"from " + p.stem + r" import", dirs_base):
0250                 continue
0251
0252             if file_can_be_removed(p.name, dirs_base):
0253                 unused.append(str(p))
0254
0255     return unused
0256
0257
0258 def main():
0259     print("\033[32mINFO\033[0m Start check_unused_files.py ...")
0260
0261     exit = 0
0262
0263     dirs_base = next(os.walk("."))[1]
0264     dirs_base.append(".")
0265     dirs_base[:] = filter_paths(dirs_base, Path("."), EXCLUDE_PATHS)
0266     dirs_base_docs = ("docs",)
0267     dirs_base_code = filter_paths(dirs_base, Path("."), dirs_base_docs)
0268
0269     exit += check_wrong_extensions(".")
0270
0271     # Collector
0272     unused_files = []
0273
0274     unused_files += find_unused_by_suffix(
0275         ".", SUFFIX_CPP, lambda p: p.name, dirs_base_code
0276     )
0277
0278     unused_files += find_unused_python_files(".", dirs_base)
0279
0280     # TODO find more reliable test for this
0281     unused_files += find_unused_by_suffix(
0282         ".", SUFFIX_DOC, lambda p: p.stem, dirs_base_docs
0283     )
0284
0285     unused_files += find_unused_by_suffix(
0286         ".", SUFFIX_IMAGE + SUFFIX_OTHER, lambda p: p.name, dirs_base
0287     )
0288
0289     if len(unused_files) != 0:
0290         print(
0291             "\n\n\033[31mERROR\033[0m "
0292             + f"The following {len(unused_files)} files seem to be unused:\n"
0293             + "\033[31m"
0294             + "\n".join(unused_files)
0295             + "\033[0m"
0296             + "\nYou have 3 options:"
0297             + "\n\t- Remove them"
0298             + "\n\t- Use them (check proper include)"
0299             + "\n\t- Modify the ignore list of this check\n"
0300         )
0301
0302         exit += 1
0303
0304     if exit == 0:
0305         print(
0306             "\n\n\033[32mINFO\033[0m Finished check_unused_files.py without any errors."
0307         )
0308
0309     return exit
0310
0311
0312 if "__main__" == __name__:
0313     sys.exit(main())