acts/CI/check_unused_files.py

0001 #!/usr/bin/env python3
0002
0003 from pathlib import Path
0004 import os
0005 import sys
0006 import subprocess
0007
0008 EXCLUDE_PATHS = (
0009     ".devcontainer",
0010     ".git",
0011     ".github",
0012     ".idea",
0013     "CI",
0014     "cmake",
0015     # Used by traccc
0016     "Detray/detectors",
0017     # CLI tools
0018     "Detray/tests/tools",
0019     "git",
0020     "Python",
0021     "Scripts",
0022     # cmake for "DD4hep-tests" looks a bit different
0023     "Tests/UnitTests/Plugins/DD4hep",
0024     "thirdparty",
0025     "white_papers/figures",
0026 )
0027 EXCLUDE_FILES = (
0028     ".gersemirc",
0029     ".gitignore",
0030     ".kodiak.toml",
0031     ".merge-sentinel.yml",
0032     ".policy.yml",
0033     ".pre-commit-config.yaml",
0034     "acts_logo_colored.svg",
0035     "CITATION.cff",
0036     "CMakeLists.txt",
0037     "CMakePresets.json",
0038     "CODE_OF_CONDUCT.md",
0039     "CODEOWNERS",
0040     "codecov.yml",
0041     "pytest.ini",
0042     "README.md",
0043     "readthedocs.yml",
0044     "sonar-project.properties",
0045     # Filename not completed in source
0046     "vertexing_event_mu20_beamspot.csv",
0047     "vertexing_event_mu20_tracks.csv",
0048     "vertexing_event_mu20_vertices_AMVF.csv",
0049     "event000000001-MuonDriftCircle.csv",
0050     "event000000001-MuonSimHit.csv",
0051     # TODO Move the following files to a better place?
0052     "Magfield.ipynb",
0053     "SolenoidField.ipynb",
0054     # TODO Add README next to the following files?
0055     "generic-input-config.json",
0056     "generic-alignment-geo.json",
0057     # TODO Mention these files somewhere?
0058     "codegen/src/codegen/sympy_common.py",
0059     "codegen/src/codegen/detray_backend.py",
0060     "CompressedIO.h",
0061     "generate_particle_data_table.py",
0062     "GeometryModule.h",
0063     "lazy_autodoc.py",
0064     "runtime_geometry_modules.md",
0065     # Files for python binding generation
0066     "acts-version-manager.js",
0067     "bugs.md",
0068     "deprecated.md",
0069     "Python/conftest.py",
0070     "serve.py",
0071     "SNIPPETS.md",
0072     "tex-mml-chtml.js",
0073     "tgeo_aux.py.in",
0074     "todo.md",
0075     # Detray python tests for auto-generated code
0076     "Detray/codegen/detray-sympy/tests/test_assumptions_D.py",
0077     "Detray/codegen/detray-sympy/tests/test_matrices.py",
0078     # Used in traccc
0079     "Detray/tests/include/detray/test/utils/perigee_stopper.hpp",
0080     "Detray/tests/include/detray/test/validation/propagation_validation.hpp",
0081     # Build-time metadata generation
0082     "Detray/python/detray/detectors/impl/definitions.py",
0083     "Detray/python/detray/detectors/impl/type_helpers.py",
0084     # Python uv files
0085     "Detray/codegen/detray-sympy/uv.lock",
0086     "Detray/python/detray/uv.lock",
0087 )
0088 SUFFIX_CPP = (
0089     ".hpp",
0090     ".cuh",
0091     ".sycl",
0092     ".hip",
0093     ".ipp",
0094     ".cpp",
0095     ".cu",
0096 )
0097 SUFFIX_IMAGE = (
0098     ".png",
0099     ".svg",
0100     ".jpg",
0101     ".gif",
0102 )
0103 SUFFIX_PYTHON = (".py",)
0104 SUFFIX_DOC = (
0105     ".md",
0106     ".rst",
0107     ".dox",
0108     ".html",
0109     ".bib",
0110 )
0111 SUFFIX_OTHER = (
0112     "",
0113     ".C",
0114     ".csv",
0115     ".css",
0116     ".gdml",
0117     ".hepmc3",
0118     ".lock",
0119     ".ico",
0120     ".in",
0121     ".ipynb",
0122     ".json",
0123     ".j2",
0124     ".onnx",
0125     ".root",
0126     ".toml",
0127     ".txt",
0128     ".yml",
0129     ".xml",
0130     ".sh",
0131 )
0132
0133
0134 def filter_paths(names, root, exclude_paths=(), exclude_files=()):
0135     """
0136     Filter names from os.walk() based on path substrings and file rules.
0137     Excludes entries if their full path matches exclude_paths or exclude_files.
0138     """
0139
0140     def keep(name):
0141         p = Path(root) / name
0142         p_str = p.as_posix()
0143         return not any(ep in p_str for ep in exclude_paths) and not any(
0144             ef in p_str if "/" in ef else p.name == ef for ef in exclude_files
0145         )
0146
0147     return [name for name in names if keep(name)]
0148
0149
0150 def file_can_be_removed(searchstring, scope):
0151     cmd = "grep -IR '" + searchstring + "' " + " ".join(scope)
0152
0153     p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
0154     output, _ = p.communicate()
0155     return output == b""
0156
0157
0158 def count_files(path="."):
0159     count = 0
0160     for root, dirs, files in os.walk(path):
0161         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0162         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0163
0164         count += len(files)
0165
0166     return count
0167
0168
0169 def check_wrong_extensions(walk_root):
0170     """
0171     Collect files with disallowed suffixes. Returns the number of problematic files.
0172     """
0173
0174     suffix_allowed = (
0175         SUFFIX_CPP + SUFFIX_IMAGE + SUFFIX_PYTHON + SUFFIX_DOC + SUFFIX_OTHER
0176     )
0177
0178     wrong_extension = []
0179     for root, dirs, files in os.walk(walk_root):
0180         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0181         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0182
0183         for f in files:
0184             p = Path(root) / f
0185             if p.suffix not in suffix_allowed:
0186                 wrong_extension.append(str(p))
0187
0188     if len(wrong_extension) != 0:
0189         print(
0190             "\n\n\033[31mERROR\033[0m "
0191             + f"The following {len(wrong_extension)} files have an unsupported extension:\n\n"
0192             + "\033[31m"
0193             + "\n".join(wrong_extension)
0194             + "\033[0m"
0195             + "\nCheck if you can change the format to one of the following:\n"
0196             + "\n".join(suffix_allowed)
0197             + "\nIf you really need that specific extension, add it to the list above.\n"
0198         )
0199
0200     return len(wrong_extension)
0201
0202
0203 def find_unused_by_suffix(walk_root, suffixes, search_key, search_scope):
0204     unused = []
0205
0206     for root, dirs, files in os.walk(walk_root):
0207         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0208         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0209
0210         for f in files:
0211             p = Path(root) / f
0212             if p.suffix in suffixes and file_can_be_removed(
0213                 search_key(p), search_scope
0214             ):
0215                 unused.append(str(p))
0216
0217     return unused
0218
0219
0220 def find_unused_python_files(walk_root, dirs_base):
0221     unused = []
0222
0223     for root, dirs, files in os.walk(walk_root):
0224         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0225         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0226
0227         for f in files:
0228             p = Path(root) / f
0229             if p.suffix not in SUFFIX_PYTHON:
0230                 continue
0231
0232             if not file_can_be_removed(r"import .*" + p.stem, dirs_base):
0233                 continue
0234
0235             if not file_can_be_removed(r"from " + p.stem + r" import", dirs_base):
0236                 continue
0237
0238             if file_can_be_removed(p.name, dirs_base):
0239                 unused.append(str(p))
0240
0241     return unused
0242
0243
0244 def main():
0245     print("\033[32mINFO\033[0m Start check_unused_files.py ...")
0246
0247     exit = 0
0248
0249     dirs_base = next(os.walk("."))[1]
0250     dirs_base.append(".")
0251     dirs_base[:] = filter_paths(dirs_base, Path("."), EXCLUDE_PATHS)
0252     dirs_base_docs = ("docs",)
0253     dirs_base_code = filter_paths(dirs_base, Path("."), dirs_base_docs)
0254
0255     exit += check_wrong_extensions(".")
0256
0257     # Collector
0258     unused_files = []
0259
0260     unused_files += find_unused_by_suffix(
0261         ".", SUFFIX_CPP, lambda p: p.name, dirs_base_code
0262     )
0263
0264     unused_files += find_unused_python_files(".", dirs_base)
0265
0266     # TODO find more reliable test for this
0267     unused_files += find_unused_by_suffix(
0268         ".", SUFFIX_DOC, lambda p: p.stem, dirs_base_docs
0269     )
0270
0271     unused_files += find_unused_by_suffix(
0272         ".", SUFFIX_IMAGE + SUFFIX_OTHER, lambda p: p.name, dirs_base
0273     )
0274
0275     if len(unused_files) != 0:
0276         print(
0277             "\n\n\033[31mERROR\033[0m "
0278             + f"The following {len(unused_files)} files seem to be unused:\n"
0279             + "\033[31m"
0280             + "\n".join(unused_files)
0281             + "\033[0m"
0282             + "\nYou have 3 options:"
0283             + "\n\t- Remove them"
0284             + "\n\t- Use them (check proper include)"
0285             + "\n\t- Modify the ignore list of this check\n"
0286         )
0287
0288         exit += 1
0289
0290     if exit == 0:
0291         print(
0292             "\n\n\033[32mINFO\033[0m Finished check_unused_files.py without any errors."
0293         )
0294
0295     return exit
0296
0297
0298 if "__main__" == __name__:
0299     sys.exit(main())