acts/CI/check_unused_files.py

0001 #!/usr/bin/env python3
0002
0003 from pathlib import Path
0004 import os
0005 import sys
0006 import subprocess
0007
0008 EXCLUDE_PATHS = (
0009     ".devcontainer",
0010     ".git",
0011     ".github",
0012     ".idea",
0013     "CI",
0014     "cmake",
0015     # Used by traccc
0016     "Detray/detectors",
0017     # CLI tools
0018     "Detray/tests/tools",
0019     "git",
0020     "Python",
0021     "Scripts",
0022     # cmake for "DD4hep-tests" looks a bit different
0023     "Tests/UnitTests/Plugins/DD4hep",
0024     "thirdparty",
0025     "white_papers/figures",
0026 )
0027 EXCLUDE_FILES = (
0028     ".gersemirc",
0029     ".gitignore",
0030     ".kodiak.toml",
0031     ".merge-sentinel.yml",
0032     ".policy.yml",
0033     ".pre-commit-config.yaml",
0034     "acts_logo_colored.svg",
0035     "CITATION.cff",
0036     "CMakeLists.txt",
0037     "CMakePresets.json",
0038     "CODE_OF_CONDUCT.md",
0039     "CODEOWNERS",
0040     "codecov.yml",
0041     "pytest.ini",
0042     "README.md",
0043     "readthedocs.yml",
0044     "sonar-project.properties",
0045     # Filename not completed in source
0046     "vertexing_event_mu20_beamspot.csv",
0047     "vertexing_event_mu20_tracks.csv",
0048     "vertexing_event_mu20_vertices_AMVF.csv",
0049     "event000000001-MuonDriftCircle.csv",
0050     "event000000001-MuonSimHit.csv",
0051     # TODO Move the following files to a better place?
0052     "Magfield.ipynb",
0053     "SolenoidField.ipynb",
0054     # TODO Add README next to the following files?
0055     "generic-input-config.json",
0056     "generic-alignment-geo.json",
0057     "odd-digi-smearing-config-notime.json",
0058     # TODO Mention these files somewhere?
0059     "codegen/src/codegen/sympy_common.py",
0060     "CompressedIO.h",
0061     "generate_particle_data_table.py",
0062     "GeometryModule.h",
0063     "lazy_autodoc.py",
0064     "runtime_geometry_modules.md",
0065     # Files for python binding generation
0066     "acts-version-manager.js",
0067     "bugs.md",
0068     "deprecated.md",
0069     "Python/conftest.py",
0070     "serve.py",
0071     "SNIPPETS.md",
0072     "tex-mml-chtml.js",
0073     "tgeo_aux.py.in",
0074     "todo.md",
0075     # Detray python tests for auto-generated code
0076     "Detray/codegen/detray-sympy/tests/test_assumptions_D.py",
0077     "Detray/codegen/detray-sympy/tests/test_matrices.py",
0078     # Used in traccc
0079     "Detray/tests/include/detray/test/utils/perigee_stopper.hpp",
0080     # Build-time metadata generation
0081     "Detray/python/detray/detectors/impl/definitions.py",
0082     "Detray/python/detray/detectors/impl/type_helpers.py",
0083     # Python uv files
0084     "Detray/codegen/detray-sympy/uv.lock",
0085     "Detray/python/detray/uv.lock",
0086     # Temporarily excluded files. TODO remove in next major release.
0087     "Core/include/Acts/EventData/detail/ParameterTraits.hpp",
0088     "Core/include/Acts/Seeding/PathSeeder.hpp",
0089     "Tests/CommonHelpers/include/ActsTests/CommonHelpers/TestSpacePoint.hpp",
0090 )
0091 SUFFIX_CPP = (
0092     ".hpp",
0093     ".cuh",
0094     ".sycl",
0095     ".hip",
0096     ".ipp",
0097     ".cpp",
0098     ".cu",
0099 )
0100 SUFFIX_IMAGE = (
0101     ".png",
0102     ".svg",
0103     ".jpg",
0104     ".gif",
0105 )
0106 SUFFIX_PYTHON = (".py",)
0107 SUFFIX_DOC = (
0108     ".md",
0109     ".rst",
0110     ".dox",
0111     ".html",
0112     ".bib",
0113 )
0114 SUFFIX_OTHER = (
0115     "",
0116     ".C",
0117     ".csv",
0118     ".css",
0119     ".gdml",
0120     ".hepmc3",
0121     ".lock",
0122     ".ico",
0123     ".in",
0124     ".ipynb",
0125     ".json",
0126     ".j2",
0127     ".onnx",
0128     ".root",
0129     ".toml",
0130     ".txt",
0131     ".yml",
0132     ".xml",
0133     ".sh",
0134 )
0135
0136
0137 def filter_paths(names, root, exclude_paths=(), exclude_files=()):
0138     """
0139     Filter names from os.walk() based on path substrings and file rules.
0140     Excludes entries if their full path matches exclude_paths or exclude_files.
0141     """
0142
0143     def keep(name):
0144         p = Path(root) / name
0145         p_str = p.as_posix()
0146         return not any(ep in p_str for ep in exclude_paths) and not any(
0147             ef in p_str if "/" in ef else p.name == ef for ef in exclude_files
0148         )
0149
0150     return [name for name in names if keep(name)]
0151
0152
0153 def file_can_be_removed(searchstring, scope):
0154     cmd = "grep -IR '" + searchstring + "' " + " ".join(scope)
0155
0156     p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
0157     output, _ = p.communicate()
0158     return output == b""
0159
0160
0161 def count_files(path="."):
0162     count = 0
0163     for root, dirs, files in os.walk(path):
0164         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0165         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0166
0167         count += len(files)
0168
0169     return count
0170
0171
0172 def check_wrong_extensions(walk_root):
0173     """
0174     Collect files with disallowed suffixes. Returns the number of problematic files.
0175     """
0176
0177     suffix_allowed = (
0178         SUFFIX_CPP + SUFFIX_IMAGE + SUFFIX_PYTHON + SUFFIX_DOC + SUFFIX_OTHER
0179     )
0180
0181     wrong_extension = []
0182     for root, dirs, files in os.walk(walk_root):
0183         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0184         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0185
0186         for f in files:
0187             p = Path(root) / f
0188             if p.suffix not in suffix_allowed:
0189                 wrong_extension.append(str(p))
0190
0191     if len(wrong_extension) != 0:
0192         print(
0193             "\n\n\033[31mERROR\033[0m "
0194             + f"The following {len(wrong_extension)} files have an unsupported extension:\n\n"
0195             + "\033[31m"
0196             + "\n".join(wrong_extension)
0197             + "\033[0m"
0198             + "\nCheck if you can change the format to one of the following:\n"
0199             + "\n".join(suffix_allowed)
0200             + "\nIf you really need that specific extension, add it to the list above.\n"
0201         )
0202
0203     return len(wrong_extension)
0204
0205
0206 def find_unused_by_suffix(walk_root, suffixes, search_key, search_scope):
0207     unused = []
0208
0209     for root, dirs, files in os.walk(walk_root):
0210         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0211         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0212
0213         for f in files:
0214             p = Path(root) / f
0215             if p.suffix in suffixes and file_can_be_removed(
0216                 search_key(p), search_scope
0217             ):
0218                 unused.append(str(p))
0219
0220     return unused
0221
0222
0223 def find_unused_python_files(walk_root, dirs_base):
0224     unused = []
0225
0226     for root, dirs, files in os.walk(walk_root):
0227         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0228         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0229
0230         for f in files:
0231             p = Path(root) / f
0232             if p.suffix not in SUFFIX_PYTHON:
0233                 continue
0234
0235             if not file_can_be_removed(r"import .*" + p.stem, dirs_base):
0236                 continue
0237
0238             if not file_can_be_removed(r"from " + p.stem + r" import", dirs_base):
0239                 continue
0240
0241             if file_can_be_removed(p.name, dirs_base):
0242                 unused.append(str(p))
0243
0244     return unused
0245
0246
0247 def main():
0248     print("\033[32mINFO\033[0m Start check_unused_files.py ...")
0249
0250     exit = 0
0251
0252     dirs_base = next(os.walk("."))[1]
0253     dirs_base.append(".")
0254     dirs_base[:] = filter_paths(dirs_base, Path("."), EXCLUDE_PATHS)
0255     dirs_base_docs = ("docs",)
0256     dirs_base_code = filter_paths(dirs_base, Path("."), dirs_base_docs)
0257
0258     exit += check_wrong_extensions(".")
0259
0260     # Collector
0261     unused_files = []
0262
0263     unused_files += find_unused_by_suffix(
0264         ".", SUFFIX_CPP, lambda p: p.name, dirs_base_code
0265     )
0266
0267     unused_files += find_unused_python_files(".", dirs_base)
0268
0269     # TODO find more reliable test for this
0270     unused_files += find_unused_by_suffix(
0271         ".", SUFFIX_DOC, lambda p: p.stem, dirs_base_docs
0272     )
0273
0274     unused_files += find_unused_by_suffix(
0275         ".", SUFFIX_IMAGE + SUFFIX_OTHER, lambda p: p.name, dirs_base
0276     )
0277
0278     if len(unused_files) != 0:
0279         print(
0280             "\n\n\033[31mERROR\033[0m "
0281             + f"The following {len(unused_files)} files seem to be unused:\n"
0282             + "\033[31m"
0283             + "\n".join(unused_files)
0284             + "\033[0m"
0285             + "\nYou have 3 options:"
0286             + "\n\t- Remove them"
0287             + "\n\t- Use them (check proper include)"
0288             + "\n\t- Modify the ignore list of this check\n"
0289         )
0290
0291         exit += 1
0292
0293     if exit == 0:
0294         print(
0295             "\n\n\033[32mINFO\033[0m Finished check_unused_files.py without any errors."
0296         )
0297
0298     return exit
0299
0300
0301 if "__main__" == __name__:
0302     sys.exit(main())