acts/CI/check_unused_files.py

0001 #!/usr/bin/env python3
0002
0003 from pathlib import Path
0004 import os
0005 import sys
0006 import subprocess
0007
0008 EXCLUDE_PATHS = (
0009     ".devcontainer",
0010     ".git",
0011     ".github",
0012     ".idea",
0013     "CI",
0014     "cmake",
0015     # Used by traccc
0016     "Detray/detectors",
0017     # CLI tools
0018     "Detray/tests/tools",
0019     "Detray/python/Core/tests",
0020     # TODO: Remove the traccc part.
0021     "Traccc",
0022     "git",
0023     "Python",
0024     "Scripts",
0025     # cmake for "DD4hep-tests" looks a bit different
0026     "Tests/UnitTests/Plugins/DD4hep",
0027     "thirdparty",
0028     "white_papers/figures",
0029 )
0030 EXCLUDE_FILES = (
0031     ".gersemirc",
0032     ".gitignore",
0033     ".kodiak.toml",
0034     ".merge-sentinel.yml",
0035     ".policy.yml",
0036     ".pre-commit-config.yaml",
0037     "acts_logo_colored.svg",
0038     "CITATION.cff",
0039     "CMakeLists.txt",
0040     "CMakePresets.json",
0041     "CODE_OF_CONDUCT.md",
0042     "CODEOWNERS",
0043     "codecov.yml",
0044     "pytest.ini",
0045     "README.md",
0046     "readthedocs.yml",
0047     "sonar-project.properties",
0048     # Filename not completed in source
0049     "vertexing_event_mu20_beamspot.csv",
0050     "vertexing_event_mu20_tracks.csv",
0051     "vertexing_event_mu20_vertices_AMVF.csv",
0052     "event000000001-MuonDriftCircle.csv",
0053     "event000000001-MuonSimHit.csv",
0054     # TODO Move the following files to a better place?
0055     "Magfield.ipynb",
0056     "SolenoidField.ipynb",
0057     # TODO Add README next to the following files?
0058     "generic-input-config.json",
0059     "generic-alignment-geo.json",
0060     # TODO Mention these files somewhere?
0061     "codegen/src/codegen/sympy_common.py",
0062     "codegen/src/codegen/detray_backend.py",
0063     "CompressedIO.h",
0064     "generate_particle_data_table.py",
0065     "GeometryModule.h",
0066     "lazy_autodoc.py",
0067     "runtime_geometry_modules.md",
0068     # Files for python binding generation
0069     "acts-version-manager.js",
0070     "bugs.md",
0071     "deprecated.md",
0072     "Python/conftest.py",
0073     "serve.py",
0074     "SNIPPETS.md",
0075     "tex-mml-chtml.js",
0076     "tgeo_aux.py.in",
0077     "todo.md",
0078     # Detray python tests for auto-generated code
0079     "Detray/codegen/detray-sympy/tests/test_assumptions_D.py",
0080     "Detray/codegen/detray-sympy/tests/test_matrices.py",
0081     # Used in traccc
0082     "Detray/tests/include/detray/test/utils/perigee_stopper.hpp",
0083     "Detray/tests/include/detray/test/validation/propagation_validation.hpp",
0084     # Build-time metadata generation
0085     "Detray/python/detray/detectors/impl/definitions.py",
0086     "Detray/python/detray/detectors/impl/type_helpers.py",
0087     # Python uv files
0088     "Detray/codegen/detray-sympy/uv.lock",
0089     "Detray/python/detray/uv.lock",
0090     # TODO: remove after file is gone
0091     "Core/include/Acts/Utilities/ProtoAxisHelpers.hpp",
0092 )
0093 SUFFIX_CPP = (
0094     ".hpp",
0095     ".cuh",
0096     ".sycl",
0097     ".hip",
0098     ".ipp",
0099     ".cpp",
0100     ".cu",
0101 )
0102 SUFFIX_IMAGE = (
0103     ".png",
0104     ".svg",
0105     ".jpg",
0106     ".gif",
0107 )
0108 SUFFIX_PYTHON = (".py",)
0109 SUFFIX_DOC = (
0110     ".md",
0111     ".rst",
0112     ".dox",
0113     ".html",
0114     ".bib",
0115 )
0116 SUFFIX_OTHER = (
0117     "",
0118     ".C",
0119     ".csv",
0120     ".css",
0121     ".gdml",
0122     ".hepmc3",
0123     ".lock",
0124     ".ico",
0125     ".in",
0126     ".ipynb",
0127     ".json",
0128     ".j2",
0129     ".onnx",
0130     ".root",
0131     ".toml",
0132     ".txt",
0133     ".yml",
0134     ".xml",
0135     ".sh",
0136 )
0137
0138
0139 def filter_paths(names, root, exclude_paths=(), exclude_files=()):
0140     """
0141     Filter names from os.walk() based on path substrings and file rules.
0142     Excludes entries if their full path matches exclude_paths or exclude_files.
0143     """
0144
0145     def keep(name):
0146         p = Path(root) / name
0147         p_str = p.as_posix()
0148         return not any(ep in p_str for ep in exclude_paths) and not any(
0149             ef in p_str if "/" in ef else p.name == ef for ef in exclude_files
0150         )
0151
0152     return [name for name in names if keep(name)]
0153
0154
0155 def file_can_be_removed(searchstring, scope):
0156     cmd = "grep -IR '" + searchstring + "' " + " ".join(scope)
0157
0158     p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
0159     output, _ = p.communicate()
0160     return output == b""
0161
0162
0163 def count_files(path="."):
0164     count = 0
0165     for root, dirs, files in os.walk(path):
0166         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0167         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0168
0169         count += len(files)
0170
0171     return count
0172
0173
0174 def check_wrong_extensions(walk_root):
0175     """
0176     Collect files with disallowed suffixes. Returns the number of problematic files.
0177     """
0178
0179     suffix_allowed = (
0180         SUFFIX_CPP + SUFFIX_IMAGE + SUFFIX_PYTHON + SUFFIX_DOC + SUFFIX_OTHER
0181     )
0182
0183     wrong_extension = []
0184     for root, dirs, files in os.walk(walk_root):
0185         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0186         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0187
0188         for f in files:
0189             p = Path(root) / f
0190             if p.suffix not in suffix_allowed:
0191                 wrong_extension.append(str(p))
0192
0193     if len(wrong_extension) != 0:
0194         print(
0195             "\n\n\033[31mERROR\033[0m "
0196             + f"The following {len(wrong_extension)} files have an unsupported extension:\n\n"
0197             + "\033[31m"
0198             + "\n".join(wrong_extension)
0199             + "\033[0m"
0200             + "\nCheck if you can change the format to one of the following:\n"
0201             + "\n".join(suffix_allowed)
0202             + "\nIf you really need that specific extension, add it to the list above.\n"
0203         )
0204
0205     return len(wrong_extension)
0206
0207
0208 def find_unused_by_suffix(walk_root, suffixes, search_key, search_scope):
0209     unused = []
0210
0211     for root, dirs, files in os.walk(walk_root):
0212         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0213         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0214
0215         for f in files:
0216             p = Path(root) / f
0217             if p.suffix in suffixes and file_can_be_removed(
0218                 search_key(p), search_scope
0219             ):
0220                 unused.append(str(p))
0221
0222     return unused
0223
0224
0225 def find_unused_python_files(walk_root, dirs_base):
0226     unused = []
0227
0228     for root, dirs, files in os.walk(walk_root):
0229         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0230         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0231
0232         for f in files:
0233             p = Path(root) / f
0234             if p.suffix not in SUFFIX_PYTHON:
0235                 continue
0236
0237             if not file_can_be_removed(r"import .*" + p.stem, dirs_base):
0238                 continue
0239
0240             if not file_can_be_removed(r"from " + p.stem + r" import", dirs_base):
0241                 continue
0242
0243             if file_can_be_removed(p.name, dirs_base):
0244                 unused.append(str(p))
0245
0246     return unused
0247
0248
0249 def main():
0250     print("\033[32mINFO\033[0m Start check_unused_files.py ...")
0251
0252     exit = 0
0253
0254     dirs_base = next(os.walk("."))[1]
0255     dirs_base.append(".")
0256     dirs_base[:] = filter_paths(dirs_base, Path("."), EXCLUDE_PATHS)
0257     dirs_base_docs = ("docs",)
0258     dirs_base_code = filter_paths(dirs_base, Path("."), dirs_base_docs)
0259
0260     exit += check_wrong_extensions(".")
0261
0262     # Collector
0263     unused_files = []
0264
0265     unused_files += find_unused_by_suffix(
0266         ".", SUFFIX_CPP, lambda p: p.name, dirs_base_code
0267     )
0268
0269     unused_files += find_unused_python_files(".", dirs_base)
0270
0271     # TODO find more reliable test for this
0272     unused_files += find_unused_by_suffix(
0273         ".", SUFFIX_DOC, lambda p: p.stem, dirs_base_docs
0274     )
0275
0276     unused_files += find_unused_by_suffix(
0277         ".", SUFFIX_IMAGE + SUFFIX_OTHER, lambda p: p.name, dirs_base
0278     )
0279
0280     if len(unused_files) != 0:
0281         print(
0282             "\n\n\033[31mERROR\033[0m "
0283             + f"The following {len(unused_files)} files seem to be unused:\n"
0284             + "\033[31m"
0285             + "\n".join(unused_files)
0286             + "\033[0m"
0287             + "\nYou have 3 options:"
0288             + "\n\t- Remove them"
0289             + "\n\t- Use them (check proper include)"
0290             + "\n\t- Modify the ignore list of this check\n"
0291         )
0292
0293         exit += 1
0294
0295     if exit == 0:
0296         print(
0297             "\n\n\033[32mINFO\033[0m Finished check_unused_files.py without any errors."
0298         )
0299
0300     return exit
0301
0302
0303 if "__main__" == __name__:
0304     sys.exit(main())