acts/CI/check_unused_files.py

0001 #!/usr/bin/env python3
0002
0003 from pathlib import Path
0004 import os
0005 import sys
0006 import subprocess
0007
0008 EXCLUDE_PATHS = (
0009     ".devcontainer",
0010     ".git",
0011     ".github",
0012     ".idea",
0013     "CI",
0014     "cmake",
0015     # Used by traccc
0016     "Detray/detectors",
0017     # CLI tools
0018     "Detray/tests/tools",
0019     "Detray/python/Io/tests",
0020     "Detray/python/Tests/tests",
0021     "Detray/python/Plugins/tests",
0022     # TODO: Remove the traccc part.
0023     "Traccc",
0024     "git",
0025     "Python",
0026     "Scripts",
0027     # cmake for "DD4hep-tests" looks a bit different
0028     "Tests/UnitTests/Plugins/DD4hep",
0029     "thirdparty",
0030     "white_papers/figures",
0031 )
0032 EXCLUDE_FILES = (
0033     ".gersemirc",
0034     ".gitignore",
0035     ".kodiak.toml",
0036     ".merge-sentinel.yml",
0037     ".policy.yml",
0038     ".pre-commit-config.yaml",
0039     "acts_logo_colored.svg",
0040     "CITATION.cff",
0041     "CMakeLists.txt",
0042     "CMakePresets.json",
0043     "CODE_OF_CONDUCT.md",
0044     "CODEOWNERS",
0045     "codecov.yml",
0046     "pytest.ini",
0047     "README.md",
0048     "readthedocs.yml",
0049     "sonar-project.properties",
0050     # Filename not completed in source
0051     "vertexing_event_mu20_beamspot.csv",
0052     "vertexing_event_mu20_tracks.csv",
0053     "vertexing_event_mu20_vertices_AMVF.csv",
0054     "event000000001-MuonDriftCircle.csv",
0055     "event000000001-MuonSimHit.csv",
0056     # TODO Move the following files to a better place?
0057     "Magfield.ipynb",
0058     "SolenoidField.ipynb",
0059     # TODO Add README next to the following files?
0060     "generic-input-config.json",
0061     "generic-alignment-geo.json",
0062     # TODO Mention these files somewhere?
0063     "codegen/src/codegen/sympy_common.py",
0064     "codegen/src/codegen/detray_backend.py",
0065     "CompressedIO.h",
0066     "generate_particle_data_table.py",
0067     "GeometryModule.h",
0068     "lazy_autodoc.py",
0069     "runtime_geometry_modules.md",
0070     # Files for python binding generation
0071     "acts-version-manager.js",
0072     "bugs.md",
0073     "deprecated.md",
0074     "Python/conftest.py",
0075     "serve.py",
0076     "SNIPPETS.md",
0077     "tex-mml-chtml.js",
0078     "tgeo_aux.py.in",
0079     "todo.md",
0080     # Detray python tests for auto-generated code
0081     "Detray/codegen/detray-sympy/tests/test_assumptions_D.py",
0082     "Detray/codegen/detray-sympy/tests/test_matrices.py",
0083     # Used in traccc
0084     "Detray/tests/include/detray/test/utils/perigee_stopper.hpp",
0085     "Detray/tests/include/detray/test/validation/propagation_validation.hpp",
0086     # Build-time metadata generation
0087     "Detray/python/detray/detectors/impl/definitions.py",
0088     "Detray/python/detray/detectors/impl/type_helpers.py",
0089     # Python uv files
0090     "Detray/codegen/detray-sympy/uv.lock",
0091     "Detray/python/detray/uv.lock",
0092     # TODO: remove after file is gone
0093     "Core/include/Acts/Utilities/ProtoAxisHelpers.hpp",
0094 )
0095 SUFFIX_CPP = (
0096     ".hpp",
0097     ".cuh",
0098     ".sycl",
0099     ".hip",
0100     ".ipp",
0101     ".cpp",
0102     ".cu",
0103 )
0104 SUFFIX_IMAGE = (
0105     ".png",
0106     ".svg",
0107     ".jpg",
0108     ".gif",
0109 )
0110 SUFFIX_PYTHON = (".py",)
0111 SUFFIX_DOC = (
0112     ".md",
0113     ".rst",
0114     ".dox",
0115     ".html",
0116     ".bib",
0117 )
0118 SUFFIX_OTHER = (
0119     "",
0120     ".C",
0121     ".csv",
0122     ".css",
0123     ".gdml",
0124     ".hepmc3",
0125     ".lock",
0126     ".ico",
0127     ".in",
0128     ".ipynb",
0129     ".json",
0130     ".j2",
0131     ".onnx",
0132     ".root",
0133     ".toml",
0134     ".txt",
0135     ".yml",
0136     ".xml",
0137     ".sh",
0138 )
0139
0140
0141 def filter_paths(names, root, exclude_paths=(), exclude_files=()):
0142     """
0143     Filter names from os.walk() based on path substrings and file rules.
0144     Excludes entries if their full path matches exclude_paths or exclude_files.
0145     """
0146
0147     def keep(name):
0148         p = Path(root) / name
0149         p_str = p.as_posix()
0150         return not any(ep in p_str for ep in exclude_paths) and not any(
0151             ef in p_str if "/" in ef else p.name == ef for ef in exclude_files
0152         )
0153
0154     return [name for name in names if keep(name)]
0155
0156
0157 def file_can_be_removed(searchstring, scope):
0158     cmd = "grep -IR '" + searchstring + "' " + " ".join(scope)
0159
0160     p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
0161     output, _ = p.communicate()
0162     return output == b""
0163
0164
0165 def count_files(path="."):
0166     count = 0
0167     for root, dirs, files in os.walk(path):
0168         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0169         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0170
0171         count += len(files)
0172
0173     return count
0174
0175
0176 def check_wrong_extensions(walk_root):
0177     """
0178     Collect files with disallowed suffixes. Returns the number of problematic files.
0179     """
0180
0181     suffix_allowed = (
0182         SUFFIX_CPP + SUFFIX_IMAGE + SUFFIX_PYTHON + SUFFIX_DOC + SUFFIX_OTHER
0183     )
0184
0185     wrong_extension = []
0186     for root, dirs, files in os.walk(walk_root):
0187         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0188         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0189
0190         for f in files:
0191             p = Path(root) / f
0192             if p.suffix not in suffix_allowed:
0193                 wrong_extension.append(str(p))
0194
0195     if len(wrong_extension) != 0:
0196         print(
0197             "\n\n\033[31mERROR\033[0m "
0198             + f"The following {len(wrong_extension)} files have an unsupported extension:\n\n"
0199             + "\033[31m"
0200             + "\n".join(wrong_extension)
0201             + "\033[0m"
0202             + "\nCheck if you can change the format to one of the following:\n"
0203             + "\n".join(suffix_allowed)
0204             + "\nIf you really need that specific extension, add it to the list above.\n"
0205         )
0206
0207     return len(wrong_extension)
0208
0209
0210 def find_unused_by_suffix(walk_root, suffixes, search_key, search_scope):
0211     unused = []
0212
0213     for root, dirs, files in os.walk(walk_root):
0214         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0215         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0216
0217         for f in files:
0218             p = Path(root) / f
0219             if p.suffix in suffixes and file_can_be_removed(
0220                 search_key(p), search_scope
0221             ):
0222                 unused.append(str(p))
0223
0224     return unused
0225
0226
0227 def find_unused_python_files(walk_root, dirs_base):
0228     unused = []
0229
0230     for root, dirs, files in os.walk(walk_root):
0231         dirs[:] = filter_paths(dirs, root, EXCLUDE_PATHS)
0232         files = filter_paths(files, root, EXCLUDE_PATHS, EXCLUDE_FILES)
0233
0234         for f in files:
0235             p = Path(root) / f
0236             if p.suffix not in SUFFIX_PYTHON:
0237                 continue
0238
0239             if not file_can_be_removed(r"import .*" + p.stem, dirs_base):
0240                 continue
0241
0242             if not file_can_be_removed(r"from " + p.stem + r" import", dirs_base):
0243                 continue
0244
0245             if file_can_be_removed(p.name, dirs_base):
0246                 unused.append(str(p))
0247
0248     return unused
0249
0250
0251 def main():
0252     print("\033[32mINFO\033[0m Start check_unused_files.py ...")
0253
0254     exit = 0
0255
0256     dirs_base = next(os.walk("."))[1]
0257     dirs_base.append(".")
0258     dirs_base[:] = filter_paths(dirs_base, Path("."), EXCLUDE_PATHS)
0259     dirs_base_docs = ("docs",)
0260     dirs_base_code = filter_paths(dirs_base, Path("."), dirs_base_docs)
0261
0262     exit += check_wrong_extensions(".")
0263
0264     # Collector
0265     unused_files = []
0266
0267     unused_files += find_unused_by_suffix(
0268         ".", SUFFIX_CPP, lambda p: p.name, dirs_base_code
0269     )
0270
0271     unused_files += find_unused_python_files(".", dirs_base)
0272
0273     # TODO find more reliable test for this
0274     unused_files += find_unused_by_suffix(
0275         ".", SUFFIX_DOC, lambda p: p.stem, dirs_base_docs
0276     )
0277
0278     unused_files += find_unused_by_suffix(
0279         ".", SUFFIX_IMAGE + SUFFIX_OTHER, lambda p: p.name, dirs_base
0280     )
0281
0282     if len(unused_files) != 0:
0283         print(
0284             "\n\n\033[31mERROR\033[0m "
0285             + f"The following {len(unused_files)} files seem to be unused:\n"
0286             + "\033[31m"
0287             + "\n".join(unused_files)
0288             + "\033[0m"
0289             + "\nYou have 3 options:"
0290             + "\n\t- Remove them"
0291             + "\n\t- Use them (check proper include)"
0292             + "\n\t- Modify the ignore list of this check\n"
0293         )
0294
0295         exit += 1
0296
0297     if exit == 0:
0298         print(
0299             "\n\n\033[32mINFO\033[0m Finished check_unused_files.py without any errors."
0300         )
0301
0302     return exit
0303
0304
0305 if "__main__" == __name__:
0306     sys.exit(main())