pyrobird/scripts/check_dist.py

0001 #!/usr/bin/env python3
0002 # This file is part of Firebird Event Display and is licensed under GPL-3.0-or-later.
0003 # See the LICENSE file in the project root for full license information.
0004 """Checks built pyrobird distributions before an upload.
0005
0006 Usage:
0007     python scripts/check_dist.py dist/                 # every .whl and .tar.gz in dist/
0008     python scripts/check_dist.py dist/pyrobird-*.whl   # specific files
0009     python scripts/check_dist.py --manifest-only       # MANIFEST.in rules only, no build needed
0010
0011 For every wheel and sdist it checks that:
0012 - the files the application needs at run time are present, including the
0013   sample event file that the documentation links to;
0014 - the license and the third-party license notices of the frontend are present;
0015 - the archive is smaller than PyPI's upload limit (100 MiB unless the
0016   project was granted more; see --max-bytes).
0017
0018 It also checks that no MANIFEST.in exclusion rule matches a required file.
0019 Exits with 1 when a check fails.
0020 """
0021 import argparse
0022 import fnmatch
0023 import os
0024 import sys
0025 import tarfile
0026 import zipfile
0027
0028 # PyPI's default upload limit per file
0029 PYPI_MAX_BYTES = 100 * 1024 * 1024
0030
0031 # Paths relative to the project root (the directory of pyproject.toml)
0032 REQUIRED_FILES = (
0033     "LICENSE",
0034     "pyrobird/__init__.py",
0035     "pyrobird/server/__init__.py",
0036     "pyrobird/data/eic_geo_process_rules.yaml",
0037     "pyrobird/server/static/index.html",
0038     "pyrobird/server/static/3rdpartylicenses.txt",
0039     "pyrobird/server/static/assets/config.jsonc",
0040     "pyrobird/server/static/assets/data/example-cherenkov.firebird.json",
0041 )
0042
0043 PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
0044
0045
0046 def archive_members(path):
0047     """Returns member paths of a wheel or sdist, relative to the project root."""
0048     if path.endswith(".whl"):
0049         with zipfile.ZipFile(path) as archive:
0050             # A wheel holds the package at its root: 'pyrobird/server/...'. License files
0051             # (pyproject license-files) sit under '<name>-<version>.dist-info/licenses/' with
0052             # their project-relative path, so 'LICENSE' maps back to 'LICENSE'.
0053             members = set()
0054             for name in archive.namelist():
0055                 if name.endswith("/"):
0056                     continue
0057                 top, _, rest = name.partition("/")
0058                 if top.endswith(".dist-info") and rest.startswith("licenses/"):
0059                     name = rest[len("licenses/"):]
0060                 members.add(name)
0061             return members
0062     if path.endswith((".tar.gz", ".tgz")):
0063         with tarfile.open(path, "r:gz") as archive:
0064             # An sdist holds the project under a top directory: 'pyrobird-1.0/pyrobird/...'
0065             return {member.name.split("/", 1)[1] for member in archive.getmembers()
0066                     if member.isfile() and "/" in member.name}
0067     raise ValueError(f"Not a wheel or an sdist: {path}")
0068
0069
0070 def check_archive(path, required=REQUIRED_FILES, max_bytes=PYPI_MAX_BYTES):
0071     """Returns a list of problems found in one archive; empty when it passes."""
0072     problems = []
0073     size = os.path.getsize(path)
0074     if size >= max_bytes:
0075         problems.append(f"{os.path.basename(path)} is {size:,} bytes; the limit is {max_bytes:,} bytes")
0076
0077     members = archive_members(path)
0078     for name in required:
0079         if name not in members:
0080             problems.append(f"{os.path.basename(path)} lacks {name}")
0081     return problems
0082
0083
0084 def manifest_exclusions(manifest_path, paths=REQUIRED_FILES):
0085     """Returns (path, rule) pairs where a MANIFEST.in exclusion rule drops one of `paths`.
0086
0087     Understands the exclusion commands of the MANIFEST.in template:
0088     exclude, recursive-exclude, global-exclude and prune. An include rule
0089     that comes after the exclusion is not taken into account.
0090     """
0091     hits = []
0092     with open(manifest_path) as manifest:
0093         rules = [line.strip() for line in manifest if line.strip() and not line.strip().startswith("#")]
0094
0095     for rule in rules:
0096         words = rule.split()
0097         command, args = words[0], words[1:]
0098         for path in paths:
0099             name = os.path.basename(path)
0100             if command == "exclude":
0101                 matched = any(fnmatch.fnmatch(path, pattern) for pattern in args)
0102             elif command == "recursive-exclude" and args:
0103                 directory = args[0].rstrip("/") + "/"
0104                 matched = path.startswith(directory) and any(fnmatch.fnmatch(name, p) for p in args[1:])
0105             elif command == "global-exclude":
0106                 matched = any(fnmatch.fnmatch(name, pattern) for pattern in args)
0107             elif command == "prune" and args:
0108                 matched = path.startswith(args[0].rstrip("/") + "/")
0109             else:
0110                 matched = False
0111             if matched:
0112                 hits.append((path, rule))
0113     return hits
0114
0115
0116 def find_archives(targets):
0117     archives = []
0118     for target in targets:
0119         if os.path.isdir(target):
0120             archives.extend(sorted(os.path.join(target, name) for name in os.listdir(target)
0121                                    if name.endswith((".whl", ".tar.gz"))))
0122         else:
0123             archives.append(target)
0124     return archives
0125
0126
0127 def main(argv=None):
0128     parser = argparse.ArgumentParser(description="Check built pyrobird distributions.")
0129     parser.add_argument("targets", nargs="*", help="Wheels, sdists, or directories that hold them")
0130     parser.add_argument("--max-bytes", type=int, default=PYPI_MAX_BYTES,
0131                         help=f"Size limit per archive (default: {PYPI_MAX_BYTES})")
0132     parser.add_argument("--manifest", default=os.path.join(PROJECT_DIR, "MANIFEST.in"),
0133                         help="MANIFEST.in to check")
0134     parser.add_argument("--manifest-only", action="store_true", help="Check MANIFEST.in rules only")
0135     args = parser.parse_args(argv)
0136
0137     problems = [f"MANIFEST.in rule '{rule}' excludes {path}"
0138                 for path, rule in manifest_exclusions(args.manifest)]
0139
0140     if not args.manifest_only:
0141         archives = find_archives(args.targets)
0142         if not archives:
0143             problems.append("No wheel or sdist to check")
0144         for archive in archives:
0145             archive_problems = check_archive(archive, max_bytes=args.max_bytes)
0146             problems.extend(archive_problems)
0147             if not archive_problems:
0148                 size = os.path.getsize(archive)
0149                 print(f"OK {os.path.basename(archive)}: {size:,} bytes, "
0150                       f"{args.max_bytes - size:,} bytes below the limit")
0151
0152     for problem in problems:
0153         print(f"FAIL {problem}")
0154     return 1 if problems else 0
0155
0156
0157 if __name__ == "__main__":
0158     sys.exit(main())