acts/CI/prune_ccache.py

0001 #!/usr/bin/env python3
0002 """Retain the newest main compiler cache per variant and archive format.
0003
0004 Dry-run by default. Only SHA-suffixed ccache keys are eligible; an uploaded,
0005 non-empty replacement must exist before an older archive can be deleted.
0006 """
0007
0008 import argparse
0009 from collections import defaultdict
0010 import json
0011 import os
0012 from pathlib import Path
0013 import re
0014 import subprocess
0015
0016 MAIN_REF = "refs/heads/main"
0017 CACHE_KEY = re.compile(r"^(ccache-.+-r\d+(?:-.+)?)-[0-9a-f]{40}$")
0018
0019
0020 def superseded_caches(caches):
0021     """Return (replacement, obsolete archives) pairs, preserving cache versions."""
0022     groups = defaultdict(list)
0023     for cache in caches:
0024         match = CACHE_KEY.fullmatch(cache["key"])
0025         if (
0026             cache["ref"] == MAIN_REF
0027             and match
0028             and cache.get("version")
0029             and cache["size_in_bytes"] > 0
0030         ):
0031             groups[(match[1], cache["version"])].append(cache)
0032     result = []
0033     for group in groups.values():
0034         ordered = sorted(group, key=lambda c: (c["created_at"], c["id"]), reverse=True)
0035         if len(ordered) > 1:
0036             result.append((ordered[0], ordered[1:]))
0037     return result
0038
0039
0040 def list_caches(repo, key="ccache-"):
0041     # Fetch all pages before any deletion, so changing page boundaries cannot
0042     # cause entries to be skipped. -X GET prevents -f switching to POST.
0043     output = subprocess.check_output(
0044         [
0045             "gh",
0046             "api",
0047             "--method",
0048             "GET",
0049             "--paginate",
0050             "--slurp",
0051             f"repos/{repo}/actions/caches",
0052             "-f",
0053             f"ref={MAIN_REF}",
0054             "-f",
0055             f"key={key}",
0056             "-f",
0057             "per_page=100",
0058         ],
0059         text=True,
0060     )
0061     return [cache for page in json.loads(output) for cache in page["actions_caches"]]
0062
0063
0064 def prune(repo, caches, apply=False):
0065     total = 0
0066     for replacement, obsolete in superseded_caches(caches):
0067         if apply:
0068             # GitHub may have evicted our replacement since the initial listing.
0069             # Recheck it before pruning this variant. A new upload is never in
0070             # the deletion list, which contains only IDs from the initial read.
0071             available = list_caches(repo, replacement["key"])
0072             if not any(
0073                 c["id"] == replacement["id"] and c["size_in_bytes"] > 0
0074                 for c in available
0075             ):
0076                 print(f"Keep older caches: replacement {replacement['id']} disappeared")
0077                 continue
0078         print(f"Keep {replacement['id']}: {replacement['key']}")
0079         for cache in obsolete:
0080             print(
0081                 f"{'Delete' if apply else 'Would delete'} {cache['id']}: {cache['key']}"
0082             )
0083             if apply:
0084                 subprocess.run(
0085                     [
0086                         "gh",
0087                         "api",
0088                         "--method",
0089                         "DELETE",
0090                         f"repos/{repo}/actions/caches/{cache['id']}",
0091                     ],
0092                     check=True,
0093                 )
0094             total += cache["size_in_bytes"]
0095     print(f"{'Removed' if apply else 'Reclaimable'} archive bytes: {total:,}")
0096     return total
0097
0098
0099 def main():
0100     parser = argparse.ArgumentParser(description=__doc__)
0101     parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY"))
0102     parser.add_argument(
0103         "--apply", action="store_true", help="Delete superseded archives"
0104     )
0105     parser.add_argument(
0106         "--snapshot", type=Path, help="Dry-run a saved cache API response"
0107     )
0108     args = parser.parse_args()
0109     if args.apply and args.snapshot:
0110         parser.error("--apply requires a fresh API listing, not --snapshot")
0111     if not args.snapshot and not args.repo:
0112         parser.error("--repo or GITHUB_REPOSITORY is required")
0113     caches = (
0114         json.loads(args.snapshot.read_text())["actions_caches"]
0115         if args.snapshot
0116         else list_caches(args.repo)
0117     )
0118     prune(args.repo, caches, args.apply)
0119
0120
0121 if __name__ == "__main__":
0122     main()