Warning, /containers/scripts/cvmfs_catalog_analysis is written in an unsupported language. File is not indexed.
0001 #!/usr/bin/env python3
0002 """Analyze .cvmfscatalog placement under a container sandbox prefix.
0003
0004 For each catalog boundary, reports how many filesystem entries (files +
0005 directories) it "owns" — i.e., entries in its subtree that are NOT delegated
0006 to a deeper nested catalog. Also flags large directories that have no catalog
0007 boundary and might benefit from one.
0008
0009 Usage:
0010 scripts/cvmfs_catalog_analysis [OPTIONS] PREFIX
0011
0012 PREFIX Root of the sandbox to analyze, e.g.:
0013 /cvmfs/singularity.opensciencegrid.org/eicweb/eic_xl:nightly
0014
0015 Options:
0016 --min-entries N Suggest adding a catalog when a non-catalog directory's
0017 owned-entry count exceeds N (default: 500)
0018 --max-entries N Warn when a catalog's owned-entry count exceeds N
0019 (default: 200000)
0020 --depth N Maximum directory depth to walk below PREFIX (default: 8)
0021 --help Show this help and exit
0022
0023 Output columns:
0024 STATUS CATALOG - directory has a .cvmfscatalog marker
0025 SUGGEST(n) - directory has no catalog but > --min-entries owned
0026 WARN(n) - catalog has > --max-entries owned entries
0027 OWNED Entries owned by this catalog (not delegated to children)
0028 TOTAL Total entries in the subtree (including delegated children)
0029 PATH Path relative to PREFIX
0030 """
0031
0032 import argparse
0033 import os
0034 import sys
0035
0036
0037 def parse_args():
0038 p = argparse.ArgumentParser(
0039 description=__doc__,
0040 formatter_class=argparse.RawDescriptionHelpFormatter,
0041 add_help=False,
0042 )
0043 p.add_argument("prefix", metavar="PREFIX")
0044 p.add_argument("--min-entries", type=int, default=500)
0045 p.add_argument("--max-entries", type=int, default=200000)
0046 p.add_argument("--depth", type=int, default=8)
0047 p.add_argument("--help", action="help")
0048 return p.parse_args()
0049
0050
0051 def walk_tree(prefix, max_depth):
0052 """Return (results, catalog_roots) for the directory tree under prefix.
0053
0054 results is a list of (abs_path, rel_path, depth, has_catalog, owned, total)
0055 where owned = entries owned by the catalog at abs_path (None if not a catalog)
0056 and total = all entries in the subtree (recursive).
0057 """
0058 prefix = os.path.realpath(prefix)
0059
0060 # First pass: collect totals with os.walk (don't follow symlinks so we
0061 # don't escape the sandbox).
0062 dir_total = {} # abs_path -> total entries in subtree
0063
0064 for root, dirs, files in os.walk(prefix, followlinks=False, onerror=lambda e: None):
0065 rel = os.path.relpath(root, prefix)
0066 depth = 0 if rel == "." else rel.count(os.sep) + 1
0067
0068 if depth > max_depth:
0069 dirs.clear()
0070 continue
0071
0072 # Exclude catalog marker files from totals so TOTAL aligns with OWNED.
0073 entries = dirs + [f for f in files if f != ".cvmfscatalog"]
0074
0075 # Accumulate totals: add our direct entries to all ancestors
0076 path = root
0077 while True:
0078 dir_total[path] = dir_total.get(path, 0) + len(entries)
0079 parent = os.path.dirname(path)
0080 if parent == path or not parent.startswith(prefix):
0081 break
0082 path = parent
0083
0084 # Second pass: for each directory, compute "owned" count.
0085 # owned = total - sum(total of direct children that are catalog roots)
0086 # We need the set of catalog roots first.
0087 catalog_roots = set()
0088 for root, dirs, files in os.walk(prefix, followlinks=False, onerror=lambda e: None):
0089 rel = os.path.relpath(root, prefix)
0090 depth = 0 if rel == "." else rel.count(os.sep) + 1
0091 if depth > max_depth:
0092 dirs.clear()
0093 continue
0094 if ".cvmfscatalog" in files:
0095 catalog_roots.add(root)
0096
0097 # For each catalog root, owned = total_in_subtree - sum(total of nested catalogs)
0098 # total_in_subtree[d] = all entries below d at any depth
0099 # For a catalog root R, subtract all nested catalog subtree totals
0100 # (but only direct nested — because indirect ones are already subtracted at
0101 # the intermediate level).
0102 #
0103 # Simpler algorithm: for each directory d, owned(d) = entries directly in d's
0104 # catalog = walk subtree of d, stop descending at any nested catalog boundary.
0105
0106 def owned_count(start):
0107 """Count entries in start's subtree that belong to start's catalog."""
0108 count = 0
0109 stack = [start]
0110 while stack:
0111 d = stack.pop()
0112 try:
0113 with os.scandir(d) as entries:
0114 for entry in entries:
0115 if entry.name == ".cvmfscatalog":
0116 continue
0117 count += 1
0118 if entry.is_dir(follow_symlinks=False):
0119 child_path = entry.path
0120 # Stop at nested catalog boundaries (other than start itself)
0121 if child_path != start and child_path in catalog_roots:
0122 pass # don't descend — owned by nested catalog
0123 else:
0124 rel_depth = child_path[len(prefix):].count(os.sep)
0125 if rel_depth <= max_depth:
0126 stack.append(child_path)
0127 except OSError:
0128 continue
0129 return count
0130
0131 # Collect results for output
0132 results = []
0133 for root, dirs, files in os.walk(prefix, followlinks=False, onerror=lambda e: None):
0134 rel = os.path.relpath(root, prefix)
0135 depth = 0 if rel == "." else rel.count(os.sep) + 1
0136 if depth > max_depth:
0137 dirs.clear()
0138 continue
0139
0140 has_catalog = ".cvmfscatalog" in files
0141 total = dir_total.get(root, 0)
0142 owned = owned_count(root) if has_catalog else None
0143 results.append((root, rel, depth, has_catalog, owned, total))
0144
0145 return results, catalog_roots
0146
0147
0148 def main():
0149 args = parse_args()
0150 prefix = os.path.realpath(args.prefix)
0151
0152 if not os.path.isdir(prefix):
0153 print(f"error: prefix not found: {prefix}", file=sys.stderr)
0154 sys.exit(1)
0155
0156 print(f"Analyzing {prefix}", file=sys.stderr)
0157 results, catalog_roots = walk_tree(prefix, args.depth)
0158
0159 # Determine which non-catalog directories are "large" (i.e., owned count >
0160 # --min-entries when treating the directory as if it had its own catalog).
0161 # We compute owned for non-catalog dirs too, lazily.
0162 def owned_count_nc(start):
0163 """Owned count treating start as a catalog boundary (for suggestion check)."""
0164 count = 0
0165 stack = [start]
0166 while stack:
0167 d = stack.pop()
0168 try:
0169 with os.scandir(d) as entries:
0170 for entry in entries:
0171 if entry.name == ".cvmfscatalog":
0172 continue
0173 count += 1
0174 if entry.is_dir(follow_symlinks=False):
0175 child_path = entry.path
0176 if child_path in catalog_roots:
0177 pass
0178 else:
0179 rel_depth = child_path[len(prefix):].count(os.sep)
0180 if rel_depth <= args.depth:
0181 stack.append(child_path)
0182 except OSError:
0183 continue
0184 return count
0185
0186 print(f"{'STATUS':<20} {'OWNED':>8} {'TOTAL':>8} PATH")
0187 print("-" * 80)
0188
0189 warned = []
0190 suggested = []
0191
0192 for root, rel, depth, has_catalog, owned, total in sorted(results, key=lambda r: r[1]):
0193 display_path = "/" + rel if rel != "." else "/"
0194
0195 if has_catalog:
0196 if owned > args.max_entries:
0197 status = f"WARN({owned})"
0198 warned.append(display_path)
0199 else:
0200 status = "CATALOG"
0201 print(f"{status:<20} {owned:>8} {total:>8} {display_path}")
0202 else:
0203 # Only report non-catalog dirs that are interesting (could benefit from catalog)
0204 # Heuristic: check if the directory's contribution to its parent catalog
0205 # is large — approximate via total minus delegated children.
0206 nc_owned = owned_count_nc(root)
0207 if nc_owned > args.min_entries:
0208 status = f"SUGGEST({nc_owned})"
0209 suggested.append((display_path, nc_owned))
0210 print(f"{status:<20} {nc_owned:>8} {total:>8} {display_path}")
0211
0212 print()
0213 print(f"Catalog roots found: {len(catalog_roots)}")
0214 if warned:
0215 print(f"\nWARNING: {len(warned)} catalog(s) exceed {args.max_entries} entries:")
0216 for p in warned:
0217 print(f" {p}")
0218 if suggested:
0219 print(f"\nSUGGESTION: {len(suggested)} directories may benefit from a catalog (> {args.min_entries} owned entries):")
0220 for p, n in sorted(suggested, key=lambda x: -x[1]):
0221 print(f" {n:>8} {p}")
0222
0223
0224 if __name__ == "__main__":
0225 main()
