Back to home page

EIC code displayed by LXR

 
 

    


File indexing completed on 2026-11-17 09:29:52

0001 #!/usr/bin/env python3
0002 # SPDX-License-Identifier: LGPL-3.0-or-later
0003 # Copyright (C) 2026 Wouter Deconinck
0004 """Map the on-disk storage taken up by the collections in a podio RNTuple file.
0005 
0006 The breakdown comes from ROOT's RNTuple treemap
0007 (ROOT::Experimental::CreateTreeMapFromRNTuple), with the podio relation,
0008 vector-member and subset-index fields grouped under the collection they belong
0009 to. If a reference file is given, every collection and field is compared to it.
0010 
0011 Writes to the output directory:
0012   storage.html         interactive treemap and per-collection table
0013   storage_treemap.svg  ROOT's own treemap rendering, colored by column type
0014   storage_summary.txt  one-line summary for the PR comment, e.g. "size Δ: +10.2 kB"
0015 
0016 Kept in sync with src/scripts/storage_map.py in https://github.com/eic/EICrecon.
0017 """
0018 import argparse
0019 import html
0020 import json
0021 import os
0022 
0023 import ROOT
0024 
0025 ROOT.gROOT.SetBatch(True)
0026 ROOT.gInterpreter.Declare("#include <ROOT/RNTupleTreeMap.hxx>")
0027 
0028 # Depth of the treemap below the collections: field, then member.
0029 MAX_DEPTH = 2
0030 
0031 
0032 def fmt_bytes(n, sign=False):
0033     for unit, scale in (("MB", 1e6), ("kB", 1e3)):
0034         if abs(n) >= scale:
0035             return f"{n / scale:{'+' if sign else ''}.1f} {unit}"
0036     return f"{n:{'+' if sign and n else ''}d} B"
0037 
0038 
0039 def fmt_rel(new, ref):
0040     if ref is None:
0041         return "new"
0042     if new is None:
0043         return "removed"
0044     if ref == 0:
0045         return "" if new == 0 else "new"
0046     return f"{(new - ref) / ref:+.1%}"
0047 
0048 
0049 def collection_of(field, collections):
0050     """Collection that a top-level podio field belongs to."""
0051     if not field.startswith("_"):
0052         return field.removesuffix("_objIdx")
0053     # Relations and vector members: _<collection>_<name>
0054     owners = [c for c in collections if field.startswith(f"_{c}_")]
0055     return max(owners, key=len) if owners else field
0056 
0057 
0058 def subfields(nodes, idx):
0059     """Field (not column) children of a treemap node, looking through the
0060     '_0' item fields that RNTuple inserts for vectors and arrays."""
0061     node = nodes[idx]
0062     for c in range(node.fChildrenIdx, node.fChildrenIdx + node.fNChildren):
0063         child = nodes[c]
0064         if str(child.fType):
0065             continue
0066         if str(child.fName) == "_0":
0067             yield from subfields(nodes, c)
0068         else:
0069             yield c
0070 
0071 
0072 def flatten(nodes, idx, path, depth, sizes):
0073     sizes[path] = nodes[idx].fSize
0074     if depth > 0:
0075         for c in subfields(nodes, idx):
0076             flatten(nodes, c, path + (str(nodes[c].fName),), depth - 1, sizes)
0077 
0078 
0079 def load(path, ntuple):
0080     """Compressed sizes keyed by (collection, field, member, ...) paths."""
0081     insp = ROOT.Experimental.RNTupleInspector.Create(ntuple, path)
0082     treemap = ROOT.Experimental.CreateTreeMapFromRNTuple(insp)
0083     nodes = treemap.fNodes
0084     top = list(subfields(nodes, 0))
0085     names = [str(nodes[i].fName) for i in top]
0086     collections = {n.removesuffix("_objIdx") for n in names if not n.startswith("_")}
0087 
0088     sizes = {(): nodes[0].fSize}
0089     uncompressed = {}
0090     for idx, field in zip(top, names):
0091         coll = collection_of(field, collections)
0092         sizes[(coll,)] = sizes.get((coll,), 0) + nodes[idx].fSize
0093         uncompressed[coll] = (
0094             uncompressed.get(coll, 0)
0095             + insp.GetFieldTreeInspector(field).GetUncompressedSize()
0096         )
0097         flatten(nodes, idx, (coll, field), MAX_DEPTH - 1, sizes)
0098 
0099     return {
0100         "file": os.path.basename(path),
0101         "file_size": os.path.getsize(path),
0102         "entries": insp.GetDescriptor().GetNEntries(),
0103         "compressed": insp.GetCompressedSize(),
0104         "uncompressed": insp.GetUncompressedSize(),
0105         "sizes": sizes,
0106         "collection_uncompressed": uncompressed,
0107         "treemap": treemap,
0108     }
0109 
0110 
0111 def write_root_svg(treemap, filename):
0112     ROOT.SetOwnership(treemap, False)
0113     canvas = ROOT.TCanvas("storage", "storage", 1600, 1000)
0114     treemap.Draw()
0115     canvas.SaveAs(filename)
0116 
0117 
0118 def write_html(new, ref, ntuple, filename):
0119     sizes = new["sizes"]
0120     ref_sizes = ref["sizes"] if ref else {}
0121 
0122     # Nodes as parallel arrays with parent indices; labels, hover text and
0123     # colors are built in the browser to keep the page small.
0124     paths = sorted(sizes)
0125     index = {path: i for i, path in enumerate(paths)}
0126     nodes = {
0127         "label": [path[-1] if path else ntuple for path in paths],
0128         "parent": [index[path[:-1]] if path else -1 for path in paths],
0129         "size": [sizes[path] for path in paths],
0130         "ref": [ref_sizes.get(path) for path in paths] if ref else None,
0131     }
0132 
0133     # Per-collection table, including collections that were removed
0134     collections = sorted({p[0] for p in sizes if len(p) == 1} | {p[0] for p in ref_sizes if len(p) == 1})
0135     rows = []
0136     for coll in collections:
0137         size = sizes.get((coll,))
0138         ref_size = ref_sizes.get((coll,)) if ref else None
0139         unc = new["collection_uncompressed"].get(coll)
0140         cells = [
0141             (coll, coll),
0142             (size or 0, fmt_bytes(size) if size is not None else "–"),
0143             (size or 0, f"{(size or 0) / new['compressed']:.2%}"),
0144             ((size or 0) / max(new["entries"], 1), fmt_bytes(round((size or 0) / max(new["entries"], 1)))),
0145             (unc / size if unc and size else 0, f"{unc / size:.1f}" if unc and size else "–"),
0146         ]
0147         if ref:
0148             delta = (size or 0) - (ref_size or 0)
0149             cells += [
0150                 (ref_size or 0, fmt_bytes(ref_size) if ref_size is not None else "–"),
0151                 (delta, fmt_bytes(delta, sign=True) if delta else ""),
0152                 (delta / ref_size if ref_size else ("Infinity" if delta else 0), fmt_rel(size, ref_size) if delta else ""),
0153             ]
0154         cls = ""
0155         if ref and size != ref_size:
0156             cls = ' class="grew"' if (size or 0) > (ref_size or 0) else ' class="shrank"'
0157         rows.append(
0158             f"<tr{cls}>"
0159             + "".join(
0160                 f'<td data-sort="{html.escape(str(key))}">{html.escape(str(text))}</td>'
0161                 for key, text in cells
0162             )
0163             + "</tr>"
0164         )
0165     headers = ["collection", "size", "share", "per event", "compression"]
0166     if ref:
0167         headers += ["reference", "Δ", "Δ (%)"]
0168 
0169     def summary_row(label, key, fmt=fmt_bytes):
0170         cells = f"<th>{label}</th><td>{fmt(new[key])}</td>"
0171         if ref:
0172             delta = new[key] - ref[key]
0173             cells += f"<td>{fmt(ref[key])}</td><td>{fmt(delta, sign=True)}</td>"
0174         return f"<tr>{cells}</tr>"
0175 
0176     summary = "".join([
0177         "<tr><th></th><th>this file</th>" + ("<th>reference</th><th>Δ</th>" if ref else "") + "</tr>",
0178         summary_row("file size", "file_size"),
0179         summary_row(f"'{html.escape(ntuple)}' compressed", "compressed"),
0180         summary_row(f"'{html.escape(ntuple)}' uncompressed", "uncompressed"),
0181         summary_row("entries", "entries", lambda n, sign=False: f"{n:{'+' if sign and n else ''}d}"),
0182     ])
0183     if ref and ref["entries"] != new["entries"]:
0184         summary += '<tr><td colspan="4" class="grew">Entry counts differ: sizes are not directly comparable.</td></tr>'
0185 
0186     title = f"Storage map of {new['file']}"
0187     reference = f"Compared to reference {html.escape(ref['file'])}. " if ref else "No reference file to compare to. "
0188     page = f"""<!DOCTYPE html>
0189 <html lang="en">
0190 <head>
0191 <meta charset="utf-8">
0192 <meta name="viewport" content="width=device-width, initial-scale=1">
0193 <title>{html.escape(title)}</title>
0194 <script src="https://cdn.jsdelivr.net/npm/plotly.js-dist-min@2.35.2/plotly.min.js"></script>
0195 <style>
0196   body {{ font-family: system-ui, sans-serif; margin: 1em 2em; color: #222; background: #fff; }}
0197   h1 {{ overflow-wrap: anywhere; }}
0198   .scroll {{ overflow-x: auto; }}
0199   table {{ border-collapse: collapse; font-size: 0.9em; }}
0200   th, td {{ padding: 2px 8px; text-align: right; border-bottom: 1px solid #eee; }}
0201   th:first-child, td:first-child {{ text-align: left; }}
0202   #collections th {{ cursor: pointer; position: sticky; top: 0; background: #f6f6f6; }}
0203   tr.grew td {{ background: #fde0dd; }}
0204   tr.shrank td {{ background: #deebf7; }}
0205   #treemap {{ height: 80vh; }}
0206 </style>
0207 </head>
0208 <body>
0209 <h1>{html.escape(title)}</h1>
0210 <p>Compressed on-disk size of the collections in the '{html.escape(ntuple)}' RNTuple, from ROOT's
0211 RNTuple treemap with podio relation, vector-member and subset fields grouped under their collection.
0212 {reference}Click a box to zoom in; click the top bar to zoom out.
0213 See also <a href="storage_treemap.svg">ROOT's treemap rendering</a>, colored by column type.</p>
0214 <div class="scroll"><table>{summary}</table></div>
0215 <div id="treemap"></div>
0216 <h2>Collections</h2>
0217 <p>Compression is uncompressed / compressed size. Click a column header to sort.</p>
0218 <div class="scroll"><table id="collections">
0219 <thead><tr>{''.join(f'<th>{h}</th>' for h in headers)}</tr></thead>
0220 <tbody>
0221 {chr(10).join(rows)}
0222 </tbody>
0223 </table></div>
0224 <script>
0225 const nodes = {json.dumps(nodes, ensure_ascii=False, separators=(",", ":")).replace("</", "<\\/")};
0226 const fmt = b => Math.abs(b) >= 1e6 ? (b / 1e6).toFixed(1) + " MB"
0227   : Math.abs(b) >= 1e3 ? (b / 1e3).toFixed(1) + " kB" : b + " B";
0228 const signed = (v, text) => (v > 0 ? "+" : "") + text;
0229 const paths = [];
0230 nodes.label.forEach((label, i) => {{
0231   const p = nodes.parent[i];
0232   paths.push(p < 0 ? label : paths[p] + "/" + label);
0233 }});
0234 const trace = {{
0235   type: "treemap",
0236   ids: nodes.label.map((_, i) => String(i)),
0237   parents: nodes.parent.map(p => p < 0 ? "" : String(p)),
0238   labels: nodes.label,
0239   values: nodes.size,
0240   branchvalues: "total",
0241   maxdepth: 3,
0242   texttemplate: "%{{label}}<br>%{{customdata[0]}}",
0243   hovertemplate: "<b>%{{customdata[1]}}</b><br>%{{customdata[0]}} (%{{percentRoot:.2%}} of total)"
0244     + (nodes.ref ? "<br>reference: %{{customdata[2]}}<br>Δ: %{{customdata[3]}}" : "") + "<extra></extra>",
0245 }};
0246 trace.customdata = nodes.size.map((s, i) => {{
0247   const r = nodes.ref ? nodes.ref[i] : null;
0248   return [fmt(s), paths[i], r === null ? "–" : fmt(r),
0249     r === null ? "new" : signed(s - r, fmt(s - r)) + (r ? " (" + signed(s - r, (100 * (s - r) / r).toFixed(1)) + "%)" : "")];
0250 }});
0251 if (nodes.ref) {{
0252   // Relative change on a log scale: 1% -> 1/3, 10% -> 2/3, 100% -> 1
0253   const scale = (s, r) => r === null || (r === 0 && s) ? 1 : r === 0 ? 0
0254     : Math.sign(s - r) * Math.min(Math.log10(1 + 100 * Math.abs((s - r) / r)) / 3, 1);
0255   trace.marker = {{
0256     colors: nodes.size.map((s, i) => scale(s, nodes.ref[i])),
0257     colorscale: "RdBu", cmin: -1, cmax: 1, cmid: 0, showscale: true,
0258     colorbar: {{
0259       title: {{text: "size Δ"}},
0260       tickvals: [-1, -2 / 3, -1 / 3, 0, 1 / 3, 2 / 3, 1],
0261       ticktext: ["≤−100%", "−10%", "−1%", "0", "+1%", "+10%", "≥+100%"],
0262     }},
0263   }};
0264 }}
0265 Plotly.newPlot("treemap", [trace], {{margin: {{t: 30, l: 0, r: 0, b: 0}}}}, {{responsive: true}});
0266 const table = document.getElementById("collections");
0267 table.querySelectorAll("th").forEach((th, col) => th.addEventListener("click", () => {{
0268   const desc = th.dataset.desc !== "true";
0269   th.dataset.desc = desc;
0270   const key = tr => {{
0271     const v = tr.children[col].dataset.sort;
0272     return col === 0 ? v : parseFloat(v);
0273   }};
0274   const rows = [...table.tBodies[0].rows].sort((a, b) =>
0275     (key(a) > key(b) ? 1 : key(a) < key(b) ? -1 : 0) * (desc ? -1 : 1));
0276   table.tBodies[0].append(...rows);
0277 }}));
0278 </script>
0279 </body>
0280 </html>
0281 """
0282     with open(filename, "w") as f:
0283         f.write(page)
0284 
0285 
0286 def main():
0287     ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
0288     ap.add_argument("file", help="podio RNTuple file")
0289     ap.add_argument("reference", nargs="?", help="podio RNTuple file to compare to")
0290     ap.add_argument("-o", "--output-dir", default=".", help="directory to write to")
0291     ap.add_argument("--ntuple", default="events", help="RNTuple to map (default: %(default)s)")
0292     args = ap.parse_args()
0293 
0294     new = load(args.file, args.ntuple)
0295     ref = load(args.reference, args.ntuple) if args.reference else None
0296 
0297     os.makedirs(args.output_dir, exist_ok=True)
0298     write_html(new, ref, args.ntuple, os.path.join(args.output_dir, "storage.html"))
0299     write_root_svg(new["treemap"], os.path.join(args.output_dir, "storage_treemap.svg"))
0300 
0301     if ref:
0302         summary = f"size Δ: {fmt_bytes(new['compressed'] - ref['compressed'], sign=True)}"
0303     else:
0304         summary = f"size: {fmt_bytes(new['compressed'])}"
0305     with open(os.path.join(args.output_dir, "storage_summary.txt"), "w") as f:
0306         f.write(summary + "\n")
0307     print(summary)
0308 
0309 
0310 if __name__ == "__main__":
0311     main()