File indexing completed on 2026-11-17 09:29:52
0001
0002
0003
0004 """Map the on-disk storage taken up by the collections in a podio RNTuple file.
0005
0006 The breakdown comes from ROOT's RNTuple treemap
0007 (ROOT::Experimental::CreateTreeMapFromRNTuple), with the podio relation,
0008 vector-member and subset-index fields grouped under the collection they belong
0009 to. If a reference file is given, every collection and field is compared to it.
0010
0011 Writes to the output directory:
0012 storage.html interactive treemap and per-collection table
0013 storage_treemap.svg ROOT's own treemap rendering, colored by column type
0014 storage_summary.txt one-line summary for the PR comment, e.g. "size Δ: +10.2 kB"
0015
0016 Kept in sync with src/scripts/storage_map.py in https://github.com/eic/EICrecon.
0017 """
0018 import argparse
0019 import html
0020 import json
0021 import os
0022
0023 import ROOT
0024
0025 ROOT.gROOT.SetBatch(True)
0026 ROOT.gInterpreter.Declare("#include <ROOT/RNTupleTreeMap.hxx>")
0027
0028
0029 MAX_DEPTH = 2
0030
0031
0032 def fmt_bytes(n, sign=False):
0033 for unit, scale in (("MB", 1e6), ("kB", 1e3)):
0034 if abs(n) >= scale:
0035 return f"{n / scale:{'+' if sign else ''}.1f} {unit}"
0036 return f"{n:{'+' if sign and n else ''}d} B"
0037
0038
0039 def fmt_rel(new, ref):
0040 if ref is None:
0041 return "new"
0042 if new is None:
0043 return "removed"
0044 if ref == 0:
0045 return "" if new == 0 else "new"
0046 return f"{(new - ref) / ref:+.1%}"
0047
0048
0049 def collection_of(field, collections):
0050 """Collection that a top-level podio field belongs to."""
0051 if not field.startswith("_"):
0052 return field.removesuffix("_objIdx")
0053
0054 owners = [c for c in collections if field.startswith(f"_{c}_")]
0055 return max(owners, key=len) if owners else field
0056
0057
0058 def subfields(nodes, idx):
0059 """Field (not column) children of a treemap node, looking through the
0060 '_0' item fields that RNTuple inserts for vectors and arrays."""
0061 node = nodes[idx]
0062 for c in range(node.fChildrenIdx, node.fChildrenIdx + node.fNChildren):
0063 child = nodes[c]
0064 if str(child.fType):
0065 continue
0066 if str(child.fName) == "_0":
0067 yield from subfields(nodes, c)
0068 else:
0069 yield c
0070
0071
0072 def flatten(nodes, idx, path, depth, sizes):
0073 sizes[path] = nodes[idx].fSize
0074 if depth > 0:
0075 for c in subfields(nodes, idx):
0076 flatten(nodes, c, path + (str(nodes[c].fName),), depth - 1, sizes)
0077
0078
0079 def load(path, ntuple):
0080 """Compressed sizes keyed by (collection, field, member, ...) paths."""
0081 insp = ROOT.Experimental.RNTupleInspector.Create(ntuple, path)
0082 treemap = ROOT.Experimental.CreateTreeMapFromRNTuple(insp)
0083 nodes = treemap.fNodes
0084 top = list(subfields(nodes, 0))
0085 names = [str(nodes[i].fName) for i in top]
0086 collections = {n.removesuffix("_objIdx") for n in names if not n.startswith("_")}
0087
0088 sizes = {(): nodes[0].fSize}
0089 uncompressed = {}
0090 for idx, field in zip(top, names):
0091 coll = collection_of(field, collections)
0092 sizes[(coll,)] = sizes.get((coll,), 0) + nodes[idx].fSize
0093 uncompressed[coll] = (
0094 uncompressed.get(coll, 0)
0095 + insp.GetFieldTreeInspector(field).GetUncompressedSize()
0096 )
0097 flatten(nodes, idx, (coll, field), MAX_DEPTH - 1, sizes)
0098
0099 return {
0100 "file": os.path.basename(path),
0101 "file_size": os.path.getsize(path),
0102 "entries": insp.GetDescriptor().GetNEntries(),
0103 "compressed": insp.GetCompressedSize(),
0104 "uncompressed": insp.GetUncompressedSize(),
0105 "sizes": sizes,
0106 "collection_uncompressed": uncompressed,
0107 "treemap": treemap,
0108 }
0109
0110
0111 def write_root_svg(treemap, filename):
0112 ROOT.SetOwnership(treemap, False)
0113 canvas = ROOT.TCanvas("storage", "storage", 1600, 1000)
0114 treemap.Draw()
0115 canvas.SaveAs(filename)
0116
0117
0118 def write_html(new, ref, ntuple, filename):
0119 sizes = new["sizes"]
0120 ref_sizes = ref["sizes"] if ref else {}
0121
0122
0123
0124 paths = sorted(sizes)
0125 index = {path: i for i, path in enumerate(paths)}
0126 nodes = {
0127 "label": [path[-1] if path else ntuple for path in paths],
0128 "parent": [index[path[:-1]] if path else -1 for path in paths],
0129 "size": [sizes[path] for path in paths],
0130 "ref": [ref_sizes.get(path) for path in paths] if ref else None,
0131 }
0132
0133
0134 collections = sorted({p[0] for p in sizes if len(p) == 1} | {p[0] for p in ref_sizes if len(p) == 1})
0135 rows = []
0136 for coll in collections:
0137 size = sizes.get((coll,))
0138 ref_size = ref_sizes.get((coll,)) if ref else None
0139 unc = new["collection_uncompressed"].get(coll)
0140 cells = [
0141 (coll, coll),
0142 (size or 0, fmt_bytes(size) if size is not None else "–"),
0143 (size or 0, f"{(size or 0) / new['compressed']:.2%}"),
0144 ((size or 0) / max(new["entries"], 1), fmt_bytes(round((size or 0) / max(new["entries"], 1)))),
0145 (unc / size if unc and size else 0, f"{unc / size:.1f}" if unc and size else "–"),
0146 ]
0147 if ref:
0148 delta = (size or 0) - (ref_size or 0)
0149 cells += [
0150 (ref_size or 0, fmt_bytes(ref_size) if ref_size is not None else "–"),
0151 (delta, fmt_bytes(delta, sign=True) if delta else ""),
0152 (delta / ref_size if ref_size else ("Infinity" if delta else 0), fmt_rel(size, ref_size) if delta else ""),
0153 ]
0154 cls = ""
0155 if ref and size != ref_size:
0156 cls = ' class="grew"' if (size or 0) > (ref_size or 0) else ' class="shrank"'
0157 rows.append(
0158 f"<tr{cls}>"
0159 + "".join(
0160 f'<td data-sort="{html.escape(str(key))}">{html.escape(str(text))}</td>'
0161 for key, text in cells
0162 )
0163 + "</tr>"
0164 )
0165 headers = ["collection", "size", "share", "per event", "compression"]
0166 if ref:
0167 headers += ["reference", "Δ", "Δ (%)"]
0168
0169 def summary_row(label, key, fmt=fmt_bytes):
0170 cells = f"<th>{label}</th><td>{fmt(new[key])}</td>"
0171 if ref:
0172 delta = new[key] - ref[key]
0173 cells += f"<td>{fmt(ref[key])}</td><td>{fmt(delta, sign=True)}</td>"
0174 return f"<tr>{cells}</tr>"
0175
0176 summary = "".join([
0177 "<tr><th></th><th>this file</th>" + ("<th>reference</th><th>Δ</th>" if ref else "") + "</tr>",
0178 summary_row("file size", "file_size"),
0179 summary_row(f"'{html.escape(ntuple)}' compressed", "compressed"),
0180 summary_row(f"'{html.escape(ntuple)}' uncompressed", "uncompressed"),
0181 summary_row("entries", "entries", lambda n, sign=False: f"{n:{'+' if sign and n else ''}d}"),
0182 ])
0183 if ref and ref["entries"] != new["entries"]:
0184 summary += '<tr><td colspan="4" class="grew">Entry counts differ: sizes are not directly comparable.</td></tr>'
0185
0186 title = f"Storage map of {new['file']}"
0187 reference = f"Compared to reference {html.escape(ref['file'])}. " if ref else "No reference file to compare to. "
0188 page = f"""<!DOCTYPE html>
0189 <html lang="en">
0190 <head>
0191 <meta charset="utf-8">
0192 <meta name="viewport" content="width=device-width, initial-scale=1">
0193 <title>{html.escape(title)}</title>
0194 <script src="https://cdn.jsdelivr.net/npm/plotly.js-dist-min@2.35.2/plotly.min.js"></script>
0195 <style>
0196 body {{ font-family: system-ui, sans-serif; margin: 1em 2em; color: #222; background: #fff; }}
0197 h1 {{ overflow-wrap: anywhere; }}
0198 .scroll {{ overflow-x: auto; }}
0199 table {{ border-collapse: collapse; font-size: 0.9em; }}
0200 th, td {{ padding: 2px 8px; text-align: right; border-bottom: 1px solid #eee; }}
0201 th:first-child, td:first-child {{ text-align: left; }}
0202 #collections th {{ cursor: pointer; position: sticky; top: 0; background: #f6f6f6; }}
0203 tr.grew td {{ background: #fde0dd; }}
0204 tr.shrank td {{ background: #deebf7; }}
0205 #treemap {{ height: 80vh; }}
0206 </style>
0207 </head>
0208 <body>
0209 <h1>{html.escape(title)}</h1>
0210 <p>Compressed on-disk size of the collections in the '{html.escape(ntuple)}' RNTuple, from ROOT's
0211 RNTuple treemap with podio relation, vector-member and subset fields grouped under their collection.
0212 {reference}Click a box to zoom in; click the top bar to zoom out.
0213 See also <a href="storage_treemap.svg">ROOT's treemap rendering</a>, colored by column type.</p>
0214 <div class="scroll"><table>{summary}</table></div>
0215 <div id="treemap"></div>
0216 <h2>Collections</h2>
0217 <p>Compression is uncompressed / compressed size. Click a column header to sort.</p>
0218 <div class="scroll"><table id="collections">
0219 <thead><tr>{''.join(f'<th>{h}</th>' for h in headers)}</tr></thead>
0220 <tbody>
0221 {chr(10).join(rows)}
0222 </tbody>
0223 </table></div>
0224 <script>
0225 const nodes = {json.dumps(nodes, ensure_ascii=False, separators=(",", ":")).replace("</", "<\\/")};
0226 const fmt = b => Math.abs(b) >= 1e6 ? (b / 1e6).toFixed(1) + " MB"
0227 : Math.abs(b) >= 1e3 ? (b / 1e3).toFixed(1) + " kB" : b + " B";
0228 const signed = (v, text) => (v > 0 ? "+" : "") + text;
0229 const paths = [];
0230 nodes.label.forEach((label, i) => {{
0231 const p = nodes.parent[i];
0232 paths.push(p < 0 ? label : paths[p] + "/" + label);
0233 }});
0234 const trace = {{
0235 type: "treemap",
0236 ids: nodes.label.map((_, i) => String(i)),
0237 parents: nodes.parent.map(p => p < 0 ? "" : String(p)),
0238 labels: nodes.label,
0239 values: nodes.size,
0240 branchvalues: "total",
0241 maxdepth: 3,
0242 texttemplate: "%{{label}}<br>%{{customdata[0]}}",
0243 hovertemplate: "<b>%{{customdata[1]}}</b><br>%{{customdata[0]}} (%{{percentRoot:.2%}} of total)"
0244 + (nodes.ref ? "<br>reference: %{{customdata[2]}}<br>Δ: %{{customdata[3]}}" : "") + "<extra></extra>",
0245 }};
0246 trace.customdata = nodes.size.map((s, i) => {{
0247 const r = nodes.ref ? nodes.ref[i] : null;
0248 return [fmt(s), paths[i], r === null ? "–" : fmt(r),
0249 r === null ? "new" : signed(s - r, fmt(s - r)) + (r ? " (" + signed(s - r, (100 * (s - r) / r).toFixed(1)) + "%)" : "")];
0250 }});
0251 if (nodes.ref) {{
0252 // Relative change on a log scale: 1% -> 1/3, 10% -> 2/3, 100% -> 1
0253 const scale = (s, r) => r === null || (r === 0 && s) ? 1 : r === 0 ? 0
0254 : Math.sign(s - r) * Math.min(Math.log10(1 + 100 * Math.abs((s - r) / r)) / 3, 1);
0255 trace.marker = {{
0256 colors: nodes.size.map((s, i) => scale(s, nodes.ref[i])),
0257 colorscale: "RdBu", cmin: -1, cmax: 1, cmid: 0, showscale: true,
0258 colorbar: {{
0259 title: {{text: "size Δ"}},
0260 tickvals: [-1, -2 / 3, -1 / 3, 0, 1 / 3, 2 / 3, 1],
0261 ticktext: ["≤−100%", "−10%", "−1%", "0", "+1%", "+10%", "≥+100%"],
0262 }},
0263 }};
0264 }}
0265 Plotly.newPlot("treemap", [trace], {{margin: {{t: 30, l: 0, r: 0, b: 0}}}}, {{responsive: true}});
0266 const table = document.getElementById("collections");
0267 table.querySelectorAll("th").forEach((th, col) => th.addEventListener("click", () => {{
0268 const desc = th.dataset.desc !== "true";
0269 th.dataset.desc = desc;
0270 const key = tr => {{
0271 const v = tr.children[col].dataset.sort;
0272 return col === 0 ? v : parseFloat(v);
0273 }};
0274 const rows = [...table.tBodies[0].rows].sort((a, b) =>
0275 (key(a) > key(b) ? 1 : key(a) < key(b) ? -1 : 0) * (desc ? -1 : 1));
0276 table.tBodies[0].append(...rows);
0277 }}));
0278 </script>
0279 </body>
0280 </html>
0281 """
0282 with open(filename, "w") as f:
0283 f.write(page)
0284
0285
0286 def main():
0287 ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
0288 ap.add_argument("file", help="podio RNTuple file")
0289 ap.add_argument("reference", nargs="?", help="podio RNTuple file to compare to")
0290 ap.add_argument("-o", "--output-dir", default=".", help="directory to write to")
0291 ap.add_argument("--ntuple", default="events", help="RNTuple to map (default: %(default)s)")
0292 args = ap.parse_args()
0293
0294 new = load(args.file, args.ntuple)
0295 ref = load(args.reference, args.ntuple) if args.reference else None
0296
0297 os.makedirs(args.output_dir, exist_ok=True)
0298 write_html(new, ref, args.ntuple, os.path.join(args.output_dir, "storage.html"))
0299 write_root_svg(new["treemap"], os.path.join(args.output_dir, "storage_treemap.svg"))
0300
0301 if ref:
0302 summary = f"size Δ: {fmt_bytes(new['compressed'] - ref['compressed'], sign=True)}"
0303 else:
0304 summary = f"size: {fmt_bytes(new['compressed'])}"
0305 with open(os.path.join(args.output_dir, "storage_summary.txt"), "w") as f:
0306 f.write(summary + "\n")
0307 print(summary)
0308
0309
0310 if __name__ == "__main__":
0311 main()