eval/summarize.py
| 1 | """Aggregate the per-case results of eval/run_test_case.py into a markdown table, optionally |
| 2 | paired against the reference numbers shipped in eval/cases/<task>_reference_metrics.json. |
| 3 | |
| 4 | python eval/summarize.py --task rope [--root DIR] [--reference] |
| 5 | python eval/summarize.py --task rope --split val --root runs/<run>/val_epoch_014 \ |
| 6 | --ref-root runs/nano_action_causal_rope_v1/val_epoch_014 # paired vs another run on the same val cases |
| 7 | |
| 8 | short = 8 observed -> 8 predicted frames (metrics on the 8 predicted frames); |
| 9 | long = mean over the 15 autoregressive windows (120 generated frames). |
| 10 | """ |
| 11 | from __future__ import annotations |
| 12 | |
| 13 | import argparse |
| 14 | import json |
| 15 | import os |
| 16 | from pathlib import Path |
| 17 | |
| 18 | REPO = Path(__file__).resolve().parents[1] |
| 19 | STREAMS = ["video", "wrist_left", "wrist_right", "tactile_left", "tactile_right"] |
| 20 | METRICS = ["psnr", "ssim", "lpips"] |
| 21 | |
| 22 | |
| 23 | def load(root: Path, man: dict) -> dict: |
| 24 | out = {} |
| 25 | for c in man["cases"]: |
| 26 | d = root / c["output_subdir"] |
| 27 | if not (d / "complete.json").is_file(): |
| 28 | continue |
| 29 | w = json.loads((d / "window_eval.json").read_text())["results"][0] |
| 30 | r = {"category": c["category"], "short": {s: {m: w["image"][s][m] for m in METRICS} for s in STREAMS}} |
| 31 | rr = d / "reference_rollout.json" |
| 32 | if c["rollout_windows"] and rr.is_file(): |
| 33 | wins = json.loads(rr.read_text())[0]["windows"] |
| 34 | r["long_mean"] = {s: {m: sum(x["image"][s][m] for x in wins) / len(wins) for m in METRICS} for s in STREAMS} |
| 35 | out[c["output_subdir"]] = r |
| 36 | return out |
| 37 | |
| 38 | |
| 39 | def mean(xs): |
| 40 | return sum(xs) / len(xs) if xs else float("nan") |
| 41 | |
| 42 | |
| 43 | def table(res: dict, hz: str, title: str) -> list[str]: |
| 44 | rows = [r for r in res.values() if hz in r] |
| 45 | lines = [f"### {title} (n={len(rows)})", "", "| stream | PSNR ↑ | SSIM ↑ | LPIPS ↓ |", "|---|---:|---:|---:|"] |
| 46 | for s in STREAMS: |
| 47 | v = [mean([r[hz][s][m] for r in rows]) for m in METRICS] |
| 48 | lines.append(f"| {s} | {v[0]:.2f} | {v[1]:.4f} | {v[2]:.4f} |") |
| 49 | return lines + [""] |
| 50 | |
| 51 | |
| 52 | def main(): |
| 53 | ap = argparse.ArgumentParser() |
| 54 | tasks = sorted(f.name[:-len("_test_cases.json")] for f in (REPO / "eval" / "cases").glob("*_test_cases.json")) |
| 55 | ap.add_argument("--task", required=True, choices=tasks) |
| 56 | ap.add_argument("--root", default=None, help="default: $ACTS_RUNS_ROOT/<run_name>/test_<ckpt stem of the manifest>") |
| 57 | ap.add_argument("--split", default="test", choices=["test", "val"]) |
| 58 | ap.add_argument("--reference", action="store_true", |
| 59 | help="also print paired deltas vs the shipped reference metrics (test split only)") |
| 60 | ap.add_argument("--ref-root", default=None, |
| 61 | help="also print paired deltas vs another result dir on the same cases (e.g. the parent run's " |
| 62 | "val_epoch_014); use this for the val split") |
| 63 | a = ap.parse_args() |
| 64 | man = json.loads((REPO / "eval" / "cases" / f"{a.task}_{a.split}_cases.json").read_text()) |
| 65 | runs = Path(os.environ.get("ACTS_RUNS_ROOT", REPO / "runs")) |
| 66 | root = Path(a.root or runs / man["run_name"] / f"{a.split}_{Path(man['checkpoint']).stem}") |
| 67 | res = load(root, man) |
| 68 | print(f"# {root.parent.name} {root.name} ({len(res)}/{len(man['cases'])} cases complete) -- {root}\n") |
| 69 | print("\n".join(table(res, "short", "short horizon: 8 -> 8 frames"))) |
| 70 | print("\n".join(table(res, "long_mean", "long horizon: mean over 15 chained windows"))) |
| 71 | refs = [] |
| 72 | if a.reference: |
| 73 | if a.split != "test": |
| 74 | raise SystemExit("--reference is the shipped TEST reference; for val use --ref-root") |
| 75 | ref_file = REPO / "eval" / "cases" / f"{a.task}_reference_metrics.json" |
| 76 | if not ref_file.is_file(): |
| 77 | raise SystemExit(f"no reference metrics for {a.task}: {ref_file}") |
| 78 | refs.append(("reference", {r["output_subdir"]: r for r in json.loads(ref_file.read_text())["cases"]})) |
| 79 | if a.ref_root: |
| 80 | rr = Path(a.ref_root) |
| 81 | refs.append((f"{rr.parent.name}/{rr.name}", load(rr, man))) |
| 82 | for name, ref in refs: |
| 83 | print(f"### paired difference vs {name} (this run − {name}), PSNR dB, n = {len(ref)} reference cases\n") |
| 84 | print("| horizon | stream | n | mean Δ | max |Δ| |\n|---|---|---:|---:|---:|") |
| 85 | for hz in ("short", "long_mean"): |
| 86 | for s in STREAMS: |
| 87 | d = [res[k][hz][s]["psnr"] - ref[k][hz][s]["psnr"] for k in res if hz in res[k] and hz in ref.get(k, {})] |
| 88 | if d: |
| 89 | print(f"| {hz} | {s} | {len(d)} | {mean(d):+.3f} | {max(abs(x) for x in d):.3f} |") |
| 90 | print() |
| 91 | |
| 92 | |
| 93 | if __name__ == "__main__": |
| 94 | main() |
| 95 | |