#!/usr/bin/env python3 """Proact-VL one-stop validation runner. Runs the official evaluation metrics on a prediction JSONL: 1. F1 (speaking-second overlap) -> f1_only.py (no API) 2. F1 + Time-Diff -> f1_timediff.py (no API) 3. LLM Score (LiveU / FinalQ) -> llm_score.py (API, optional) 4. CC win-rate vs a baseline -> llm_judge.py (API, optional) 5. PAUC (progressive area under curve) -> pauc.py (API, optional) Example: python -m proactvl_eval.run_validation \ --pred results/pred.jsonl \ --reference results/anns/all_in_one_val_proactive.jsonl \ --asr_jsonl results/anns/all_in_one.jsonl \ --baseline_id gemini2.5-pro --baseline_jsonl results/baseline/captions/gemini-2.5-pro.jsonl \ --judge_model gpt-4o --judge_limit 50 API metrics are skipped automatically when no OPENAI_API_KEY is set (override with --force_llm_metrics). """ import argparse import json import os import subprocess import sys from pathlib import Path ROOT = Path(__file__).resolve().parent.parent def run_script(script: str, args: list, env_extra: dict = None) -> None: cmd = [sys.executable, "-m", f"proactvl_eval.{script}", *args] env = dict(os.environ) if env_extra: env.update(env_extra) print(f"\n{'=' * 70}\n$ {' '.join(cmd)}\n{'=' * 70}") subprocess.run(cmd, cwd=str(ROOT), env=env, check=True) def load_json(path: Path): with open(path, encoding="utf-8") as f: return json.load(f) def load_jsonl(path: Path): rows = [] with open(path, encoding="utf-8") as f: for line in f: line = line.strip() if line: rows.append(json.loads(line)) return rows def mean(vals): return sum(vals) / len(vals) if vals else float("nan") def summarize(out_dir: Path, model_id: str, do_cc: bool, do_pauc: bool): """Parse metric outputs and print one consolidated table.""" rows = [] f1 = out_dir / "f1.json" if f1.exists(): d = load_json(f1) m = d["overall"]["overlap_micro"] rows.append(("F1 (micro)", f"{m['f1'] * 100:.2f} (P={m['precision'] * 100:.1f} R={m['recall'] * 100:.1f}, n={m['sample_count']})")) ft = out_dir / "f1_timediff.json" if ft.exists(): d = load_json(ft) t = d["overall"]["timediff"] f1m = d["overall"]["overlap_micro"]["f1"] rows.append(("F1+TimeDiff (micro)", f"F1={f1m * 100:.2f} TimeDiff avg={t['avg']:.3f} (n={t['count']})")) liveu = out_dir / "llm_score" / f"llm_score_liveu_{model_id}.jsonl" if liveu.exists(): rows_ = load_jsonl(liveu) for k in ("Time", "Rate", "TextU"): vals = [r["liveu_score"][k] for r in rows_ if k in r.get("liveu_score", {})] rows.append((f"LiveU {k}", f"{mean(vals):.2f} (n={len(vals)})")) finalq = out_dir / "llm_score" / f"llm_score_finalq_{model_id}.jsonl" if finalq.exists(): rows_ = load_jsonl(finalq) for k in ("Fidelity", "Continuity", "Substance"): vals = [r["finalq_score"][k] for r in rows_ if k in r.get("finalq_score", {})] rows.append((f"FinalQ {k}", f"{mean(vals):.2f} (n={len(vals)})")) if do_cc: cc = out_dir / "cc" / f"llm_judge_{model_id}.jsonl" if cc.exists(): rows_ = load_jsonl(cc) a = sum(1 for r in rows_ if r.get("ab_winner") == model_id) b = sum(1 for r in rows_ if r.get("ba_winner") == model_id) n = max(len(rows_), 1) rows.append(("CC win-rate", f"A={a / n * 100:.1f}% B={b / n * 100:.1f}% (n={n})")) if do_pauc: pauc = out_dir / "pauc" / f"pauc_{model_id}.json" if pauc.exists(): d = load_json(pauc) for omega, stat in (("overall", d.get("overall") or {})): m_auc = stat.get("mean_auc", {}).get("0.5") if m_auc is not None: rows.append(("PAUC (omega=0.5)", f"{m_auc * 100:.2f}")) if not any(r[0].startswith("PAUC") for r in rows): rows.append(("PAUC", "see " + str(pauc))) print(f"\n{'#' * 70}\n# Validation summary ({model_id})\n{'#' * 70}") for name, val in rows: print(f" {name:<20} {val}") if not rows: print(" No metric outputs found.") def main(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("--pred", required=True, help="Prediction JSONL ({video_id, dataset_name, begin, end, idx, tag, pred})") p.add_argument("--reference", required=True, help="Proactive reference JSONL with per-second labels (e.g. all_in_one_val_proactive.jsonl)") p.add_argument("--model_id", default="model", help="Model id used in outputs") p.add_argument("--out_dir", default="./eval_out", help="Output directory") # LLM-judged metrics (optional, need an OpenAI-compatible API key) p.add_argument("--asr_jsonl", default=None, help="Annotation/ASR JSONL for LLM Score & CC") p.add_argument("--baseline_id", default=None, help="Baseline id for CC win-rate") p.add_argument("--baseline_jsonl", default=None, help="Baseline prediction JSONL for CC win-rate") p.add_argument("--judge_model", default=os.getenv("OPENAI_JUDGE_MODEL", "gpt-5.1_2025-11-13")) p.add_argument("--judge_limit", type=int, default=100, help="Max new PAUC judge requests (-1 = all)") p.add_argument("--concurrency", type=int, default=8) p.add_argument("--skip_f1", action="store_true") p.add_argument("--skip_llm_metrics", action="store_true", help="Skip all API-based metrics") p.add_argument("--force_llm_metrics", action="store_true") args = p.parse_args() out_dir = Path(args.out_dir) out_dir.mkdir(parents=True, exist_ok=True) env = {"OPENAI_JUDGE_MODEL": args.judge_model} need_api = not args.skip_llm_metrics and (args.asr_jsonl or args.force_llm_metrics) if need_api and not os.getenv("OPENAI_API_KEY") and not args.force_llm_metrics: print("OPENAI_API_KEY not set -> running deterministic metrics only (F1, F1+TimeDiff).") need_api = False do_cc = need_api and bool(args.baseline_jsonl and args.baseline_id and args.asr_jsonl) if not args.skip_f1: run_script("f1_only", [args.pred, "--reference", args.reference, "--output", str(out_dir / "f1.json"), "--verbose"], env) run_script("f1_timediff", [args.pred, "--reference", args.reference, "--output", str(out_dir / "f1_timediff.json"), "--alpha", "0.2", "--verbose"], env) if need_api: if not args.asr_jsonl: print("--asr_jsonl is required for LLM metrics; skipping them.") else: if not args.skip_f1: pass run_script("llm_score", ["--model_id", args.model_id, "--prediction_jsonl", args.pred, "--asr_jsonl", args.asr_jsonl, "--output_dir", str(out_dir / "llm_score"), "--num_workers", str(args.concurrency)], env) if do_cc: run_script("llm_judge", ["--model_id", args.model_id, "--prediction_jsonl", args.pred, "--num_workers", str(args.concurrency), "--baseline_id", args.baseline_id, "--baseline_jsonl", args.baseline_jsonl, "--asr_jsonl", args.asr_jsonl, "--output_dir", str(out_dir / "cc")], env) pauc_out = out_dir / "pauc" / f"pauc_{args.model_id}.json" pauc_out.parent.mkdir(parents=True, exist_ok=True) run_script("pauc", ["--func", "one_step", "--pred_fname", args.pred, "--gold_fname", args.reference, "--output_fname", str(pauc_out), "--openai_model", args.judge_model, "--concurrency", str(args.concurrency), "--start_score", "0", "--judge_limit", str(args.judge_limit), "--resume"], env) summarize(out_dir, args.model_id, do_cc, need_api) print(f"\nAll outputs under: {out_dir.resolve()}") if __name__ == "__main__": main()