Proact-VL-Washing/anthro_test/proactvl_eval/run_validation.py

#!/usr/bin/env python3
"""Proact-VL one-stop validation runner.

Runs the official evaluation metrics on a prediction JSONL:
  1. F1 (speaking-second overlap)            -> f1_only.py        (no API)
  2. F1 + Time-Diff                           -> f1_timediff.py    (no API)
  3. LLM Score (LiveU / FinalQ)              -> llm_score.py      (API, optional)
  4. CC win-rate vs a baseline               -> llm_judge.py     (API, optional)
  5. PAUC (progressive area under curve)     -> pauc.py          (API, optional)

Example:
  python -m proactvl_eval.run_validation \
      --pred results/pred.jsonl \
      --reference results/anns/all_in_one_val_proactive.jsonl \
      --asr_jsonl results/anns/all_in_one.jsonl \
      --baseline_id gemini2.5-pro --baseline_jsonl results/baseline/captions/gemini-2.5-pro.jsonl \
      --judge_model gpt-4o --judge_limit 50

API metrics are skipped automatically when no OPENAI_API_KEY is set
(override with --force_llm_metrics).
"""

import argparse
import json
import os
import subprocess
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent


def run_script(script: str, args: list, env_extra: dict = None) -> None:
    cmd = [sys.executable, "-m", f"proactvl_eval.{script}", *args]
    env = dict(os.environ)
    if env_extra:
        env.update(env_extra)
    print(f"\n{'=' * 70}\n$ {' '.join(cmd)}\n{'=' * 70}")
    subprocess.run(cmd, cwd=str(ROOT), env=env, check=True)


def load_json(path: Path):
    with open(path, encoding="utf-8") as f:
        return json.load(f)


def load_jsonl(path: Path):
    rows = []
    with open(path, encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if line:
                rows.append(json.loads(line))
    return rows


def mean(vals):
    return sum(vals) / len(vals) if vals else float("nan")


def summarize(out_dir: Path, model_id: str, do_cc: bool, do_pauc: bool):
    """Parse metric outputs and print one consolidated table."""
    rows = []

    f1 = out_dir / "f1.json"
    if f1.exists():
        d = load_json(f1)
        m = d["overall"]["overlap_micro"]
        rows.append(("F1 (micro)", f"{m['f1'] * 100:.2f}  (P={m['precision'] * 100:.1f} R={m['recall'] * 100:.1f}, n={m['sample_count']})"))

    ft = out_dir / "f1_timediff.json"
    if ft.exists():
        d = load_json(ft)
        t = d["overall"]["timediff"]
        f1m = d["overall"]["overlap_micro"]["f1"]
        rows.append(("F1+TimeDiff (micro)", f"F1={f1m * 100:.2f}  TimeDiff avg={t['avg']:.3f} (n={t['count']})"))

    liveu = out_dir / "llm_score" / f"llm_score_liveu_{model_id}.jsonl"
    if liveu.exists():
        rows_ = load_jsonl(liveu)
        for k in ("Time", "Rate", "TextU"):
            vals = [r["liveu_score"][k] for r in rows_ if k in r.get("liveu_score", {})]
            rows.append((f"LiveU {k}", f"{mean(vals):.2f} (n={len(vals)})"))

    finalq = out_dir / "llm_score" / f"llm_score_finalq_{model_id}.jsonl"
    if finalq.exists():
        rows_ = load_jsonl(finalq)
        for k in ("Fidelity", "Continuity", "Substance"):
            vals = [r["finalq_score"][k] for r in rows_ if k in r.get("finalq_score", {})]
            rows.append((f"FinalQ {k}", f"{mean(vals):.2f} (n={len(vals)})"))

    if do_cc:
        cc = out_dir / "cc" / f"llm_judge_{model_id}.jsonl"
        if cc.exists():
            rows_ = load_jsonl(cc)
            a = sum(1 for r in rows_ if r.get("ab_winner") == model_id)
            b = sum(1 for r in rows_ if r.get("ba_winner") == model_id)
            n = max(len(rows_), 1)
            rows.append(("CC win-rate", f"A={a / n * 100:.1f}% B={b / n * 100:.1f}% (n={n})"))

    if do_pauc:
        pauc = out_dir / "pauc" / f"pauc_{model_id}.json"
        if pauc.exists():
            d = load_json(pauc)
            for omega, stat in (("overall", d.get("overall") or {})):
                m_auc = stat.get("mean_auc", {}).get("0.5")
                if m_auc is not None:
                    rows.append(("PAUC (omega=0.5)", f"{m_auc * 100:.2f}"))
            if not any(r[0].startswith("PAUC") for r in rows):
                rows.append(("PAUC", "see " + str(pauc)))

    print(f"\n{'#' * 70}\n# Validation summary ({model_id})\n{'#' * 70}")
    for name, val in rows:
        print(f"  {name:<20} {val}")
    if not rows:
        print("  No metric outputs found.")


def main():
    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    p.add_argument("--pred", required=True, help="Prediction JSONL ({video_id, dataset_name, begin, end, idx, tag, pred})")
    p.add_argument("--reference", required=True, help="Proactive reference JSONL with per-second labels (e.g. all_in_one_val_proactive.jsonl)")
    p.add_argument("--model_id", default="model", help="Model id used in outputs")
    p.add_argument("--out_dir", default="./eval_out", help="Output directory")
    # LLM-judged metrics (optional, need an OpenAI-compatible API key)
    p.add_argument("--asr_jsonl", default=None, help="Annotation/ASR JSONL for LLM Score & CC")
    p.add_argument("--baseline_id", default=None, help="Baseline id for CC win-rate")
    p.add_argument("--baseline_jsonl", default=None, help="Baseline prediction JSONL for CC win-rate")
    p.add_argument("--judge_model", default=os.getenv("OPENAI_JUDGE_MODEL", "gpt-5.1_2025-11-13"))
    p.add_argument("--judge_limit", type=int, default=100, help="Max new PAUC judge requests (-1 = all)")
    p.add_argument("--concurrency", type=int, default=8)
    p.add_argument("--skip_f1", action="store_true")
    p.add_argument("--skip_llm_metrics", action="store_true", help="Skip all API-based metrics")
    p.add_argument("--force_llm_metrics", action="store_true")
    args = p.parse_args()

    out_dir = Path(args.out_dir)
    out_dir.mkdir(parents=True, exist_ok=True)
    env = {"OPENAI_JUDGE_MODEL": args.judge_model}

    need_api = not args.skip_llm_metrics and (args.asr_jsonl or args.force_llm_metrics)
    if need_api and not os.getenv("OPENAI_API_KEY") and not args.force_llm_metrics:
        print("OPENAI_API_KEY not set -> running deterministic metrics only (F1, F1+TimeDiff).")
        need_api = False
    do_cc = need_api and bool(args.baseline_jsonl and args.baseline_id and args.asr_jsonl)

    if not args.skip_f1:
        run_script("f1_only", [args.pred, "--reference", args.reference,
                               "--output", str(out_dir / "f1.json"), "--verbose"], env)
        run_script("f1_timediff", [args.pred, "--reference", args.reference,
                                   "--output", str(out_dir / "f1_timediff.json"),
                                   "--alpha", "0.2", "--verbose"], env)

    if need_api:
        if not args.asr_jsonl:
            print("--asr_jsonl is required for LLM metrics; skipping them.")
        else:
            if not args.skip_f1:
                pass
            run_script("llm_score", ["--model_id", args.model_id,
                                     "--prediction_jsonl", args.pred,
                                     "--asr_jsonl", args.asr_jsonl,
                                     "--output_dir", str(out_dir / "llm_score"),
                                     "--num_workers", str(args.concurrency)], env)
            if do_cc:
                run_script("llm_judge", ["--model_id", args.model_id,
                                          "--prediction_jsonl", args.pred,
                                          "--num_workers", str(args.concurrency),
                                          "--baseline_id", args.baseline_id,
                                          "--baseline_jsonl", args.baseline_jsonl,
                                          "--asr_jsonl", args.asr_jsonl,
                                          "--output_dir", str(out_dir / "cc")], env)
            pauc_out = out_dir / "pauc" / f"pauc_{args.model_id}.json"
            pauc_out.parent.mkdir(parents=True, exist_ok=True)
            run_script("pauc", ["--func", "one_step",
                                "--pred_fname", args.pred,
                                "--gold_fname", args.reference,
                                "--output_fname", str(pauc_out),
                                "--openai_model", args.judge_model,
                                "--concurrency", str(args.concurrency),
                                "--start_score", "0",
                                "--judge_limit", str(args.judge_limit),
                                "--resume"], env)

    summarize(out_dir, args.model_id, do_cc, need_api)
    print(f"\nAll outputs under: {out_dir.resolve()}")


if __name__ == "__main__":
    main()