Proact-VL-Washing/anthro_test/proactvl_eval/llm_score.py

import os, openai, json, functools, argparse
try:
    from datasets import load_dataset  # optional, only needed by some official flows
except ImportError:
    load_dataset = None
try:
    from proactvl.utils.multiprocessor import local_mt
except ImportError:
    try:
        from proactvl_eval.multiprocessor import local_mt
    except ImportError:
        from multiprocessor import local_mt
import re
from typing import Dict, Union

def extract_scores(text: str) -> Dict[str, Union[int, float]]:
    # Each line should look like: key: value
    pattern = re.compile(r'^\s*([^:\n]+?)\s*:\s*([+-]?\d+(?:\.\d+)?)\s*$', re.M)
    out = {}
    for k, v in pattern.findall(text):
        v_num = float(v)
        out[k.strip()] = int(v_num) if v_num.is_integer() else v_num
    return out

# ========================= Initialize GPT Client =========================
model_id = ''
try:
    from evaluation.gaming.gpt_client_utils import GPTClientFactory
except ImportError:
    try:
        from proactvl_eval.gpt_client_utils import GPTClientFactory
    except ImportError:
        from gpt_client_utils import GPTClientFactory


deployment_name = os.getenv("OPENAI_JUDGE_MODEL", "gpt-5.1_2025-11-13")

try:
    gpt = GPTClientFactory().build()
except Exception as e:
    raise RuntimeError(f"Error: failed to initialize GPT client. {e}") from e
# ======================== End Initialize GPT Client =========================

video_event_id_to_gt_asr, video_start_end_to_video_event_id, video_event_id_to_model_pred = {}, {}, {}
video_event_id_to_dataset_name = {}
video_event_id_to_tag = {}
video_event_id_to_context = {}
video_event_id_to_gt_asr_clip = {}
video_event_id_to_model_pred_clip = {}

def judge_liveu(context, model_response, ground_truth):
    prompt = r"""Task Description:
You are an expert in evaluating the **Delivery Dynamics** of real-time, second-by-second streaming commentary for video clips.
Assess the model based on scenario constraints, timing/rhythm, and real-time listenability.

You will be provided:
1) Context Text (previous commentary or background context; may also contain prior lines from the same commentator)
2) Label Timeline Text (time-aligned reference timeline; includes all on-clip information such as [USER] questions, [SPEAKER*] commentary, and the current commentator’s lines; use mainly as a reference for rhythm and salient windows)
3) Prediction Timeline Text (the current commentator’s output to be evaluated)

Speaker tags (apply to all inputs):
- [ASSISTANT] = the current commentator’s lines (may appear in Context as prior commentary; and appears in Label/Prediction as on-clip current commentator)
- [USER] = user questions (reference/context only; may appear in Label Timeline)
- [SPEAKER*] = other commentators (reference/context only; may appear in Label Timeline)

What to score:
- ONLY score the content in Prediction Timeline Text.
- Any tagged lines in Context or Label Timeline are reference/context signals only and must NOT be scored.

Scenario & speaking constraints (infer from Label Timeline):
- Solo: normal streaming commentary flow.
- Multi-commentator: generally avoid speaking when [SPEAKER*] is speaking (avoid overlap), unless clearly necessary for a key moment. A brief 1-second overlap is acceptable; penalize overlap mainly when it is sustained (e.g., 2+ consecutive seconds), frequent (repeated across the clip), or clearly disruptive.
- Guidance: when [USER] asks a question, provide step-by-step guidance; avoid "one huge dump" of info.

Important notes:
- Prediction may be event-triggered: it may speak only on some seconds; missing seconds are silence. "SILENCE"/empty = silence.
- Label Timeline is a reference only; no verbatim match required. Use it mainly to judge speaking rhythm, salient windows, and whether Prediction contradicts key events.
- Subjective/expressive commentary is acceptable (brief opinions, reactions, humor, hype, atmosphere-building), as long as it does not contradict key events and does not cause disruptive chattering.

Scoring rules:
- Output integer scores in [1..10] (no 0).
- Score bands: 10 excellent; 7–9 good; 4–6 mixed; 2–3 very poor but somewhat usable; 1 unusable.

Dimension 1. Time (Synchronicity: timing + coverage + overlap etiquette):
Scoring criteria:
- 10: Excellent rhythm; enters within salient windows; reacts promptly to key moments and [USER] questions; stays silent when appropriate; strong coverage; overlap is rare and not sustained/disruptive.
- 7-9: Generally good timing; minor drift or small gaps; occasional brief overlap is acceptable; sustained overlap is rare.
- 4-6: Unstable rhythm; noticeable late/early speaking; multiple unnecessary lines or noticeable gaps; partial coverage; overlap becomes noticeable (e.g., repeated or sometimes sustained).
- 2-3: Poor; frequently misses obvious speaking moments or speaks when silence/etiquette requires not speaking; overlap is frequent or often sustained/disruptive.
- 1: Unusable timing; chaotic speaking/silence patterns or near-constant disruptive overlap that breaks the live experience.

Coverage tolerance rule (apply to all bands):
- Window shift is acceptable: if Label indicates a salient speaking window (e.g., 3–7s), speaking in any reasonably overlapping window such as 4–6, 4–7, 4–8, or 2–5 can be considered acceptable (exact boundaries not required).

Dimension 2. Rate (Cadence: density + pacing + anti-dump):
Scoring criteria:
- 10: Perfect real-time pacing; short and listenable per-second/burst outputs; no dense dumps; minimal filler; high signal-to-noise; Guidance answers delivered step-by-step (not a single large dump).
- 7-9: Mostly well paced; occasional slightly long bursts or mild redundancy/info-clumping, but still listenable.
- 4-6: Noticeable pacing issues; multiple long/dense bursts, repetitive filler, or fragmented delivery; Guidance sometimes turns into a dump; signal-to-noise often low.
- 2-3: Very poor pacing; frequent extremely long/dense outputs OR clearly under-speaking such that salient moments are missed.
- 1: Unusable pacing; consistently unlistenable due to extreme length/density or severe fragmentation.

Dimension 3. TextU (Streaming usability: spoken-form clarity):
Scoring criteria:
- 10: Consistently clear and well-formed as live speech; natural spoken wording; minimal grammar issues; easy to follow in real time; Guidance is step-by-step and actionable in delivery.
- 7-9: Mostly clear; minor awkwardness; occasional light fragmentation but overall usable as live speech.
- 4-6: Mixed; frequent awkward/incomplete phrasing or noticeable fragmentation; live usability is noticeably harmed.
- 2-3: Poor; many lines hard to understand, overly fragmented, or off-scenario (e.g., does not engage a [USER] question in Guidance); live usability is low.
- 1: Unusable; mostly incoherent/garbled or consistently impossible to follow as live speech.

STRICT output format (no extra characters):
Time: <1-10>
Rate: <1-10>
TextU: <1-10>

Context: {context}
Label Timeline: {label_timeline}
Prediction Timeline: {prediction_timeline}
""".format(context=context, label_timeline=ground_truth, prediction_timeline=model_response)
    return prompt

def judge_finalq(context, model_response, ground_truth):
    prompt = r"""Task Description:
You are an expert in evaluating the **Narrative Integrity** and overall quality of consolidated video commentary.
Assess the final script’s readability, coherence, and usefulness, while only penalizing direct event contradictions.

You will be provided:
1) Context Text (previous commentary or background context; may also contain prior lines from the same commentator)
2) Label Final Text (raw ASR final text from the original video; reference only)
3) Prediction Text (the final model output to be evaluated)

Speaker definitions (apply to Context, Label Final, and Prediction):
- [ASSISTANT] = current commentator’s lines (may appear in Context as prior commentary)
- [USER] = user questions (reference/context only; may appear in Context and/or Label Final)
- [SPEAKER*] = other commentators (reference/context only; may appear in Context and/or Label Final)

What to score:
- ONLY score the content in Prediction Text.
- Any tagged lines in Context / Label Final are reference/context signals only and must NOT be scored.

Evaluation principles:
1) Scenario alignment: in Multi-commentator mode, check whether [ASSISTANT] complements [SPEAKER*] logically; in Guidance mode, check whether the [USER] query is actually addressed/resolved.
2) Directional compatibility: use Label Final as a reference for major events; do NOT require word-by-word match; penalize **direct contradictions** (major event reversal, incompatible outcomes, or clearly wrong situation claims).
3) Fidelity (lightweight): expressive commentary is allowed, but avoid unjustified concrete specifics (e.g., names, numbers, causes, outcomes) that would mislead; major invented events that conflict with Label should be penalized.

Scoring rules:
- Output integer scores in [1..10] (no 0).
- Score bands: 10 excellent; 7–9 good; 4–6 mixed; 2–3 very poor but somewhat usable; 1 unusable.

Dimension 1. Fidelity (Event compatibility / non-contradiction):
Criteria: compatibility with Label Final and avoidance of misleading concrete claims.
- 10: Fully compatible; captures core events accurately; no major contradictions; any added specifics remain non-misleading and do not conflict with Context/Label.
- 7-9: Generally compatible; core events align; some added detail/emphasis is acceptable and does not contradict the situation.
- 4-6: Mixed; includes notable mismatches or unjustified concrete specifics that could mislead; weakens trust.
- 2-3: Largely incompatible; many statements contradict the reference or introduce incompatible events/claims.
- 1: Unusable; completely unrelated or fundamentally opposite to the reference facts.

Dimension 2. Continuity (Coherence with Context, co-hosts, and user thread):
Criteria: logical flow and referential integrity across entities, stance, and conversation.
- 10: Seamless continuity; consistent topics/entities/stance; complements [SPEAKER*] logically; clear referents (no confusing pronouns); smooth transitions; avoids obvious stitched repetition (e.g., repeating the same conclusion multiple times).
- 7-9: Mostly consistent; follows the narrative direction; minor awkwardness in transitions or small continuity slips; limited repetition.
- 4-6: Noticeable logic drift; repeats points already established; confusing references to entities; clunky transitions; coherence is weakened.
- 2-3: Poor; ignores [USER] questions in Guidance or contradicts important prior context; heavy repetition/looping; hard to follow logically.
- 1: Unusable; fundamental breakdown of context awareness or narrative logic.

Dimension 3. Substance (Text quality + usefulness + redundancy control):
Criteria: informational value, guidance effectiveness, and consolidated-script usability (structure, readability, low redundancy).
- 10: High value; clear structure; strong readability; minimal redundancy; provides helpful insights; directly and effectively addresses [USER] queries when present; adds value beyond generic filler.
- 7-9: Clear and informative; engaging voice; minor redundancy; covers the [USER] request reasonably well; mostly avoids clichés and empty looping.
- 4-6: Mediocre; generic/cliché; partially answers [USER] queries; noticeable repetition or bloated sections; usefulness is limited.
- 2-3: Low value; hollow/robotic; fails to address the [USER] or provides disjoint segments; heavy looping/redundancy; mostly filler.
- 1: Unusable due to severe readability failure or near-total lack of substance.

STRICT output format (no extra characters):
Fidelity: <1-10>
Continuity: <1-10>
Substance: <1-10>

Context: {context}
Label Final: {label_final}
Prediction: {prediction}
""".format(context=context, label_final=ground_truth, prediction=model_response)
    return prompt

def llm_score(video_event_id):
    model_pred_clip = video_event_id_to_model_pred_clip[video_event_id]
    model_pred = video_event_id_to_model_pred[video_event_id]
    gt_asr_clip = video_event_id_to_gt_asr_clip[video_event_id]
    gt_asr = video_event_id_to_gt_asr[video_event_id]
    context = video_event_id_to_context[video_event_id]
    score_map = {}
    prompt_liveu = judge_liveu(context, model_pred_clip, gt_asr_clip)


    prompt_final = judge_finalq(context, model_pred, gt_asr)
    # score_map_finalq = {}
    while True:
        try:
            resp = gpt.chat.completions.create(
                # model='gpt-4o-2024-08-06',
                # model='gpt-4.1',
                model=deployment_name,
                messages=[{"role": "user", "content": [{'type': 'text', 'text': prompt_liveu}]}],
                seed=42,
                temperature=0,
            ).choices[0].message.content
            liveU_score = extract_scores(resp)
            if 'Time' in liveU_score and 'Rate' in liveU_score and 'TextU' in liveU_score:
                break
            else:
                print(f'Incomplete scores received: {liveU_score}, retrying...')
        except Exception as e:
            print('Failed to get response...', e)

    while True:
        try:
            resp_final = gpt.chat.completions.create(
                # model='gpt-4o-2024-08-06',
                # model='gpt-4.1',
                model=deployment_name,
                messages=[{"role": "user", "content": [{'type': 'text', 'text': prompt_final}]}],
                seed=42,
                temperature=0,
            ).choices[0].message.content
            finalQ_score = extract_scores(resp_final)
            if 'Fidelity' in finalQ_score and 'Continuity' in finalQ_score and 'Substance' in finalQ_score:
                break
            else:
                print(f'Incomplete scores received: {finalQ_score}, retrying...')
        except Exception as e:
            print('Failed to get response...', e)
    # Merge the two scoring results
    score_map.update(liveU_score)
    score_map.update(finalQ_score)
    return score_map

def judge(video_event_id, model_id):
    gt_asr = video_event_id_to_gt_asr[video_event_id]
    dataset_name = video_event_id_to_dataset_name[video_event_id]
    tag = video_event_id_to_tag[video_event_id]
    return {
        'video_event_id': video_event_id, 
        'score': llm_score(video_event_id), 
        'dataset_name': dataset_name,
        'tag': tag
    }

def merge_commentary(commentary_dict):
    sorted_seconds = sorted(commentary_dict.keys(), key=int)
    merged_commentary = ' '.join([commentary_dict[sec].replace(' ...', '') for sec in sorted_seconds])
    return merged_commentary.strip()

def pred2standard(pred):
    sorted_seconds = sorted(pred.keys(), key=int)
    asrs = []
    asr_lines = []
    for sec in sorted_seconds:
        line = pred[sec]
        # Remove trailing ' ...'
        if line.endswith(' ...'):
            line = line[:-4]
        if line.strip() == '':
            continue
        else:
            asr_lines.append(f'[t={sec}] {line}')
            asrs.append(line)
    return '\n'.join(asr_lines), ' '.join(asrs)

def gt2standard(annotations):
    active_speaker = annotations['active_speaker']['name']
    asrs = []
    asr_lines = []
    for item in annotations['annotations']:
        # if item['speaker'] != active_speaker:
        #     continue
        sec = item['start']
        line = item['text'] if 'text' in item else item['query']

        if line.strip() == '':
            continue
        else:
            if item['speaker'] == active_speaker:
                
                asr_lines.append(f'[t={sec}][ASSISTANT]:{line}')
            elif item['speaker'] == 'user':
                asr_lines.append(f'[t={sec}][USER]:{line}')
            else:
                asr_lines.append(f'[t={sec}][{item["speaker"]}]:{line}')
            asrs.append(line)
    return '\n'.join(asr_lines), ' '.join(asrs)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--model_id', type=str, required=True, help='Model name to compare against baseline')
    parser.add_argument('--prediction_jsonl', type=str, required=True, help='Path to model predictions in JSONL format')
    parser.add_argument('--output_dir', type=str, default='evaluation/gaming/judges/', help='Directory to save judgment results')
    parser.add_argument('--num_workers', type=int, default=1)
    parser.add_argument('--asr_jsonl', type=str, default=None, help='Path to ASR transcriptions in JSONL format')
    args = parser.parse_args()

    model_id = args.model_id
    prediction_jsonl = args.prediction_jsonl
    output_dir = args.output_dir


    os.makedirs(output_dir, exist_ok=True)
    save_path = os.path.join(output_dir, f'llm_score_{model_id}.jsonl')

    # prediction_jsonl = 'evaluation/gaming/gaming/Qwen2.5-Omni-7B.jsonl'
    prediction_jsonl = args.prediction_jsonl



    for line in open(prediction_jsonl):
        datum = json.loads(line)
        # if datum['dataset_name'] not in args.dataset_name_list:
        #     continue
        dataset_name = datum['dataset_name']
        if dataset_name == 'ego4d_goal_step':
            dataset_name = 'ego4d'
        video_event_id = f'{datum["video_id"]}_{datum["begin"]}_{datum["end"]}_{dataset_name}_{datum["idx"]}'
        if video_event_id in video_event_id_to_model_pred:
            print(f'Warning: duplicate video_event_id {video_event_id} in predictions')
        pred_clip, pred = pred2standard(datum['pred'])
        video_event_id_to_model_pred_clip[video_event_id] = pred_clip
        video_event_id_to_model_pred[video_event_id] = pred
        video_event_id_to_dataset_name[video_event_id] = datum['dataset_name']
        video_event_id_to_tag[video_event_id] = datum['tag']

    for line in open(args.asr_jsonl):
        datum = json.loads(line)
        video_event_id = f'{os.path.basename(datum["video_path"])}_{datum["video_begin"]}_{datum["video_end"]}_{datum["dataset_name"]}_{datum["idx"]}'
        pred_clip, pred = gt2standard(datum)
        video_event_id_to_gt_asr_clip[video_event_id] = pred_clip
        video_event_id_to_gt_asr[video_event_id] = pred
        video_event_id_to_context[video_event_id] = datum['history'] if 'history' in datum else ''

    # assert len(video_event_id_to_model_pred) == len(video_event_id_to_gt_asr), "Mismatch between predictions and ASR ground truths"
    if len(video_event_id_to_model_pred) != len(video_event_id_to_context):
        print(f'Length of predictions: {len(video_event_id_to_model_pred)}, Length of contexts: {len(video_event_id_to_context)}')
        # Iterate and print missing video_event_id values
        missing_ids = set(video_event_id_to_context.keys()) - set(video_event_id_to_model_pred.keys())
        print(f'Number of missing contexts: {len(missing_ids)}')
        for missing_id in missing_ids:
            print(f'Missing context for video_event_id: {missing_id}')
        # raise ValueError("Mismatch between predictions and contexts")
    winner_results = local_mt(
        video_event_id_to_model_pred_clip.keys(),
        functools.partial(judge, model_id=model_id), 
        desc=f'LLM Score Call gpt4o for {model_id}', 
        num_workers=args.num_workers
    )
    
    # For liveu, aggregate results by game, by tag, and overall
    with open(os.path.join(output_dir, f'llm_score_liveu_{model_id}.jsonl'), 'w') as f:
        for winner_result in winner_results:
            liveu_score = {k: v for k, v in winner_result['score'].items() if k in ['Time', 'Rate', 'TextU']}
            output_dict = {
                'video_event_id': winner_result['video_event_id'],
                'liveu_score': liveu_score,
                'dataset_name': winner_result['dataset_name'],
                'tag': winner_result['tag']
            }
            f.write(json.dumps(output_dict) + '\n')
    # For finalq, aggregate results by game, by tag, and overall
    with open(os.path.join(output_dir, f'llm_score_finalq_{model_id}.jsonl'), 'w') as f:
        for winner_result in winner_results:
            finalq_score = {k: v for k, v in winner_result['score'].items() if k in ['Fidelity', 'Continuity', 'Substance']}
            output_dict = {
                'video_event_id': winner_result['video_event_id'],
                'finalq_score': finalq_score,
                'dataset_name': winner_result['dataset_name'],
                'tag': winner_result['tag']
            }
            f.write(json.dumps(output_dict) + '\n')
    # Aggregate liveu_score and finalq_score by game, by tag, then overall
    summary_by_game = {}
    for winner_result in winner_results:
        dataset_name = winner_result['dataset_name']
        if dataset_name not in summary_by_game:
            summary_by_game[dataset_name] = {
                'liveu_score': {'Time': 0, 'Rate': 0, 'TextU': 0},
                'finalq_score': {'Fidelity': 0, 'Continuity': 0, 'Substance': 0},
                'count': 0
            }
        liveu_score = {k: v for k, v in winner_result['score'].items() if k in ['Time', 'Rate', 'TextU']}
        finalq_score = {k: v for k, v in winner_result['score'].items() if k in ['Fidelity', 'Continuity', 'Substance']}
        for k in liveu_score:
            summary_by_game[dataset_name]['liveu_score'][k] += liveu_score[k]
        for k in finalq_score:
            summary_by_game[dataset_name]['finalq_score'][k] += finalq_score[k]
        summary_by_game[dataset_name]['count'] += 1
    # summary by game: output mean scores
    for dataset_name, summary in summary_by_game.items():
        count = summary['count']
        liveu_avg = {k: v / count for k, v in summary['liveu_score'].items()}
        finalq_avg = {k: v / count for k, v in summary['finalq_score'].items()}
        output = f'Average LiveU Score for {model_id} on {dataset_name}: ' + ', '.join([f'{k}: {v:.2f}' for k, v in liveu_avg.items()])
        print(output)
        with open(os.path.join(output_dir, 'log.txt'), 'a') as f:
            f.write(output + '\n')
        output = f'Average FinalQ Score for {model_id} on {dataset_name}: ' + ', '.join([f'{k}: {v:.2f}' for k, v in finalq_avg.items()])
        print(output)
        with open(os.path.join(output_dir, 'log.txt'), 'a') as f:
            f.write(output + '\n')
    # summary by tag
    summary_by_tag = {}
    for winner_result in winner_results:
        tag = winner_result['tag']
        if tag not in summary_by_tag:
            summary_by_tag[tag] = {
                'liveu_score': {'Time': 0, 'Rate': 0, 'TextU': 0},
                'finalq_score': {'Fidelity': 0, 'Continuity': 0, 'Substance': 0},
                'count': 0
            }
        liveu_score = {k: v for k, v in winner_result['score'].items() if k in ['Time', 'Rate', 'TextU']}
        finalq_score = {k: v for k, v in winner_result['score'].items() if k in ['Fidelity', 'Continuity', 'Substance']}
        for k in liveu_score:
            summary_by_tag[tag]['liveu_score'][k] += liveu_score[k]
        for k in finalq_score:
            summary_by_tag[tag]['finalq_score'][k] += finalq_score[k]
        summary_by_tag[tag]['count'] += 1
    # summary by tag: output mean scores
    for tag, summary in summary_by_tag.items():
        count = summary['count']
        liveu_avg = {k: v / count for k, v in summary['liveu_score'].items()}
        finalq_avg = {k: v / count for k, v in summary['finalq_score'].items()}
        output = f'Average LiveU Score for {model_id} on tag {tag}: ' + ', '.join([f'{k}: {v:.2f}' for k, v in liveu_avg.items()])
        print(output)
        with open(os.path.join(output_dir, 'log.txt'), 'a') as f:
            f.write(output + '\n')
        output = f'Average FinalQ Score for {model_id} on tag {tag}: ' + ', '.join([f'{k}: {v:.2f}' for k, v in finalq_avg.items()])
        print(output)
        with open(os.path.join(output_dir, 'log.txt'), 'a') as f:
            f.write(output + '\n')
    # overall average
    total_count = len(winner_results)
    total_liveu_score = {'Time': 0, 'Rate': 0, 'TextU': 0}
    total_finalq_score = {'Fidelity': 0, 'Continuity': 0, 'Substance': 0}
    for winner_result in winner_results:
        liveu_score = {k: v for k, v in winner_result['score'].items() if k in ['Time', 'Rate', 'TextU']}
        finalq_score = {k: v for k, v in winner_result['score'].items() if k in ['Fidelity', 'Continuity', 'Substance']}
        for k in liveu_score:
            total_liveu_score[k] += liveu_score[k]
        for k in finalq_score:
            total_finalq_score[k] += finalq_score[k]
    overall_liveu_avg = {k: v / total_count for k, v in total_liveu_score.items()}
    overall_finalq_avg = {k: v / total_count for k, v in total_finalq_score.items()}
    output = f'Overall Average LiveU Score for {model_id}: ' + ', '.join([f'{k}: {v:.2f}' for k, v in overall_liveu_avg.items()])
    print(output)
    with open(os.path.join(output_dir, 'log.txt'), 'a') as f:
        f.write(output + '\n')
    output = f'Overall Average FinalQ Score for {model_id}: ' + ', '.join([f'{k}: {v:.2f}' for k, v in overall_finalq_avg.items()])
    print(output)
    with open(os.path.join(output_dir, 'log.txt'), 'a') as f:
        f.write(output + '\n')