"""grade.py — the exact grading logic behind results_per_task.csv. Self-contained, stdlib only, no network, no LLM judge. Concatenation of the graders actually used by the run scripts: - Everyday-mix : datasets/mixed_public/scorer.py (verbatim, below) - AIME 2025 : extract_aime_answer() - GPQA Diamond : extract_gpqa_answer() - LiveCodeBench: not a text match — generated code is executed against the public LCB test cases; a task counts correct only if every test passes. See lcb_grade_note() for the exact procedure. Reproduce Everyday-mix end to end: python grade.py --selftest # replays answer keys through the grader (163/163) Known limitation, stated rather than hidden: _norm() strips to ASCII, so a non-Latin gold answer normalizes to the empty string and alias_match can only succeed via a Latin alias. One task (E1-Q-trivia-008, Arabic answer key) relies on this; its published alias list contains "Syria", which is what models actually answer. Any grader change here must be re-run against the full CSV. """ from __future__ import annotations import re _LETTERS = "ABCDEFGHIJ" def extract_letter(text: str) -> str | None: if not text: return None t = text.strip() # Strongest signal: explicit "ANSWER: X" / "answer is X" near the end. pats = [ r"answer\s*(?:is|:)?\s*\(?([A-J])\)?\b", r"\b([A-J])\)?\s*$", # trailing lone letter r"\*\*\s*([A-J])\s*\*\*", # **B** r"\boption\s*\(?([A-J])\)?\b", ] # search from the END (final answer usually last) by scanning reversed lines lines = [l for l in t.splitlines() if l.strip()] for line in reversed(lines): for p in pats: m = re.search(p, line, re.IGNORECASE) if m: return m.group(1).upper() # last resort: any standalone capital letter A-J in the whole text (last hit) hits = re.findall(r"\b([A-J])\b", t) return hits[-1].upper() if hits else None def extract_number(text: str) -> str | None: if not text: return None t = text.strip() # Prefer explicit "ANSWER: " m = re.search(r"answer\s*[:=]?\s*\$?\(?(-?[\d,]+(?:\.\d+)?)\)?", t, re.IGNORECASE) if not m: nums = re.findall(r"-?\d[\d,]*(?:\.\d+)?", t) if not nums: return None cand = nums[-1] else: cand = m.group(1) cand = cand.replace(",", "") # normalize trailing .0 try: f = float(cand) return str(int(f)) if f == int(f) else str(f) except ValueError: return None def _norm(s: str) -> str: """Normalize for alias matching: lowercase, strip articles/punct/space.""" import unicodedata s = unicodedata.normalize("NFKD", s).encode("ascii", "ignore").decode() s = s.lower().strip() s = re.sub(r"\b(a|an|the)\b", " ", s) s = re.sub(r"[^a-z0-9 ]", " ", s) s = re.sub(r"\s+", " ", s).strip() return s def score(answer_text: str, answer_key: str, scoring: str, aliases: list[str] | None = None) -> dict: if scoring == "exact_match_letter": got = extract_letter(answer_text) ok = (got is not None and got == answer_key.strip().upper()) elif scoring == "exact_match_number": got = extract_number(answer_text) key = answer_key.replace(",", "") try: ok = (got is not None and float(got) == float(key)) except ValueError: ok = (got == key) elif scoring == "yes_no": # Binary yes/no judgment. Read the first explicit yes/no token. t = (answer_text or "").lower() m = re.search(r"\b(yes|no|true|false)\b", t) got = None if m: tok = m.group(1) got = "yes" if tok in ("yes", "true") else "no" ok = (got is not None and got == answer_key.strip().lower()) return {"correct": 1.0 if ok else 0.0, "extracted": got, "key": answer_key} elif scoring == "alias_match": # Open-ended factual QA: correct if any gold alias appears as a # whole-token substring of the (normalized) model answer. Pure string # matching against the dataset's published alias set — NO LLM judge. got = (answer_text or "").strip() na = _norm(got) cand = list(aliases or []) if answer_key: cand.append(answer_key) ok = False for a in cand: an = _norm(a) if not an: continue if re.search(r"(?:^| )" + re.escape(an) + r"(?:$| )", na): ok = True break return {"correct": 1.0 if ok else 0.0, "extracted": got[:80], "key": answer_key} else: raise ValueError(f"unknown scoring mode: {scoring}") return {"correct": 1.0 if ok else 0.0, "extracted": got, "key": answer_key} # --------------------------------------------------------------------------- # AIME 2025 — integer answer 0-999 # --------------------------------------------------------------------------- _BOX_INT = re.compile(r"\\boxed\{\s*(-?\d+)\s*\}") _ANS_INT = re.compile(r"(?:final answer|answer is|answer:)\s*\$?\\?(?:boxed\{)?\s*(-?\d+)", re.IGNORECASE) def extract_aime_answer(text: str): """Return the integer the model committed to, or None.""" if not text: return None boxes = _BOX_INT.findall(text) if boxes: try: return int(boxes[-1]) except ValueError: pass m = list(_ANS_INT.finditer(text)) if m: try: return int(m[-1].group(1)) except ValueError: pass nums = re.findall(r"\b(\d{1,3})\b", text) # last resort if nums: try: return int(nums[-1]) except ValueError: pass return None # --------------------------------------------------------------------------- # GPQA Diamond — multiple choice A-D # --------------------------------------------------------------------------- _BOX_LET = re.compile(r"\\boxed\{\s*([A-Da-d])\s*\}") _ANS_LET = re.compile(r"(?:final answer|answer is|answer:)\s*\(?([A-Da-d])\)?", re.IGNORECASE) def extract_gpqa_answer(text: str): if not text: return None boxes = _BOX_LET.findall(text) if boxes: return boxes[-1].upper() m = list(_ANS_LET.finditer(text)) if m: return m[-1].group(1).upper() tail = text[-400:] # last resort lets = re.findall(r"\(([A-D])\)", tail) if lets: return lets[-1] return None def lcb_grade_note() -> str: return ( "LiveCodeBench is graded by execution, not string match:\n" " 1. extract the last fenced code block from the model output\n" " 2. run it in a subprocess against every public LCB test case for that\n" " problem, stdin -> stdout, 10s wall clock per case\n" " 3. compare stdout after stripping trailing whitespace per line\n" " 4. correct = 1.0 only if ALL cases pass; any timeout/exception = 0.0\n" "task_id in results_per_task.csv is the upstream LCB problem file " "(e.g. abc374_e.json), so the public test cases can be fetched directly." ) # --------------------------------------------------------------------------- def _selftest(): """Replay every Everyday-mix answer key through the grader. Feeds each task's own gold answer back in as if the model had produced it. A correct grader must score 163/163; anything less is an extraction bug. """ import json as _json import os as _os path = _os.path.join(_os.path.dirname(_os.path.abspath(__file__)), "everyday_mix.jsonl") tasks = [_json.loads(l) for l in open(path, encoding="utf-8")] ok = 0 for t in tasks: key = t["answer_key"] if t["scoring"] == "alias_match": # open-ended QA: the grader matches gold aliases inside the answer # text, so replay a bare gold string rather than a prefixed one. asc = [a for a in (t.get("answer_aliases") or []) if a.isascii()] synthetic = max(asc, key=len) if asc else key else: synthetic = f"Reasoning omitted.\nANSWER: {key}" r = score(synthetic, key, t["scoring"], t.get("answer_aliases")) ok += int(r["correct"] == 1.0) print(f"everyday_mix grader selftest: {ok}/{len(tasks)}") return ok == len(tasks) if __name__ == "__main__": import sys if "--selftest" in sys.argv: sys.exit(0 if _selftest() else 1) print(__doc__) print(lcb_grade_note())