#!/usr/bin/env python3
"""
Did the shorter answers survive? Three blind gradings, one per arm pair.

Anything can make output shorter. The only question worth asking is whether the
answer still contains what a reader needed. Each pair is graded blind, with the
order swapped on odd-numbered pairs.
"""

import csv
import statistics as st
from collections import Counter, defaultdict

FILES = [
    ("A vs B", "quality-grades-AB.csv", "the block, LENGTH line removed"),
    ("B vs C", "quality-grades-BC.csv", "the LENGTH line, acting alone"),
    ("A vs C", "quality-grades.csv", "both, the deployed state"),
]

print("=" * 74)
print("DID THE SHORTER ANSWERS SURVIVE?".center(74))
print("=" * 74)
print(f"{'pair':<8}{'isolates':<34}{'n':>4}{'ok':>6}{'minor':>7}{'MATERIAL':>13}")
print("-" * 74)

data = {}
for label, path, what in FILES:
    rows = list(csv.DictReader(open(path)))
    data[label] = rows
    c = Counter(r["verdict"] for r in rows)
    n = len(rows)
    print(f"{label:<8}{what:<34}{n:>4}{c['complete']:>6}{c['minor_loss']:>7}"
          f"{c['material_loss']:>7} ({c['material_loss']/n*100:.0f}%)")

print()
print("=" * 74)
print("WHERE IT BREAKS: the 6 compliance questions")
print("=" * 74)
for label in ("A vs B", "B vs C", "A vs C"):
    pol = [r for r in data[label] if r["category"] == "policy"]
    m = sum(1 for r in pol if r["verdict"] == "material_loss")
    bar = "#" * m + "." * (len(pol) - m)
    print(f"  {label}   [{bar}]  {m} of {len(pol)} lost something a reader needed")

rows = [r for r in csv.DictReader(open("token-measure-results.csv"))
        if r["category"] == "policy"]
byarm = defaultdict(list)
for r in rows:
    byarm[r["arm"]].append(int(r["output_tokens"]))
print()
print("  mean output tokens on those same 6 questions:")
print(f"     A {st.mean(byarm['A']):.0f}      B {st.mean(byarm['B']):.0f}      "
      f"C {st.mean(byarm['C']):.0f}")
print()
print("  The brevity line is not what breaks them. The block is.")