"""Score one corpus extraction validation for PB-RAG-10-03.

Reads the hand-scored pages and answers (CSV), applies the decision table and writes the results record.
It only reads the files you give it. The thresholds are team defaults: change them in one place, below.

    python score_validation.py pages.csv questions.csv --holdout pages-holdout.csv --docs 60 --failed-docs 3 --pages-billed 1840 --cost ${RUN_COST}
"""


import argparse
import csv
import re
import sys
from collections import Counter, defaultdict


BAR = {"table_fidelity": 90, "figure_coverage": 80, "answer_accuracy": 85}   # team defaults: Go at or above
FLOOR = 70                 # team default: a measure below this is a No-go
MAX_ERROR_PCT = 5          # team default: documents with errors or warnings
MAX_HOLDOUT_GAP = 10       # team default: points the hold-out may score below the sample
MAX_SCORER_GAP = 5         # team default: points two scorers may differ
MIN_DOCUMENTS = 50         # team default: documents in the sample, so it has variety
MIN_PAGES = 20             # team default: scored pages for each document type
MIN_WORST_PAGES = 3        # team default: scored pages for each worst-case type (name starts "worst-")
MIN_QUESTIONS, MIN_PER_KIND = 30, 10


PAGE_COLS = ["doc_type", "page", "table_cells", "table_cells_right", "figures", "figures_found",
             "numbers", "numbers_right"]
PAIRS = [("table_cells", "table_cells_right"), ("figures", "figures_found"), ("numbers", "numbers_right")]
ANSWER_COLS = ["id", "kind", "result"]
RESULTS = {"right", "partly", "wrong", "no_answer"}
SKIPPABLE = {"tables", "figures", "holdout"}
ALL = "(all)"


def pct(part, whole):
    """Percentage to one decimal place, or None when there is nothing to measure."""
    return round(100 * part / whole, 1) if whole else None


def read_csv(path, columns):
    with open(path, newline="", encoding="utf-8-sig") as f:
        reader = csv.DictReader(f)
        missing = [c for c in columns if c not in (reader.fieldnames or [])]
        if missing:
            raise ValueError(f"{path}: missing column(s): {', '.join(missing)}")
        return [dict(r, _line=n) for n, r in enumerate(reader, start=2)]


def whole_number(row, col, path):
    """Read a whole number that is not negative, or say which line is wrong."""
    try:
        value = int(row[col])
    except (TypeError, ValueError):
        raise ValueError(f"{path} line {row['_line']}: {col} must be a whole number") from None
    if value < 0:
        raise ValueError(f"{path} line {row['_line']}: {col} must not be negative")
    return value


def page_numbers(row, path):
    nums = {c: whole_number(row, c, path) for c in PAGE_COLS[2:]}
    for total, right in PAIRS:
        if nums[right] > nums[total]:
            raise ValueError(f"{path} line {row['_line']}: {right} is more than {total}")
    return nums


def score_pages(rows, path="pages.csv"):
    """Add up the page scores for each document type and for everything together."""
    totals = defaultdict(lambda: defaultdict(int))
    for row in rows:
        kind = (row["doc_type"] or "").strip()
        if not kind:
            raise ValueError(f"{path} line {row['_line']}: doc_type is empty")
        for key in (kind, ALL):
            totals[key]["pages"] += 1
            for col, n in page_numbers(row, path).items():
                totals[key][col] += n
    return {k: dict(v) for k, v in totals.items()}


def page_metrics(t):
    """Turn the totals into percentages. A measure with nothing to count is None, not 100."""
    return {"table_fidelity": pct(t.get("table_cells_right", 0), t.get("table_cells", 0)),
            "figure_coverage": pct(t.get("figures_found", 0), t.get("figures", 0)),
            "number_accuracy": pct(t.get("numbers_right", 0), t.get("numbers", 0))}


def answer_points(row, path):
    """A right answer scores 1, a partly right one 0.5. For a question the documents cannot answer, only 'no_answer' is right."""
    result = (row["result"] or "").strip().lower()
    if result not in RESULTS:
        raise ValueError(f"{path} line {row['_line']}: result must be one of {sorted(RESULTS)}")
    if row["kind"].strip() == "not_in_docs":
        return result, 1.0 if result == "no_answer" else 0.0
    return result, {"right": 1.0, "partly": 0.5}.get(result, 0.0)


def score_answers(rows, path="questions.csv"):
    scored = [answer_points(r, path) for r in rows]
    counts = Counter(result for result, _ in scored)
    return {"total": len(rows), "right": counts["right"], "partly": counts["partly"],
            "wrong": counts["wrong"], "no_answer": counts["no_answer"],
            "kinds": dict(Counter(r["kind"].strip() for r in rows)),
            "answer_accuracy": pct(sum(p for _, p in scored), len(rows))}


def thin_types(by_type):
    """Document types with fewer scored pages than the minimum."""
    out = []
    for kind, t in sorted(by_type.items()):
        need = MIN_WORST_PAGES if kind.startswith("worst-") else MIN_PAGES
        if kind != ALL and t["pages"] < need:
            out.append(f"{kind}: {t['pages']} scored pages, need {need}")
    return out


def evidence_gaps(by_type, answers, skip):
    """Reasons the evidence is too thin to decide. An empty list means it is enough."""
    gaps = thin_types(by_type) if by_type else ["no pages scored"]
    if answers["total"] < MIN_QUESTIONS:
        gaps.append(f"{answers['total']} questions, need {MIN_QUESTIONS}")
    for kind, flag in (("table", "tables"), ("figure", "figures")):
        if flag not in skip and answers["kinds"].get(kind, 0) < MIN_PER_KIND:
            gaps.append(f"fewer than {MIN_PER_KIND} {kind} questions")
    return gaps


def measure_gaps(metrics, skip):
    """Reasons a measure is missing, or the run or the scorers disagree too much to trust."""
    need = {"table_fidelity": "tables", "figure_coverage": "figures"}
    return [f"{k} not measured" for k in BAR if metrics.get(k) is None and need.get(k) not in skip]


def weak_types(by_type, skip):
    """Types that miss the bar even if the whole sample passes. These become conditions."""
    out = []
    for kind, t in sorted(by_type.items()):
        for key, value in page_metrics(t).items():
            skipped = {"table_fidelity": "tables", "figure_coverage": "figures"}.get(key) in skip
            if kind != ALL and value is not None and not skipped and value < BAR.get(key, 0):
                out.append(f"{kind}: {key} {value}%")
    return out


def decide(metrics, gaps, notes):
    """Return (decision, reasons) from the decision table in the playbook."""
    if gaps:
        return "not_decided", gaps
    scored = {k: v for k, v in metrics.items() if k in BAR and v is not None}
    low = [f"{k} {v}% is below {FLOOR}%" for k, v in scored.items() if v < FLOOR]
    if low:
        return "no_go", low
    under = [f"{k} {v}% is below {BAR[k]}%" for k, v in scored.items() if v < BAR[k]]
    if under or notes:
        return "go_with_conditions", under + notes
    return "go", []


def add_sample_args(ap):
    ap.add_argument("pages")
    ap.add_argument("questions")
    ap.add_argument("--holdout", help="hold-out pages, same columns as pages")
    ap.add_argument("--skip", default="", help="comma list of: tables, figures, holdout (not in scope)")
    ap.add_argument("--corpus", default="${CORPUS_ID}")
    ap.add_argument("--path", default="content_understanding")
    ap.add_argument("--api-version", default="2026-04-01")
    ap.add_argument("--preview-features", default="")
    ap.add_argument("--scorer-gap", type=float, help="points between two scorers on the re-scored pages")
    ap.add_argument("--out", default="extraction-results.yaml")


def parse_args(argv=None):
    ap = argparse.ArgumentParser()
    add_sample_args(ap)
    ap.add_argument("--docs", type=int, required=True)
    ap.add_argument("--failed-docs", type=int, required=True, help="documents with an error or a warning")
    ap.add_argument("--pages-billed", type=int, required=True)
    ap.add_argument("--cost", type=float, required=True, help="run cost from the real bill, in your currency")
    ap.add_argument("--budget-per-1000", type=float, help="agreed budget for 1,000 pages")
    ap.add_argument("--minutes", type=int, help="processing time for the run")
    return ap.parse_args(argv)


def run_checks(args, by_type, answers):
    """Collect everything that blocks a decision (gaps) or limits a Go (notes)."""
    skip = {s for s in args.skip.split(",") if s}
    metrics = {**page_metrics(by_type.get(ALL, {})), "answer_accuracy": answers["answer_accuracy"]}
    err = pct(args.failed_docs, args.docs)
    per_1000 = round(args.cost * 1000 / args.pages_billed, 2) if args.pages_billed else None
    gaps = evidence_gaps(by_type, answers, skip) + measure_gaps(metrics, skip)
    if args.docs < MIN_DOCUMENTS:
        gaps.append(f"{args.docs} documents in the sample, need {MIN_DOCUMENTS}")
    if err is None or err > MAX_ERROR_PCT:
        gaps.append(f"documents with errors or warnings: {err}%, investigate first")
    if args.scorer_gap is not None and args.scorer_gap > MAX_SCORER_GAP:
        gaps.append(f"scorers differ by {args.scorer_gap} points: fix the scoring rule and re-score")
    over = args.budget_per_1000 is not None and per_1000 is not None and per_1000 > args.budget_per_1000
    return skip, metrics, err, per_1000, gaps, over


def holdout_gap(args, metrics, skip):
    """Points the hold-out scores below the sample on table fidelity, or None when it was not given."""
    if args.holdout is None:
        return None, ([] if "holdout" in skip else ["no hold-out result"])
    held = page_metrics(score_pages(read_csv(args.holdout, PAGE_COLS), args.holdout).get(ALL, {}))
    if held["table_fidelity"] is None or metrics["table_fidelity"] is None:
        return None, []
    gap = round(metrics["table_fidelity"] - held["table_fidelity"], 1)
    return gap, ([f"hold-out is {gap} points worse than the sample"] if gap > MAX_HOLDOUT_GAP else [])


def record_dict(args, by_type, answers, parts, outcome):
    """Lay the results out in the order of the results record."""
    metrics, err, per_1000, gap = parts
    types = {k: v["pages"] for k, v in sorted(by_type.items()) if k != ALL}
    return {"corpus": args.corpus,
            "sample": {"documents": args.docs, "pages": by_type.get(ALL, {}).get("pages", 0), "types": types},
            "extraction": {"path": args.path, "api_version": args.api_version,
                           "preview_features": [p for p in args.preview_features.split(",") if p],
                           "failed_documents_pct": err},
            "quality": {**metrics, "holdout_gap_points": gap, "scorer_gap_points": args.scorer_gap},
            "answers": answers,
            "cost": {"pages_billed": args.pages_billed, "run_cost": args.cost, "per_1000_pages": per_1000,
                     "budget_per_1000_pages": args.budget_per_1000, "processing_minutes": args.minutes},
            "decision": outcome[0], "reasons": outcome[1], "reviewer": None}


def build_record(args):
    by_type = score_pages(read_csv(args.pages, PAGE_COLS), args.pages)
    answers = score_answers(read_csv(args.questions, ANSWER_COLS), args.questions)
    skip, metrics, err, per_1000, gaps, over = run_checks(args, by_type, answers)
    gap, held_gaps = holdout_gap(args, metrics, skip)
    notes = weak_types(by_type, skip) + (["cost is over the budget: revisit the path or the scope"] if over else [])
    outcome = decide(metrics, gaps + held_gaps, notes)
    return record_dict(args, by_type, answers, (metrics, err, per_1000, gap), outcome)


def scalar(v):
    if v is None:
        return "null"
    if isinstance(v, bool):
        return "true" if v else "false"
    if isinstance(v, (int, float)):
        return str(v)
    return '"' + str(v).replace("\\", "\\\\").replace('"', '\\"') + '"'


def to_yaml(value, depth=0):
    """Write a dict of scalars, dicts and lists of text as YAML. No other library is needed."""
    pad, lines = "  " * depth, []
    for key, item in value.items():
        name = key if re.fullmatch(r"[A-Za-z0-9_.-]+", key) else scalar(key)
        if isinstance(item, dict) and item:
            lines += [f"{pad}{name}:", to_yaml(item, depth + 1)]
        elif isinstance(item, list) and item:
            lines += [f"{pad}{name}:"] + [f"{pad}  - {scalar(x)}" for x in item]
        else:
            lines.append(f"{pad}{name}: " + ("[]" if item == [] else "{}" if item == {} else scalar(item)))
    return "\n".join(lines)


def main(argv=None):
    args = parse_args(argv)
    try:
        record = build_record(args)
    except (ValueError, OSError) as err:
        print("Cannot score:", err, file=sys.stderr)
        sys.exit(2)
    with open(args.out, "w", encoding="utf-8") as f:
        f.write(to_yaml(record) + "\n")
    print(f"Decision: {record['decision']}")
    print(*[f"  - {r}" for r in record["reasons"]], sep="\n")
    print(f"Written: {args.out}")


if __name__ == "__main__":
    main()