project/capstone_reference.py

"""Visible reference comparison, not a hidden benchmark or a generated answer."""

import argparse
import json

from methods import DATA, coverage, select_context
from retrieval import Index, evaluate, load_documents


def capstone_inputs():
    documents = load_documents(DATA / "corpus.json") + load_documents(DATA / "capstone_extension.json")
    index = Index(documents)  # Also rejects duplicate IDs across both files.
    cases = json.loads((DATA / "capstone_questions.json").read_text(encoding="utf-8"))
    return index, cases, len(documents)


def run_capstone(k=2, max_quote_chars=400, selector=select_context):
    index, cases, total = capstone_inputs()
    retrieval = evaluate(index, cases, k)
    context_rows = []
    for case in cases:
        hits = index.search(case["question"], k)
        result = selector(hits, max_quote_chars)
        selected = [entry["id"] for entry in result["packet"]["evidence"]]
        context_rows.append({"id": case["id"], "relevant_ids": case["relevant_ids"],
                             "retrieved_ids": [hit.document.id for hit in hits],
                             "selected_ids": selected,
                             "labeled_evidence_coverage": coverage(selected, case["relevant_ids"]),
                             "budget_skipped_ids": result["budget_skipped_ids"],
                             "conflicting_source_versions": result["conflicting_source_versions"],
                             "packet_has_evidence": bool(selected)})
    answerable_rows = [row for row in context_rows if row["relevant_ids"]]
    return {"kind": "synthetic_visible_development_capstone", "model_used": False,
            "configuration": {"k": k, "max_quote_chars": max_quote_chars},
            "source_record_count": total, "eligible_passage_count": len(index.documents),
            "retrieval": retrieval, "context_rows": context_rows,
            "mean_packet_evidence_coverage": sum(row["labeled_evidence_coverage"] for row in answerable_rows)
                                             / len(answerable_rows),
            "limit": "No generated answers or actual reservations; all cases were visible during authoring."}


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--k", type=int, default=2)
    parser.add_argument("--max-quote-chars", type=int, default=400)
    arguments = parser.parse_args()
    try:
        result = run_capstone(arguments.k, arguments.max_quote_chars)
    except ValueError as error:
        parser.error(str(error))
    print(json.dumps(result, indent=2, ensure_ascii=False))


if __name__ == "__main__":
    main()