"""Build the claim-traceable research.json manifest from released artifacts."""

from __future__ import annotations

import hashlib
import json
from pathlib import Path

HERE = Path(__file__).resolve().parent


def digest(relative: str) -> str:
    return hashlib.sha256((HERE / relative).read_bytes()).hexdigest()


def output(
    output_id: str,
    kind: str,
    path: str,
    command: str,
    notes: str,
    environment: str = "Python 3.13; NumPy; Matplotlib; standard library",
) -> dict:
    return {
        "id": output_id,
        "kind": kind,
        "path": path,
        "sha256": digest(path),
        "command": command,
        "environment": environment,
        "notes": notes,
    }


def main() -> None:
    analysis_command = "python run_realworld_study.py --raw-dir <verified-source-directory>"
    manifest = {
        "schemaVersion": "1.0",
        "id": "semantic-trace-crossfit-tau2",
        "title": "Cross-Domain Evaluation of Semantic Trace Summaries: A Leakage-Controlled Study of 4,448 τ-Bench Trajectories",
        "deliverableMode": "report",
        "question": {"path": "inputs/question.md", "sha256": digest("inputs/question.md")},
        "sources": [
            {
                "id": "tau-bench-paper",
                "title": "tau-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
                "url": "https://arxiv.org/abs/2406.12045",
                "accessed": "2026-08-03",
                "notes": "Primary benchmark paper; labels use task-specific database and policy evaluation.",
            },
            {
                "id": "tau-bench-repository",
                "title": "Sierra Research tau2-bench repository",
                "url": "https://github.com/sierra-research/tau2-bench/tree/165848b9a78cead7ca7fe7c89c688b58e6501219",
                "accessed": "2026-08-03",
                "notes": "Official code, leaderboard metadata, submission format, and public trajectory links.",
            },
            {
                "id": "claude-submission",
                "title": "Claude Sonnet 4.5 standard tau-bench submission",
                "url": "https://github.com/sierra-research/tau2-bench/blob/165848b9a78cead7ca7fe7c89c688b58e6501219/web/leaderboard/public/submissions/claude-sonnet-4-5_sierra_2026-02-26/submission.json",
                "accessed": "2026-08-03",
                "notes": "Four public trials per task; exact raw file hashes are in study_config.json.",
            },
            {
                "id": "glm-submission",
                "title": "GLM-5 standard tau-bench submission",
                "url": "https://github.com/sierra-research/tau2-bench/blob/165848b9a78cead7ca7fe7c89c688b58e6501219/web/leaderboard/public/submissions/glm-5-think_sierra_2026-03-02/submission.json",
                "accessed": "2026-08-03",
                "notes": "Four public trials per task; official notes count infrastructure errors as failures.",
            },
            {
                "id": "gpt-submission",
                "title": "GPT-5.2 high standard tau-bench submission",
                "url": "https://github.com/sierra-research/tau2-bench/blob/165848b9a78cead7ca7fe7c89c688b58e6501219/web/leaderboard/public/submissions/gpt-5-2_sierra_2026-02-26/submission.json",
                "accessed": "2026-08-03",
                "notes": "Four public trials per task; exact raw file hashes are in study_config.json.",
            },
            {
                "id": "qwen-submission",
                "title": "Qwen3.5-397B standard tau-bench submission",
                "url": "https://github.com/sierra-research/tau2-bench/blob/165848b9a78cead7ca7fe7c89c688b58e6501219/web/leaderboard/public/submissions/qwen3.5-397b-a17b-think_sierra_2026-03-02/submission.json",
                "accessed": "2026-08-03",
                "notes": "Four public trials per task; exact raw file hashes are in study_config.json.",
            },
            {
                "id": "calibration-paper",
                "title": "On Calibration of Modern Neural Networks",
                "url": "https://proceedings.mlr.press/v70/guo17a.html",
                "accessed": "2026-08-03",
                "notes": "Primary methodological background for post-hoc logistic calibration; not evidence for this package's numerical result.",
            },
        ],
        "outputs": [
            output(
                "trajectory-features",
                "data",
                "results/trajectory_features.csv",
                analysis_command,
                "Privacy-reduced numeric feature table with 4,448 trajectories.",
            ),
            output(
                "crossfit-predictions",
                "data",
                "results/crossfit_predictions.csv",
                analysis_command,
                "One strictly cross-fitted prediction per trajectory, cohort, and method.",
            ),
            output(
                "split-inventory",
                "table",
                "results/split_inventory.csv",
                analysis_command,
                "Auditable task-fold and agent-model holdout blocks.",
            ),
            output(
                "method-metrics",
                "table",
                "results/method_metrics.csv",
                analysis_command,
                "Brier, log-loss, and AUROC summaries.",
            ),
            output(
                "paired-contrasts",
                "table",
                "results/paired_contrasts.csv",
                analysis_command,
                "Task-cluster bootstrap intervals for paired Brier differences.",
            ),
            output(
                "source-checks",
                "log",
                "results/source_checks.csv",
                analysis_command,
                "Executed hash, count, trial, missing-reward, and pass-rate audit.",
            ),
            output(
                "summary",
                "other",
                "results/summary.json",
                analysis_command,
                "Machine-readable headline results and evidence boundary.",
            ),
            output(
                "crossfit-figure",
                "figure",
                "figures/crossfit_semantic_evidence.pdf",
                analysis_command,
                "Vector primary-result figure.",
            ),
            output(
                "posthoc-recalibrated-predictions",
                "data",
                "results/posthoc_recalibrated_predictions.csv",
                "python -B analyze_transport_calibration.py",
                "Nested out-of-model, out-of-fold predictions for the trace and semantic representations.",
            ),
            output(
                "posthoc-calibration-parameters",
                "table",
                "results/posthoc_calibration_parameters.csv",
                "python -B analyze_transport_calibration.py",
                "All 64 fitted logistic calibration intercepts and slopes with overlap audits.",
            ),
            output(
                "posthoc-transport-cells",
                "table",
                "results/posthoc_transport_cells.csv",
                "python -B analyze_transport_calibration.py",
                "Exhaustive frozen and recalibrated four-model by three-domain failure map.",
            ),
            output(
                "posthoc-transport-summary",
                "other",
                "results/posthoc_transport_summary.json",
                "python -B analyze_transport_calibration.py",
                "Machine-readable secondary role, guard, global contrasts, cell counts, and limitations.",
            ),
            output(
                "posthoc-transport-figure",
                "figure",
                "figures/posthoc_transport_calibration.pdf",
                "python -B analyze_transport_calibration.py",
                "Vector slope graph and exhaustive signed cell maps.",
            ),
            output(
                "paper-pdf",
                "other",
                "paper.pdf",
                "python ../../build_papers.py --paper semantic-state-diffs",
                "Compiled release paper copied from the clean LaTeX build after log validation.",
                environment="Python 3.13 orchestrator; existing TinyTeX/latexmk toolchain",
            ),
        ],
        "claims": [
            {
                "id": "independent-outcome",
                "kind": "empirical",
                "statement": "Tau-bench evaluates customer-service trajectories against task-specific database and policy outcomes; this study uses the released reward rather than a label generated by its semantic rule.",
                "status": "supported",
                "sourceRefs": [
                    {"sourceId": "tau-bench-paper", "locator": "Abstract and Sections 2-3 (environment, tools, database-state evaluation)"},
                    {"sourceId": "tau-bench-repository", "locator": "README overview and docs/task-schema.md evaluation documentation"},
                ],
                "outputIds": [],
                "dependsOn": [],
                "caveats": ["The outcome is an automated benchmark oracle, not blinded human adjudication or production behavior."],
            },
            {
                "id": "cohort-composition",
                "kind": "computational",
                "statement": "The integrity-gated cohort contains 4,448 trajectories from four agents, 278 domain-task clusters, three state-changing domains, and four trials per agent-task pair.",
                "status": "reproduced",
                "sourceRefs": [],
                "outputIds": ["trajectory-features", "source-checks"],
                "dependsOn": [],
                "caveats": ["The four agents are a source-consistent convenience cohort, not a random sample of all agents."],
            },
            {
                "id": "leakage-control",
                "kind": "computational",
                "statement": "Every released prediction is produced by a fit containing neither its domain-task cluster nor its agent model.",
                "status": "reproduced",
                "sourceRefs": [],
                "outputIds": ["split-inventory", "crossfit-predictions"],
                "dependsOn": [],
                "caveats": ["Domains remain shared between training and test; this estimates new-task/new-model transport within the studied domains."],
            },
            {
                "id": "primary-negative",
                "kind": "computational",
                "statement": "Adding the compact semantic summary changes Brier score by +0.0057 relative to the interaction trace, with a task-cluster bootstrap 95% interval from -0.0002 to +0.0117; positive is worse.",
                "status": "reproduced",
                "sourceRefs": [],
                "outputIds": ["method-metrics", "paired-contrasts", "crossfit-figure"],
                "dependsOn": [],
                "caveats": ["The feature lexicon was fixed before fitting but was exploratory rather than preregistered."],
            },
            {
                "id": "missing-reward-sensitivity",
                "kind": "computational",
                "statement": "After removing 56 trajectories without numeric rewards, semantic-minus-trace Brier is +0.0010 with a 95% interval from -0.0045 to +0.0065.",
                "status": "reproduced",
                "sourceRefs": [],
                "outputIds": ["paired-contrasts", "summary"],
                "dependsOn": [],
                "caveats": ["The sensitivity conditions on trajectories with numeric released rewards and does not rerun failed infrastructure."],
            },
            {
                "id": "nested-calibration-guard",
                "kind": "computational",
                "statement": "All 64 post-hoc calibrator fits exclude both the held agent model and the held task fold.",
                "status": "reproduced",
                "sourceRefs": [{"sourceId": "calibration-paper", "locator": "Post-hoc calibration background; the two-axis exclusion is specific to this study"}],
                "outputIds": ["posthoc-calibration-parameters", "posthoc-recalibrated-predictions", "posthoc-transport-summary"],
                "dependsOn": [],
                "caveats": ["The calibration protocol was selected after the primary analysis and uses outcomes from other models and task folds."],
            },
            {
                "id": "calibration-does-not-reverse",
                "kind": "computational",
                "statement": "Nested recalibration narrows semantic-minus-trace Brier from +0.0057 to +0.0033, with a task-cluster interval from -0.0001 to +0.0066, while worsening absolute Brier for both methods.",
                "status": "reproduced",
                "sourceRefs": [],
                "outputIds": ["posthoc-recalibrated-predictions", "posthoc-transport-summary", "posthoc-transport-figure"],
                "dependsOn": ["nested-calibration-guard"],
                "caveats": ["This is post-hoc secondary analysis, not a replacement confirmatory estimand."],
            },
            {
                "id": "opposed-transport",
                "kind": "computational",
                "statement": "Semantic features have lower recalibrated Brier in six of 12 exhaustive model-domain cells; seven cell intervals exclude zero, with three favoring semantics and four favoring the trace.",
                "status": "reproduced",
                "sourceRefs": [],
                "outputIds": ["posthoc-transport-cells", "posthoc-transport-summary", "posthoc-transport-figure"],
                "dependsOn": ["nested-calibration-guard"],
                "caveats": ["Cell intervals are descriptive failure mapping and are not adjusted confirmatory subgroup tests."],
            },
            {
                "id": "bounded-conclusion",
                "kind": "synthetic",
                "statement": "This generic, task-agnostic semantic summary does not show transportable predictive value in the studied cohort; task-conditioned semantic contracts remain an open hypothesis.",
                "status": "supported",
                "sourceRefs": [],
                "outputIds": [],
                "dependsOn": ["primary-negative", "missing-reward-sensitivity", "leakage-control", "independent-outcome", "calibration-does-not-reverse", "opposed-transport"],
                "caveats": ["The conclusion is limited to four released agents, three tau-bench domains, this representation, and retrospective benchmark outcomes."],
            },
        ],
        "report": {
            "path": "outputs/report.md",
            "sha256": digest("outputs/report.md"),
            "status": "draft",
        },
        "approvals": {
            "scope": {
                "status": "approved",
                "approvedAt": "2026-08-03",
                "note": "User explicitly requested replacing circular studies with defensible original research and authorized major rewrites.",
            },
            "final": {
                "status": "pending",
                "approvedAt": None,
                "note": "Final wording and publication approval remain with the user.",
            },
        },
    }
    (HERE / "research.json").write_text(
        json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8"
    )


if __name__ == "__main__":
    main()
