rea/experiments/analysis-efficiency/native-export-costs.py

"""Check equivalent export questions and count explicitly projected tokenizer costs."""
import argparse
import hashlib
import json
from pathlib import Path
import statistics

import tiktoken


def compact(value):
    return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False)


def inspect(path, encoding):
    report = json.loads((path / "report.json").read_text())
    catalog = json.loads((path / "catalog.json").read_text())
    transcript = json.loads((path / "transcript.json").read_text())
    scope = next(json.loads(line) for line in reversed(path.with_suffix(".log").read_text().splitlines())
                 if line.startswith('{"resource_scope":'))
    observed = scope.get("payload_resource_observation") or {}
    events = observed.get("resource_events", {})
    assert report["status"] == "passed" and report.get("runtime_identity_sha256")
    assert not report["cleanup_failures"] and not report["receipt_failures"]
    assert scope["abort"] is None and scope.get("payload_resource_observation_verified") is True
    assert observed.get("payload_exit_code") == 0 and observed.get("payload_cleanup_verified") is True
    assert all(events.get("memory.events", {}).get(key) == 0 for key in ["high", "max", "oom", "oom_kill"])
    assert events.get("pids.events", {}).get("max") == 0
    assert len(catalog["tools"]) == 220
    count = lambda text: len(encoding.encode(text, disallowed_special=()))
    directory = [{key: tool[key] for key in ["name", "description", "inputSchema"]} for tool in catalog["tools"]]
    question = (f"For {report['artifact']['path']}, report the export count and every version/alias of "
                f"{', '.join(dict.fromkeys(row['name'] for row in report['answer']['rows']))}, including addresses, "
                "sizes, types, binding and symbol-version facts. Check that __rea_export_oracle_missing__ "
                "has no recorded match, preserve source coverage and export the complete Evidence. "
                "Report static declarations without claiming runtime execution.")
    arguments = sum(count(compact({"name": event["name"], "arguments": event["input"]})) for event in transcript)
    replies = sum(count(block["text"]) for event in transcript for block in event["reply"]["content"] if block["type"] == "text")
    inputs = count(compact(directory))
    raw_catalog = count(compact(catalog))
    task_directory = [tool for tool in directory if tool["name"] in ["inspect_native_exports", "inspect_analysis_view", "export_evidence_bundle"]]
    return {"case": path.name, "lane": report["lane"], "artifact": report["artifact"],
            "answer": report["answer"], "parent_evidence_id": report["parent_evidence_id"],
            "source_report_sha256": report["source_report_sha256"],
            "runtime_identity_sha256": report["runtime_identity_sha256"],
            "catalog_sha256": report["catalog_sha256"], "catalog_tools": len(catalog["tools"]),
            "buffer_bytes": report["buffer_bytes"], "calls": report["calls"], "phases": report["phases"],
            "wall_seconds": scope["elapsed_seconds"], "cpu_seconds": observed["cpu_stat"]["usage_usec"] / 1e6,
            "kernel_family_peak_bytes": observed["memory_peak_bytes"],
            "events_at_payload_exit": events, "payload_cleanup_verified": True,
            "final_whole_scope_events": scope.get("resource_events_final"),
            "token_proxy": {"question": count(question), "input_catalog_once": inputs, "arguments": arguments,
                            "text_replies": replies, "combined": count(question) + inputs + arguments + replies,
                            "only_three_task_tools": count(question) + count(compact(task_directory)) + arguments + replies,
                            "complete_transport_catalog": raw_catalog},
            "receipt_sha256": {file.name: hashlib.sha256(file.read_bytes()).hexdigest() for file in
                               [path / "report.json", path / "catalog.json", path / "transcript.json", path / "evidence.json", path.with_suffix(".log")]}}


def summarize(rows):
    result = {}
    for artifact in sorted({row["artifact"]["sha256"] for row in rows}):
        selected = [row for row in rows if row["artifact"]["sha256"] == artifact]
        assert len({row["parent_evidence_id"] for row in selected}) == 1
        assert len({row["source_report_sha256"] for row in selected}) == 1
        assert all(row["answer"] == selected[0]["answer"] for row in selected)
        result[artifact] = {}
        for lane in ["complete", "selected"]:
            samples = [row for row in selected if row["lane"] == lane]
            assert len(samples) == 3
            metrics = {key: [row[key] for row in samples] for key in ["wall_seconds", "cpu_seconds", "kernel_family_peak_bytes"]}
            metrics["projected_tokens"] = [row["token_proxy"]["combined"] for row in samples]
            metrics["mcp_call_count"] = [len(row["calls"]) for row in samples]
            metrics["mcp_reply_bytes"] = [sum(call["reply_bytes"] for call in row["calls"]) for row in samples]
            result[artifact][lane] = {metric: {"median": statistics.median(values), "minimum": min(values), "maximum": max(values)} for metric, values in metrics.items()}
    return result


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument("--cohort", type=Path, required=True)
    parser.add_argument("cases", type=Path, nargs="+")
    args = parser.parse_args()
    manifest = json.loads((args.cohort / "manifest.json").read_text())
    index = json.loads((args.cohort / "index.json").read_text())
    assert len(index) == 12 and all(case["exit"] == 0 and case["status"] == "passed" for case in index)
    assert {path.resolve() for path in args.cases} == {(args.cohort / case["name"]).resolve() for case in index}
    assert len(args.cases) == len(index)
    for file, digest in manifest["driver_files"].items():
        assert hashlib.sha256(Path(file).read_bytes()).hexdigest() == digest, file
    runtimes = {}
    for case in index:
        path = Path(case["runtime"]) / "runtime-identity.json"
        data = path.read_bytes()
        identity = json.loads(data)
        assert identity["source_clean"] is True
        runtimes[hashlib.sha256(data).hexdigest()] = {
            "source_revision": identity["source_revision"],
            "compiled_files": len(identity["compiled_files"]),
            "entrypoints": len(identity["entrypoints"]),
        }
    encoding = tiktoken.get_encoding("cl100k_base")
    rows = [inspect(path, encoding) for path in args.cases]
    assert all(row["runtime_identity_sha256"] in runtimes for row in rows)
    receipt = {"tiktoken": tiktoken.__version__, "encoding": encoding.name, "actual_model_usage": None,
               "cohort": manifest, "commands": index, "runtime_identities": runtimes,
               "projection": "Scripted SDK tasks, not an actual model run. The estimate includes a fixed task description, every discovered tool name/description/input schema once, every tool argument and text reply, including Evidence export. Complete raw SDK directory token costs and a three-tool-only alternative are reported separately; output schemas are not included in the primary model-context projection. Host injection, history replay, inference quality and billable usage are unmeasured.",
               "resources": "CPU quota 100%; launcher binds clients/servers to CPUs 0-1 (the outer observer starts on CPU15). The guard receipt affinity is initial placement, not child affinity; a live cohort observation records the override. 3 GiB early stop/4 GiB hard/no swap. Fresh client/server/HOME per task, Node compile cache disabled, OS caches unflushed. Later retained pages are warm queries; no separate warm-server cohort. Whole-family costs include client/oracle comparisons, source hashing, runtime binding, serialization and receipts. Tokenizer processing itself is outside these measured tasks. Verified counters are at payload exit while the observer remains alive; unavailable final whole-scope counters remain null.",
               "summary": summarize(rows), "recordings": rows}
    args.output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")


if __name__ == "__main__":
    main()