diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index eec6ac9b43..a54a035e91 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -352,6 +352,35 @@ application and semantic coverage. The result carries a digest of the selected facts. Actual serialized size determines MCP transport admission: reduce the page size or export retained Evidence if the view is too large. +`compare_source_to_bundle` accepts the committed historical graph inline or as +`reference: {"kind":"file","path":"/absolute/reference.json"}`. The shared +reader checks resource bounds and the graph's root commitment on every read. File-backed +comparison Evidence records `parameters.reference_source`: the exact +`requested_path` and the reader's absolute selected `path`, preserving POSIX +`..`/symlink semantics. Normalized comparison facts and mapping identities agree +with inline input; Evidence IDs distinguish caller-selected file origins. +Omitting `view` preserves the complete comparison. For a compact initial reply, +select `view: {"kind":"summary"}` or a `source-to-bundle` page; the session retains +the complete comparison before delivery. Later calls use `inspect_analysis_view` +with the returned `parent_evidence_id`. + +A `source-to-bundle` view selects `collection: "mappings"` or `"candidates"` with +exact `source_paths`, `mapping_ids`, `statuses`, `fields`, `offset` and `limit`. +Omitted filters select all; empty arrays select none. The default mapping fields +include status, confidence, source digest/language and candidate/mapped counts. +Candidate defaults include kind, score and confidence; request `signals` for exact +supporting values. Mapping identity and `source_index` always remain; candidates +also retain `current_node_id`, `candidate_index` and whether they were actually +mapped. Indices refer to the complete comparison's item array and that item's +candidate array. All classifications are computed before selection. The original +`source_coverage` remains separate from page `coverage`; exhausting a page never +turns unknown correspondence into absence or proves behavioral equivalence. + +The CLI accepts the same reference-file alternative (relative paths resolve from +the working directory). Save its complete `--json` Evidence, then use +`inspect-analysis-view` with an `evidence-file` source and the same view. Neither +adapter executes the target or repeats analysis for retained queries. + Completed `inspect_artifact` and atomic `inventory_artifact` Evidence also support summary and inventory views. A summary reports the manifest's expected counts separately from available record counts, observed entry kinds, inventory diff --git a/experiments/analysis-efficiency/source-to-bundle-costs.py b/experiments/analysis-efficiency/source-to-bundle-costs.py new file mode 100644 index 0000000000..fc9ffce1d9 --- /dev/null +++ b/experiments/analysis-efficiency/source-to-bundle-costs.py @@ -0,0 +1,34 @@ +import argparse, json, statistics +from pathlib import Path +import tiktoken +parser=argparse.ArgumentParser() +parser.add_argument('root', type=Path) +parser.add_argument('--pattern', default='paired-*') +parser.add_argument('--output', type=Path) +args=parser.parse_args() +root=args.root +enc=tiktoken.get_encoding('cl100k_base') +compact=lambda v: json.dumps(v,ensure_ascii=False,separators=(',',':')) +count=lambda v: len(enc.encode(v,disallowed_special=())) +results=[] +for case in sorted(root.glob(args.pattern)): + if not case.is_dir(): continue + report=json.loads((case/'report.json').read_text()) + assert report['status']=='observed' and report['cleanup_verified'] + scope=next(json.loads(line) for line in reversed(case.with_name(case.name+'.log').read_text().splitlines()) if line.startswith('{"resource_scope":')) + observed=scope['payload_resource_observation'] + assert scope['abort'] is None and scope['payload_resource_observation_verified'] and observed['payload_cleanup_verified'] and observed['payload_exit_code']==0 + catalog=json.loads((case/'catalog.json').read_text()) + transcript=json.loads((case/'transcript.json').read_text()) + inputs=[{k: t[k] for k in ['name','description','inputSchema']} for t in catalog['tools']] + arg=sum(count(compact({'name':r['name'],'arguments':r['arguments']})) for r in transcript) + text=sum(count(b['text']) for r in transcript for b in r['reply']['content'] if b['type']=='text') + results.append({'case':case.name,'mode':report['mode'],'output_schema_profile':report.get('output_schema_profile','full'),'runtime_revision':report['runtime_revision'],'runtime_worktree_status':report['runtime_worktree_status'],'token_estimates':{'input_catalog_once':count(compact(inputs)),'arguments':arg,'text_replies':text,'whole_task':count(compact(inputs))+arg+text},'actual_model_usage':None,'wall_seconds':scope['elapsed_seconds'],'cpu_seconds':observed['cpu_stat']['usage_usec']/1e6,'peak_process_family_bytes_including_page_cache':observed['memory_peak_bytes'],'calls':report['calls'],'retained_records':report['retained_records'],'proof':json.loads((case/'independent-proof.json').read_text()),'resource_events':observed['resource_events'],'cleanup_verified':True}) +summary={} +for mode in ['complete','selected']: + rows=[r for r in results if r['mode']==mode] + summary[mode]={key:statistics.median(r[key] for r in rows) for key in ['wall_seconds','cpu_seconds','peak_process_family_bytes_including_page_cache']} + summary[mode]['whole_task_tokens_estimated']=[r['token_estimates']['whole_task'] for r in rows] +out={'status':'passed','tokenizer':'cl100k_base, tiktoken 0.12.0; estimates only; one input catalog plus arguments plus MCP text replies; not actual model usage','cold_warm':'Fresh server process and isolated HOME each run; alternating complete/selected, selected/complete, complete/selected. OS page caches are warm/uncontrolled; not an OS-cold claim.','tarball_sha256':'f2517c466b50424b661a27e48039f15db7783e5822f46b5c7d39f20104ac67a9','preparation_scope':'Reference-source import is performed once outside these repeated SDK trials and reported separately. Both workflows use the explicit existing semantic cap of one for the static correspondence question; full semantic coverage is not claimed.','summary':summary,'runs':results} +(args.output or root/'costs.json').write_text(json.dumps(out,indent=2)+'\n') +print(json.dumps(summary,indent=2)) diff --git a/experiments/analysis-efficiency/source-to-bundle-oracle.py b/experiments/analysis-efficiency/source-to-bundle-oracle.py new file mode 100644 index 0000000000..4deb26a3ae --- /dev/null +++ b/experiments/analysis-efficiency/source-to-bundle-oracle.py @@ -0,0 +1,63 @@ +"""Verify published source/map facts independently of REA's parsers.""" +import argparse +import collections +import hashlib +import json +from pathlib import Path, PurePosixPath + +parser = argparse.ArgumentParser() +parser.add_argument("case", type=Path) +parser.add_argument("reference", type=Path) +args = parser.parse_args() +root = args.case.parent +bundle = json.loads((args.case / "evidence.json").read_text()) +reference = json.loads(args.reference.read_text()) +oracle = json.loads((root / "oracle.json").read_text()) +application = next(r for r in bundle["records"] if r["operation"] == "analyze_javascript_application") +comparison = next(r for r in bundle["records"] if r["operation"] == "compare_source_to_bundle")["normalized_result"] +expected = {(r["map"], r["source_index"]): r for r in oracle["rows"]} +nodes = [n for n in application["normalized_result"]["graph"]["nodes"] if n["kind"] == "source-module"] +assert len(nodes) == len(expected) == 99 +by_path = collections.defaultdict(set) +by_basename = collections.defaultdict(set) +seen = set() +for node in nodes: + observation = node["observations"][0] + location = observation["source_map_reference"] + key = (PurePosixPath(location["map_path"]).name, observation["properties"]["source_index"]) + row = expected[key] + assert key not in seen + seen.add(key) + assert node["identity"]["original_source"] == row["map_source"] == location["source_name"] + assert node["identity"]["source_sha256"] == row["content_sha256"] == observation["properties"]["content_sha256"] + assert node["identity"]["source_map_sha256"] == hashlib.sha256((root / "artifact/package/dist" / row["map"]).read_bytes()).hexdigest() + path = "src/" + row["source_path"] + by_path[path].add(node["node_id"]) + by_basename[PurePosixPath(path).name].add(node["node_id"]) +actual_files = {p.relative_to(root / "reference").as_posix(): hashlib.sha256(p.read_bytes()).hexdigest() + for p in (root / "reference/src").rglob("*") if p.is_file()} +entries = {e["path"]: e for e in reference["entries"] if e["kind"] == "file"} +assert set(entries) == set(actual_files) +assert all(entries[p]["sha256"] == sha for p, sha in actual_files.items()) +items = {item["source_path"]: item for item in comparison["items"]} +assert set(items) == set(actual_files) +candidate_count = 0 +for path, item in items.items(): + assert item["source_sha256"] == actual_files[path] + candidates = {candidate["current_node_id"] for candidate in item["candidates"]} + assert candidates == by_basename[PurePosixPath(path).name], path + candidate_count += len(candidates) + assert set(item["current_node_ids"]) == by_path[path] + assert item["status"] == ("split" if by_path[path] else "unknown") + assert all(signal["kind"] != "exact-source-digest" + for candidate in item["candidates"] for signal in candidate["signals"]) +assert candidate_count == comparison["coverage"]["candidate_evaluations"] == 210 +assert comparison["coverage"]["status"] == "partial" +proof = {"status": "passed", "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": len(actual_files), "embedded_nodes": len(nodes), "candidate_relations": candidate_count, + "strong_location_rows": sum(bool(by_path[p]) for p in actual_files), + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."} +with (args.case / "independent-proof.json").open("x") as output: + json.dump(proof, output, indent=2) + output.write("\n") +print(json.dumps(proof)) diff --git a/experiments/analysis-efficiency/source-to-bundle-output-off.json b/experiments/analysis-efficiency/source-to-bundle-output-off.json new file mode 100644 index 0000000000..f74c849cb2 --- /dev/null +++ b/experiments/analysis-efficiency/source-to-bundle-output-off.json @@ -0,0 +1,548 @@ +{ + "status": "passed", + "tokenizer": "cl100k_base, tiktoken 0.12.0; estimates only; one input catalog plus arguments plus MCP text replies; not actual model usage", + "cold_warm": "Fresh server process and isolated HOME each run; alternating complete/selected, selected/complete, complete/selected. OS page caches are warm/uncontrolled; not an OS-cold claim.", + "tarball_sha256": "f2517c466b50424b661a27e48039f15db7783e5822f46b5c7d39f20104ac67a9", + "preparation_scope": "Reference-source import is performed once outside these repeated SDK trials and reported separately. Both workflows use the explicit existing semantic cap of one for the static correspondence question; full semantic coverage is not claimed.", + "summary": { + "complete": { + "wall_seconds": 11.597, + "cpu_seconds": 11.243998, + "peak_process_family_bytes_including_page_cache": 659202048, + "whole_task_tokens_estimated": [152668, 152668, 152668] + }, + "selected": { + "wall_seconds": 10.89, + "cpu_seconds": 10.569455, + "peak_process_family_bytes_including_page_cache": 656396288, + "whole_task_tokens_estimated": [54841, 54841, 54841] + } + }, + "runs": [ + { + "case": "off-paired-1-complete", + "mode": "complete", + "output_schema_profile": "off", + "runtime_revision": "01788bac863ac786bfbb648cad69b11e23b40a89", + "runtime_worktree_status": "", + "token_estimates": { + "input_catalog_once": 47987, + "arguments": 37273, + "text_replies": 67408, + "whole_task": 152668 + }, + "actual_model_usage": null, + "wall_seconds": 12.215, + "cpu_seconds": 11.872135, + "peak_process_family_bytes_including_page_cache": 650854400, + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 7275.801749, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 81.34299699999974, + "is_error": false, + "reply_bytes": 427799 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 74.90714800000023, + "is_error": false, + "reply_bytes": 342 + } + ], + "retained_records": [ + { + "evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe", + "operation": "compare_source_to_bundle" + }, + { + "evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f", + "operation": "analyze_javascript_application" + }, + { + "evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78", + "operation": "record_unknown" + }, + { + "evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089", + "operation": "inspect_analysis_view" + } + ], + "proof": { + "status": "passed", + "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": 180, + "embedded_nodes": 99, + "candidate_relations": 210, + "strong_location_rows": 33, + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim." + }, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "cleanup_verified": true + }, + { + "case": "off-paired-2-selected", + "mode": "selected", + "output_schema_profile": "off", + "runtime_revision": "543b273d7adf581f6fd52c2938020207e455d3df", + "runtime_worktree_status": " M experiments/analysis-efficiency/source-to-bundle-costs.py\n M experiments/analysis-efficiency/source-to-bundle.mjs\n M src/cli/sourceToBundleCommand.ts\n M tests/acceptance/applications/applicationWorkflowCli.test.ts\n M tests/boundary/mcp/sourceToBundleViewsMcp.test.ts\n", + "token_estimates": { + "input_catalog_once": 48100, + "arguments": 298, + "text_replies": 6443, + "whole_task": 54841 + }, + "actual_model_usage": null, + "wall_seconds": 11.196, + "cpu_seconds": 10.7977, + "peak_process_family_bytes_including_page_cache": 650358784, + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 6273.303819000001, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 81.66274700000031, + "is_error": false, + "reply_bytes": 13199 + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 20.52900899999986, + "is_error": false, + "reply_bytes": 19393 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 83.38822599999912, + "is_error": false, + "reply_bytes": 342 + } + ], + "retained_records": [ + { + "evidence_id": "ev_02b198af5997beaf606bf300768cc1ac8cd608fcd0860acbe9b791dbd5bae5dc", + "operation": "inspect_analysis_view" + }, + { + "evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe", + "operation": "compare_source_to_bundle" + }, + { + "evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f", + "operation": "analyze_javascript_application" + }, + { + "evidence_id": "ev_95d8fe5e75c019d86fc17bafeea32445e82eec06e0ec07d0ecdafd1e8bf8ef34", + "operation": "inspect_analysis_view" + }, + { + "evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78", + "operation": "record_unknown" + }, + { + "evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089", + "operation": "inspect_analysis_view" + } + ], + "proof": { + "status": "passed", + "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": 180, + "embedded_nodes": 99, + "candidate_relations": 210, + "strong_location_rows": 33, + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim." + }, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "cleanup_verified": true + }, + { + "case": "off-paired-3-selected", + "mode": "selected", + "output_schema_profile": "off", + "runtime_revision": "543b273d7adf581f6fd52c2938020207e455d3df", + "runtime_worktree_status": " M experiments/analysis-efficiency/source-to-bundle-costs.py\n M experiments/analysis-efficiency/source-to-bundle.mjs\n M src/cli/sourceToBundleCommand.ts\n M tests/acceptance/applications/applicationWorkflowCli.test.ts\n M tests/boundary/mcp/sourceToBundleViewsMcp.test.ts\n", + "token_estimates": { + "input_catalog_once": 48100, + "arguments": 298, + "text_replies": 6443, + "whole_task": 54841 + }, + "actual_model_usage": null, + "wall_seconds": 10.89, + "cpu_seconds": 10.569455, + "peak_process_family_bytes_including_page_cache": 656396288, + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 6153.662525000002, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 78.89196699999957, + "is_error": false, + "reply_bytes": 13199 + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 20.370968999999604, + "is_error": false, + "reply_bytes": 19393 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 81.88148700000056, + "is_error": false, + "reply_bytes": 342 + } + ], + "retained_records": [ + { + "evidence_id": "ev_02b198af5997beaf606bf300768cc1ac8cd608fcd0860acbe9b791dbd5bae5dc", + "operation": "inspect_analysis_view" + }, + { + "evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe", + "operation": "compare_source_to_bundle" + }, + { + "evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f", + "operation": "analyze_javascript_application" + }, + { + "evidence_id": "ev_95d8fe5e75c019d86fc17bafeea32445e82eec06e0ec07d0ecdafd1e8bf8ef34", + "operation": "inspect_analysis_view" + }, + { + "evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78", + "operation": "record_unknown" + }, + { + "evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089", + "operation": "inspect_analysis_view" + } + ], + "proof": { + "status": "passed", + "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": 180, + "embedded_nodes": 99, + "candidate_relations": 210, + "strong_location_rows": 33, + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim." + }, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "cleanup_verified": true + }, + { + "case": "off-paired-4-complete", + "mode": "complete", + "output_schema_profile": "off", + "runtime_revision": "01788bac863ac786bfbb648cad69b11e23b40a89", + "runtime_worktree_status": "", + "token_estimates": { + "input_catalog_once": 47987, + "arguments": 37273, + "text_replies": 67408, + "whole_task": 152668 + }, + "actual_model_usage": null, + "wall_seconds": 11.597, + "cpu_seconds": 11.243998, + "peak_process_family_bytes_including_page_cache": 659202048, + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 6342.5286989999995, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 90.04261700000097, + "is_error": false, + "reply_bytes": 427799 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 97.75641600000017, + "is_error": false, + "reply_bytes": 342 + } + ], + "retained_records": [ + { + "evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe", + "operation": "compare_source_to_bundle" + }, + { + "evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f", + "operation": "analyze_javascript_application" + }, + { + "evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78", + "operation": "record_unknown" + }, + { + "evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089", + "operation": "inspect_analysis_view" + } + ], + "proof": { + "status": "passed", + "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": 180, + "embedded_nodes": 99, + "candidate_relations": 210, + "strong_location_rows": 33, + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim." + }, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "cleanup_verified": true + }, + { + "case": "off-paired-5-complete", + "mode": "complete", + "output_schema_profile": "off", + "runtime_revision": "01788bac863ac786bfbb648cad69b11e23b40a89", + "runtime_worktree_status": "", + "token_estimates": { + "input_catalog_once": 47987, + "arguments": 37273, + "text_replies": 67408, + "whole_task": 152668 + }, + "actual_model_usage": null, + "wall_seconds": 10.899, + "cpu_seconds": 10.597762, + "peak_process_family_bytes_including_page_cache": 663457792, + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 6138.704318, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 87.58297700000003, + "is_error": false, + "reply_bytes": 427799 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 76.88620699999956, + "is_error": false, + "reply_bytes": 342 + } + ], + "retained_records": [ + { + "evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe", + "operation": "compare_source_to_bundle" + }, + { + "evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f", + "operation": "analyze_javascript_application" + }, + { + "evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78", + "operation": "record_unknown" + }, + { + "evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089", + "operation": "inspect_analysis_view" + } + ], + "proof": { + "status": "passed", + "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": 180, + "embedded_nodes": 99, + "candidate_relations": 210, + "strong_location_rows": 33, + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim." + }, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "cleanup_verified": true + }, + { + "case": "off-paired-6-selected", + "mode": "selected", + "output_schema_profile": "off", + "runtime_revision": "543b273d7adf581f6fd52c2938020207e455d3df", + "runtime_worktree_status": " M experiments/analysis-efficiency/source-to-bundle-costs.py\n M experiments/analysis-efficiency/source-to-bundle.mjs\n M src/cli/sourceToBundleCommand.ts\n M tests/acceptance/applications/applicationWorkflowCli.test.ts\n M tests/boundary/mcp/sourceToBundleViewsMcp.test.ts\n", + "token_estimates": { + "input_catalog_once": 48100, + "arguments": 298, + "text_replies": 6443, + "whole_task": 54841 + }, + "actual_model_usage": null, + "wall_seconds": 10.802, + "cpu_seconds": 10.466533, + "peak_process_family_bytes_including_page_cache": 660979712, + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 6126.120589999999, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 79.81148600000051, + "is_error": false, + "reply_bytes": 13199 + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 18.2459889999991, + "is_error": false, + "reply_bytes": 19393 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 82.41327699999965, + "is_error": false, + "reply_bytes": 342 + } + ], + "retained_records": [ + { + "evidence_id": "ev_02b198af5997beaf606bf300768cc1ac8cd608fcd0860acbe9b791dbd5bae5dc", + "operation": "inspect_analysis_view" + }, + { + "evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe", + "operation": "compare_source_to_bundle" + }, + { + "evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f", + "operation": "analyze_javascript_application" + }, + { + "evidence_id": "ev_95d8fe5e75c019d86fc17bafeea32445e82eec06e0ec07d0ecdafd1e8bf8ef34", + "operation": "inspect_analysis_view" + }, + { + "evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78", + "operation": "record_unknown" + }, + { + "evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089", + "operation": "inspect_analysis_view" + } + ], + "proof": { + "status": "passed", + "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": 180, + "embedded_nodes": 99, + "candidate_relations": 210, + "strong_location_rows": 33, + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim." + }, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "cleanup_verified": true + } + ], + "baseline_revision": "01788bac8", + "configuration_scope": "Both cohorts explicitly use existing REA_MCP_OUTPUT_SCHEMA_PROFILE=off. REA still canonical-validates successful results; clients do not receive output schemas. Defaults remain unchanged. Pilot off-1/off-2 and warm calibration excluded.", + "implementation_files_sha256": { + "src/application/javascript/SourceToBundleReference.ts": "00f590540b1d8a1755badd7318a4f65b472671e505989c98338f2c63dbe5f7fe", + "src/cli/applicationCommands.ts": "7a436590d7d8f38c059b1ef9a008a20b6021eb04a6caafb72d473149f8762a00", + "src/cli/sourceToBundleCommand.ts": "943ee98cd4abb6674a8f3d90d84d28744ff5a262200a57a45fc1a5b7fba9d105", + "src/contracts/applicationToolContracts.ts": "3b910f1ad1baf13b428d2d108a0020b6235a06dc0bd2bfa2f4b3024ccf57100c", + "src/contracts/javascript/applicationWorkflowInputContracts.ts": "4256f7de82fabf3cc946ca607b8429defa4e8b9f6d3f4be935c20f70741c2d42", + "src/domain/analysisView/analysisView.ts": "56107291635491f3606d57fb85a2b2b175a4689d291d3855a899cb201203148a", + "src/domain/analysisView/sourceToBundleView.ts": "a8cf9a894a4d0383ae109cfc6671c3ba9aa2b75b2b95e229b1297e55f8393de5", + "src/server/createServer.ts": "400764b1ef9bad9d7250cc179736f20daf4cadcde622bbea34e30861b9b23e04", + "src/server/registerApplicationTools/compareSourceToBundle.ts": "8796d139d011ccc89fecfc01dc0fea48b57d6da3823f5148120e34f868a9e0a0", + "src/server/registerApplicationTools/helpers.ts": "c49ec1a9a28d531696012af5d731818deab3fe59f71f4bdcceb3fbb79736824a", + "src/server/registerApplicationTools/types.ts": "959dbfc7162048da2bf477aaa545cb10c766f4e64a25ec5ff370d37244fc0eb7" + } +} diff --git a/experiments/analysis-efficiency/source-to-bundle-prepare.py b/experiments/analysis-efficiency/source-to-bundle-prepare.py new file mode 100644 index 0000000000..7d88534e3f --- /dev/null +++ b/experiments/analysis-efficiency/source-to-bundle-prepare.py @@ -0,0 +1,51 @@ +"""Prepare a pinned public source/bundle corpus without extracting or executing package scripts.""" +import argparse +import hashlib +import json +from pathlib import Path, PurePosixPath +import tarfile + +parser = argparse.ArgumentParser() +parser.add_argument('tarball', type=Path) +parser.add_argument('output', type=Path) +args = parser.parse_args() +expected = 'f2517c466b50424b661a27e48039f15db7783e5822f46b5c7d39f20104ac67a9' +assert hashlib.sha256(args.tarball.read_bytes()).hexdigest() == expected +args.output.mkdir() +bundles = ['redux-toolkit.browser.mjs', 'redux-toolkit.legacy-esm.js', 'redux-toolkit.modern.mjs'] +selected = {f'package/dist/{name}{suffix}' for name in bundles for suffix in ['', '.map']} +with tarfile.open(args.tarball) as archive: + for member in archive: + if not member.isfile(): + continue + path = PurePosixPath(member.name) + if path.is_absolute() or '..' in path.parts: + raise ValueError('Unsafe archive member') + if not (member.name.startswith('package/src/') or member.name in selected): + continue + data = archive.extractfile(member).read() + destination = args.output / 'artifact' / str(path) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(data) + if member.name.startswith('package/src/'): + layout = args.output / 'reference' / str(path.relative_to('package')) + else: + layout = args.output / 'artifact/shipped' / str(path.relative_to('package')) + layout.parent.mkdir(parents=True, exist_ok=True) + layout.write_bytes(data) +rows = [] +for name in sorted(bundles): + document = json.loads((args.output / 'artifact/package/dist' / (name + '.map')).read_text()) + assert len(document['sources']) == len(document['sourcesContent']) == 33 + for index, (source, content) in enumerate(zip(document['sources'], document['sourcesContent'])): + assert source.startswith('../src/') + relative = source.removeprefix('../src/') + embedded_sha = hashlib.sha256(content.encode()).hexdigest() + source_sha = hashlib.sha256((args.output / 'reference/src' / relative).read_bytes()).hexdigest() + rows.append({'map': name + '.map', 'source_index': index, 'source_path': relative, + 'map_source': source, 'content_sha256': embedded_sha, + 'source_sha256': source_sha, 'exact': embedded_sha == source_sha}) +assert len(list((args.output / 'reference/src').rglob('*.*'))) == 180 +assert len(rows) == 99 and not any(row['exact'] for row in rows) +(args.output / 'oracle.json').write_text(json.dumps({'independent_method': 'Python JSON and SHA-256 over published map sourcesContent versus published source files; no REA parser.', 'rows': rows}, indent=2) + '\n') +print(json.dumps({'tarball_sha256': expected, 'source_files': 180, 'embedded_nodes': 99, 'executed_target_code': False})) diff --git a/experiments/analysis-efficiency/source-to-bundle.json b/experiments/analysis-efficiency/source-to-bundle.json new file mode 100644 index 0000000000..398671eba1 --- /dev/null +++ b/experiments/analysis-efficiency/source-to-bundle.json @@ -0,0 +1,916 @@ +{ + "status": "passed", + "tokenizer": "cl100k_base, tiktoken 0.12.0; estimates only; one input catalog plus arguments plus MCP text replies; not actual model usage", + "cold_warm": "Fresh server process and isolated HOME each run; alternating complete/selected, selected/complete, complete/selected. OS page caches are warm/uncontrolled; not an OS-cold claim.", + "tarball_sha256": "f2517c466b50424b661a27e48039f15db7783e5822f46b5c7d39f20104ac67a9", + "preparation_scope": "Reference-source import is performed once outside these repeated SDK trials and reported separately. Both workflows use the explicit existing semantic cap of one for the static correspondence question; full semantic coverage is not claimed.", + "summary": { + "complete": { + "wall_seconds": 11.899, + "cpu_seconds": 11.551076, + "peak_process_family_bytes_including_page_cache": 698986496, + "whole_task_tokens_estimated": [152592, 152592, 152592] + }, + "selected": { + "wall_seconds": 14.733, + "cpu_seconds": 14.376874, + "peak_process_family_bytes_including_page_cache": 996917248, + "whole_task_tokens_estimated": [54765, 54765, 54765] + } + }, + "runs": [ + { + "case": "paired-1-complete", + "mode": "complete", + "runtime_revision": "aa51e4343963fdd0b97ec2ebdc3c41579fd9cb38", + "runtime_worktree_status": "", + "token_estimates": { + "input_catalog_once": 47913, + "arguments": 37272, + "text_replies": 67407, + "whole_task": 152592 + }, + "actual_model_usage": null, + "wall_seconds": 12.618, + "cpu_seconds": 12.259151, + "peak_process_family_bytes_including_page_cache": 699125760, + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 6728.166864999999, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 205.6020669999998, + "is_error": false, + "reply_bytes": 427799 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 77.03590900000017, + "is_error": false, + "reply_bytes": 334 + } + ], + "retained_records": [ + { + "evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe", + "operation": "compare_source_to_bundle" + }, + { + "evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f", + "operation": "analyze_javascript_application" + }, + { + "evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78", + "operation": "record_unknown" + }, + { + "evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089", + "operation": "inspect_analysis_view" + } + ], + "proof": { + "status": "passed", + "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": 180, + "embedded_nodes": 99, + "candidate_relations": 210, + "strong_location_rows": 33, + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim." + }, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "cleanup_verified": true + }, + { + "case": "paired-2-selected", + "mode": "selected", + "runtime_revision": "aa51e4343963fdd0b97ec2ebdc3c41579fd9cb38", + "runtime_worktree_status": " M src/cli/applicationCommands.ts\n M src/contracts/applicationToolContracts.ts\n M src/contracts/javascript/applicationWorkflowInputContracts.ts\n M src/domain/analysisView/analysisView.ts\n M src/server/createServer.ts\n M src/server/registerApplicationTools/compareSourceToBundle.ts\n M src/server/registerApplicationTools/helpers.ts\n M src/server/registerApplicationTools/types.ts\n M tests/acceptance/applications/applicationWorkflowCli.test.ts\n?? experiments/analysis-efficiency/source-to-bundle.mjs\n?? src/application/javascript/SourceToBundleReference.ts\n?? src/cli/sourceToBundleCommand.ts\n?? src/domain/analysisView/sourceToBundleView.ts\n?? tests/boundary/mcp/sourceToBundleViewsMcp.test.ts\n", + "token_estimates": { + "input_catalog_once": 48026, + "arguments": 297, + "text_replies": 6442, + "whole_task": 54765 + }, + "actual_model_usage": null, + "wall_seconds": 15.327, + "cpu_seconds": 14.954805, + "peak_process_family_bytes_including_page_cache": 997220352, + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 7057.07386, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 261.767695999999, + "is_error": false, + "reply_bytes": 13199 + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2391.5776320000004, + "is_error": false, + "reply_bytes": 19393 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 179.67298699999992, + "is_error": false, + "reply_bytes": 334 + } + ], + "retained_records": [ + { + "evidence_id": "ev_02b198af5997beaf606bf300768cc1ac8cd608fcd0860acbe9b791dbd5bae5dc", + "operation": "inspect_analysis_view" + }, + { + "evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe", + "operation": "compare_source_to_bundle" + }, + { + "evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f", + "operation": "analyze_javascript_application" + }, + { + "evidence_id": "ev_95d8fe5e75c019d86fc17bafeea32445e82eec06e0ec07d0ecdafd1e8bf8ef34", + "operation": "inspect_analysis_view" + }, + { + "evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78", + "operation": "record_unknown" + }, + { + "evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089", + "operation": "inspect_analysis_view" + } + ], + "proof": { + "status": "passed", + "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": 180, + "embedded_nodes": 99, + "candidate_relations": 210, + "strong_location_rows": 33, + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim." + }, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "cleanup_verified": true + }, + { + "case": "paired-3-selected", + "mode": "selected", + "runtime_revision": "aa51e4343963fdd0b97ec2ebdc3c41579fd9cb38", + "runtime_worktree_status": " M src/cli/applicationCommands.ts\n M src/contracts/applicationToolContracts.ts\n M src/contracts/javascript/applicationWorkflowInputContracts.ts\n M src/domain/analysisView/analysisView.ts\n M src/server/createServer.ts\n M src/server/registerApplicationTools/compareSourceToBundle.ts\n M src/server/registerApplicationTools/helpers.ts\n M src/server/registerApplicationTools/types.ts\n M tests/acceptance/applications/applicationWorkflowCli.test.ts\n?? experiments/analysis-efficiency/source-to-bundle-oracle.py\n?? experiments/analysis-efficiency/source-to-bundle.mjs\n?? src/application/javascript/SourceToBundleReference.ts\n?? src/cli/sourceToBundleCommand.ts\n?? src/domain/analysisView/sourceToBundleView.ts\n?? tests/boundary/mcp/sourceToBundleViewsMcp.test.ts\n", + "token_estimates": { + "input_catalog_once": 48026, + "arguments": 297, + "text_replies": 6442, + "whole_task": 54765 + }, + "actual_model_usage": null, + "wall_seconds": 14.117, + "cpu_seconds": 13.83662, + "peak_process_family_bytes_including_page_cache": 996917248, + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 6560.107647999999, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 248.57295700000032, + "is_error": false, + "reply_bytes": 13199 + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2105.5653280000006, + "is_error": false, + "reply_bytes": 19393 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 163.21972800000003, + "is_error": false, + "reply_bytes": 334 + } + ], + "retained_records": [ + { + "evidence_id": "ev_02b198af5997beaf606bf300768cc1ac8cd608fcd0860acbe9b791dbd5bae5dc", + "operation": "inspect_analysis_view" + }, + { + "evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe", + "operation": "compare_source_to_bundle" + }, + { + "evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f", + "operation": "analyze_javascript_application" + }, + { + "evidence_id": "ev_95d8fe5e75c019d86fc17bafeea32445e82eec06e0ec07d0ecdafd1e8bf8ef34", + "operation": "inspect_analysis_view" + }, + { + "evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78", + "operation": "record_unknown" + }, + { + "evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089", + "operation": "inspect_analysis_view" + } + ], + "proof": { + "status": "passed", + "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": 180, + "embedded_nodes": 99, + "candidate_relations": 210, + "strong_location_rows": 33, + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim." + }, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "cleanup_verified": true + }, + { + "case": "paired-4-complete", + "mode": "complete", + "runtime_revision": "aa51e4343963fdd0b97ec2ebdc3c41579fd9cb38", + "runtime_worktree_status": "", + "token_estimates": { + "input_catalog_once": 47913, + "arguments": 37272, + "text_replies": 67407, + "whole_task": 152592 + }, + "actual_model_usage": null, + "wall_seconds": 11.899, + "cpu_seconds": 11.551076, + "peak_process_family_bytes_including_page_cache": 687702016, + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 6484.389229, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 216.41979700000047, + "is_error": false, + "reply_bytes": 427799 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 83.66305800000009, + "is_error": false, + "reply_bytes": 334 + } + ], + "retained_records": [ + { + "evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe", + "operation": "compare_source_to_bundle" + }, + { + "evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f", + "operation": "analyze_javascript_application" + }, + { + "evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78", + "operation": "record_unknown" + }, + { + "evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089", + "operation": "inspect_analysis_view" + } + ], + "proof": { + "status": "passed", + "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": 180, + "embedded_nodes": 99, + "candidate_relations": 210, + "strong_location_rows": 33, + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim." + }, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "cleanup_verified": true + }, + { + "case": "paired-5-complete", + "mode": "complete", + "runtime_revision": "aa51e4343963fdd0b97ec2ebdc3c41579fd9cb38", + "runtime_worktree_status": "", + "token_estimates": { + "input_catalog_once": 47913, + "arguments": 37272, + "text_replies": 67407, + "whole_task": 152592 + }, + "actual_model_usage": null, + "wall_seconds": 11.608, + "cpu_seconds": 11.240981, + "peak_process_family_bytes_including_page_cache": 698986496, + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 6322.663412, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 206.36909699999887, + "is_error": false, + "reply_bytes": 427799 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 84.77780899999925, + "is_error": false, + "reply_bytes": 334 + } + ], + "retained_records": [ + { + "evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe", + "operation": "compare_source_to_bundle" + }, + { + "evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f", + "operation": "analyze_javascript_application" + }, + { + "evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78", + "operation": "record_unknown" + }, + { + "evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089", + "operation": "inspect_analysis_view" + } + ], + "proof": { + "status": "passed", + "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": 180, + "embedded_nodes": 99, + "candidate_relations": 210, + "strong_location_rows": 33, + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim." + }, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "cleanup_verified": true + }, + { + "case": "paired-6-selected", + "mode": "selected", + "runtime_revision": "aa51e4343963fdd0b97ec2ebdc3c41579fd9cb38", + "runtime_worktree_status": " M src/cli/applicationCommands.ts\n M src/contracts/applicationToolContracts.ts\n M src/contracts/javascript/applicationWorkflowInputContracts.ts\n M src/domain/analysisView/analysisView.ts\n M src/server/createServer.ts\n M src/server/registerApplicationTools/compareSourceToBundle.ts\n M src/server/registerApplicationTools/helpers.ts\n M src/server/registerApplicationTools/types.ts\n M tests/acceptance/applications/applicationWorkflowCli.test.ts\n?? experiments/analysis-efficiency/source-to-bundle-oracle.py\n?? experiments/analysis-efficiency/source-to-bundle-prepare.py\n?? experiments/analysis-efficiency/source-to-bundle.mjs\n?? src/application/javascript/SourceToBundleReference.ts\n?? src/cli/sourceToBundleCommand.ts\n?? src/domain/analysisView/sourceToBundleView.ts\n?? tests/boundary/mcp/sourceToBundleViewsMcp.test.ts\n", + "token_estimates": { + "input_catalog_once": 48026, + "arguments": 297, + "text_replies": 6442, + "whole_task": 54765 + }, + "actual_model_usage": null, + "wall_seconds": 14.733, + "cpu_seconds": 14.376874, + "peak_process_family_bytes_including_page_cache": 992952320, + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 6604.5753270000005, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 257.93621600000006, + "is_error": false, + "reply_bytes": 13199 + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2186.2845859999998, + "is_error": false, + "reply_bytes": 19393 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 170.3188169999994, + "is_error": false, + "reply_bytes": 334 + } + ], + "retained_records": [ + { + "evidence_id": "ev_02b198af5997beaf606bf300768cc1ac8cd608fcd0860acbe9b791dbd5bae5dc", + "operation": "inspect_analysis_view" + }, + { + "evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe", + "operation": "compare_source_to_bundle" + }, + { + "evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f", + "operation": "analyze_javascript_application" + }, + { + "evidence_id": "ev_95d8fe5e75c019d86fc17bafeea32445e82eec06e0ec07d0ecdafd1e8bf8ef34", + "operation": "inspect_analysis_view" + }, + { + "evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78", + "operation": "record_unknown" + }, + { + "evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089", + "operation": "inspect_analysis_view" + } + ], + "proof": { + "status": "passed", + "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": 180, + "embedded_nodes": 99, + "candidate_relations": 210, + "strong_location_rows": 33, + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim." + }, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "cleanup_verified": true + } + ], + "implementation_files_sha256": { + "src/application/javascript/SourceToBundleReference.ts": "00f590540b1d8a1755badd7318a4f65b472671e505989c98338f2c63dbe5f7fe", + "src/cli/applicationCommands.ts": "7a436590d7d8f38c059b1ef9a008a20b6021eb04a6caafb72d473149f8762a00", + "src/cli/sourceToBundleCommand.ts": "70887d0a060c2b315df2eeaabb5889f5e47e61da79dbc9ca076e49b8858c3571", + "src/contracts/applicationToolContracts.ts": "3b910f1ad1baf13b428d2d108a0020b6235a06dc0bd2bfa2f4b3024ccf57100c", + "src/contracts/javascript/applicationWorkflowInputContracts.ts": "4256f7de82fabf3cc946ca607b8429defa4e8b9f6d3f4be935c20f70741c2d42", + "src/domain/analysisView/analysisView.ts": "56107291635491f3606d57fb85a2b2b175a4689d291d3855a899cb201203148a", + "src/domain/analysisView/sourceToBundleView.ts": "a8cf9a894a4d0383ae109cfc6671c3ba9aa2b75b2b95e229b1297e55f8393de5", + "src/server/createServer.ts": "400764b1ef9bad9d7250cc179736f20daf4cadcde622bbea34e30861b9b23e04", + "src/server/registerApplicationTools/compareSourceToBundle.ts": "8796d139d011ccc89fecfc01dc0fea48b57d6da3823f5148120e34f868a9e0a0", + "src/server/registerApplicationTools/helpers.ts": "c49ec1a9a28d531696012af5d731818deab3fe59f71f4bdcceb3fbb79736824a", + "src/server/registerApplicationTools/types.ts": "959dbfc7162048da2bf477aaa545cb10c766f4e64a25ec5ff370d37244fc0eb7" + }, + "separate_preparation": { + "reference_import": { + "wall_seconds": 4.757, + "cpu_seconds": 4.602731, + "peak_process_family_bytes_including_page_cache": 383381504, + "cleanup_verified": true + }, + "application_default": { + "wall_seconds": 25.824, + "cpu_seconds": 25.433681, + "peak_process_family_bytes_including_page_cache": 1100005376, + "cleanup_verified": true + }, + "application_explicit_semantic_one": { + "wall_seconds": 12.933, + "cpu_seconds": 12.600848, + "peak_process_family_bytes_including_page_cache": 731721728, + "cleanup_verified": true + }, + "semantic_scope": "Comparison facts exactly agree except lineage across default and semantic-one; full semantic coverage is not equivalent. These two one-run SDK observations are not repeatable speedup evidence." + }, + "native_cli_control": { + "wall_seconds": 2.842, + "cpu_seconds": 2.669896, + "peak_process_family_bytes_including_page_cache": 374730752, + "cleanup_verified": true, + "comparison_bytes": 205231, + "external_three_row_projection_bytes": 22108, + "scope": "Public CLI full Evidence exactly equals MCP. An external Python projection is already possible; raw SDK context reduction does not establish actual model savings over code composition. Byte sizes are not token counts." + }, + "warm_continuation_calibration": { + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 6546.054383000001, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 250.82817400000022, + "is_error": false, + "reply_bytes": 13199 + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2071.093723, + "is_error": false, + "reply_bytes": 19393 + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 71.82136900000114, + "is_error": false, + "reply_bytes": 19393 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 97.2455460000001, + "is_error": false, + "reply_bytes": 332 + } + ], + "scope": "One additional fresh-process trial, excluded from paired whole-task medians. Second inspect_analysis_view returns identical Evidence with an already reused SDK validator; full parent remains retained.", + "wall_seconds": 14.218, + "cpu_seconds": 13.858723, + "peak_process_family_bytes_including_page_cache": 993660928, + "cleanup_verified": true + }, + "subsequent_cli_error_envelope_fix": { + "file": "src/cli/sourceToBundleCommand.ts", + "sha256": "943ee98cd4abb6674a8f3d90d84d28744ff5a262200a57a45fc1a5b7fba9d105", + "scope": "Preserves the previous error: Application workflow failed field around finishAnalysisCommand failures. Original paired MCP measurements predate this CLI-only negative-result fix. Actual CLI wrong-application output now exactly equals the baseline, including all projected diagnostic fields. MCP producer/view implementation is unchanged." + }, + "subsequent_review_guards": { + "scope": "Additional post-review checks, excluded from the original paired medians. Shared JSON authentication headroom now precedes graph validation; cancellation dispatch follows validation and all pure comparison/view computation before any session writes. Candidate pages now have cross-source boundary regression coverage. Matching/scoring and complete comparison Evidence are unchanged.", + "implementation_files_sha256": { + "src/application/JsonAuthenticationHeadroom.ts": "9f3cd40e0dc204d30a1c979c9e18535344963edbd2e75c47f633fa72a6447f96", + "src/application/analysisView/evidenceFileSource.ts": "5b7adedb474de0ac9a1576f46a801db8232d473189689129ed3aece13322a42a", + "src/application/javascript/SourceToBundleReference.ts": "a65aa4b5853001d133a4af385dc70c051144dfc7e4e22cd0b919bce5fa65ccc4", + "src/cli/applicationCommands.ts": "7a436590d7d8f38c059b1ef9a008a20b6021eb04a6caafb72d473149f8762a00", + "src/cli/sourceToBundleCommand.ts": "943ee98cd4abb6674a8f3d90d84d28744ff5a262200a57a45fc1a5b7fba9d105", + "src/contracts/applicationToolContracts.ts": "3b910f1ad1baf13b428d2d108a0020b6235a06dc0bd2bfa2f4b3024ccf57100c", + "src/contracts/javascript/applicationWorkflowInputContracts.ts": "4256f7de82fabf3cc946ca607b8429defa4e8b9f6d3f4be935c20f70741c2d42", + "src/domain/analysisView/analysisView.ts": "56107291635491f3606d57fb85a2b2b175a4689d291d3855a899cb201203148a", + "src/domain/analysisView/sourceToBundleView.ts": "a8cf9a894a4d0383ae109cfc6671c3ba9aa2b75b2b95e229b1297e55f8393de5", + "src/server/createServer.ts": "400764b1ef9bad9d7250cc179736f20daf4cadcde622bbea34e30861b9b23e04", + "src/server/registerApplicationTools/compareSourceToBundle.ts": "a5a3f125c9fb31b39fe50ae9135b9f3d76a2ce08e245fcb0a352ec550bef0083", + "src/server/registerApplicationTools/helpers.ts": "c49ec1a9a28d531696012af5d731818deab3fe59f71f4bdcceb3fbb79736824a", + "src/server/registerApplicationTools/types.ts": "959dbfc7162048da2bf477aaa545cb10c766f4e64a25ec5ff370d37244fc0eb7" + }, + "regressions": { + "shared_headroom_and_evidence_reader_tests": 7, + "first_post_review_guard_suites": 12, + "final_cancellation_and_cross_source_page_suites": 65, + "negative_control_disabled_guards": { + "expected_failed_tests": 3, + "scope": "All three new guard regressions fail when the admission and cancellation gates are temporarily disabled; production source restored before successful checks." + } + }, + "verification": [ + { + "profile": "full", + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 6483.442064, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 252.73620699999992, + "is_error": false, + "reply_bytes": 13199 + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2205.9964070000005, + "is_error": false, + "reply_bytes": 19393 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 85.7624880000003, + "is_error": false, + "reply_bytes": 332 + } + ], + "wall_seconds": 14.217, + "cpu_seconds": 13.843447, + "peak_process_family_bytes_including_page_cache": 976711680, + "complete_parent_byte_equal_to_original": true, + "whole_task_cl100k_estimate": 54835, + "independent_proof": { + "status": "passed", + "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": 180, + "embedded_nodes": 99, + "candidate_relations": 210, + "strong_location_rows": 33, + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim." + }, + "cleanup_verified": true, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + }, + { + "profile": "off", + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 6463.359534, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 94.13822900000014, + "is_error": false, + "reply_bytes": 13199 + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 17.32437199999913, + "is_error": false, + "reply_bytes": 19393 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 79.37489800000003, + "is_error": false, + "reply_bytes": 330 + } + ], + "wall_seconds": 11.297, + "cpu_seconds": 10.913648, + "peak_process_family_bytes_including_page_cache": 656244736, + "complete_parent_byte_equal_to_original": true, + "whole_task_cl100k_estimate": 54835, + "independent_proof": { + "status": "passed", + "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": 180, + "embedded_nodes": 99, + "candidate_relations": 210, + "strong_location_rows": 33, + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim." + }, + "cleanup_verified": true, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + } + ] + }, + "subsequent_reference_provenance": { + "scope": "Final post-review verification, excluded from historical paired medians. File calls now preserve reference_source kind/path/requested_path from the same reader. Full Evidence identity intentionally distinguishes chosen file origins; inline output and normalized comparison facts are unchanged. Read-I/O cancellations report the comparison operation and preserve their cause.", + "implementation_files_sha256": { + "src/application/JsonAuthenticationHeadroom.ts": "9f3cd40e0dc204d30a1c979c9e18535344963edbd2e75c47f633fa72a6447f96", + "src/application/JsonFiles.ts": "dfc6eaf9b6d0b7f8d4e12d2c39f9e0c5288a29a16fae0451f4c7be1ee2e31f65", + "src/application/analysisView/evidenceFileSource.ts": "5b7adedb474de0ac9a1576f46a801db8232d473189689129ed3aece13322a42a", + "src/application/javascript/JavaScriptApplicationWorkflowService.ts": "657e36d2a20aee2722c76d6494d092db2d06ce5cfa6e9ba0506bcf13aa4e9d1d", + "src/application/javascript/SourceToBundleReference.ts": "8206198666a249a6d545ec79684b9788b7937603a7d9968ffba007ce9abe30f7", + "src/cli/applicationCommands.ts": "7a436590d7d8f38c059b1ef9a008a20b6021eb04a6caafb72d473149f8762a00", + "src/cli/sourceToBundleCommand.ts": "b905a11c377c159658c1b77e9ecfde2a2961604f1366cfc67509341f4a924fa9", + "src/contracts/applicationToolContracts.ts": "3b910f1ad1baf13b428d2d108a0020b6235a06dc0bd2bfa2f4b3024ccf57100c", + "src/contracts/javascript/applicationWorkflowInputContracts.ts": "4256f7de82fabf3cc946ca607b8429defa4e8b9f6d3f4be935c20f70741c2d42", + "src/domain/analysisView/analysisView.ts": "56107291635491f3606d57fb85a2b2b175a4689d291d3855a899cb201203148a", + "src/domain/analysisView/sourceToBundleView.ts": "a8cf9a894a4d0383ae109cfc6671c3ba9aa2b75b2b95e229b1297e55f8393de5", + "src/server/createServer.ts": "400764b1ef9bad9d7250cc179736f20daf4cadcde622bbea34e30861b9b23e04", + "src/server/registerApplicationTools/compareSourceToBundle.ts": "e14f07f6114f7ce4ec7d97034713074125d3133c876f12a8656a9322017882ee", + "src/server/registerApplicationTools/helpers.ts": "c49ec1a9a28d531696012af5d731818deab3fe59f71f4bdcceb3fbb79736824a", + "src/server/registerApplicationTools/types.ts": "959dbfc7162048da2bf477aaa545cb10c766f4e64a25ec5ff370d37244fc0eb7" + }, + "regressions": { + "mcp_profiles_and_source_guards_passed": 6, + "cli_and_shared_json_ownership_selected_tests_passed": 29, + "negative_control_without_provenance_and_cancellation_attribution": 3 + }, + "verification": [ + { + "profile": "full", + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 6576.780192, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 264.797908999999, + "is_error": false, + "reply_bytes": 13199 + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2090.7180249999983, + "is_error": false, + "reply_bytes": 19393 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 164.20850099999916, + "is_error": false, + "reply_bytes": 330 + } + ], + "wall_seconds": 14.417, + "cpu_seconds": 14.027529, + "peak_process_family_bytes_including_page_cache": 995631104, + "normalized_comparison_byte_equal_to_original": true, + "file_provenance_recorded": { + "kind": "file", + "path": "/tmp/rea-perf-source-bundle-20261021/reference-main-aa51.json", + "requested_path": "/tmp/rea-perf-source-bundle-20261021/reference-main-aa51.json" + }, + "evidence_id_differs_due_to_file_provenance": true, + "whole_task_cl100k_estimate": 54896, + "independent_proof": { + "status": "passed", + "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": 180, + "embedded_nodes": 99, + "candidate_relations": 210, + "strong_location_rows": 33, + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim." + }, + "cleanup_verified": true, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + }, + { + "profile": "off", + "calls": [ + { + "name": "analyze_javascript_application", + "elapsed_ms": 6315.5019010000005, + "is_error": false, + "reply_bytes": 21651 + }, + { + "name": "compare_source_to_bundle", + "elapsed_ms": 91.56914900000083, + "is_error": false, + "reply_bytes": 13199 + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 24.95524800000021, + "is_error": false, + "reply_bytes": 19393 + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 81.6187800000007, + "is_error": false, + "reply_bytes": 328 + } + ], + "wall_seconds": 11.104, + "cpu_seconds": 10.819556, + "peak_process_family_bytes_including_page_cache": 658169856, + "normalized_comparison_byte_equal_to_original": true, + "file_provenance_recorded": { + "kind": "file", + "path": "/tmp/rea-perf-source-bundle-20261021/reference-main-aa51.json", + "requested_path": "/tmp/rea-perf-source-bundle-20261021/reference-main-aa51.json" + }, + "evidence_id_differs_due_to_file_provenance": true, + "whole_task_cl100k_estimate": 54896, + "independent_proof": { + "status": "passed", + "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.", + "source_files": 180, + "embedded_nodes": 99, + "candidate_relations": 210, + "strong_location_rows": 33, + "limitation": "Static multiple-location projections; no semantic-equivalence or removal claim." + }, + "cleanup_verified": true, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + } + ], + "native_cli": { + "legacy_inline_evidence_equal_to_baseline": true, + "file_cli_mcp_evidence_identical_for_same_origin": true, + "normalized_comparison_equal_to_baseline": true, + "selected_file_parent_resolves": true + } + } +} diff --git a/experiments/analysis-efficiency/source-to-bundle.md b/experiments/analysis-efficiency/source-to-bundle.md new file mode 100644 index 0000000000..e6682b1703 --- /dev/null +++ b/experiments/analysis-efficiency/source-to-bundle.md @@ -0,0 +1,210 @@ +# Selected source-to-bundle facts + +Issue #6112. Baseline: `main@aa51e4343963fdd0b97ec2ebdc3c41579fd9cb38`. +Measured implementation: the production-file SHA-256 manifest in +[source-to-bundle.json](source-to-bundle.json). These are working-tree +measurements, not a claim that the baseline commit already contains the feature. +The original cohort predates the CLI error-envelope and review guard fixes, +pinned separately in the JSON. The CLI invalid-application output was restored +exactly. Post-guard SDK tasks verified both output profiles before the later +reference-file provenance fix. Final tasks preserve normalized comparison facts +while full Evidence IDs now commit the selected file origin. Historical paired +medians are not presented as new measurements of the later guard changes. + +## Analyst question and correctness + +For `src/createSlice.ts`, `src/createReducer.ts` and +`src/query/core/buildThunks.ts`, report source correspondence and candidate counts, +then inspect the exact supporting candidate signals for the two split files. +Keep the complete comparison available. This is static location correspondence; +it does not establish behavioral equivalence or removal. + +Public corpus: [Redux Toolkit 2.2.7 npm tarball](https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.2.7.tgz), +1,184,767 bytes; SHA-256 +`f2517c466b50424b661a27e48039f15db7783e5822f46b5c7d39f20104ac67a9`. +The published `src/` and three root `dist/` bundles/maps are copied byte-for-byte +into separate roots, retaining the `src/` and `dist/` prefixes. No target code or +package scripts execute. REA runs its own static analysis helpers. + +An independent Python JSON/SHA-256/path oracle checks all 180 source-file hashes, +99 embedded source nodes and 210 candidate relations in every completed task. +All 99 embedded source strings differ from the published source bytes; path +correspondence must not become an exact-source or semantic-equivalence claim. +The complete report has 33 split and 147 unknown rows. The selected files have +3, 3 and 0 candidates respectively. Selected mapping/candidate identities, +original indices, signals, mapped flags and source coverage agree with the full +retained comparison. Global classification runs before selection. + +## Completed-task costs + +Linux, Node 24.18.0, one CPU, 4 GiB hard / 3 GiB early-stop process-family guard, +192-task ceiling and no swap. Six SDK tasks alternate complete/selected, +selected/complete, complete/selected. Each starts a fresh server with an isolated +HOME. OS page caches are warm/uncontrolled: these are process-cold observations, +not OS-cold measurements. All six exit successfully, release their processes and +record no OOM, resource stop or task-limit event. + +Each task includes catalog discovery, application preparation, comparison and +full Evidence export. Selected tasks additionally query candidate signals via +`inspect_analysis_view`. Both cohorts use the existing explicit semantic-node +cap of one because the question only needs source/map correspondence. This is +not equivalent semantic-graph coverage; the omission metadata remains visible. + +| Whole task | Existing full inline graph/report | File graph + selected mappings/candidates | +| ----------------------------------------- | --------------------------------: | ----------------------------------------: | +| Input catalog, cl100k estimate | 47,913 | 48,026 | +| All call arguments, estimate | 37,272 | 297 | +| All MCP text replies, estimate | 67,407 | 6,442 | +| Total estimated context | 152,592 | 54,765 | +| Wall seconds, median | 11.899 | 14.733 | +| Process-family CPU seconds, median | 11.551 | 14.377 | +| Kernel process-family peak MiB, median | 666.6 | 950.7 | +| Comparison reply bytes, text + structured | 427,799 | 13,199 | +| Additional candidate reply bytes | 0 | 19,393 | + +Context estimates fall 64.1%, including the slightly larger catalog and extra +call. Tokenizer: `tiktoken==0.12.0`, `cl100k_base`; one name/description/input-schema +catalog plus every argument and MCP text reply. Output schemas, transport +framing, model prompts/reasoning and actual billed usage are not included. +There was no model invocation; these are not actual model token savings. + +This workflow trades extra cold-client time/memory for substantially smaller +context. Comparison itself takes approximately 206–216 ms before and 249–262 ms +with file validation and selected delivery. The first candidate-view call takes +2.1–2.4 seconds. A separate calibration repeats the identical candidate query: +2,071 ms first, 72 ms second, with identical Evidence. This is consistent with +one-time SDK validator work; it is not a general speedup or a server-only profile. +No comparison cache was added: the measured matching computation is already +small, and a file cache would need additional freshness/identity rules. + +Reference import is a separate one-time public CLI step: 4.757 s wall, +4.603 s CPU, 365.6 MiB peak. Add that same preparation cost to either first task; +download/extraction and compilation are excluded. An additional baseline with +default semantic coverage retains 22,090 semantic nodes and exports 42,870,148 +bytes; explicit cap one retains one and exports 2,896,937 bytes. All comparison +facts agree after excluding expected Evidence lineage. Those single-run +observations demonstrate question-specific existing task selection, not a new +semantic-analysis optimization or equal-coverage performance claim. + +## Existing output-schema selection + +A second six-task cohort explicitly selects the existing +`REA_MCP_OUTPUT_SCHEMA_PROFILE=off` for **both** workflows, with baseline +`main@01788bac8`. REA still validates successful results against the canonical +contracts and returns complete structured content; clients do not receive output +schemas to compile. This is an explicit caller configuration, not a new default. +The independent oracle and selected/full fact checks pass in every run. + +| Whole task, output profile off | Full graph/report | Selected facts | +| -------------------------------------- | ----------------: | -------------: | +| Total estimated context | 152,668 | 54,841 | +| Wall seconds, median | 11.597 | 10.890 | +| Process-family CPU seconds, median | 11.244 | 10.569 | +| Kernel process-family peak MiB, median | 628.7 | 626.0 | +| Candidate query milliseconds | not needed | 18–21 | + +The context reduction remains approximately 64%. Whole-task ranges overlap +(complete 10.899–12.215 s; selected 10.802–11.196 s), so the apparent median +improvement does not establish a stable speedup. Unlike the default full-output +cohort, this configured workflow avoids the extra cold SDK validator cost and +has similar task memory/time. Two initial pilot cases are excluded. All six +observed runs, explicit configuration and implementation hashes are retained in +[source-to-bundle-output-off.json](source-to-bundle-output-off.json). + +## Existing CLI/code-composition control + +The public CLI on the same saved application/reference produces complete Evidence +identical to MCP (205,231 output bytes). The measured CLI comparison step is +2.842 s wall, 2.670 s CPU, 357.4 MiB process-family peak. A separate Python +projection of the three rows already yields 22,108 bytes. A native agent can +redirect full JSON to a file and select facts in code, so raw SDK savings do not +establish savings over every native client. These byte sizes are not token counts; +actual native-agent context and external projection CPU are unmeasured. + +The new CLI reference-file form preserves the normalized facts and records its +file origin. Its saved Evidence +can be queried by `inspect-analysis-view` using the same source-to-bundle +projector as MCP. The interface supplies reusable, authenticated facts and +relationships instead of requiring each caller to maintain an ad-hoc projection. + +## Reproduce + +Use pinned Node/npm, `npm ci` and `npm run build:cached` for baseline and feature +worktrees. Install dependencies once per lockfile; do not run npm through a shared +node_modules symlink. Run heavy steps serially with the checked-in resource guard. +Set absolute paths for `BASE_RUNTIME`, `FEATURE_RUNTIME` and `LAB`. + +```sh +curl -fL 'https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.2.7.tgz' -o /tmp/toolkit.tgz +python3 experiments/analysis-efficiency/source-to-bundle-prepare.py /tmp/toolkit.tgz "$LAB" +python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \ + node "$BASE_RUNTIME/scripts/rea.mjs" import-reference-source "$LAB/reference" --json \ + > "$LAB/reference.json" 2> "$LAB/reference-import.log" +``` + +For each of the six ordered trials, use a new output directory; set `RUNTIME` to +the baseline for `MODE=complete`, feature for `MODE=selected`. Capture both stdout +and stderr in the adjacent `.log` so the resource receipt remains available. + +```sh +python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \ + node experiments/analysis-efficiency/source-to-bundle.mjs \ + --runtime "$RUNTIME" --corpus "$LAB" --reference "$LAB/reference.json" \ + --output "$LAB/paired-1-complete" --mode "$MODE" \ + > "$LAB/paired-1-complete.log" 2>&1 +python3 experiments/analysis-efficiency/source-to-bundle-oracle.py \ + "$LAB/paired-1-complete" "$LAB/reference.json" +uv run --no-project --with tiktoken==0.12.0 python \ + experiments/analysis-efficiency/source-to-bundle-costs.py "$LAB" +``` + +Use a separate output name and `--mode selected --warm-continuation` for the +validator calibration; exclude it from paired medians. For the configured +cohort, add `--output-profile off`, use new `off-paired-*` directories and the +01788bac8 control, then run the cost reader with +`--pattern 'off-paired-*' --output "$LAB/costs-off.json"`. Retain the tarball, +reference, transcripts, bundles, independent proofs and guard receipts locally. +Never commit target binaries, dependency trees, credentials or generated builds. + +## Review guards and regression controls + +The file reader admits JSON assembly; a shared application guard additionally +reserves live heap headroom before schema validation clones data and computes the +root commitment. Evidence-file and historical-graph readers use the same owner, +with their original operation-specific diagnostics. Validation yields before +checking a pending cancellation, including validation failures. MCP computes its +selected view before yielding and committing any application source, comparison, +residual unknown or view to the session. + +Seven shared headroom/Evidence-reader tests pass. A separate 65-test focused run +covers file cancellation, graph paths, actual SDK cancellation during validation +and comparison, and cross-source pages with candidate counts 2/1/2. The latter +checks original indices, totals and next offsets across boundaries and filters. +All three new guard regressions fail in a negative control with those gates +temporarily removed, then pass after restoration. Memory admission is an estimate; +the reported hard process-family ceilings come from the experiment guard. + +Post-guard public-package SDK tasks, before the later provenance fix, retained all +180 source-file / 99 embedded-node / 210 relation oracle checks and byte-identical +complete comparison Evidence. Their receipts, source hashes and costs are under +`subsequent_review_guards` in the main JSON; they are excluded from paired medians. + +## Reference-file provenance and cancellation attribution + +The final file form records `parameters.reference_source` with `kind: file`, the +exact caller `requested_path` and the reader's absolute selected `path`. The +shared file owner supplies that path without a second interpretation; POSIX +`..` and symlink semantics remain intact. Identical graphs in different chosen +files have the same normalized facts/mapping IDs and distinct Evidence IDs. +Inline legacy Evidence remains identical. Actual public-package CLI file +Evidence equals MCP file Evidence for the same origin; relative CLI paths also +have maintained coverage. + +Reader cancellation is reattributed to `compare_source_to_bundle`, preserving the +original cause. Six MCP/profile/guard tests and 29 selected CLI/shared-JSON +ownership tests pass. Removing provenance recording and cancellation attribution +makes three tests fail as expected. Additional Evidence-reader/file cancellation +checks pass. Final real-package tasks estimate 54,896 cl100k tokens in both output +profiles, preserve all independent oracle facts, and retain the new provenance; +these are additional verification runs, not new paired medians. The main JSON's +`subsequent_reference_provenance` records final source hashes and receipts. diff --git a/experiments/analysis-efficiency/source-to-bundle.mjs b/experiments/analysis-efficiency/source-to-bundle.mjs new file mode 100644 index 0000000000..475af68609 --- /dev/null +++ b/experiments/analysis-efficiency/source-to-bundle.mjs @@ -0,0 +1,251 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +const { values } = parseArgs({ + options: { + runtime: { type: "string" }, + corpus: { type: "string" }, + reference: { type: "string" }, + output: { type: "string" }, + mode: { type: "string" }, + "output-profile": { type: "string", default: "full" }, + "warm-continuation": { type: "boolean", default: false }, + }, +}); +for (const key of ["runtime", "corpus", "reference", "output"]) + assert.ok( + values[key] && resolve(values[key]) === values[key], + key + " must be absolute", + ); +assert.ok(["complete", "selected"].includes(values.mode)); +assert.ok(["full", "off"].includes(values["output-profile"])); +const root = values.corpus, + runtime = values.runtime, + output = values.output; +await mkdir(output); +const reference = JSON.parse(await readFile(values.reference, "utf8")); +const sha = (value) => createHash("sha256").update(value).digest("hex"); +const revision = execFileSync("git", ["rev-parse", "HEAD"], { + cwd: runtime, + encoding: "utf8", +}).trim(); +const sourceState = execFileSync("git", ["status", "--porcelain"], { + cwd: runtime, + encoding: "utf8", +}); +const transport = new StdioClientTransport({ + command: process.execPath, + args: [join(runtime, "scripts/rea.mjs"), "mcp"], + cwd: output, + env: { + PATH: process.env.PATH, + HOME: output, + REA_LOG_LEVEL: "silent", + REA_MCP_OUTPUT_SCHEMA_PROFILE: values["output-profile"], + UV_THREADPOOL_SIZE: "1", + NODE_OPTIONS: "--max-old-space-size=2048", + NODE_DISABLE_COMPILE_CACHE: "1", + }, + stderr: "pipe", +}); +const client = new Client({ name: "source-bundle-baseline", version: "1" }); +let stderr = ""; +transport.stderr?.on("data", (chunk) => { + stderr += chunk; +}); +const report = { + status: "incomplete", + runtime_revision: revision, + runtime_worktree_status: sourceState, + mode: values.mode, + output_schema_profile: values["output-profile"], + reference_sha256: sha(await readFile(values.reference)), + node_version: process.version, + calls: [], + cleanup_verified: false, +}; +const transcript = []; +const began = performance.now(); +try { + await client.connect(transport); + const catalog = await client.listTools(); + await writeFile(join(output, "catalog.json"), JSON.stringify(catalog)); + const definitions = new Map(catalog.tools.map((t) => [t.name, t])); + const call = async (name, args) => { + const start = performance.now(); + const reply = await client.callTool( + { name, arguments: args }, + { toolDefinition: definitions.get(name) }, + ); + transcript.push({ name, arguments: args, reply }); + report.calls.push({ + name, + elapsed_ms: performance.now() - start, + is_error: reply.isError === true, + reply_bytes: Buffer.byteLength(JSON.stringify(reply)), + }); + return reply; + }; + const analyzed = await call("analyze_javascript_application", { + input_path: join(root, "artifact/shipped"), + detail: "summary", + max_semantic_nodes: 1, + max_semantic_nodes_per_file: 1, + }); + assert.notEqual(analyzed.isError, true, JSON.stringify(analyzed)); + report.preparation_summary = analyzed.structuredContent.normalized_result; + const parentId = report.preparation_summary.parent_evidence_id; + assert.ok(parentId); + const compared = await call("compare_source_to_bundle", { + reference: + values.mode === "complete" + ? reference + : { kind: "file", path: values.reference }, + application: { kind: "retained-evidence", evidence_id: parentId }, + ...(values.mode === "complete" + ? {} + : { + view: { + kind: "source-to-bundle", + collection: "mappings", + source_paths: [ + "src/createSlice.ts", + "src/createReducer.ts", + "src/query/core/buildThunks.ts", + ], + }, + }), + }); + assert.notEqual(compared.isError, true, JSON.stringify(compared)); + if (values.mode === "selected") { + const selected = compared.structuredContent.normalized_result; + assert.equal(selected.items.length, 3); + assert.deepEqual( + selected.items.map((r) => [r.source_path, r.status]), + [ + ["src/createReducer.ts", "split"], + ["src/createSlice.ts", "split"], + ["src/query/core/buildThunks.ts", "unknown"], + ], + ); + const candidates = await call("inspect_analysis_view", { + source: { + kind: "retained-evidence", + evidence_id: selected.parent_evidence_id, + }, + view: { + kind: "source-to-bundle", + collection: "candidates", + source_paths: ["src/createSlice.ts", "src/createReducer.ts"], + fields: ["score", "confidence", "signals"], + }, + }); + assert.notEqual(candidates.isError, true, JSON.stringify(candidates)); + assert.equal( + candidates.structuredContent.normalized_result.items.length, + 6, + ); + if (values["warm-continuation"]) { + const repeated = await call( + "inspect_analysis_view", + transcript.at(-1).arguments, + ); + assert.deepEqual(repeated, candidates); + } + } + report.comparison_reply = compared.structuredContent; + const exported = await call("export_evidence_bundle", { + path: join(output, "evidence.json"), + }); + assert.notEqual(exported.isError, true); + const bundle = JSON.parse( + await readFile(join(output, "evidence.json"), "utf8"), + ); + const comparison = bundle.records.find( + (r) => r.operation === "compare_source_to_bundle", + ); + report.retained_records = bundle.records.map((r) => ({ + evidence_id: r.evidence_id, + operation: r.operation, + })); + if (comparison) { + report.comparison_id = comparison.normalized_result.comparison_id; + report.comparison_summary = comparison.normalized_result.summary; + report.coverage = comparison.normalized_result.coverage; + report.selected_items = comparison.normalized_result.items.filter((r) => + [ + "src/createSlice.ts", + "src/createReducer.ts", + "src/query/core/buildThunks.ts", + ].includes(r.source_path), + ); + report.complete_comparison_bytes = Buffer.byteLength( + JSON.stringify(comparison), + ); + } + report.status = "observed"; + assert.equal(report.comparison_summary.split, 33); + assert.equal(report.comparison_summary.unknown, 147); + const views = transcript.filter( + (r) => + ["compare_source_to_bundle", "inspect_analysis_view"].includes(r.name) && + r.reply.structuredContent.normalized_result.parent_evidence_id, + ); + for (const view of views) { + const result = view.reply.structuredContent.normalized_result; + assert.equal(result.parent_evidence_id, comparison.evidence_id); + assert.deepEqual( + result.source_coverage, + comparison.normalized_result.coverage, + ); + for (const row of result.items) { + const original = comparison.normalized_result.items[row.source_index]; + assert.equal(row.mapping_id, original.mapping_id); + assert.equal(row.source_path, original.source_path); + if (result.view.collection === "candidates") { + const candidate = original.candidates[row.candidate_index]; + assert.equal(row.current_node_id, candidate.current_node_id); + assert.deepEqual(row.signals, candidate.signals); + assert.equal( + row.mapped, + original.current_node_ids.includes(candidate.current_node_id), + ); + } else { + assert.equal(row.status, original.status); + assert.equal(row.candidate_count, original.candidates.length); + } + } + } +} finally { + await client.close(); + await transport.close(); + report.cleanup_verified = transport.pid === null; + report.elapsed_ms = performance.now() - began; + await writeFile( + join(output, "report.json"), + JSON.stringify(report, null, 2) + "\n", + ); + await writeFile(join(output, "transcript.json"), JSON.stringify(transcript)); + await writeFile(join(output, "server.log"), stderr); +} +console.log( + JSON.stringify({ + status: report.status, + calls: report.calls, + comparison_summary: report.comparison_summary, + coverage: report.coverage, + selected: report.selected_items?.map((r) => ({ + path: r.source_path, + status: r.status, + candidates: r.candidates.length, + })), + complete_comparison_bytes: report.complete_comparison_bytes, + cleanup_verified: report.cleanup_verified, + }), +); diff --git a/src/application/JsonAuthenticationHeadroom.test.ts b/src/application/JsonAuthenticationHeadroom.test.ts new file mode 100644 index 0000000000..a78897061e --- /dev/null +++ b/src/application/JsonAuthenticationHeadroom.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { + estimateJsonValueBytes, + requireAuthenticationHeadroom, +} from "./JsonAuthenticationHeadroom.js"; +describe("JSON authentication headroom", () => { + it("estimates parsed JSON at no less than its UTF-16 text", () => { + const text = "x".repeat(1000); + expect(estimateJsonValueBytes(text)).toBeGreaterThanOrEqual(2000); + expect( + estimateJsonValueBytes({ outer: [text, { inner: text }] }), + ).toBeGreaterThan(2 * estimateJsonValueBytes(text)); + }); + + it("refuses authentication when live heap headroom cannot hold its copies", () => { + const value = { normalized_result: "y".repeat(10_000) }; + expect( + requireAuthenticationHeadroom(value, { + operation: "compare_source_to_bundle", + path: "/tmp/capture.json", + description: "Cannot authenticate graph", + availableBytes: 1024, + }), + ).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisResourceConstraintError", + resource: "memory", + reportedLimits: { + input_path: "/tmp/capture.json", + available_heap_bytes: 1024, + }, + }, + }); + expect( + requireAuthenticationHeadroom(value, { + operation: "compare_source_to_bundle", + path: "/tmp/capture.json", + description: "Cannot authenticate graph", + availableBytes: 1 << 30, + }), + ).toEqual({ ok: true, value: undefined }); + }); +}); diff --git a/src/application/JsonAuthenticationHeadroom.ts b/src/application/JsonAuthenticationHeadroom.ts new file mode 100644 index 0000000000..36c161978e --- /dev/null +++ b/src/application/JsonAuthenticationHeadroom.ts @@ -0,0 +1,58 @@ +import { getHeapStatistics } from "node:v8"; +import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { err, ok, type Result } from "../domain/result.js"; + +/** Authentication clones parsed JSON and hashes its canonical commitment. */ +const AUTHENTICATION_COPIES = 2; + +/** Conservative bytes a parsed JSON value occupies: UTF-16 text plus container and slot overhead. */ +export const estimateJsonValueBytes = (value: unknown): number => { + let bytes = 0; + const stack: unknown[] = [value]; + while (stack.length > 0) { + const item = stack.pop(); + if (typeof item === "string") bytes += 64 + item.length * 2; + else if (Array.isArray(item)) { + bytes += 128 + item.length * 16; + for (const entry of item) stack.push(entry); + } else if (typeof item === "object" && item !== null) { + bytes += 128; + for (const [key, entry] of Object.entries(item)) { + bytes += 64 + key.length * 2; + stack.push(entry); + } + } else bytes += 16; + } + return bytes; +}; + +/** Refuse before authentication when live heap headroom cannot hold its validation and commitment working copies. */ +export const requireAuthenticationHeadroom = ( + value: unknown, + options: { + readonly operation: string; + readonly path: string; + readonly description: string; + readonly availableBytes?: number; + }, +): Result => { + const availableBytes = + options.availableBytes ?? getHeapStatistics().total_available_size; + const valueBytes = estimateJsonValueBytes(value); + const requiredBytes = valueBytes * AUTHENTICATION_COPIES; + if (requiredBytes <= availableBytes) return ok(undefined); + return err( + new AnalysisResourceConstraintError( + options.operation, + "memory", + options.description, + { + input_path: options.path, + projected_value_bytes: valueBytes, + authentication_bytes: requiredBytes, + available_heap_bytes: availableBytes, + }, + ), + ); +}; diff --git a/src/application/JsonFiles.ts b/src/application/JsonFiles.ts index 1ae2b2b32b..fa3729e07a 100644 --- a/src/application/JsonFiles.ts +++ b/src/application/JsonFiles.ts @@ -47,7 +47,7 @@ export interface TextWriteCancellation { type TextWriteResult = Result<{ readonly path: string; readonly bytes: number }, Failure>; -/** Read JSON data from a regular file at the caller-supplied path. */ +/** Read JSON data without changing the caller's existing data-only contract. */ export const readJsonFile = async ( path: string, signal?: AbortSignal, @@ -57,11 +57,33 @@ export const readJsonFile = async ( unknown, EvidenceFileError | AnalysisResourceConstraintError | AnalysisCancelledError > +> => { + const read = await readJsonFileWithSource(path, signal, resources); + return read.ok ? ok(read.value.data) : read; +}; + +/** Read data and the absolute selected path at the same owning regular-file boundary. */ +export const readJsonFileWithSource = ( + path: string, + signal?: AbortSignal, + resources?: ArtifactResourceScope, +): Promise< + Result< + { readonly data: unknown; readonly path: string }, + EvidenceFileError | AnalysisResourceConstraintError | AnalysisCancelledError + > > => readJsonFileInput( path, - (selected) => - readJsonInputFile(selected, "read_evidence_file", signal, resources), + async (selected) => { + const read = await readJsonInputFile( + selected, + "read_evidence_file", + signal, + resources, + ); + return read.ok ? ok({ data: read.value, path: selected }) : read; + }, signal, ); diff --git a/src/application/analysisView/evidenceFileSource.test.ts b/src/application/analysisView/evidenceFileSource.test.ts index ac593127e0..b9a71de605 100644 --- a/src/application/analysisView/evidenceFileSource.test.ts +++ b/src/application/analysisView/evidenceFileSource.test.ts @@ -10,41 +10,9 @@ import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; import { JsonInputPhysicalReadFailure } from "../JsonInputFile.js"; import { failFileReadAfterAbort } from "../../../tests/support/failFileReadAfterAbort.js"; import { holdFirstFileRead } from "../../../tests/support/holdFirstFileRead.js"; -import { - estimateJsonValueBytes, - readEvidenceFileSource, - requireAuthenticationHeadroom, -} from "./evidenceFileSource.js"; +import { readEvidenceFileSource } from "./evidenceFileSource.js"; describe("evidence-file source", () => { - it("estimates parsed JSON at no less than its UTF-16 text", () => { - const text = "x".repeat(1000); - expect(estimateJsonValueBytes(text)).toBeGreaterThanOrEqual(2000); - expect( - estimateJsonValueBytes({ outer: [text, { inner: text }] }), - ).toBeGreaterThan(2 * estimateJsonValueBytes(text)); - }); - - it("refuses authentication when live heap headroom cannot hold its copies", () => { - const value = { normalized_result: "y".repeat(10_000) }; - expect( - requireAuthenticationHeadroom(value, "/tmp/capture.json", 1024), - ).toMatchObject({ - ok: false, - error: { - _tag: "AnalysisResourceConstraintError", - resource: "memory", - reportedLimits: { - input_path: "/tmp/capture.json", - available_heap_bytes: 1024, - }, - }, - }); - expect( - requireAuthenticationHeadroom(value, "/tmp/capture.json", 1 << 30), - ).toEqual({ ok: true, value: undefined }); - }); - it("refuses a cancelled request before reading and reads within the caller's resource scope", async () => { const controller = new AbortController(); const reason = new JsonInputPhysicalReadFailure( diff --git a/src/application/analysisView/evidenceFileSource.ts b/src/application/analysisView/evidenceFileSource.ts index a4755234b6..303e6726cb 100644 --- a/src/application/analysisView/evidenceFileSource.ts +++ b/src/application/analysisView/evidenceFileSource.ts @@ -1,12 +1,9 @@ -import { getHeapStatistics } from "node:v8"; +import { requireAuthenticationHeadroom } from "../JsonAuthenticationHeadroom.js"; import { z } from "zod"; import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; -import { - AnalysisCancelledError, - AnalysisResourceConstraintError, -} from "../../domain/analysisErrorCore.js"; +import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; import { INSPECT_ANALYSIS_VIEW_OPERATION } from "../../domain/analysisView/analysisViewErrors.js"; import { err, ok, type Result } from "../../domain/result.js"; import { parseEvidence, type Evidence } from "../../domain/evidence.js"; @@ -23,54 +20,6 @@ export interface EvidenceFileReadOptions { readonly resources?: ArtifactResourceScope; } -/** Authentication clones the parsed Evidence and hashes its canonical JSON. */ -const AUTHENTICATION_COPIES = 2; - -/** Conservative bytes a parsed JSON value occupies: UTF-16 text plus container and slot overhead. */ -export const estimateJsonValueBytes = (value: unknown): number => { - let bytes = 0; - const stack: unknown[] = [value]; - while (stack.length > 0) { - const item = stack.pop(); - if (typeof item === "string") bytes += 64 + item.length * 2; - else if (Array.isArray(item)) { - bytes += 128 + item.length * 16; - for (const entry of item) stack.push(entry); - } else if (typeof item === "object" && item !== null) { - bytes += 128; - for (const [key, entry] of Object.entries(item)) { - bytes += 64 + key.length * 2; - stack.push(entry); - } - } else bytes += 16; - } - return bytes; -}; - -/** Refuse before authentication when live heap headroom cannot hold its copies of the parsed value. */ -export const requireAuthenticationHeadroom = ( - value: unknown, - path: string, - availableBytes: number = getHeapStatistics().total_available_size, -): Result => { - const valueBytes = estimateJsonValueBytes(value); - const requiredBytes = valueBytes * AUTHENTICATION_COPIES; - if (requiredBytes <= availableBytes) return ok(undefined); - return err( - new AnalysisResourceConstraintError( - INSPECT_ANALYSIS_VIEW_OPERATION, - "memory", - "Insufficient heap headroom to authenticate the Evidence file", - { - input_path: path, - projected_value_bytes: valueBytes, - authentication_bytes: requiredBytes, - available_heap_bytes: availableBytes, - }, - ), - ); -}; - /** Read and authenticate one Evidence file under the request's cancellation and resource scope. */ export const readEvidenceFileSource = async ( path: string, @@ -99,7 +48,11 @@ export const readEvidenceFileSource = async ( cause: signal.reason, }), ); - const headroom = requireAuthenticationHeadroom(loaded.value, path); + const headroom = requireAuthenticationHeadroom(loaded.value, { + operation: INSPECT_ANALYSIS_VIEW_OPERATION, + path, + description: "Insufficient heap headroom to authenticate the Evidence file", + }); if (!headroom.ok) return headroom; try { return ok(parseEvidence(loaded.value)); diff --git a/src/application/javascript/JavaScriptApplicationWorkflowService.ts b/src/application/javascript/JavaScriptApplicationWorkflowService.ts index b76259801a..c04703d611 100644 --- a/src/application/javascript/JavaScriptApplicationWorkflowService.ts +++ b/src/application/javascript/JavaScriptApplicationWorkflowService.ts @@ -1,3 +1,4 @@ +import type { ResolvedSourceToBundleReference } from "./SourceToBundleReference.js"; import { z } from "zod"; import { AnalysisProtocolError } from "../../domain/analysisErrorCore.js"; @@ -184,7 +185,8 @@ export const compareApplicationVersionsEvidenceValidated = ( /** Compare source and bundle inputs already parsed by a trusted adapter. */ export const compareSourceToBundleEvidenceValidated = ( - input: z.output, + input: ResolvedSourceToBundleReference & + Pick, "application">, ): Result => { const operation = "compare_source_to_bundle"; const applicationResult = parseApplicationGraphEvidence(input.application, [ @@ -209,6 +211,9 @@ export const compareSourceToBundleEvidenceValidated = ( { reference_root_sha256: input.reference.root_sha256, application_evidence_id: application.evidence.evidence_id, + ...(input.reference_source === undefined + ? {} + : { reference_source: input.reference_source }), }, result, ), diff --git a/src/application/javascript/SourceToBundleReference.ts b/src/application/javascript/SourceToBundleReference.ts new file mode 100644 index 0000000000..c6596a7837 --- /dev/null +++ b/src/application/javascript/SourceToBundleReference.ts @@ -0,0 +1,93 @@ +import { requireAuthenticationHeadroom } from "../JsonAuthenticationHeadroom.js"; +import { setImmediate } from "node:timers/promises"; +import type { z } from "zod"; +import type { compareSourceToBundleCliInputSchema } from "../../contracts/javascript/applicationWorkflowInputContracts.js"; +import { + historicalSourceGraphSchema, + type HistoricalSourceGraph, +} from "../../domain/referenceSourceGraph.js"; +import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; +import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { readJsonFileWithSource } from "../JsonFiles.js"; + +/** Historical facts plus the file-origin metadata supplied by the reader, if any. */ +export type ResolvedSourceToBundleReference = { + readonly reference: HistoricalSourceGraph; + readonly reference_source?: { + readonly kind: "file"; + readonly path: string; + readonly requested_path: string; + }; +}; + +/** Resolve one caller-selected historical graph using the shared bounded regular-file reader. No file cache. */ +export const resolveSourceToBundleReference = async ( + reference: z.output["reference"], + signal?: AbortSignal, + resources?: ArtifactResourceScope, +): Promise> => { + const cancelled = () => signal?.aborted === true; + if (cancelled()) + return err( + new AnalysisCancelledError("compare_source_to_bundle", { + cause: signal?.reason, + }), + ); + if (!("kind" in reference)) return ok({ reference }); + const read = await readJsonFileWithSource(reference.path, signal, resources); + if (!read.ok) { + return read.error instanceof AnalysisCancelledError + ? err( + new AnalysisCancelledError("compare_source_to_bundle", { + cause: read.error.cause, + }), + ) + : read; + } + await setImmediate(); + if (cancelled()) + return err( + new AnalysisCancelledError("compare_source_to_bundle", { + cause: signal?.reason, + }), + ); + const headroom = requireAuthenticationHeadroom(read.value.data, { + operation: "compare_source_to_bundle", + path: read.value.path, + description: + "Insufficient heap headroom to authenticate the historical source graph", + }); + if (!headroom.ok) return headroom; + const parsed = historicalSourceGraphSchema.safeParse(read.value.data); + // Dispatch aborts queued while synchronous validation and commitment hashing ran. + await setImmediate(); + if (cancelled()) + return err( + new AnalysisCancelledError("compare_source_to_bundle", { + cause: signal?.reason, + }), + ); + if (!parsed.success) + return err( + analysisInputErrorFromIssues( + "compare_source_to_bundle", + parsed.error.issues.map((issue) => ({ + ...issue, + path: ["reference", ...issue.path], + })), + { reference: read.value.data }, + { cause: parsed.error }, + ), + ); + return ok({ + reference: parsed.data, + reference_source: { + kind: "file", + path: read.value.path, + requested_path: reference.path, + }, + }); +}; diff --git a/src/cli/applicationCommands.ts b/src/cli/applicationCommands.ts index 75c63e09ee..4561c0f7df 100644 --- a/src/cli/applicationCommands.ts +++ b/src/cli/applicationCommands.ts @@ -14,7 +14,7 @@ import type { Logger } from "pino"; import { traceApplicationFeatureInputSchema } from "../domain/javascript/javascriptFeatureTraceSchemas.js"; import { traceJavaScriptSemanticsInputSchema } from "../domain/javascript/javascriptSemanticTraceSchemas.js"; import { compareApplicationVersionsInputSchema } from "../domain/javascript/javascriptApplicationVersionComparisonSchemas.js"; -import { compareSourceToBundleInputSchema } from "../domain/javascript/sourceToBundleComparisonSchemas.js"; +import { registerSourceToBundleCommand } from "./sourceToBundleCommand.js"; import { compareJavaScriptExportShapesInputSchema } from "../domain/javascript/javascriptExportShapeComparisonSchemas.js"; import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; import { reconstructionObligationLedgerInputSchema } from "../domain/reconstructionObligationLedgerSchemas.js"; @@ -65,19 +65,7 @@ export const registerApplicationCommands = ( return compareApplicationVersionsEvidenceValidated(input); }, }); - registerJsonCommand({ - cli, - logger, - name: CLI_COMMANDS.compareSourceToBundle, - description: - "Compare committed historical source with authenticated application Evidence", - inputSchema: compareSourceToBundleInputSchema, - workflow: async (input) => { - const { compareSourceToBundleEvidenceValidated } = - await import("../application/javascript/JavaScriptApplicationWorkflowService.js"); - return compareSourceToBundleEvidenceValidated(input); - }, - }); + registerSourceToBundleCommand(cli, logger); registerJsonCommand({ cli, logger, diff --git a/src/cli/sourceToBundleCommand.ts b/src/cli/sourceToBundleCommand.ts new file mode 100644 index 0000000000..39eb4e0d5a --- /dev/null +++ b/src/cli/sourceToBundleCommand.ts @@ -0,0 +1,74 @@ +import { z } from "incur"; +import type { Logger } from "pino"; +import type { CliInstance } from "./types.js"; +import { CLI_COMMANDS } from "../cliCommandNames.js"; +import { parseCliJsonInput } from "../cliJsonInput.js"; +import { logCliCommand } from "../cliLogging.js"; +import { compareSourceToBundleCliInputSchema } from "../contracts/javascript/applicationWorkflowInputContracts.js"; +import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { withCommandCancellation } from "./commandCancellation.js"; +import { finishAnalysisCommand } from "./finishAnalysisCommand.js"; + +/** CLI file transport delegates graph authentication and comparison to the same owners as MCP. */ +export const registerSourceToBundleCommand = ( + cli: CliInstance, + logger: Logger, +): void => { + const name = CLI_COMMANDS.compareSourceToBundle; + cli.command(name, { + description: + "Compare historical source (inline graph or reference {kind:file,path}) with authenticated application Evidence. Save the complete JSON to query with inspect-analysis-view.", + args: z.object({ + inputJson: z.string().describe("Inline workflow JSON or JSON file path"), + }), + run: ({ args }) => + withCommandCancellation((signal) => + logCliCommand(logger, name, async () => { + const input = await parseCliJsonInput(args.inputJson, name, signal); + if (!input.ok) return input.error; + const parsed = compareSourceToBundleCliInputSchema.safeParse( + input.value, + ); + if (!parsed.success) + return { + error: "Application workflow failed", + ...projectAnalysisError( + analysisInputErrorFromIssues( + name, + parsed.error.issues, + input.value, + ), + ), + }; + const resources = new ArtifactResourceScope(); + const result = await finishAnalysisCommand( + async () => { + const { resolveSourceToBundleReference } = + await import("../application/javascript/SourceToBundleReference.js"); + const reference = await resolveSourceToBundleReference( + parsed.data.reference, + signal, + resources, + ); + if (!reference.ok) return reference; + const { compareSourceToBundleEvidenceValidated } = + await import("../application/javascript/JavaScriptApplicationWorkflowService.js"); + return compareSourceToBundleEvidenceValidated({ + ...reference.value, + application: parsed.data.application, + }); + }, + resources, + "javascript-application-workflow", + "compare_source_to_bundle", + signal, + ); + return "evidence_id" in result + ? result + : { error: "Application workflow failed", ...result }; + }), + ), + }); +}; diff --git a/src/contracts/applicationToolContracts.ts b/src/contracts/applicationToolContracts.ts index ade29e9600..d4f9555367 100644 --- a/src/contracts/applicationToolContracts.ts +++ b/src/contracts/applicationToolContracts.ts @@ -1,3 +1,8 @@ +import { z } from "zod"; +import { + sourceToBundleSummaryViewResultSchema, + sourceToBundlePageViewResultSchema, +} from "../domain/analysisView/analysisView.js"; import { applicationVersionComparisonResultSchema } from "../domain/javascript/javascriptApplicationVersionComparisonSchemas.js"; import { applicationFeatureTraceResultSchema } from "../domain/javascript/javascriptFeatureTraceSchemas.js"; import { javaScriptSemanticTraceResultSchema } from "../domain/javascript/javascriptSemanticTraceSchemas.js"; @@ -52,7 +57,11 @@ const comparisonOutputSchema = evidenceResultOf( applicationVersionComparisonResultSchema, ); const sourceToBundleOutputSchema = evidenceResultOf( - sourceToBundleComparisonResultSchema, + z.union([ + sourceToBundleComparisonResultSchema, + sourceToBundleSummaryViewResultSchema, + sourceToBundlePageViewResultSchema, + ]), ); const exportShapeComparisonOutputSchema = evidenceResultOf( javaScriptExportShapeComparisonResultSchema, @@ -144,7 +153,7 @@ export const APPLICATION_TOOL_CONTRACTS = [ name: "compare_source_to_bundle", ...toolContractMetadata("compare_source_to_bundle"), description: - "Compare a cryptographically committed HistoricalSourceGraph with authenticated JavaScript Application Graph Evidence supplied inline or by exact same-session retained reference. Uses explicit exact-digest, source-map path, current-path, suffix, and basename signals with stable weights. Classifies unchanged, modified, removed, split, merged, duplicated, and unknown; incomplete coverage and ambiguous weak signals never become absence or forced matches.", + "Compare a cryptographically committed HistoricalSourceGraph supplied inline or as reference {kind:file,path:absolute-local-JSON-path} with authenticated JavaScript Application Graph Evidence supplied inline or by exact same-session retained reference. Omitted view preserves the full report. Optional view {kind:summary} or {kind:source-to-bundle,collection:mappings|candidates,source_paths?,mapping_ids?,statuses?,fields?,offset?,limit?} retains the full parent and returns selected facts. Query it again using inspect_analysis_view. Empty filters select none; identity fields always remain. Candidate signals are opt-in; classifications use the full comparison. Page coverage never upgrades source coverage. Uses explicit exact-digest, source-map path, current-path, suffix, and basename signals with stable weights. Classifies unchanged, modified, removed, split, merged, duplicated, and unknown; incomplete coverage and ambiguous weak signals never become absence or forced matches.", kind: "application", inputSchema: compareSourceToBundleRequestSchema, outputSchema: sourceToBundleOutputSchema, diff --git a/src/contracts/javascript/applicationWorkflowInputContracts.ts b/src/contracts/javascript/applicationWorkflowInputContracts.ts index 4b50621ed0..53773c2d1a 100644 --- a/src/contracts/javascript/applicationWorkflowInputContracts.ts +++ b/src/contracts/javascript/applicationWorkflowInputContracts.ts @@ -1,3 +1,9 @@ +import { + absoluteLocalPathStringSchema, + localPathStringSchema, +} from "../../domain/localPath.js"; +import { sourceToBundleViewRequestSchema } from "../../domain/analysisView/sourceToBundleView.js"; +import { evidenceSchema } from "../../domain/evidence.js"; import { z } from "zod"; import { evidenceInputSchema } from "../evidenceInputContracts.js"; @@ -40,14 +46,34 @@ export const compareApplicationVersionsRequestSchema = z.strictObject({ right: evidenceInputSchema, }); -const compareSourceToBundleFacts = { - reference: compareSourceToBundleInputSchema.shape.reference, -} as const; - -/** Historical-source comparison accepting inline or same-session application Evidence. */ +const referenceFile = z.strictObject({ + kind: z.literal("file"), + path: localPathStringSchema, +}); +const sourceToBundleSelection = { + view: z + .union([ + z.strictObject({ kind: z.literal("summary") }), + sourceToBundleViewRequestSchema, + ]) + .optional(), +}; +/** Historical graph JSON is a transport alternative to the inline graph, authenticated by the same graph parser. */ +export const compareSourceToBundleCliInputSchema = z.strictObject({ + reference: z.union([ + compareSourceToBundleInputSchema.shape.reference, + referenceFile, + ]), + application: evidenceSchema, +}); +/** The MCP host accepts absolute local paths; omitted view preserves the full report. */ export const compareSourceToBundleRequestSchema = z.strictObject({ - ...compareSourceToBundleFacts, + reference: z.union([ + compareSourceToBundleInputSchema.shape.reference, + referenceFile.extend({ path: absoluteLocalPathStringSchema }), + ]), application: evidenceInputSchema, + ...sourceToBundleSelection, }); const compareJavaScriptExportShapesFacts = { diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index bcf6f93c39..eb982680b2 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -1,3 +1,9 @@ +import { + sourceToBundleViewRequestSchema, + sourceToBundleViewItemsSchema, + sourceToBundleViewContextSchema, + projectSourceToBundleView, +} from "./sourceToBundleView.js"; import { gpuCodeObjectsViewRequestSchema, gpuCodeObjectsSummarySchema, @@ -290,6 +296,7 @@ const nativeViewSchema = z.strictObject({ /** Caller-selected projection of already completed analysis Evidence. */ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ nativeFunctionMatchingViewRequestSchema, + sourceToBundleViewRequestSchema, summaryViewSchema, goBinaryViewRequestSchema, rustBinaryViewRequestSchema, @@ -669,8 +676,25 @@ export const nativeFunctionMatchingSummaryViewResultSchema = z.strictObject({ summary: nativeFunctionMatchingViewContextSchema, }); +export const sourceToBundleSummaryViewResultSchema = z.strictObject({ + ...summaryViewResultBase, + artifact: z.null(), + parent_operation: z.literal("compare_source_to_bundle"), + summary: sourceToBundleViewContextSchema, +}); +export const sourceToBundlePageViewResultSchema = z.strictObject({ + ...viewResultBase, + ...sourceToBundleViewContextSchema.shape, + kind: z.literal("source-to-bundle"), + artifact: z.null(), + parent_operation: z.literal("compare_source_to_bundle"), + view: sourceToBundleViewRequestSchema, + items: sourceToBundleViewItemsSchema, +}); + /** Projected facts for one selected view of retained analysis Evidence. */ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ + sourceToBundlePageViewResultSchema, z.strictObject({ ...viewResultBase, ...nativeFunctionMatchingViewContextSchema.shape, @@ -736,6 +760,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ gpuCodeObjectsSummaryViewResultSchema, nativeInventorySummaryViewResultSchema, nativeFunctionMatchingSummaryViewResultSchema, + sourceToBundleSummaryViewResultSchema, ]), z.strictObject({ ...viewResultBase, @@ -1026,6 +1051,7 @@ const retainedViewProjectors = new Map([ ["inspect_model_artifact", projectModelArtifactView], ["inspect_gpu_code_objects", projectGpuCodeObjectsView], ["match_native_functions", projectNativeFunctionMatchingView], + ["compare_source_to_bundle", projectSourceToBundleView], ["inspect_artifact", projectArtifactInventoryView], ["inventory_artifact", projectArtifactInventoryView], ]); diff --git a/src/domain/analysisView/sourceToBundleView.ts b/src/domain/analysisView/sourceToBundleView.ts new file mode 100644 index 0000000000..7f7541d07e --- /dev/null +++ b/src/domain/analysisView/sourceToBundleView.ts @@ -0,0 +1,265 @@ +import { z } from "zod"; +import type { JsonValue } from "../jsonValue.js"; +import { + sourceToBundleComparisonResultSchema, + type SourceToBundleComparisonResult, +} from "../javascript/sourceToBundleComparisonSchemas.js"; +import { isImmutableJsonSnapshot } from "../immutableJson.js"; +import { err, ok, type Result } from "../result.js"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import type { + AnalysisViewParent, + AnalysisViewRequest, + UnsignedAnalysisView, +} from "./analysisView.js"; +import { analysisViewInputError } from "./analysisViewErrors.js"; +import { parseAnalysisViewResult } from "./analysisViewParent.js"; +import { + boundedViewMessages, + completeWithinViewCoverage, + pageViewCoverage, +} from "./analysisViewCoverage.js"; +import { selectViewRows } from "./selectViewRows.js"; + +const mapping = sourceToBundleComparisonResultSchema.shape.items.element; +const candidate = mapping.shape.candidates.element; +const identity = mapping + .pick({ mapping_id: true, source_path: true }) + .extend({ source_index: z.number().int().nonnegative() }); +const mappingRow = mapping + .omit({ candidates: true, current_node_ids: true }) + .extend({ + source_index: identity.shape.source_index, + candidate_count: z.number().int().nonnegative(), + mapped_node_count: z.number().int().nonnegative(), + }); +const candidateRow = candidate.extend({ + ...identity.shape, + candidate_index: z.number().int().nonnegative(), + mapped: z.boolean(), +}); +const page = { + kind: z.literal("source-to-bundle"), + source_paths: z.array(z.string()).optional(), + mapping_ids: z.array(mapping.shape.mapping_id).optional(), + statuses: z.array(mapping.shape.status).optional(), + offset: z.number().int().nonnegative().default(0), + limit: z.number().int().positive().default(64), +}; +/** Select completed mapping facts or their candidate relationships without reclassifying a subset. */ +export const sourceToBundleViewRequestSchema = z.discriminatedUnion( + "collection", + [ + z.strictObject({ + ...page, + collection: z.literal("mappings"), + fields: z.array(mappingRow.keyof()).optional(), + }), + z.strictObject({ + ...page, + collection: z.literal("candidates"), + fields: z.array(candidateRow.keyof()).optional(), + }), + ], +); +export const sourceToBundleViewItemsSchema = z.union([ + z.array( + mappingRow + .partial() + .required({ mapping_id: true, source_path: true, source_index: true }), + ), + z.array( + candidateRow.partial().required({ + mapping_id: true, + source_path: true, + source_index: true, + candidate_index: true, + current_node_id: true, + mapped: true, + }), + ), +]); +export const sourceToBundleViewContextSchema = + sourceToBundleComparisonResultSchema + .pick({ + comparison_id: true, + reference: true, + application: true, + scoring: true, + summary: true, + evidence_links: true, + }) + .extend({ + source_coverage: sourceToBundleComparisonResultSchema.shape.coverage, + unmapped_current_node_count: z.number().int().nonnegative(), + }); + +const checked = new WeakMap(); +const checkedReport = ( + parent: AnalysisViewParent, +): Result => { + const source = parent.normalizedResult; + const reusable = + typeof source === "object" && + source !== null && + parent.sealedResult === true && + isImmutableJsonSnapshot(source); + const cached = reusable ? checked.get(source) : undefined; + if (cached !== undefined) return ok(cached); + const parsed = parseAnalysisViewResult( + sourceToBundleComparisonResultSchema, + parent, + ); + if (!parsed.ok) return parsed; + const report = reusable + ? (source as SourceToBundleComparisonResult) + : parsed.value; + if (reusable) checked.set(source, report); + return ok(report); +}; +const selected = (values: readonly string[] | undefined) => + values === undefined ? undefined : new Set(values); +const columns = ( + row: object, + fields: readonly string[], +): Record => { + const source = row as Record; + return Object.fromEntries(fields.map((field) => [field, source[field]!])); +}; + +/** Page facts owned by a complete comparison; page exhaustion does not upgrade source coverage. */ +export const projectSourceToBundleView = ( + parent: AnalysisViewParent, + view: AnalysisViewRequest, +): Result => { + if (view.kind !== "summary" && view.kind !== "source-to-bundle") + return err( + analysisViewInputError([ + { + path: ["view"], + reason: "invalid_value", + message: + "Source-to-bundle comparisons support summary and source-to-bundle pages", + }, + ]), + ); + const parsed = checkedReport(parent); + if (!parsed.ok) return parsed; + const report = parsed.value; + const { + items: _items, + unmapped_current_node_ids, + coverage: source_coverage, + limitations: _limitations, + ...facts + } = report; + const context = { + ...facts, + source_coverage, + unmapped_current_node_count: unmapped_current_node_ids.length, + }; + const shared = { + parent_evidence_id: parent.evidenceId, + parent_operation: "compare_source_to_bundle" as const, + parent_digest: parent.evidenceId.slice(3), + artifact: null, + limitations: boundedViewMessages( + [...parent.limitations, ...report.limitations], + parent.evidenceId, + "limitations", + ), + unknowns: [ + "Candidate correspondence is not behavioral equivalence. Exhausting a selected page does not establish absence outside the examined inputs.", + ], + }; + if (view.kind === "summary") + return ok({ + kind: "summary", + view, + ...shared, + summary: context, + coverage: completeWithinViewCoverage(1, 1), + }); + const paths = selected(view.source_paths), + ids = selected(view.mapping_ids), + statuses = selected(view.statuses); + const matches = (row: SourceToBundleComparisonResult["items"][number]) => + (paths === undefined || paths.has(row.source_path)) && + (ids === undefined || ids.has(row.mapping_id)) && + (statuses === undefined || statuses.has(row.status)); + const fields = + view.fields ?? + (view.collection === "mappings" + ? [ + "status", + "confidence", + "source_sha256", + "source_language", + "candidate_count", + "mapped_node_count", + ] + : ["current_node_kind", "score", "confidence"]); + let items: z.output, total: number; + if (view.collection === "mappings") { + const page = selectViewRows( + report.items, + view, + matches, + (row, source_index) => ({ + ...columns( + { + ...row, + source_index, + candidate_count: row.candidates.length, + mapped_node_count: row.current_node_ids.length, + }, + fields, + ), + mapping_id: row.mapping_id, + source_path: row.source_path, + source_index, + }), + ); + items = page.rows; + total = page.total; + } else { + items = []; + total = 0; + for (const [source_index, row] of report.items.entries()) { + if (!matches(row)) continue; + const start = Math.max(0, view.offset - total); + const end = Math.min( + row.candidates.length, + start + view.limit - items.length, + ); + if (start < end) { + const mapped = new Set(row.current_node_ids); + for ( + let candidate_index = start; + candidate_index < end; + candidate_index++ + ) { + const candidate = row.candidates[candidate_index]!; + items.push({ + ...columns(candidate, fields), + mapping_id: row.mapping_id, + source_path: row.source_path, + source_index, + candidate_index, + current_node_id: candidate.current_node_id, + mapped: mapped.has(candidate.current_node_id), + }); + } + } + total += row.candidates.length; + } + } + return ok({ + kind: "source-to-bundle", + view, + ...shared, + ...context, + items, + coverage: pageViewCoverage(view.offset, items.length, total), + }); +}; diff --git a/src/server/createServer.ts b/src/server/createServer.ts index c2768c5f90..fa98728744 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -1012,6 +1012,7 @@ const registerObservationTools = ({ options.optionalProviderLoadFailures?.javascriptRuntimeObservation, }); registerApplicationTools(server, { + artifactResources, evidenceById, logger, recordEvidence, diff --git a/src/server/registerApplicationTools/compareSourceToBundle.ts b/src/server/registerApplicationTools/compareSourceToBundle.ts index 5ca65ee1dc..30a766e124 100644 --- a/src/server/registerApplicationTools/compareSourceToBundle.ts +++ b/src/server/registerApplicationTools/compareSourceToBundle.ts @@ -1,3 +1,11 @@ +import { setImmediate } from "node:timers/promises"; +import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; +import { ok } from "../../domain/result.js"; +import { resolveSourceToBundleReference } from "../../application/javascript/SourceToBundleReference.js"; +import { + requireAnalysisViewRetention, + inspectAnalysisViewValidated, +} from "../../application/analysisView/AnalysisViewService.js"; import type { EvidenceMcpServer } from "../EvidenceMcpServer.js"; import { runAdmittedToolOperation } from "../admittedToolOperation.js"; import { resolveApplicationEvidenceRequest } from "../../application/EvidenceInputResolver.js"; @@ -8,7 +16,7 @@ import { applicationToolContract } from "../../contracts/applicationToolContract import { sourceToBundleComparisonResultSchema } from "../../domain/javascript/sourceToBundleComparisonSchemas.js"; import { logToolExecution } from "../toolLogging.js"; import { toolRegistrationOptions } from "../toolRegistrationOptions.js"; -import { recordResult } from "./helpers.js"; +import { recordApplicationResult } from "./helpers.js"; import type { ApplicationToolRegistration } from "./types.js"; const contract = applicationToolContract("compare_source_to_bundle"); @@ -28,13 +36,31 @@ export const registerCompareSourceToBundleTool = ( contract.name, context.mcpReq.signal, async () => { + if (input.view !== undefined) { + const retention = requireAnalysisViewRetention( + contract.name, + options, + ); + if (!retention.ok) + return server.delivery.toCallToolResult(retention, contract); + } const resolved = resolveApplicationEvidenceRequest( input, options.evidenceById, ); if (!resolved.ok) return server.delivery.toCallToolResult(resolved, contract); - const parsed = resolved.value; + const reference = await resolveSourceToBundleReference( + resolved.value.reference, + context.mcpReq.signal, + options.artifactResources, + ); + if (!reference.ok) + return server.delivery.toCallToolResult(reference, contract); + const parsed = { + ...reference.value, + application: resolved.value.application, + }; const result = await logToolExecution( options.logger, contract.name, @@ -43,23 +69,59 @@ export const registerCompareSourceToBundleTool = ( ); if (!result.ok) return server.delivery.toCallToolResult(result, contract); + const parent = result.value; + const comparison = sourceToBundleComparisonResultSchema.parse( + parent.normalized_result, + ); + // Stage the projection against the owned parent before committing any session facts. + const selected = + input.view === undefined + ? ok(parent) + : inspectAnalysisViewValidated( + { + source: { + kind: "retained-evidence", + evidence_id: parent.evidence_id, + }, + view: input.view, + }, + (id) => + id === parent.evidence_id + ? parent + : options.evidenceById?.(id), + ); + await setImmediate(); + if (context.mcpReq.signal.aborted) + return server.delivery.toErrorToolResult( + new AnalysisCancelledError(contract.name, { + cause: context.mcpReq.signal.reason, + }), + contract.returnsTargetContent, + ); + if (!selected.ok) + return server.delivery.toCallToolResult(selected, contract); const recorded = recordSessionEvidenceSources( options.recordEvidence, [parsed.application], ); if (!recorded.ok) return server.delivery.toCallToolResult(recorded, contract); - const comparison = sourceToBundleComparisonResultSchema.parse( - result.value.normalized_result, - ); - return recordResult( - { ...options, delivery: server.delivery }, - contract, - result.value, + const saved = recordApplicationResult( + options, + parent, comparison.summary.unknown > 0 ? "source-to-bundle-comparison" : undefined, ); + if (saved !== undefined && !saved.ok) + return server.delivery.toCallToolResult(saved, contract); + return server.delivery.toEvidenceToolResult( + selected.value, + contract, + input.view === undefined + ? saved + : options.recordEvidence?.(selected.value), + ); }, ), ); diff --git a/src/server/registerApplicationTools/helpers.ts b/src/server/registerApplicationTools/helpers.ts index dd881887bd..747ef1636f 100644 --- a/src/server/registerApplicationTools/helpers.ts +++ b/src/server/registerApplicationTools/helpers.ts @@ -14,16 +14,26 @@ export const recordResult = ( | "javascript-export-shape" | "source-to-bundle-comparison", ) => { - const recorded = - unknownKind === undefined - ? options.recordEvidence?.(evidence) - : options.recordEvidenceWithUnknown?.( - evidence, - unknownRegistration(evidence, unknownKind), - ); + const recorded = recordApplicationResult(options, evidence, unknownKind); return options.delivery.toEvidenceToolResult(evidence, contract, recorded); }; +/** Record the complete parent before choosing a delivery projection. */ +export const recordApplicationResult = ( + options: ApplicationToolRegistration, + evidence: Evidence, + unknownKind?: + | "application-version-comparison" + | "javascript-export-shape" + | "source-to-bundle-comparison", +) => + unknownKind === undefined + ? options.recordEvidence?.(evidence) + : options.recordEvidenceWithUnknown?.( + evidence, + unknownRegistration(evidence, unknownKind), + ); + const unknownRegistration = ( evidence: Evidence, kind: diff --git a/src/server/registerApplicationTools/types.ts b/src/server/registerApplicationTools/types.ts index 27fbf70c73..a0979aae58 100644 --- a/src/server/registerApplicationTools/types.ts +++ b/src/server/registerApplicationTools/types.ts @@ -1,3 +1,4 @@ +import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; import type { EvidenceWriter, EvidenceReader, @@ -8,6 +9,7 @@ import type { WithAdmittedAnalysis } from "../analysisAdmission.js"; /** Shared services for registering JavaScript application graph workflows. */ export interface ApplicationToolRegistration { + readonly artifactResources?: ArtifactResourceScope; readonly withAdmittedAnalysis?: WithAdmittedAnalysis; readonly logger: Logger; readonly evidenceById: EvidenceReader["evidenceById"] | undefined; diff --git a/tests/acceptance/applications/applicationWorkflowCli.test.ts b/tests/acceptance/applications/applicationWorkflowCli.test.ts index 3ecb567882..6354a604fe 100644 --- a/tests/acceptance/applications/applicationWorkflowCli.test.ts +++ b/tests/acceptance/applications/applicationWorkflowCli.test.ts @@ -834,6 +834,7 @@ describe("application workflow CLI parity", () => { }), "--json", ]); + await assertSourceFileView(sourceCompared); expect(sourceCompared).toMatchObject({ operation: "compare_source_to_bundle", predicate_type: "rea.source-to-bundle-comparison", @@ -1849,3 +1850,67 @@ const runCliWithRequestFile = async ( await writeFile(requestPath, JSON.stringify(request)); return runCli([command, requestPath, "--json"]); }; + +const assertSourceFileView = async (sourceCompared: unknown) => { + const wrongApplication = await runCli([ + "compare-source-to-bundle", + JSON.stringify({ + reference: SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE.reference, + application: sourceCompared, + }), + "--json", + ]); + expect(wrongApplication).toMatchObject({ + error: "Application workflow failed", + }); + + const sourceViewRoot = await createTestTempDirectory("rea-cli-source-view-"); + const referencePath = join(sourceViewRoot, "reference.json"); + const sourceComparisonPath = join(sourceViewRoot, "comparison.json"); + await writeFile( + referencePath, + JSON.stringify(SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE.reference), + ); + const requestedPath = relative(process.cwd(), referencePath); + const fileParent = parseEvidence( + await runCli([ + "compare-source-to-bundle", + JSON.stringify({ + reference: { kind: "file", path: requestedPath }, + application: JAVASCRIPT_FEATURE_TRACE_EXAMPLE.application, + }), + "--json", + ]), + ); + expect(fileParent.normalized_result).toEqual( + parseEvidence(sourceCompared).normalized_result, + ); + expect(fileParent.parameters.reference_source).toEqual({ + kind: "file", + path: absoluteCallerPath(requestedPath), + requested_path: requestedPath, + }); + await writeFile(sourceComparisonPath, JSON.stringify(fileParent)); + const selectedSource = await runCli([ + "inspect-analysis-view", + JSON.stringify({ + source: { kind: "evidence-file", path: sourceComparisonPath }, + view: { + kind: "source-to-bundle", + collection: "mappings", + source_paths: ["src/main.ts"], + fields: ["status"], + }, + }), + "--json", + ]); + expect(selectedSource).toMatchObject({ + normalized_result: { + parent_operation: "compare_source_to_bundle", + items: [ + { source_path: "src/main.ts", source_index: 0, status: "removed" }, + ], + coverage: { total: 1, exhausted: true }, + }, + }); +}; diff --git a/tests/boundary/filesystem/sourceToBundlePaths.test.ts b/tests/boundary/filesystem/sourceToBundlePaths.test.ts index d546605def..cdc65261a2 100644 --- a/tests/boundary/filesystem/sourceToBundlePaths.test.ts +++ b/tests/boundary/filesystem/sourceToBundlePaths.test.ts @@ -1,3 +1,5 @@ +import { resolveSourceToBundleReference } from "../../../src/application/javascript/SourceToBundleReference.js"; +import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it } from "vitest"; @@ -810,3 +812,38 @@ it.each([ ).toEqual({ kind: "suffix", path: expected }); }, ); + +it("cancels historical graph file reads and rejects malformed JSON through the owning reader", async () => { + const root = await createTestTempDirectory("rea-reference-read-"); + const path = join(root, "reference.json"); + await writeFile(path, "{malformed"); + const resources = new ArtifactResourceScope(); + try { + const malformed = await resolveSourceToBundleReference( + { kind: "file", path }, + undefined, + resources, + ); + expect(malformed).toMatchObject({ + ok: false, + error: { _tag: "EvidenceFileError", reason: "invalid-json" }, + }); + for (const abortBefore of [true, false]) { + const controller = new AbortController(); + if (abortBefore) controller.abort("cancelled test read"); + const pending = resolveSourceToBundleReference( + { kind: "file", path }, + controller.signal, + resources, + ); + if (!abortBefore) controller.abort("cancelled test read"); + expect(await pending).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + } + } finally { + await resources.close(); + } + expect(resources.hasPending).toBe(false); +}); diff --git a/tests/boundary/mcp/sourceToBundleViewsMcp.test.ts b/tests/boundary/mcp/sourceToBundleViewsMcp.test.ts new file mode 100644 index 0000000000..bcdae16a93 --- /dev/null +++ b/tests/boundary/mcp/sourceToBundleViewsMcp.test.ts @@ -0,0 +1,379 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it, onTestFinished } from "vitest"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { sourceToBundleComparisonResultSchema } from "../../../src/domain/javascript/sourceToBundleComparisonSchemas.js"; +import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { importReferenceSource } from "../../support/referenceSourceResourceScope.js"; +import { analyzeJavaScriptApplication } from "../../support/javascriptApplicationScope.js"; + +it.each(["full", "compact"] as const)( + "selects real source-map correspondence with stable identities and full parent retention (%s)", + async (profile) => { + const { root, path, reference, session, client, inputs, call } = + await comparisonFixture(profile); + const parent = await call("compare_source_to_bundle", inputs); + const full = sourceToBundleComparisonResultSchema.parse( + parent.normalized_result, + ); + expect(full.summary).toMatchObject({ split: 1, unknown: 2 }); + const fileParent = await call("compare_source_to_bundle", { + ...inputs, + reference: { kind: "file", path }, + }); + expect(fileParent.normalized_result).toEqual(parent.normalized_result); + expect(fileParent.parameters.reference_source).toEqual({ + kind: "file", + path, + requested_path: path, + }); + expect(fileParent.evidence_id).not.toBe(parent.evidence_id); + const secondPath = join(root, "same-graph.json"); + await writeFile(secondPath, JSON.stringify(reference.value)); + const secondParent = await call("compare_source_to_bundle", { + ...inputs, + reference: { kind: "file", path: secondPath }, + }); + expect(secondParent.normalized_result).toEqual(parent.normalized_result); + expect(secondParent.evidence_id).not.toBe(fileParent.evidence_id); + const summary = await call("compare_source_to_bundle", { + ...inputs, + view: { kind: "summary" }, + }); + expect(summary.normalized_result).toMatchObject({ + kind: "summary", + parent_evidence_id: parent.evidence_id, + summary: { summary: full.summary, source_coverage: full.coverage }, + }); + const { tools } = await client.listTools(); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const comparisonTool = tools.find( + (tool) => tool.name === "compare_source_to_bundle", + )!; + const viewTool = tools.find( + (tool) => tool.name === "inspect_analysis_view", + )!; + for (const tool of [comparisonTool, viewTool]) { + expect(ajv.validateSchema(tool.inputSchema)).toBe(true); + const outputSchema: Record = tool.outputSchema!; + expect(ajv.validateSchema(outputSchema)).toBe(true); + } + const request = { + kind: "source-to-bundle", + collection: "mappings", + source_paths: ["src/a.js"], + fields: ["status", "candidate_count"], + limit: 1, + }; + expect( + ajv.compile(comparisonTool.inputSchema)({ ...inputs, view: request }), + ).toBe(true); + const selected = await call("compare_source_to_bundle", { + ...inputs, + reference: { kind: "file", path }, + view: request, + }); + const outputSchema: Record = comparisonTool.outputSchema!; + expect(ajv.compile(outputSchema)(selected)).toBe(true); + const page = analysisViewResultSchema.parse(selected.normalized_result); + expect(page).toMatchObject({ + kind: "source-to-bundle", + parent_evidence_id: fileParent.evidence_id, + source_coverage: full.coverage, + items: [ + { + mapping_id: full.items[0]!.mapping_id, + source_index: 0, + source_path: "src/a.js", + status: "split", + candidate_count: 2, + }, + ], + coverage: { total: 1, exhausted: true }, + }); + expect(session.evidenceById(fileParent.evidence_id)).toEqual(fileParent); + const source = { + kind: "retained-evidence", + evidence_id: fileParent.evidence_id, + }; + await assertRetainedPages(call, source, request, full); + await writeFile( + path, + JSON.stringify({ ...reference.value, root_sha256: "0".repeat(64) }), + ); + expect( + ( + await client.callTool({ + name: "compare_source_to_bundle", + arguments: { ...inputs, reference: { kind: "file", path } }, + }) + ).isError, + ).toBe(true); + for (const value of [ + { kind: "file", path: "relative.json" }, + { kind: "file", path: root }, + { kind: "file", path: join(root, "missing.json") }, + ]) + expect( + ( + await client.callTool({ + name: "compare_source_to_bundle", + arguments: { ...inputs, reference: value }, + }) + ).isError, + ).toBe(true); + expect( + ( + await client.callTool({ + name: "inspect_analysis_view", + arguments: { source, view: { ...request, fields: ["not_a_field"] } }, + }) + ).isError, + ).toBe(true); + }, +); + +const comparisonFixture = async (profile: "full" | "compact") => { + const root = await createTestTempDirectory("rea-source-view-"); + const previous = join(root, "previous"), + current = join(root, "current"); + await mkdir(join(previous, "src"), { recursive: true }); + await mkdir(join(previous, "src/other"), { recursive: true }); + await writeFile( + join(previous, "src/other/a.js"), + "export const unrelated = 3;\n", + ); + await mkdir(join(current, "dist"), { recursive: true }); + await writeFile(join(previous, "src/a.js"), "export const value = 1;\n"); + await writeFile( + join(previous, "src/unseen.js"), + "export const unseen = 1;\n", + ); + for (const name of ["one", "two"]) { + await writeFile( + join(current, `dist/${name}.js`), + `var value = 2;\n//# sourceMappingURL=${name}.js.map\n`, + ); + await writeFile( + join(current, `dist/${name}.js.map`), + JSON.stringify({ + version: 3, + file: `${name}.js`, + sources: ["../src/a.js"], + sourcesContent: ["export const value = 2;\n"], + names: [], + mappings: "AAAA", + }), + ); + } + await writeFile(join(previous, "src/b.js"), "export const other = 1;\n"); + await writeFile( + join(current, "dist/three.js"), + "var other = 2;\n//# sourceMappingURL=three.js.map\n", + ); + await writeFile( + join(current, "dist/three.js.map"), + JSON.stringify({ + version: 3, + file: "three.js", + sources: ["../src/b.js"], + sourcesContent: ["export const other = 2;\n"], + names: [], + mappings: "AAAA", + }), + ); + const reference = await importReferenceSource({ + root: previous, + caller: "source-view-test", + policy: { secretPatterns: [] }, + }); + if (!reference.ok) throw reference.error; + const application = await analyzeJavaScriptApplication({ + input_path: current, + max_semantic_nodes: 1, + max_semantic_nodes_per_file: 1, + }); + if (!application.ok) throw application.error; + const path = join(root, "reference.json"); + await writeFile(path, JSON.stringify(reference.value)); + const session = createTestBinarySession(() => { + throw new Error("Views must not open a native provider"); + }); + const server = createServer( + { kind: "session", session }, + { + environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: profile }, + }, + ); + const client = new Client({ name: "source-view", version: "1" }); + onTestFinished(async () => { + await client.close(); + await server.close(); + await session.close(); + }); + const [a, b] = InMemoryTransport.createLinkedPair(); + await server.connect(b); + await client.connect(a); + expect(session.recordEvidence(application.value).ok).toBe(true); + const inputs = { + reference: reference.value, + application: { + kind: "retained-evidence", + evidence_id: application.value.evidence_id, + }, + }; + const call = async (name: string, args: Record) => { + const reply = await client.callTool({ name, arguments: args }); + expect(reply.isError, JSON.stringify(reply)).not.toBe(true); + return parseEvidence(reply.structuredContent); + }; + return { root, path, reference, session, client, inputs, call }; +}; + +const assertRetainedPages = async ( + call: Awaited>["call"], + source: Record, + request: Record, + full: ReturnType, +) => { + const view = async (value: Record) => + analysisViewResultSchema.parse( + (await call("inspect_analysis_view", { source, view: value })) + .normalized_result, + ); + for (const selector of [ + { source_paths: [] }, + { mapping_ids: [] }, + { statuses: [] }, + { source_paths: ["Src/a.js"] }, + { offset: 100 }, + ]) + expect( + await view({ + kind: "source-to-bundle", + collection: "mappings", + ...selector, + }), + ).toMatchObject({ items: [], coverage: { exhausted: true } }); + const identity = await view({ ...request, fields: [] }); + expect(identity).toMatchObject({ + items: [ + { + mapping_id: full.items[0]!.mapping_id, + source_path: "src/a.js", + source_index: 0, + }, + ], + }); + if (identity.kind !== "source-to-bundle") throw new Error("Expected page"); + expect(Object.keys(identity.items[0]!)).toEqual([ + "mapping_id", + "source_path", + "source_index", + ]); + for (let offset = 0; offset < 2; offset++) { + const candidate = full.items[0]!.candidates[offset]!; + expect( + await view({ + kind: "source-to-bundle", + collection: "candidates", + mapping_ids: [full.items[0]!.mapping_id], + fields: ["signals"], + offset, + limit: 1, + }), + ).toMatchObject({ + items: [ + { + mapping_id: full.items[0]!.mapping_id, + candidate_index: offset, + current_node_id: candidate.current_node_id, + mapped: full.items[0]!.current_node_ids.includes( + candidate.current_node_id, + ), + signals: candidate.signals, + }, + ], + coverage: { total: 2, next_offset: offset === 0 ? 1 : null }, + }); + } + await assertCandidateBoundaryPages(view); + const weak = await view({ + kind: "source-to-bundle", + collection: "candidates", + source_paths: ["src/other/a.js"], + fields: ["confidence"], + }); + expect(weak).toMatchObject({ + items: [ + { source_path: "src/other/a.js", mapped: false, confidence: "low" }, + { source_path: "src/other/a.js", mapped: false, confidence: "low" }, + ], + coverage: { total: 2, exhausted: true }, + }); + expect(await view({ kind: "summary" })).toMatchObject({ + summary: { + summary: full.summary, + source_coverage: full.coverage, + unmapped_current_node_count: full.unmapped_current_node_ids.length, + }, + }); +}; + +const assertCandidateBoundaryPages = async ( + view: ( + value: Record, + ) => Promise>, +) => { + const boundary = await view({ + kind: "source-to-bundle", + collection: "candidates", + fields: [], + offset: 1, + limit: 3, + }); + expect(boundary).toMatchObject({ + items: [ + { + source_path: "src/a.js", + source_index: 0, + candidate_index: 1, + mapped: true, + }, + { + source_path: "src/b.js", + source_index: 1, + candidate_index: 0, + mapped: true, + }, + { + source_path: "src/other/a.js", + source_index: 2, + candidate_index: 0, + mapped: false, + }, + ], + coverage: { total: 5, next_offset: 4, exhausted: false }, + }); + expect( + await view({ + kind: "source-to-bundle", + collection: "candidates", + source_paths: ["src/a.js", "src/b.js"], + fields: [], + offset: 1, + limit: 2, + }), + ).toMatchObject({ + items: [ + { source_path: "src/a.js", source_index: 0, candidate_index: 1 }, + { source_path: "src/b.js", source_index: 1, candidate_index: 0 }, + ], + coverage: { total: 3, next_offset: null, exhausted: true }, + }); +}; diff --git a/tests/process-global/sourceToBundleReferenceGuards.test.ts b/tests/process-global/sourceToBundleReferenceGuards.test.ts new file mode 100644 index 0000000000..d217ee9704 --- /dev/null +++ b/tests/process-global/sourceToBundleReferenceGuards.test.ts @@ -0,0 +1,209 @@ +import { holdFirstFileRead } from "../support/holdFirstFileRead.js"; +import { AnalysisCancelledError } from "../../src/domain/analysisErrorCore.js"; +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { setImmediate } from "node:timers"; +import { afterEach, expect, it, onTestFinished, vi } from "vitest"; +import { resolveSourceToBundleReference } from "../../src/application/javascript/SourceToBundleReference.js"; +import { requireAuthenticationHeadroom } from "../../src/application/JsonAuthenticationHeadroom.js"; +import * as workflow from "../../src/application/javascript/JavaScriptApplicationWorkflowService.js"; +import { historicalSourceGraphSchema } from "../../src/domain/referenceSourceGraph.js"; +import { ArtifactResourceScope } from "../../src/artifacts/ArtifactResourceScope.js"; +import { SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE } from "../../src/contracts/javascript/javascriptApplicationWorkflowExamples.js"; +import { createServer } from "../../src/server/createServer.js"; +import { ToolResultDelivery } from "../../src/server/toolResult.js"; +import { createTestBinarySession } from "../fixtures/binarySession.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +vi.mock("../../src/application/JsonAuthenticationHeadroom.js", { spy: true }); +vi.mock( + "../../src/application/javascript/JavaScriptApplicationWorkflowService.js", + { spy: true }, +); +afterEach(() => vi.restoreAllMocks()); + +it("reserves graph validation/hash memory before the owning parser allocates", async () => { + const root = await createTestTempDirectory("rea-source-headroom-"); + const path = join(root, "graph.json"); + await writeFile( + path, + JSON.stringify(SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE.reference), + ); + const actual = await vi.importActual< + typeof import("../../src/application/JsonAuthenticationHeadroom.js") + >("../../src/application/JsonAuthenticationHeadroom.js"); + vi.mocked(requireAuthenticationHeadroom).mockImplementationOnce( + (value, options) => + actual.requireAuthenticationHeadroom(value, { + ...options, + availableBytes: 0, + }), + ); + const parser = vi.spyOn(historicalSourceGraphSchema, "safeParse"); + const resources = new ArtifactResourceScope(); + try { + expect( + await resolveSourceToBundleReference( + { kind: "file", path }, + undefined, + resources, + ), + ).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisResourceConstraintError", + operation: "compare_source_to_bundle", + resource: "memory", + }, + }); + expect(parser).not.toHaveBeenCalled(); + } finally { + await resources.close(); + } + expect(resources.hasPending).toBe(false); +}); + +it("attributes file-I/O cancellation to the comparison and preserves its exact cause", async () => { + const root = await createTestTempDirectory("rea-reference-io-cancel-"); + const path = join(root, "graph.json"); + await writeFile( + path, + JSON.stringify(SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE.reference), + ); + const held = await holdFirstFileRead(path); + const controller = new AbortController(); + const cause = new Error("cancel historical graph read"); + const resources = new ArtifactResourceScope(); + try { + const pending = resolveSourceToBundleReference( + { kind: "file", path }, + controller.signal, + resources, + ); + await held.firstRead; + controller.abort(cause); + held.release(); + const result = await pending; + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + operation: "compare_source_to_bundle", + }, + }); + if (result.ok) throw new Error("Expected cancelled read"); + expect(result.error.cause).toBe(cause); + } finally { + held.release(); + await resources.close(); + } + expect(resources.hasPending).toBe(false); +}); + +it.each(["validation", "comparison"] as const)( + "dispatches cancellation during %s before committing session facts", + async (stage) => { + const { client, session, path } = await connectedComparison(); + const controller = new AbortController(); + const before = session.exportEvidenceBundle(); + if (stage === "validation") { + const parse = historicalSourceGraphSchema.safeParse; + vi.spyOn(historicalSourceGraphSchema, "safeParse").mockImplementationOnce( + (value) => { + const parsed = parse(value); + setImmediate(() => + controller.abort(new Error("cancel during graph validation")), + ); + return parsed; + }, + ); + } else { + const actual = await vi.importActual( + "../../src/application/javascript/JavaScriptApplicationWorkflowService.js", + ); + vi.mocked( + workflow.compareSourceToBundleEvidenceValidated, + ).mockImplementationOnce((input) => { + const compared = actual.compareSourceToBundleEvidenceValidated(input); + setImmediate(() => + controller.abort(new Error("cancel during graph comparison")), + ); + return compared; + }); + } + let settle: (() => void) | undefined; + const settled = new Promise((resolve) => { + settle = resolve; + }); + const errorDelivery = ToolResultDelivery.prototype.toErrorToolResult; + vi.spyOn( + ToolResultDelivery.prototype, + "toErrorToolResult", + ).mockImplementation(function ( + this: ToolResultDelivery, + error, + targetContent, + ) { + const result = errorDelivery.call(this, error, targetContent); + if ( + error instanceof AnalysisCancelledError && + error.operation === "compare_source_to_bundle" + ) + settle?.(); + return result; + }); + const evidenceDelivery = ToolResultDelivery.prototype.toEvidenceToolResult; + vi.spyOn( + ToolResultDelivery.prototype, + "toEvidenceToolResult", + ).mockImplementation(function (this: ToolResultDelivery, ...args) { + const result = evidenceDelivery.apply(this, args); + if (args[1].name === "compare_source_to_bundle") settle?.(); + return result; + }); + const pending = client + .callTool( + { + name: "compare_source_to_bundle", + arguments: { + ...SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE, + reference: { kind: "file", path }, + view: { kind: "summary" }, + }, + }, + { signal: controller.signal }, + ) + .then( + () => null, + (cause: unknown) => cause, + ); + expect(await pending).toBeInstanceOf(Error); + await settled; + expect(session.exportEvidenceBundle()).toEqual(before); + await expect(client.ping()).resolves.toEqual({}); + }, +); + +const connectedComparison = async () => { + const root = await createTestTempDirectory("rea-source-commit-cancel-"); + const path = join(root, "graph.json"); + await writeFile( + path, + JSON.stringify(SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE.reference), + ); + const session = createTestBinarySession(() => { + throw new Error("Comparison must not start a native provider"); + }); + const server = createServer({ kind: "session", session }); + const client = new Client({ name: "source-cancel", version: "1" }); + onTestFinished(async () => { + await client.close(); + await server.close(); + await session.close(); + }); + const [a, b] = InMemoryTransport.createLinkedPair(); + await server.connect(b); + await client.connect(a); + return { client, session, path }; +};