diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index a8fb74d053..67a7c66c01 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -497,7 +497,10 @@ declared by their tools. Artifact extraction materializes the selected regular files into a fresh REA-chosen temporary directory and reports its path. `analyze_javascript_application` reads the supplied local directory or ASAR path -and returns its result and Evidence inline. +and returns its result and Evidence inline. Its owned analysis worker and private +temporary source files are declared as `launchesProcess` and `writesFilesystem`, +including for a single JavaScript source. This internal analysis does not execute +the supplied application code. ## Ghidra first-query deadlines and recovery diff --git a/docs/verification/javascript-analysis-effects-2026-10-15.json b/docs/verification/javascript-analysis-effects-2026-10-15.json new file mode 100644 index 0000000000..c7e0f8eb0a --- /dev/null +++ b/docs/verification/javascript-analysis-effects-2026-10-15.json @@ -0,0 +1,249 @@ +{ + "base_revision": "e598b4656ad90d59ede2d10454a0ac34f3813558", + "scope": "Canonical effect metadata only; real static Obsidian reads, no target execution", + "baseline_source_sha256": { + "src/contracts/toolEffects.ts": "39505b5f1501768515229a37ff67e8a1a512faedfb6669aee7a690a2ac3ac633", + "src/javascript/analysis/JavaScriptAnalysisWorker.ts": "49ea4e6344060fae3529ffe6e14bfadc534132067cfa5c370263dc7476375be0", + "scripts/verify/javascript/module-references.mjs": "9d5f0b8b35ed2ebb4591f9dc3baea0fc116a0c3fd7d8e5ce30ec476c84671ed1" + }, + "final_source_sha256": { + "src/contracts/toolEffects.ts": "feb66f1b71c4a6849d1730227c9c74e4efbb67b4537d75fd41ce9ff615c25753", + "src/javascript/analysis/JavaScriptAnalysisWorker.ts": "49ea4e6344060fae3529ffe6e14bfadc534132067cfa5c370263dc7476375be0", + "scripts/verify/javascript/module-references.mjs": "9d5f0b8b35ed2ebb4591f9dc3baea0fc116a0c3fd7d8e5ce30ec476c84671ed1", + "tests/boundary/mcp/electronMcp.test.ts": "e483bec8512cfeafa7f951c8faf25e1aecd6e6987ea74aead14230501912928d", + "tests/boundary/mcp/toolSchemaValidity.test.ts": "ab6d9ba37035a6ff9279b820aa8660c50a3ef9f31d25f838c0ba40929148e89c" + }, + "cohorts": { + "baseline": { + "metadata": { + "tool": "analyze_javascript_application", + "advertised_annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + }, + "session_effects": ["mutatesSession", "idempotent"], + "session_annotations": ["idempotent"], + "catalog_tool_count": 207 + }, + "metadata_probe_resources": { + "elapsed_seconds": 10.711, + "cpu_seconds": 10.503415, + "peak_cgroup_bytes": 842891264, + "payload_cleanup_verified": true, + "events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + }, + "real_workflow": { + "artifact": { + "path": "/tmp/rea-codemode-experiment-20261015/artifacts/obsidian-1.14.4.asar", + "sha256": "146ef8470d8cbb4a6db209bdc19bf2bad13350444186c0fd93c31087b142b92f" + }, + "parent_evidence_id": "ev_73ef06e3b9b944f54c5257f7960279127d8518d2a73cd8618bd045dbe44e6b5a", + "samples": [ + { + "round": 0, + "mode": "references", + "elapsed_ms": 109.19401700000162, + "calls": 1, + "structured_reply_bytes": 22158, + "facts_sha256": "d3be5e91d2ec3fd9a95851cfbbdef9c168bb4c7153f78810b8b235e005401299", + "matching_modules": 23, + "coverage": { + "status": "page", + "examined": 23, + "total": 23, + "next_offset": null, + "exhausted": true + } + } + ], + "token_costs": { + "tokenizer": "cl100k_base", + "tiktoken_version": "0.12.0", + "costs": { + "references": { + "selected_catalog": 7977, + "full_catalog": 195522, + "common_request": 53, + "common_summary": 2675, + "requests": 102, + "structured_replies": 5741, + "selected_catalog_task_total": 16548, + "full_catalog_task_total": 204093 + } + } + }, + "cli_parity": { + "evidence_id": "ev_b3586dfdf7d4f01157f8f2978326aaa7103987110f2df4fa5bf3331ff2898749", + "facts_sha256": "d3be5e91d2ec3fd9a95851cfbbdef9c168bb4c7153f78810b8b235e005401299", + "exact_cli_mcp_view_parity": true + } + }, + "real_workflow_resources": { + "elapsed_seconds": 30.47, + "cpu_seconds": 29.939807, + "peak_cgroup_bytes": 1551282176, + "payload_cleanup_verified": true, + "events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + } + }, + "final": { + "metadata": { + "tool": "analyze_javascript_application", + "advertised_annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": true + }, + "session_effects": [ + "mutatesSession", + "writesFilesystem", + "launchesProcess", + "idempotent" + ], + "session_annotations": ["idempotent", "open_world"], + "catalog_tool_count": 207 + }, + "metadata_probe_resources": { + "elapsed_seconds": 10.909, + "cpu_seconds": 10.664997, + "peak_cgroup_bytes": 853958656, + "payload_cleanup_verified": true, + "events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + }, + "real_workflow": { + "artifact": { + "path": "/tmp/rea-codemode-experiment-20261015/artifacts/obsidian-1.14.4.asar", + "sha256": "146ef8470d8cbb4a6db209bdc19bf2bad13350444186c0fd93c31087b142b92f" + }, + "parent_evidence_id": "ev_73ef06e3b9b944f54c5257f7960279127d8518d2a73cd8618bd045dbe44e6b5a", + "samples": [ + { + "round": 0, + "mode": "references", + "elapsed_ms": 110.17946600000141, + "calls": 1, + "structured_reply_bytes": 22158, + "facts_sha256": "d3be5e91d2ec3fd9a95851cfbbdef9c168bb4c7153f78810b8b235e005401299", + "matching_modules": 23, + "coverage": { + "status": "page", + "examined": 23, + "total": 23, + "next_offset": null, + "exhausted": true + } + } + ], + "token_costs": { + "tokenizer": "cl100k_base", + "tiktoken_version": "0.12.0", + "costs": { + "references": { + "selected_catalog": 7977, + "full_catalog": 195522, + "common_request": 53, + "common_summary": 2675, + "requests": 102, + "structured_replies": 5741, + "selected_catalog_task_total": 16548, + "full_catalog_task_total": 204093 + } + } + }, + "cli_parity": { + "evidence_id": "ev_b3586dfdf7d4f01157f8f2978326aaa7103987110f2df4fa5bf3331ff2898749", + "facts_sha256": "d3be5e91d2ec3fd9a95851cfbbdef9c168bb4c7153f78810b8b235e005401299", + "exact_cli_mcp_view_parity": true + } + }, + "real_workflow_resources": { + "elapsed_seconds": 31.059, + "cpu_seconds": 30.663163, + "peak_cgroup_bytes": 1584574464, + "payload_cleanup_verified": true, + "events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + } + } + }, + "conditions": "Separate fresh MCP sessions, uncontrolled warm filesystem/build and shared-host scheduling. One CPU, 8 GiB hard cap, 6 GiB early stop, no swap. Resource counters are at payload exit with the observer alive, not final whole-scope counters; memory includes charged file cache, not summed RSS. Whole real-verifier resources include analysis, export, CLI parity and tokenizer; metadata probe resources are separate. Static BPE uses name/description/inputSchema catalog projections plus requests/structured replies, excludes annotations, metadata-probe calls, rendering/replay/reasoning/final-answer/model usage. No performance improvement or equivalence bound established by one before/after pair.", + "regressions": { + "baseline": "20 executed: 2 expected metadata failures, 18 passed", + "final": "27 passed after the test ownership cleanup across canonical effects and MCP/catalog boundaries", + "interim_correction": "26 passed before metadata assertions were split into their own target-free case to satisfy function-size lint" + }, + "selected_input": { + "input_path": "/tmp/rea-codemode-experiment-20261015/artifacts/obsidian-1.14.4.asar", + "detail": "summary", + "source_paths": ["package.json", "main.js"] + }, + "selected_view": { + "kind": "javascript", + "collection": "modules", + "projection": "references", + "where": { + "path": "main.js", + "kind": "javascript-module" + } + }, + "observer_sources_sha256": { + "resource-run.py": "7728862171415e7dee2c11af2ead2b9a54265fccaf82a7113956d36b29065e33", + "resource_payload.py": "6f089e645459eba80c5d73e39c21606c78173a44daf49fabc3b88c4631ce51c6" + }, + "runtime": { + "platform": "Linux x64", + "node": "24.18.0", + "npm": "11.16.0", + "tiktoken": "0.12.0" + } +} diff --git a/docs/verification/javascript-analysis-effects-2026-10-15.md b/docs/verification/javascript-analysis-effects-2026-10-15.md new file mode 100644 index 0000000000..c1cbdeceed --- /dev/null +++ b/docs/verification/javascript-analysis-effects-2026-10-15.md @@ -0,0 +1,121 @@ +# JavaScript analysis effect metadata verification + +Tracks #3867 under #4027. The canonical effect record omitted existing internal +worker launches and source staging. `CapabilityInventory` and SDK registration +already consume that owner directly; no adapter override is required. The +[worker](../../src/javascript/analysis/JavaScriptAnalysisWorker.ts) creates a +private runtime root, stages `.source` files and launches an owned REA worker. +These are source-derived facts; the analyzed application code is not executed. + +## Executed results + +Public stdio MCP before/after: + +| Surface | Before | After | +| ---------------------------- | ------------------------------ | ------------------------------------------ | +| `tools/list` `openWorldHint` | `false` | `true` | +| `binary_session` effects | `mutatesSession`, `idempotent` | also `writesFilesystem`, `launchesProcess` | +| Session annotation flags | `idempotent` | also `open_world` | +| Advertised tools | 207 | 207 | + +Regression assertions failed on the original contract: **2 failed, +18 passed**. After correction and separating the target-free metadata case, **27 passed** across canonical effects, public +Electron/JavaScript MCP and SDK-converted catalog validity. The tests retain the +non-destructive, idempotent hints and exact session effects; no foreign target +execution is declared. + +The existing real verifier analyzed official Obsidian 1.14.4 with `package.json` +and `main.js` selected, then read the typed module references and verified exact +public CLI/MCP parity. Both revisions returned the same parent/view Evidence +identities and 23-row fact digest. Query replies remained 22,158 bytes and the +selected-schema task remained 16,548 `cl100k_base` BPE tokens (204,093 with the full +name/description/inputSchema catalog projection). These are static counts, not +model billing; annotations, the separate metadata probe, host rendering/replay, +reasoning and final answers are outside this projection. No hosted agent was run. + +| Complete real verifier | Before | After | +| ---------------------- | ------------: | ------------: | +| Elapsed seconds | 30.470 | 31.059 | +| CPU seconds | 29.939807 | 30.663163 | +| Peak cgroup bytes | 1,551,282,176 | 1,584,574,464 | + +The scope includes shared analysis, temporary export, CLI parity and tokenizer. +The metadata-only probe separately used 10.711/10.909 s wall, +10.503415/10.664997 s CPU and 842,891,264/853,958,656 B peak cgroup memory. +All scopes used one CPU, an 8 GiB hard cap, 6 GiB early stop and no swap; cleanup +was verified with no OOM. Memory includes charged file cache, not summed RSS. +Counters were observed at payload exit while the observer remained alive. +Fresh MCP sessions do not establish an OS-cold cache. Shared-host scheduling and +filesystem/build warmness were uncontrolled; one pair establishes neither a +performance improvement nor an equivalence bound. Linux x64 static ASAR analysis +was executed; live Electron and other platforms were not exercised locally. +[Compact records and source hashes](javascript-analysis-effects-2026-10-15.json) +preserve these distinctions without archiving the downloadable application. + +## Reproduce + +Use pinned Node/npm, `npm ci`, then `npm run build:cached`. Run the following as +an ES module from the checkout root (`node --input-type=module < probe.mjs`): + +```javascript +import assert from "node:assert/strict"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { toolAvailabilityInventory } from "./dist/contracts/toolOutputSchemaPrimitives.js"; +import { closeVerifierResources } from "./scripts/lib/verifier-cleanup.mjs"; + +const client = new Client({ + name: "javascript-effect-verification", + version: "1", +}); +const transport = new StdioClientTransport({ + command: process.execPath, + args: ["scripts/rea.mjs", "mcp"], + stderr: "pipe", +}); +let primaryError; +try { + await client.connect(transport); + transport.stderr?.resume(); + const { tools } = await client.listTools(); + const tool = tools.find( + ({ name }) => name === "analyze_javascript_application", + ); + assert.ok(tool); + const response = await client.callTool({ + name: "binary_session", + arguments: {}, + }); + assert.notEqual(response.isError, true); + const inventory = toolAvailabilityInventory.parse( + response.structuredContent?.result?.tool_availability, + ); + const group = inventory.groups.find(({ names }) => names.includes(tool.name)); + assert.ok(group); + console.log( + JSON.stringify({ + tool: tool.name, + advertised_annotations: tool.annotations, + session_effects: group.effects, + session_annotations: group.annotations, + catalog_tool_count: tools.length, + }), + ); +} catch (cause) { + primaryError = cause; + throw cause; +} finally { + await closeVerifierResources( + [() => client.close(), () => transport.close()], + primaryError, + ); +} +``` + +For the real workflow, follow the artifact acquisition/digest instructions in +[module-reference verification](javascript-module-references-2026-10-15.md). +Run its existing command with `--mode references --rounds 1 --cli-parity` and the +same selected source paths and tokenizer. That verifier is unchanged by this +fix. The preserved 8 GiB cgroup observer and its source hashes are documented in +the same guide. The machine-readable records separate metadata-probe resources +from the complete real-workflow resources. diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index 7fe152e9bd..216326ee18 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -647,7 +647,11 @@ export const TOOL_EFFECTS: Readonly> = { }), list_javascript_runtime_targets: browserEvidence, observe_javascript_runtime: browserEvidence, - analyze_javascript_application: evidence, + analyze_javascript_application: effects({ + mutatesSession: true, + writesFilesystem: true, + launchesProcess: true, + }), inspect_electron_fuses: evidence, export_web_scripts: effects({ mutatesSession: true, diff --git a/tests/boundary/mcp/electronMcp.test.ts b/tests/boundary/mcp/electronMcp.test.ts index 72d9fdaf67..92ec605e05 100644 --- a/tests/boundary/mcp/electronMcp.test.ts +++ b/tests/boundary/mcp/electronMcp.test.ts @@ -25,6 +25,7 @@ import { writeElectronBoundaryFixture } from "../../fixtures/electronBoundaryApp import { createElectronActiveObservationFixtureResult } from "../../../src/domain/javascript/electronActiveObservation.fixture.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { AsarArtifactReader } from "../../../src/artifacts/AsarArtifactReader.js"; +import { toolAvailabilityInventory } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; const browsers: FakeCdpBrowser[] = []; const resources: Array<{ close(): Promise }> = []; @@ -138,6 +139,47 @@ it("lets MCP JavaScript analysis continue only with recorded untrusted ASAR byte }); }); +it("advertises owned JavaScript analysis effects without foreign target execution", async () => { + const session = createTestBinarySession(() => ({ + execute: () => Promise.resolve(observed(null)), + close: () => Promise.resolve(resultOk(null)), + })); + const server = createServer({ kind: "session", session }); + const client = new Client({ + name: "javascript-analysis-effects", + version: "1", + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + resources.push(client, server, session); + await server.connect(serverTransport); + await client.connect(clientTransport); + + const status = await client.callTool({ + name: "binary_session", + arguments: {}, + }); + expect(status.isError).not.toBe(true); + const { result } = z + .object({ + result: z.object({ tool_availability: toolAvailabilityInventory }), + }) + .parse(status.structuredContent); + const analysisAvailability = result.tool_availability.groups.find( + ({ names }) => names.includes("analyze_javascript_application"), + ); + expect(analysisAvailability?.effects.toSorted()).toEqual([ + "idempotent", + "launchesProcess", + "mutatesSession", + "writesFilesystem", + ]); + expect(analysisAvailability?.annotations.toSorted()).toEqual([ + "idempotent", + "open_world", + ]); +}); + it("exposes endpoint-scoped Electron discovery and inspection as Evidence", async () => { const root = await createTestTempDirectory("rea-electron-mcp-"); temporary.push(root); diff --git a/tests/boundary/mcp/toolSchemaValidity.test.ts b/tests/boundary/mcp/toolSchemaValidity.test.ts index 63c8d7abf8..8716aa21a5 100644 --- a/tests/boundary/mcp/toolSchemaValidity.test.ts +++ b/tests/boundary/mcp/toolSchemaValidity.test.ts @@ -129,6 +129,12 @@ function expectKnownAuthorityHints(tools: readonly ToolSchemas[]): void { idempotentHint: false, openWorldHint: false, }, + analyze_javascript_application: { + readOnlyHint: false, + destructiveHint: false, + idempotentHint: true, + openWorldHint: true, + }, capture_process_scenario: { readOnlyHint: false, destructiveHint: true,