From abcb1b2cd33dd272fd47c797d49c8808959fda76 Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Sun, 25 Oct 2026 17:39:17 +0000 Subject: [PATCH] test(native): verify configurable matching with real LLVM inventories --- .../references/native-and-artifacts.md | 7 + docs/native-function-matching.md | 5 +- .../native-inventory-capture.mjs | 276 ++++++++++++ .../native-inventory-profile.json | 34 ++ .../native-matching-capacity.md | 177 ++++++++ .../native-matching-capacity.mjs | 413 ++++++++++++++++++ .../native-matching-elf-oracle.py | 23 + 7 files changed, 933 insertions(+), 2 deletions(-) create mode 100644 experiments/analysis-efficiency/native-inventory-capture.mjs create mode 100644 experiments/analysis-efficiency/native-inventory-profile.json create mode 100644 experiments/analysis-efficiency/native-matching-capacity.md create mode 100644 experiments/analysis-efficiency/native-matching-capacity.mjs create mode 100644 experiments/analysis-efficiency/native-matching-elf-oracle.py diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index d2f9a57b5e..d6a2f229ba 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -13,6 +13,13 @@ Select columns, batch IDs and page individual facts; aliases remain unresolved. Unpaired observations do not prove additions/removals. Existing matched dossier pairs include inputs for `compare_functions`; otherwise analyze just chosen pairs. +For a matching representation refusal, inspect `details.limit` and raise +`native.matching_representation_bytes` with `update_analysis_settings` before +retrying the unchanged MCP request. CLI launch seed: +`REA_NATIVE_MATCHING_REPRESENTATION_BYTES`. The default is 96 MiB; the policy +range is 1 byte to 1 GiB and does not bound process RSS. Keep the complete +inventories: splitting them changes global name/body uniqueness. + ## Apple app resources Use `open_bundle_resources` with the selected app path, then `inspect_plist`, diff --git a/docs/native-function-matching.md b/docs/native-function-matching.md index 2813279d28..0f3ca743fe 100644 --- a/docs/native-function-matching.md +++ b/docs/native-function-matching.md @@ -126,8 +126,8 @@ charged for the copies their authentication or loading creates. An exact session reference to retained Evidence is borrowed without a copy, so only what matching derives from it is charged; the inventory's own Evidence ID is charged once, not per function. Selected observations are charged in full. -Measured under the 96 MiB default with Ghidra-shaped rows, about 20-character names and no -observations, about 22,000 rows per side fit through retained +Measured under the 96 MiB default with Ghidra-shaped rows, about 20-character +names and no observations, about 22,000 rows per side fit through retained `list_procedures` references, and about 10,000 through inline Evidence or CLI `source_bundle` inputs. Work yields to cancellation. No worker/provider concurrency is introduced; this budget is not @@ -144,6 +144,7 @@ by matching only, and a runtime value repairs it. A refusal carries allowance lets matching retain more in the REA process. The 1 GiB maximum is an REA policy, not a host-capacity claim, and peak heap during a request can exceed the allowance. + Summary delivery happens after complete matching Evidence is constructed and retained. It reduces the initial reply, but does not avoid that construction or raise the allowance. Repeated provenance remains diff --git a/experiments/analysis-efficiency/native-inventory-capture.mjs b/experiments/analysis-efficiency/native-inventory-capture.mjs new file mode 100644 index 0000000000..8e8d370fbe --- /dev/null +++ b/experiments/analysis-efficiency/native-inventory-capture.mjs @@ -0,0 +1,276 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { createWriteStream } from "node:fs"; +import { mkdir, mkdtemp, open, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { finished } from "node:stream/promises"; +import { parseArgs } from "node:util"; +import { pathToFileURL } from "node:url"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +let parseEvidence, createEvidenceBundle; +import { createRetainedQueryCaller } from "./retained-query-calls.mjs"; +import { finalizeRetainedQueryReceipt } from "./retained-query-finalization.mjs"; +const { values } = parseArgs({ + options: Object.fromEntries( + ["runtime", "target", "output", "ghidra", "java", "profile"].map((key) => [ + key, + { type: "string" }, + ]), + ), +}); +for (const key of [ + "runtime", + "target", + "output", + "ghidra", + "java", + "profile", +]) { + assert.ok(values[key], `Missing --${key}`); + values[key] = resolve(values[key]); +} +const target = values.target, + output = values.output, + repo = values.runtime; +await mkdir(output); +const report = { + target, + profile_scope: + "Ghidra-reported function identities and defined strings under explicit disabled-analyzer profile; no claim of default-analysis or function-body completeness", + calls: [], + }, + transcript = []; +let account, + transport, + serverLog, + serverLogFinished, + client, + call, + opened = false; +const start = performance.now(); +try { + ({ parseEvidence } = await import( + pathToFileURL(join(repo, "dist/domain/evidence.js")).href + )); + ({ createEvidenceBundle } = await import( + pathToFileURL(join(repo, "dist/domain/evidenceBundle.js")).href + )); + report.sha256 = createHash("sha256") + .update(await readFile(target)) + .digest("hex"); + const disabled = JSON.parse(await readFile(values.profile, "utf8")).names; + report.disabled_analyzers = disabled; + account = await mkdtemp(join(tmpdir(), "rea-large-native-inventory-")); + transport = new StdioClientTransport({ + command: process.execPath, + args: [join(repo, "scripts/rea.mjs"), "mcp"], + cwd: repo, + env: { + HOME: account, + PATH: process.env.PATH, + NODE_OPTIONS: "--max-old-space-size=3072", + GHIDRA_INSTALL_DIR: values.ghidra, + JAVA_HOME: values.java, + GHIDRA_HEADLESS_MAXMEM: "3072M", + REA_GHIDRA_STARTUP_TIMEOUT_MS: "600000", + REA_GHIDRA_DISABLED_ANALYZERS: disabled.join(","), + REA_MCP_INPUT_SCHEMA_PROFILE: "compact", + REA_LOG_LEVEL: "silent", + }, + stderr: "pipe", + }); + serverLog = createWriteStream(join(output, "server.log")); + serverLogFinished = finished(serverLog); + serverLogFinished.catch(() => {}); + transport.stderr?.pipe(serverLog); + client = new Client({ + name: "large-native-inventory-verifier", + version: "1", + }); + call = createRetainedQueryCaller(client, (receipt, event) => { + report.calls.push(receipt); + transcript.push(event); + }); + await client.connect(transport); + const catalog = await client.listTools(); + await writeFile(join(output, "catalog.json"), JSON.stringify(catalog)); + const defs = new Map(catalog.tools.map((t) => [t.name, t])); + const original = client.callTool.bind(client); + client.callTool = (input, options) => + original(input, { + ...options, + timeout: 900000, + toolDefinition: defs.get(input.name), + }); + const raw = async (name, args, phase) => { + const r = await call(name, args, phase); + if (r.isError) { + const e = JSON.parse(r.content.find((c) => c.type === "text").text).error; + throw new Error( + name + + ": " + + e.code + + ": " + + (e.details?.diagnostics?.failure_message ?? e.message), + ); + } + return r; + }; + const good = async (name, args, phase) => + parseEvidence((await raw(name, args, phase)).structuredContent); + await raw("open_binary", { path: target, provider_id: "ghidra" }, "open"); + opened = true; + const procedures = await good( + "list_procedures", + { detail: "summary" }, + "functions", + ); + report.procedures = procedures.normalized_result; + const full = await call("list_strings", {}, "complete-strings"); + report.complete_strings_refused = full.isError === true; + if (full.isError) { + const error = JSON.parse( + full.content.find((c) => c.type === "text").text, + ).error; + assert.equal(error.code, "resource_constraint"); + assert.equal(error.details.reported_limits.boundary, "mcp-response"); + report.overflow_evidence_id = + error.details.reported_limits.evidence_reference.evidence_id; + } + const parentId = + report.overflow_evidence_id ?? + parseEvidence(full.structuredContent).evidence_id; + const strings = await good( + "inspect_analysis_view", + { + source: { kind: "retained-evidence", evidence_id: parentId }, + view: { kind: "summary" }, + }, + "summary-strings", + ); + const summary = strings.normalized_result; + report.strings = summary; + if (report.overflow_evidence_id) + assert.equal(summary.parent_evidence_id, report.overflow_evidence_id); + const selected = []; + const total = summary.summary.row_count; + for (const offset of [...new Set([0, Math.max(0, total - 3)])]) { + const e = await good( + "inspect_analysis_view", + { + source: { + kind: "retained-evidence", + evidence_id: summary.parent_evidence_id, + }, + view: { kind: "native-inventory", offset, limit: 3, fields: ["value"] }, + }, + "selected-strings", + ); + assert.equal(e.normalized_result.coverage.total, total); + selected.push(...e.normalized_result.rows); + } + report.byte_oracles = []; + const file = await open(target, "r"); + try { + for (const row of selected) { + const mapping = await good( + "address_to_file_offset", + { address: row.record.address }, + "string-file-coordinate", + ); + const offset = mapping.normalized_result.file_offset; + assert.ok(Number.isSafeInteger(offset)); + const expected = Buffer.from(row.record.value, "utf8"), + bytes = Buffer.alloc(expected.length); + const read = await file.read(bytes, 0, bytes.length, offset); + assert.equal(read.bytesRead, bytes.length); + const encoding = + row.record.string?.encoding ?? row.record.decoding?.encoding; + if (["utf-8", "UTF-8", "ascii", "ASCII", "US-ASCII"].includes(encoding)) + assert.deepEqual(bytes, expected); + report.byte_oracles.push({ + source_index: row.source_index, + address: row.record.address, + file_offset: offset, + encoding, + utf8_bytes_match: bytes.equals(expected), + bytes_sha256: createHash("sha256").update(bytes).digest("hex"), + }); + } + } finally { + await file.close(); + } + await raw( + "export_evidence_bundle", + { path: join(output, "complete-evidence.json") }, + "export", + ); + await raw("close_binary", {}, "close"); + opened = false; + const bundle = JSON.parse( + await readFile(join(output, "complete-evidence.json"), "utf8"), + ); + const source = parseEvidence( + bundle.records.find((e) => e.evidence_id === summary.parent_evidence_id), + ); + assert.equal(source.subject.digest.sha256, report.sha256); + assert.equal(source.normalized_result.length, total); + for (const row of selected) + assert.equal( + row.record.value, + source.normalized_result[row.source_index].value, + ); + const fn = parseEvidence( + bundle.records.find( + (e) => e.evidence_id === report.procedures.parent_evidence_id, + ), + ); + assert.equal(fn.subject.digest.sha256, report.sha256); + report.procedure_count = fn.normalized_result.length; + assert.deepEqual( + fn.analysis_profile.parameters.analysis_options.disabled_analyzers, + disabled, + ); + assert.equal( + fn.evidence_links.length, + 0, + "This capture expects a self-contained inventory; retain its full source closure otherwise.", + ); + await writeFile( + join(output, "functions.evidence.json"), + JSON.stringify(createEvidenceBundle([fn])), + { flag: "wx" }, + ); + report.profile_digest = fn.analysis_profile.digest; + report.source_string_count = source.normalized_result.length; + report.status = "passed"; +} catch (cause) { + report.status = "failed"; + report.failure = String(cause?.stack ?? cause); + process.exitCode = 1; +} finally { + if (opened) + try { + const r = await call("close_binary", {}, "cleanup"); + assert.notEqual(r.isError, true); + } catch (cause) { + report.close_failure = String(cause); + process.exitCode = 1; + } + report.elapsed_ms = performance.now() - start; + if ( + !(await finalizeRetainedQueryReceipt({ + report, + client, + transport, + account, + serverLog, + serverLogFinished, + output, + transcript, + })) + ) + process.exitCode = 1; +} diff --git a/experiments/analysis-efficiency/native-inventory-profile.json b/experiments/analysis-efficiency/native-inventory-profile.json new file mode 100644 index 0000000000..0bc6077c58 --- /dev/null +++ b/experiments/analysis-efficiency/native-inventory-profile.json @@ -0,0 +1,34 @@ +{ + "description": "Ghidra 12.1.4 Linux x86-64 read-only inventory verification profile; ASCII Strings remains enabled. Default-analysis and complete function-body coverage are not claimed.", + "names": [ + "Condense Filler Bytes", + "Apply Data Archives", + "DWARF", + "Data Reference", + "ELF Scalar Operand References", + "Embedded Media", + "External Symbol Resolver", + "Non-Returning Functions - Known", + "Create Address Tables", + "Call-Fixup Installer", + "Disassemble Entry Points", + "External Entry References", + "Subroutine References", + "Shared Return Calls", + "Stack", + "X86 Function Callee Purge", + "GCC Exception Handlers", + "Aggressive Instruction Finder", + "Call Convention ID", + "Decompiler Parameter ID", + "Decompiler Switch Analysis", + "Function ID", + "Demangler GNU", + "x86 Constant Reference Analyzer", + "Reference", + "Non-Returning Functions - Discovered", + "Function Start Search", + "Function Start Search After Code", + "Function Start Pre Search" + ] +} diff --git a/experiments/analysis-efficiency/native-matching-capacity.md b/experiments/analysis-efficiency/native-matching-capacity.md new file mode 100644 index 0000000000..3be1fdfeed --- /dev/null +++ b/experiments/analysis-efficiency/native-matching-capacity.md @@ -0,0 +1,177 @@ +# Native matching capacity verification + +The runtime setting changes admission headroom, not matching semantics. The +96 MiB default is preserved. This experiment checks the real CLI and stdio MCP +with synthetic Ghidra-shaped rows and a real pair of stripped LLVM inventories. +It is not evidence of faster matching or behavioral equivalence. + +The control is main `14631f698`; candidate runtime is `115df89c9`, based on +Kaoru0822's #7112 (`87d3d6022`, `0600c7d11`) with main's #7108 provenance +accounting retained. Imported commits retain Kaoru's authorship and their +Claude Opus 5.5 co-author trailers. + +## Synthetic capacity + +Each side has 20,000 or 50,000 unique imported names, original source indices, +explicit symbol/procedure metadata and different addresses. Provider identity +and limitations explicitly label these as synthetic; no native provider ran. +The known answer is every row paired by its unique exact name, with no +left-only, right-only or unresolved groups. Verification authenticates complete +exports and checks every member against its original inventory identity, +index, address and name. MCP also reads first/last selected pages. + +CLI uses two source bundles and writes the complete matching/source closure. +The control refuses both sizes at 96 MiB. The candidate succeeds with the +536870912-byte launch seed. MCP imports the same records, tries the default, +raises the runtime setting to 512 MiB and repeats matching twice. The 20k +retained case already fits the default; the 50k case is refused then recovers +without restarting or replacing inputs. Matching Evidence IDs remain stable +across allowance changes and repeated successful requests. + +| Lane / rows per side | Outcome | Whole elapsed s | CPU s | Family peak GiB | Text token estimate | +| -------------------- | ------- | --------------: | ----: | --------------: | ------------------: | +| mcp-20000 | matched | 25.89 | 30.77 | 1.77 | 938,961 | +| mcp-50000 | matched | 71.74 | 81.87 | 2.31 | 938,209 | +| cli-20000 | matched | 27.55 | 31.04 | 1.02 | 1,166 | +| cli-50000 | matched | 63.61 | 70.37 | 1.53 | 1,138 | +| main-cli-20000 | refused | 10.45 | 12.28 | 0.72 | 364 | +| main-cli-50000 | refused | 17.85 | 19.17 | 0.90 | 357 | + +These whole-harness costs include input preparation/authentication, client and +catalog work, matching, output publication and offline correctness checks. +MCP rows include the initial default attempt, settings change, two raised-budget +matches, two selected pages and export. CLI rows include one fresh invocation +and its complete export. The lanes do different amounts of validation/work; +these numbers are not an adapter speed comparison. A refused control does not +complete the matching task, so no completed-task speedup can be inferred. + +The first successful MCP match is 3.535 s at 20k (under the default, before +raising it), and 12.393 s at 50k (first raised-budget call). Last repeated calls +are 3.216 s and 13.640 s. Original receipt phase labels `cold`/`warm` denoted +raised-budget call order; the published driver now uses `first-raised` and +`repeat-raised` to avoid mislabelling the already-warm 20k case. Every final +whole harness starts a fresh process; OS file caches are warm. CLI matching +invocations, including complete publication, are 19.546 s / 47.712 s; the extra +whole-harness cost is visible in the table. + +Token estimates use cl100k_base over one raw catalog discovery, every argument +and every text reply, including refusal, settings, paging and export. CLI counts +its invocation arguments, request JSON and actual stdout. Local Evidence file +contents are not sent to the agent; their loading and checking are included in +resource costs. Catalog discovery dominates MCP. Initialization, JSON-RPC +framing, prompts, host rendering and replay are excluded; actual model usage +was not measured. The matching summary keeps the full report out of the MCP +frame; the complete 50k export is about 85 MiB and remains accessible locally. + +## Real stripped inventories + +The unmodified, stripped Linux x86-64 LLVM 14 and 15 shared libraries produce +43,643 and 45,230 Ghidra procedure observations. Target digests: + +- LLVM 14: `436887791de0478d72c8323be99df69d6d0cf82745e5abec79d5e0374f4df560`. +- LLVM 15: `e45650cba881293ba3b6a0e7241920fc48fa4a522ca6dfda72dc94f5c54e44b0`. + +Both MCP and CLI match the same complete inputs at 512 MiB: 34,008 exact-symbol +correspondences, 9,635 left-only observations and 11,222 right-only observations. +Every source row appears exactly once, with its original inventory ID, source +index, address and name. Matching Evidence IDs are identical between adapters +for both synthetic sizes and this real pair. The default retained request is +refused. Left/right-only observations do not establish actual product additions +or removals; inventory discovery completeness remains unknown. + +An independent GNU `nm --without-symbol-versions --dynamic --no-demangle +--format=posix` comparison corroborates 32,935 matched raw dynamic symbol names. +The other 1,073 are not corroborated by that raw-name comparison (examples include +`::lstat64`); no raw-name equivalence is claimed for that remainder. +The independent check costs 1.86 s elapsed/CPU and 0.34 GiB sampled RSS, in +addition to the matching harness. No byte fingerprints or decompilation were +supplied, and no behavioral-equivalence claim is made. + +| Real matching lane | Whole elapsed s | CPU s | Family peak GiB | Text token estimate | +| ------------------ | --------------: | ----: | --------------: | ------------------: | +| real-mcp | 56.04 | 62.63 | 2.11 | 943,676 | +| real-cli | 74.04 | 79.28 | 1.36 | 2,482 | + +The capture phase is measured separately: LLVM 14 takes 280.14 s elapsed, +366.63 s CPU and 2.88 GiB family RSS; LLVM 15 takes 278.39 s, 358.03 s and +3.11 GiB. These are actual REA/Ghidra MCP captures with Ghidra 12.1.4/JDK +21.0.12.1 and the explicitly committed 29-disabled-analyzer profile in +`native-inventory-profile.json`, with ASCII Strings enabled. This deliberately +narrow profile does not establish default-analysis or full body coverage. +The captures also validate 177,830/193,493 defined strings, retained summaries +and first/last pages after complete delivery exceeds 10 MiB. Six selected UTF-8 +observations per target match independently read original-file bytes, including +non-default `.shstrtab` address-space observations. Exports are authenticated +before selecting unchanged, self-contained procedure Evidence into the two +function-only bundles; no Ghidra fact is reconstructed or relabelled. + +End to end, sum both capture phases, the chosen matching lane and the independent +symbol check. Sequential peak RSS is the maximum of the phases. The actual +workflow used separate capture and matching connections, so its token accounting +must include three catalog discoveries and every capture RPC, not merely the +matching row above. Capture/discovery totals are reported below. Failed early +capture attempts (600 s SDK timeout, unregistered delayed-analyzer name, and a +driver exporting after session close) remain failed verification attempts; +all cleaned up without OOM. The final driver exports before closing, then +performs memory-intensive offline checks after the Java process has exited. + +Capture reproduction (provide an appropriate process-family supervisor): + +```sh +node experiments/analysis-efficiency/native-inventory-capture.mjs \ + --runtime /absolute/candidate --target /absolute/libLLVM.so \ + --output /tmp/llvm-capture --ghidra /absolute/ghidra --java /absolute/jdk \ + --profile experiments/analysis-efficiency/native-inventory-profile.json +``` + +Use each capture's `functions.evidence.json` as the matching driver's input. + +| Capture | Catalog tokens | Arguments + replies | Total token estimate | +| ---------------------------- | -------------: | ------------------: | -------------------: | +| rea-6426-llvm14-inventory-v4 | 932,545 | 14,685 | 947,230 | +| rea-6406-llvm15-inventory | 932,629 | 14,663 | 947,292 | + +Both captures + real-mcp + independent symbol check: 616.42 s elapsed, 789.15 s CPU, 2,838,198 estimated text tokens; sequential sampled family peak 3.11 GiB. + +Both captures + real-cli + independent symbol check: 634.43 s elapsed, 805.81 s CPU, 1,897,004 estimated text tokens; sequential sampled family peak 3.11 GiB. + +Early failed capture attempts, excluded from those successful-run totals: + +| Attempt | Elapsed s | CPU s | Peak family GiB | Outcome | +| ---------------------------- | --------: | -----: | --------------: | ---------------------- | +| rea-6426-llvm14-inventory | 610.34 | 920.41 | 2.86 | SDK timeout | +| rea-6426-llvm14-inventory-v2 | 39.26 | 66.81 | 2.16 | unregistered analyzer | +| rea-6426-llvm14-inventory-v3 | 217.65 | 285.06 | 3.12 | driver export ordering | + +## Reproduce + +Use Node 24.22.1, the pinned npm install, and built runtime worktrees. Supply +new output directories. The driver keeps the catalog, actual RPC transcript, +phase receipts, complete matching bundle and verification results. + +```sh +node experiments/analysis-efficiency/native-matching-capacity.mjs \ + --runtime /absolute/candidate --output /tmp/matching-50k \ + --adapter mcp --rows 50000 --allowance 536870912 +node experiments/analysis-efficiency/native-matching-capacity.mjs \ + --runtime /absolute/candidate --output /tmp/matching-cli \ + --adapter cli --left /tmp/matching-50k/left.evidence.json \ + --right /tmp/matching-50k/right.evidence.json --allowance 536870912 +``` + +For real data, replace the two paths with authenticated one-inventory bundles +captured from the desired targets; no provider is started by the matcher. +Omit allowance and use `--refusal` when a baseline refusal is the expected oracle. +Do not split inventories: that changes global uniqueness. + +All measured runs used two CPU affinity slots and a supervisor with an 8 GiB +summed process-family RSS ceiling and a 4 GiB available-host-memory floor. +Node old space was 3 GiB. No resource stop or OOM occurred. RSS is summed every +100 ms, can count shared pages repeatedly and can miss short peaks. CPU includes +all waited descendants. The driver does not itself enforce a family limit; +apply a supervisor appropriate to the host. The 1 GiB setting maximum is a +representation policy, not a process-memory or host-capacity guarantee. + +The small CLI harness pilot failed because it treated the documented +`{summary, evidence_bundle}` CLI envelope as bare Evidence; its failed receipt +was retained. The named envelope was corrected before all capacity measurements. diff --git a/experiments/analysis-efficiency/native-matching-capacity.mjs b/experiments/analysis-efficiency/native-matching-capacity.mjs new file mode 100644 index 0000000000..45121435eb --- /dev/null +++ b/experiments/analysis-efficiency/native-matching-capacity.mjs @@ -0,0 +1,413 @@ +/** Real CLI/SDK capacity investigations; synthetic inputs are explicitly labelled. */ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createWriteStream } from "node:fs"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { finished } from "node:stream/promises"; +import { parseArgs, promisify } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { createRetainedQueryCaller } from "./retained-query-calls.mjs"; +import { finalizeRetainedQueryReceipt } from "./retained-query-finalization.mjs"; +const { values: v } = parseArgs({ + options: { + ...Object.fromEntries( + [ + "runtime", + "left", + "right", + "output", + "adapter", + "allowance", + "rows", + ].map((k) => [k, { type: "string" }]), + ), + refusal: { type: "boolean", default: false }, + }, +}); +for (const k of ["runtime", "output"]) { + assert.ok(v[k]); + v[k] = resolve(v[k]); +} +assert.ok(["cli", "mcp"].includes(v.adapter)); +await mkdir(v.output); +let account, client, transport, serverLog, serverLogFinished, call; +const report = { + adapter: v.adapter, + runtime: v.runtime, + synthetic_rows: v.rows ? Number(v.rows) : null, + allowance: v.allowance ? Number(v.allowance) : null, + calls: [], + runs: [], + }, + transcript = []; +const start = performance.now(); +try { + const load = (p) => import(pathToFileURL(join(v.runtime, p)).href); + const { createEvidence, parseEvidence } = await load( + "dist/domain/evidence.js", + ); + const { createEvidenceBundle, parseEvidenceBundle } = await load( + "dist/domain/evidenceBundle.js", + ); + const { nativeFunctionMatchingSchema } = await load( + "dist/domain/native/functionMatching.js", + ); + report.revision = ( + await promisify(execFile)("git", ["-C", v.runtime, "rev-parse", "HEAD"]) + ).stdout.trim(); + const sources = []; + for (const [index, key] of ["left", "right"].entries()) { + if (v.rows) { + const count = Number(v.rows); + assert.ok(Number.isSafeInteger(count) && count > 0); + const source = createEvidence( + { + path: `/synthetic/${key}.elf`, + sha256: String(index + 1).repeat(64), + format: "elf", + architecture: "x86_64", + }, + { + id: "fixture.capacity", + name: "Synthetic capacity fixture", + version: "1", + }, + { + operation: "list_procedures", + parameters: {}, + result: Array.from({ length: count }, (_, i) => ({ + address: `0x${(0x400000 + index * 0x400000 + i * 16).toString(16)}`, + value: `subsystem_function_${i}`, + symbol: { + primary: true, + dynamic: false, + external: false, + type: "Function", + source: "imported", + }, + procedure: { external: false, thunk: false, thunk_target: null }, + })), + limitations: [ + "Synthetic Ghidra-shaped inventory; no native provider ran.", + ], + }, + ); + v[key] = join(v.output, `${key}.evidence.json`); + await writeFile(v[key], JSON.stringify(createEvidenceBundle([source])), { + flag: "wx", + }); + sources.push(source); + } else { + assert.ok(v[key], `Missing --${key}`); + v[key] = resolve(v[key]); + const bundle = parseEvidenceBundle( + JSON.parse(await readFile(v[key], "utf8")), + ); + const inventories = bundle.records.filter( + (e) => e.operation === "list_procedures", + ); + assert.equal(inventories.length, 1); + sources.push(inventories[0]); + } + } + const request = { + left: { kind: "retained-evidence", evidence_id: sources[0].evidence_id }, + right: { kind: "retained-evidence", evidence_id: sources[1].evidence_id }, + detail: "summary", + }; + report.input_kind = sources.every((e) => e.provider.id === "fixture.capacity") + ? "synthetic" + : "captured-provider"; + if (report.input_kind === "synthetic") + report.synthetic_rows = sources[0].normalized_result.length; + report.sources = sources.map((e, i) => ({ + path: v[["left", "right"][i]], + evidence_id: e.evidence_id, + subject: e.subject, + provider: e.provider, + profile_digest: e.analysis_profile?.digest ?? null, + rows: e.normalized_result.length, + })); + account = await mkdtemp(join(tmpdir(), "rea-matching-capacity-")); + const environment = { + HOME: account, + PATH: process.env.PATH, + NODE_OPTIONS: "--max-old-space-size=3072", + REA_LOG_LEVEL: "silent", + REA_MCP_INPUT_SCHEMA_PROFILE: "compact", + }; + const errorOf = (reply) => + JSON.parse(reply.content.find((c) => c.type === "text").text).error; + let parentId, + bundlePath = join(v.output, "matching.evidence.json"); + if (v.adapter === "cli") { + if (v.allowance) + environment.REA_NATIVE_MATCHING_REPRESENTATION_BYTES = v.allowance; + const path = join(v.output, "request.json"); + await writeFile(path, JSON.stringify(request)); + const args = [ + join(v.runtime, "scripts/rea.mjs"), + "match-native-functions", + path, + "--source_bundle", + v.left, + "--source_bundle", + v.right, + "--evidence_output", + bundlePath, + "--json", + ]; + let result, + exit = 0; + const at = performance.now(); + try { + result = await promisify(execFile)(process.execPath, args, { + cwd: v.runtime, + env: environment, + maxBuffer: 16 * 1024 ** 2, + timeout: 300000, + }); + } catch (cause) { + if (cause.code !== 1) throw cause; + result = cause; + exit = 1; + } + await writeFile(join(v.output, "cli.stdout.json"), result.stdout); + await writeFile(join(v.output, "cli.stderr.log"), result.stderr ?? ""); + const reply = JSON.parse(result.stdout); + report.runs.push({ + phase: "cold", + elapsed_ms: performance.now() - at, + exit_code: exit, + }); + transcript.push({ + name: "match-native-functions", + input: request, + args, + reply, + }); + if (exit) { + assert.equal(v.refusal, true); + assert.equal(reply.code, "resource_constraint"); + assert.equal(reply.retryable, false); + report.refusal = reply; + report.outcome = "refused"; + } else { + assert.equal(v.refusal, false); + const e = parseEvidence(reply.summary); + assert.equal(reply.evidence_bundle.path, bundlePath); + parentId = e.normalized_result.parent_evidence_id; + report.runs[0].summary = e.normalized_result.summary; + report.outcome = "matched"; + } + } else { + transport = new StdioClientTransport({ + command: process.execPath, + args: [join(v.runtime, "scripts/rea.mjs"), "mcp"], + cwd: v.runtime, + env: environment, + stderr: "pipe", + }); + serverLog = createWriteStream(join(v.output, "server.log")); + serverLogFinished = finished(serverLog); + serverLogFinished.catch(() => {}); + transport.stderr?.pipe(serverLog); + client = new Client({ name: "native-matching-capacity", version: "1" }); + call = createRetainedQueryCaller(client, (receipt, event) => { + report.calls.push(receipt); + transcript.push(event); + }); + await client.connect(transport); + const catalog = await client.listTools(); + await writeFile(join(v.output, "catalog.json"), JSON.stringify(catalog)); + const definitions = new Map(catalog.tools.map((t) => [t.name, t])); + const original = client.callTool.bind(client); + client.callTool = (input, options) => + original(input, { + ...options, + timeout: 300000, + toolDefinition: definitions.get(input.name), + }); + const good = async (name, input, phase) => { + const r = await call(name, input, phase); + if (r.isError) { + const e = errorOf(r); + throw new Error(`${name}: ${e.code}: ${e.message}`); + } + return r; + }; + for (const side of ["left", "right"]) + await good("import_evidence_bundle", { path: v[side] }, `import:${side}`); + const initial = await call("match_native_functions", request, "default"); + if (initial.isError) { + const e = errorOf(initial); + assert.equal(e.code, "resource_constraint"); + assert.equal(e.retryable, false); + report.refusal = e; + report.default_admitted = false; + } else { + report.default_admitted = true; + parentId = parseEvidence(initial.structuredContent).normalized_result + .parent_evidence_id; + } + if (v.allowance) { + await good( + "update_analysis_settings", + { + changes: [ + { + setting: "native.matching_representation_bytes", + action: "set", + value: Number(v.allowance), + }, + ], + }, + "raise-budget", + ); + for (const phase of ["first-raised", "repeat-raised"]) { + const at = performance.now(); + const e = parseEvidence( + (await good("match_native_functions", request, phase)) + .structuredContent, + ); + const id = e.normalized_result.parent_evidence_id; + if (parentId) assert.equal(id, parentId); + parentId = id; + report.runs.push({ + phase, + elapsed_ms: performance.now() - at, + summary: e.normalized_result.summary, + }); + } + report.outcome = "matched"; + } else if (initial.isError) { + assert.equal(v.refusal, true); + report.outcome = "refused"; + } else { + assert.equal(v.refusal, false); + report.outcome = "matched"; + } + if (report.outcome === "matched") { + report.selected = []; + for (const offset of [ + ...new Set([0, Math.max(0, sources[0].normalized_result.length - 2)]), + ]) { + const e = parseEvidence( + ( + await good( + "inspect_analysis_view", + { + source: { kind: "retained-evidence", evidence_id: parentId }, + view: { + kind: "function-matches", + collection: "functions", + sides: ["left"], + fields: ["name"], + offset, + limit: 2, + }, + }, + "selected-correspondences", + ) + ).structuredContent, + ); + assert.equal( + e.normalized_result.coverage.total, + sources[0].normalized_result.length, + ); + for (const row of e.normalized_result.items) { + const original = sources[0].normalized_result[row.source_index]; + assert.equal(row.inventory_evidence_id, sources[0].evidence_id); + assert.equal(row.address, original.address); + assert.equal(row.name, original.value); + } + report.selected.push(...e.normalized_result.items); + } + await good("export_evidence_bundle", { path: bundlePath }, "export"); + // Release the server's retained representation before offline verification. + await client.close(); + await transport.close(); + } + } + if (report.outcome === "matched") { + const bundle = parseEvidenceBundle( + JSON.parse(await readFile(bundlePath, "utf8")), + ); + const evidence = bundle.records.find((e) => e.evidence_id === parentId); + assert.ok(evidence); + const result = nativeFunctionMatchingSchema.parse( + evidence.normalized_result, + ); + assert.deepEqual( + [result.left.evidence_id, result.right.evidence_id], + sources.map((e) => e.evidence_id), + ); + const seen = [new Set(), new Set()]; + for (const group of result.matches) + for (const [side, key] of ["left", "right"].entries()) + for (const member of group[key]) { + const source = sources[side]; + assert.equal(member.inventory_evidence_id, source.evidence_id); + assert.ok(!seen[side].has(member.source_index)); + seen[side].add(member.source_index); + const row = source.normalized_result[member.source_index]; + assert.ok(row); + assert.equal(member.address, row.address); + assert.equal(member.name, row.value); + assert.ok(member.evidence_links.includes(source.evidence_id)); + if (report.synthetic_rows !== null) { + assert.equal(group.state, "matched"); + assert.equal(group.basis, "exact-symbol"); + assert.equal(group.left.length, 1); + assert.equal(group.right.length, 1); + assert.equal( + group.left[0].source_index, + group.right[0].source_index, + ); + } + } + assert.deepEqual( + seen.map((s) => s.size), + sources.map((e) => e.normalized_result.length), + ); + if (report.synthetic_rows !== null) + assert.deepEqual(result.summary, { + matched: report.synthetic_rows, + only_left: 0, + only_right: 0, + unresolved: 0, + }); + else assert.ok(result.summary.matched > 0); + for (const source of sources) + assert.ok( + bundle.records.some((e) => e.evidence_id === source.evidence_id), + ); + report.parent_evidence_id = parentId; + report.summary = result.summary; + report.verified_members = seen.map((s) => s.size); + } + report.status = "passed"; +} catch (cause) { + report.status = "failed"; + report.failure = String(cause?.stack ?? cause); + process.exitCode = 1; +} finally { + report.elapsed_ms = performance.now() - start; + if ( + !(await finalizeRetainedQueryReceipt({ + report, + client, + transport, + account, + serverLog, + serverLogFinished, + output: v.output, + transcript, + })) + ) + process.exitCode = 1; +} diff --git a/experiments/analysis-efficiency/native-matching-elf-oracle.py b/experiments/analysis-efficiency/native-matching-elf-oracle.py new file mode 100644 index 0000000000..b3c29c7de4 --- /dev/null +++ b/experiments/analysis-efficiency/native-matching-elf-oracle.py @@ -0,0 +1,23 @@ +import json,subprocess,argparse,hashlib +p=argparse.ArgumentParser();p.add_argument("--left-target",required=True);p.add_argument("--right-target",required=True);p.add_argument("--matching-report",required=True);p.add_argument("--matching-bundle",required=True);args=p.parse_args() +paths=[args.left_target,args.right_target] +receipt=json.load(open(args.matching_report)) +for i,path in enumerate(paths): + h=hashlib.sha256() + with open(path,'rb') as f: + for chunk in iter(lambda:f.read(1024*1024),b''):h.update(chunk) + assert h.hexdigest()==receipt['sources'][i]['subject']['digest']['sha256'] +names=[] +for p in paths: + r=subprocess.run(['nm','--without-symbol-versions','--dynamic','--no-demangle','--format=posix',p],capture_output=True,text=True,check=True,timeout=20) + rows=[line.split() for line in r.stdout.splitlines() if line] + assert all(2<=len(row)<=4 and len(row[1])==1 for row in rows) + names.append({row[0] for row in rows}) +report=receipt;bundle=json.load(open(args.matching_bundle));source=next(e for e in bundle['records'] if e['evidence_id']==report['parent_evidence_id']);matches=[m for m in source['normalized_result']['matches'] if m['state']=='matched'];common=names[0]&names[1] +verified=[];other=[] +for m in matches: + assert m['basis']=='exact-symbol' and len(m['left'])==len(m['right'])==1 + a=m['left'][0];b=m['right'][0];assert a['name']==b['name'] + (verified if a['name'] in common else other).append(a['name']) +r={'command':['nm','--without-symbol-versions','--dynamic','--no-demangle','--format=posix'],'source_files':paths,'common_raw_dynamic_names':len(common),'matched_correspondences':len(matches),'raw_name_corroborated':len(verified),'provider_only_names':len(other),'provider_only_examples':other[:8]} +print(json.dumps(r,indent=2))