From 2653b194b695879dcb1d290feca56301282594bc Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sun, 11 Oct 2026 03:13:56 +0800 Subject: [PATCH] feat(skills)!: improve Codex onboarding and task recovery (#1613) Guide task-centered recovery from observed client, registration, PATH and provider blockers. Install the recovery reference, document exact JavaScript module selectors, and require clarification before searching for an unnamed app. Validate public setup in isolated Codex accounts and preserve failed or unrecognized tool outcomes in the existing evaluator. Refs: #1587 BREAKING CHANGE: Optional verify:agent reports now use schemaVersion 4. Consumers must accept expectedFirstTool: null and targetClarificationPassed for missing-target scenarios. Production CLI/MCP contracts are unchanged. --- .../skills/reverse-engineer-anything/SKILL.md | 89 +++---- .../references/connection-and-recovery.md | 127 ++++++++++ .../references/javascript-applications.md | 89 ++++++- docs/testing.md | 51 +++- scripts/verify-agent-experience.mjs | 229 +++++++++++++++--- src/application/SetupSkill.ts | 1 + src/evaluation/AgentEvaluationReport.ts | 17 +- src/evaluation/CodexAgentEval.ts | 72 ++++-- src/generatedPackageMetadata.ts | 2 +- tests/boundary/filesystem/setupSkill.test.ts | 4 +- .../model-evals/agentEvaluationReport.test.ts | 23 ++ .../model-evals/codexAgentEval.test.ts | 142 +++++++++++ 12 files changed, 720 insertions(+), 126 deletions(-) create mode 100644 .agents/skills/reverse-engineer-anything/references/connection-and-recovery.md diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index f87a4e1103..7e35c0db04 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -2,7 +2,7 @@ name: reverse-engineer-anything description: Reverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. metadata: - version: "36" + version: "37" --- # REA @@ -25,57 +25,18 @@ a skill installed from repository main may describe capabilities absent from an older npm release. Keep complete inline Evidence when that server does not advertise retained references. -When tools are absent or registration is stale: +If tools are absent, a call is blocked by the client, or a required provider is +unavailable, read [connection and recovery](references/connection-and-recovery.md). +Diagnose the observed blocker, continue useful supported work, and resume the +original task after repair. Client approval, shell PATH, registration and engine +availability have different remedies; avoid repeating setup for all of them. -1. Diagnose without changing files. For Codex, run - `npx -y rea-agents@latest doctor --client codex --json`. Substitute the current - supported client: `claude_code`, `claude_desktop`, `codex`, `cursor`, - `gemini_cli`, `windsurf`, `devin`, `opencode`, `antigravity`, `copilot_cli`, - `commandcode`, `qwen_code`, `vscode`, `grok_build`, `omp`, `pi`, `hermes`, or `grok_bot`. `grok_bot` has no local - file registration: doctor reports the Grok Bot chat step, and setup does not - write `mcp.json`. If the client is unknown, use `doctor --json` and - inspect its registration results before choosing a setup scope. -2. Distinguish the reason. Missing, malformed, or stale registration needs a - scoped configuration repair. An aligned registration with no tools in the - active session needs a restart/reconnection; doctor cannot prove that the - current agent has connected. A missing provider affects only tasks requiring - that provider: static JavaScript inspection needs neither Hopper nor Ghidra, - and Android inspection has separate bring-your-own JADX/Java prerequisites. -3. For a configuration repair, prepare the read-only plan: - `npx -y rea-agents@latest setup --client codex --dry-run --json`. - Use the current client's ID, show its exact proposed paths, backups, and - changes, and obtain approval before setup writes configuration or installs - Hopper. Setup normally installs the matching bundled skill too; include that - replacement in the reviewed plan. After approval, apply the same scope with - `npx -y rea-agents@latest setup --client codex --yes`. Add `--install-hopper` - only if that separate installation was needed and explicitly approved. -4. Restart/reconnect the affected agent. Verify that REA tools actually appear - in the session, then resume the original investigation. If they remain - absent, inspect the client's MCP launch error rather than repeating setup. - -REA setup never installs or upgrades Node.js, npm, Homebrew, Java, Ghidra, IDA, -JADX, Binwalk, or Unblob. Use existing prerequisites; do not install unrelated -software to repair -MCP registration. For an unsupported client, use manual stdio registration -with a version-pinned `rea-agents` package or continue through the CLI. - -A concrete CLI fallback for an operator-supplied JavaScript tree or ASAR is: - -```bash -npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json > app-evidence.json -jq -c '{source: {kind: "inline", evidence: .}, view: {kind: "summary"}}' app-evidence.json > app-view.json -npx -y rea-agents@latest inspect-analysis-view app-view.json -``` - -No MCP registration or native engine is required. The CLI returns the complete -Evidence record directly, and a real application can produce hundreds of -megabytes, so save it to a file rather than reading it into context. Start from -the summary view, then request a module page -(`{"kind": "page", "collection": "modules", "offset": 0, "limit": 32}`) or one -module the same way, or pass the saved Evidence inline to -`trace-application-feature`. Read graph, limitations, and unknowns with the -same care as an MCP result. This fallback does not establish that MCP is -configured. Native CLI tasks still require their selected engine. +Prefer an existing `rea` CLI. When none is installed and REA package acquisition +is within the authorized scope, use `npx -y rea-agents@latest` in its place. +Configuration repair starts with a read-only setup plan; show paths, backups and +skill replacement before applying authorized changes. Preserve existing approval +and target-execution boundaries. REA setup never installs or upgrades unrelated +software, including Node.js, npm, Homebrew, Java, Ghidra, IDA, JADX, Binwalk or Unblob. ## Route the target first @@ -140,9 +101,11 @@ after opening; IDA does not supply `binary_overview`. Consult session availabili when composing broader workflows. Modern direct callers and unsupported dossier facets remain unknown; live IDA results are not replayed from snapshots. -If the app is missing, ask which app to inspect. Resolve a human-readable app -name to one clear installed artifact when possible; ask only when matches are -ambiguous. Never choose an example app on the user's behalf. +If the user says only "my app" without a name, path, or target already selected +for this task, ask which app before searching for artifacts. Paths in unrelated +saved reports and nearby test fixtures do not select the user's target. Resolve +an actual human-readable app name to one clear installed artifact when possible; +ask when matches are ambiguous. Never choose an example app on the user's behalf. In a target-free session, use `open_binary` to bind any archive/package or native target whose analysis tool operates on the active target. Do not call an @@ -157,7 +120,11 @@ For a JavaScript application or ELF layout whose size is unknown or large, call `analyze_javascript_application` or `inspect_binary_layout` with `"detail": "summary"`, then inspect the retained `parent_evidence_id` with `inspect_analysis_view`. -Do not repeat an identical tool call. Make a focused follow-up only when the +Page module identities and select the returned `node_id` before requesting an +item; a single file path can represent several graph roles. Do not repeat an +identical analysis or rejected call without a changed input or resolved blocker. +Recheck session state when a lifecycle change makes that status useful. +Make a focused follow-up only when the returned result leaves a specific question unanswered. If MCP reports `resource_constraint` with `details.resource: "transport"`, call @@ -168,6 +135,14 @@ session to a selected path. The analysis remains complete in the ledger; avoid repeating it merely to request the same oversized response. Complete CLI JSON output also streams without a single MCP frame. +A client can also truncate a successful tool result without a REA transport +error. Treat that preview as incomplete; use the retained parent ID for smaller +views or export when the full record is needed. Do not parse the preview as +canonical JSON or repeat the producer. For JavaScript, read the +[module-view and trace recipes](references/javascript-applications.md) before +drilling into a large graph; a broad bidirectional trace can expand unrelated +containment relationships even after summary analysis. + Every conclusion must distinguish observations, inferences, and unknowns. Cite Evidence IDs, preserve limitations and incomplete coverage, and never imply that static analysis observed execution. Runtime requests execute the declared @@ -175,7 +150,9 @@ target and lifecycle; do not broaden the target or action beyond those fields. Within the user's requested investigation, call available analysis tools directly. REA does not require permission grants or per-call approval flags. -Follow the declared request scope and the host's actual access requirements. +The agent client can independently require approval. Follow the declared request +scope and the host's actual access requirements; use the recovery guide for an +observed client-policy rejection. ## Plan broader investigations diff --git a/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md b/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md new file mode 100644 index 0000000000..5e5f6251da --- /dev/null +++ b/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md @@ -0,0 +1,127 @@ +# Connection and task recovery + +Read this guide only for an observed connection, command, approval or provider +blocker. Keep the user's target and question in view. Explain what prevented the +operation and the next useful step; after repair, continue the investigation. + +Commands below use an installed `rea`. If none is available, use the package +invocation in `SKILL.md` when acquiring REA is authorized. Follow the connected +server's schemas and the installed CLI's help rather than assuming repository +main and an older release expose the same arguments. + +## Choose the remedy from the observed failure + +| Observation | Meaning and next step | +| --------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| No REA tools in the active session | Inspect registration with `rea doctor --client codex --json`. A skill installation alone does not register MCP. | +| Missing, malformed or stale registration | Prepare a scoped setup plan; apply only authorized changes with backups. | +| Aligned registration, but no connected tools | Restart/reconnect the client. Doctor checks registration; it cannot establish the current agent's connection. Inspect the client's launch error if reconnection fails. | +| `MCP tool call requires approval, but approval policy is never` | Codex rejected the call before REA executed it. Do not install an engine or repeat setup to solve this client-policy decision. See the approval section below. | +| REA returns `provider_unavailable` or an unavailable tool entry | Read that operation's provider diagnostics or `binary_session.result.tool_availability`. Resolve only the prerequisite needed for the task; a complete tool catalog does not imply every operation is available. | +| `rea` is missing in a child shell, or Node is incompatible | Inspect the actual shell PATH and launcher/runtime. A prefix installation may be absent from a login shell's PATH; this does not establish a broken MCP registration. | + +If the client cannot launch its shell/sandbox helper or read the installed skill, +inspect its launch diagnostic. For example, Codex can refuse helper aliases when +CODEX_HOME is under `/tmp`; use an authorized profile outside the OS temporary +directory. REA setup cannot repair the client's executor. Preserve the sandbox +instead of silently disabling it to make an analysis command work. + +Static JavaScript and ordinary ZIP inventory need neither Hopper nor Ghidra. +HAR inspection is offline and does not require a live browser. Native +decompilation, Android analysis and firmware extraction have their own +bring-your-own prerequisites. Use supported inventory/static evidence when it +answers part of the question, and name the unavailable evidence separately. +Do not substitute another target. Label observations from ordinary shell tools +as such; only actual REA output supplies REA Evidence. + +## Repair registration when needed + +For Codex: + +```bash +rea doctor --client codex --json +rea setup --client codex --dry-run --json +``` + +Use the current supported client's ID: `claude_code`, `claude_desktop`, `codex`, +`cursor`, `gemini_cli`, `windsurf`, `devin`, `opencode`, `antigravity`, +`copilot_cli`, `commandcode`, `qwen_code`, `vscode`, `grok_build`, `omp`, `pi`, +`hermes`, or `grok_bot`. If unknown, inspect `doctor --json` before choosing a +scope. `grok_bot` uses a chat registration step rather than local `mcp.json`. +For unsupported clients, use version-pinned manual stdio registration or CLI. + +Show the plan's exact paths, backups, changes and bundled-skill replacement. +Obtain approval for writes when that concrete scope is not already authorized, +then apply the same scope with `rea setup --client codex --yes`. Add +`--install-hopper` only when separately needed and explicitly approved. Do not +install or upgrade unrelated prerequisites. Restart/reconnect, verify tools +actually appear, and resume the original task. If they remain absent, inspect +the launch failure rather than looping through setup. + +## Codex tool approval is separate from REA readiness + +REA tools have no permission-grant handshake. Codex can still require approval +because an operation creates owned resources, launches a helper or has cleanup +effects. An advertised tool or successful doctor result does not override that +client policy. `workspace-write` alone does not resolve an MCP approval +rejection, and `approval_policy = "never"` does not mean every MCP tool is +preapproved. + +When interactive approval is available, use the client's approval flow for the +requested operation. In an unattended run, continue through an authorized CLI +workflow if the execution environment permits it, or report the blocked +operation and the approval needed to proceed. Do not retry the same rejected +call, silently broaden approvals, or rewrite MCP effects as read-only. + +For an operator who explicitly wants to preapprove named operations in a +disposable Codex profile, current Codex supports per-tool configuration. For +example, opening and closing an analysis session can be scoped independently: + +```toml +[mcp_servers.rea.tools.open_binary] +approval_mode = "approve" + +[mcp_servers.rea.tools.close_binary] +approval_mode = "approve" +``` + +Use the actual server ID and only the operations whose effects were approved; +this example does not approve inspection, extraction or runtime execution. +Show the proposed config edit and preserve unrelated entries and backups before +writing. Managed policy can constrain client settings. Consult the +[Codex configuration reference](https://learn.chatgpt.com/docs/config-file/config-reference) +for the installed client's supported settings. This is an optional recovery +choice, not a prerequisite for every REA task. + +## Continue through an installed CLI + +Check the command environment only if the launcher fails: + +```bash +command -v rea +command -v node +node --version +rea --version +``` + +Use the public launcher at its known installation path with a supported existing +Node on PATH when the child shell loses a prefix installation. Avoid internal +`dist/cli.js` entrypoints or global toolchain changes. MCP may have its own valid +launcher even when a shell cannot resolve `rea`. + +For a user-supplied JavaScript tree or ASAR, select output files in the authorized +workspace and save complete Evidence before reading a view: + +```bash +rea analyze-javascript-application /absolute/path/to/app --json > app-evidence.json +jq -c '{source: {kind: "inline", evidence: .}, view: {kind: "summary"}}' app-evidence.json > app-view.json +rea inspect-analysis-view app-view.json +``` + +These commands do not register MCP or install an analysis engine. CLI JavaScript +JSON is complete and can be very large; redirect it rather than loading it into +context. Use an existing JSON processor if `jq` is absent, without installing +one just for this recipe. Inspect a module page or item using the same inline +source and the [JavaScript recipes](javascript-applications.md). Preserve graph +coverage, limitations and unknowns. Native CLI analysis still requires its +selected engine, and all CLI operations retain the host's execution permissions. diff --git a/.agents/skills/reverse-engineer-anything/references/javascript-applications.md b/.agents/skills/reverse-engineer-anything/references/javascript-applications.md index 0ee112a511..01b49a8887 100644 --- a/.agents/skills/reverse-engineer-anything/references/javascript-applications.md +++ b/.agents/skills/reverse-engineer-anything/references/javascript-applications.md @@ -1,20 +1,90 @@ # JavaScript and Electron application artifacts Use `analyze_javascript_application` directly on the operator-supplied ASAR or -extracted tree. The complete result, graph, and Evidence context are returned -inline when they fit the MCP frame. Source-map contents are part of the static analysis. Start from its -findings, coverage, unknowns, and graph context, then make a focused follow-up -only when a specific question remains unanswered. +extracted tree. When size is unknown or large and the connected schema advertises +`detail`, request `detail: "summary"`. This returns an `inspect_analysis_view` +Evidence record with a distinct ID and a `normalized_result.parent_evidence_id` +for the complete retained application analysis. Source-map contents are part of +that static analysis. Read findings, coverage, limitations and unknowns, then +follow a specific unanswered question. Older servers without `detail` return +the complete application Evidence inline when it fits. + +## Read a module without repeating analysis + +Use the actual returned parent ID, on the same connection, as `source.evidence_id`. +The following IDs are placeholders to replace with returned values. When the +module's node ID is not already known, page identities before selecting an item. +Pages return only node IDs, paths and roles; choose a useful page size and follow +`coverage.next_offset` when another page matters: + +```json +{ + "source": { + "kind": "retained-evidence", + "evidence_id": "ev_ACTUAL_PARENT_ID" + }, + "view": { "kind": "page", "collection": "modules", "offset": 0, "limit": 16 } +} +``` + +Select the relevant page entry by its exact `node_id` and recorded role: + +```json +{ + "source": { + "kind": "retained-evidence", + "evidence_id": "ev_ACTUAL_PARENT_ID" + }, + "view": { + "kind": "item", + "collection": "modules", + "selector": { "node_id": "jag_node_ACTUAL_NODE_ID" } + } +} +``` + +Module pages include asset, bundled and source module roles, so one path can +identify several entries. Use `selector.path` only when uniqueness is established; +the returned node ID avoids needing a complete inventory to prove uniqueness. On ambiguity, +select the matching node ID from the page or the error's candidates. Follow +`next_offset` when another page matters. An item exposes that module node's +recorded observations, identity, hashes, exports and source ranges; it does not +include every child syntax observation, related edge or original source file. +An absent item fact is not evidence of absence from the whole application. + +## Trace only the remaining question + +Use `trace_application_feature` on the parent application Evidence for one +literal node ID, route, string, API, IPC channel, module, or native export. +Select `incoming` for relationships leading to the seed, `outgoing` for those +leaving it, and `both` when the question needs both. Trace traversal includes +containment relationships, so even a narrow seed can expand many nodes; a +bidirectional trace can reach unrelated sibling modules. A module item or page +is preferable when its observations already answer the question. Do not treat +all reached nodes as participants in the feature. + +Reuse retained parent Evidence when the server advertises it, or supply complete +Evidence inline. Include Hopper or Ghidra Evidence only when its artifact digest +matches exactly. + +## Recover an oversized display An oversized response reports `resource_constraint` and `details.resource: "transport"`. Reuse `details.reported_limits.evidence_reference` with `inspect_analysis_view` for a -summary, one module, or a stable page of module identities. Use -`trace_application_feature` once a module, route, or string seed is known. +summary, one module, or a stable page of module identities. `export_evidence_bundle` writes the complete canonical session to a caller-selected file. These workflows keep the original Evidence and coverage; a broad follow-up can also exceed framing. +A client can truncate a successful result inside its text envelope without a +REA `resource_constraint`. Treat a truncation notice or incomplete JSON as a +display limitation; do not claim the full record was examined. Reuse the known +parent ID for smaller views, or export complete Evidence to a selected file +while the session remains open. If the parent ID was lost from the preview, +use the prior analysis summary or explicit ledger entries to recover it; do +not invent an ID or rerun analysis just to reproduce the same display. + BrowserWindow preferences, preload and contextBridge surfaces, IPC registrations, utility processes, and native binding requests are static syntax observations. Only a unique exact literal IPC channel match supports an inferred @@ -22,13 +92,6 @@ pairing. Dynamic or ambiguous channels remain unresolved. A requested `.node` member is not a verified native export. Never claim runtime reachability, registration, defaults, or policy enforcement from static analysis. -Use `trace_application_feature` on existing application Evidence for one -literal node ID, route, string, API, IPC channel, module, or native export. -Choose a direction and reuse the returned application Evidence ID when the -connected server advertises retained references, or supply complete Evidence -inline. Include -Hopper or Ghidra Evidence only when its artifact digest matches exactly. - For version comparison, analyze each version once, then call `compare_application_versions`. Accept only its digest, source-map, structural fingerprint, or non-module semantic matches. Module ordinals and minified names diff --git a/docs/testing.md b/docs/testing.md index ecb2600b6e..b6a4a40364 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1053,6 +1053,39 @@ real local Codex CLI with: npm run verify:agent ``` +The lane runs scenarios sequentially in standalone Codex (`--no-daemon`) with a +disposable HOME and CODEX_HOME under `~/.cache/rea-agent-evaluations/`. Set +`REA_AGENT_EVAL_ROOT` to choose another parent directory outside the OS temporary +directory: current Codex refuses helper aliases under `/tmp`, preventing shell +commands and skill reads even if MCP works. Each owned run directory is removed +after verification unless fixture retention is requested. The lane +installs the packaged skill and MCP registration through public `rea setup`, +rather than copying a skill into a project that also inherits the caller's +skills and configuration. Only authentication is copied, when present, and it +is removed even with `REA_AGENT_EVAL_KEEP_FIXTURES=true`. No user configuration, +rules, plugins or history is copied; shell snapshots and additional agents are +disabled. Existing API-key environment authentication remains available. +The native missing-provider scenario preapproves only `open_binary` and +`close_binary` in the disposable client so it reaches REA's actual provider +boundary instead of stopping at Codex approval. This does not validate a real +native engine or authorize arbitrary runtime execution. + +Select a smaller run with `REA_AGENT_EVAL_SCENARIOS`, a comma-separated list of +scenario IDs. `asar-zh` exercises the packaged desktop rubric from a Chinese +request. `javascript-module-view` exercises summary analysis, module paging and +item inspection in a tree with unrelated modules; its routing and text checks +remain heuristic, so review the retained transcript for selector choice and +coverage claims. Set `REA_CODEX_CLI` to an installed Codex executable and +`REA_AGENT_EVAL_MODEL` to an account-supported model. Use an external resource +limit for the whole process tree when testing on a constrained host; a Node +heap limit alone does not bound client and child-process memory. + +`missing-target` leaves desktop fixtures visible without selecting one in the +request. It requires zero REA calls and a structured request for an app name or +artifact path. Its `expectedFirstTool` is `null`, and +`targetClarificationPassed` grades that limited routing outcome independently +of analysis prose heuristics. Analyzing a nearby example cannot pass this case. + The packaged desktop and parser-comparison scenarios now assess a closed set of known fixture claims. Each final answer must be a strict JSON manifest containing exactly the requested claim IDs, values, Evidence IDs, and Evidence authority and @@ -1077,9 +1110,21 @@ navigation-context, and address-context scenarios have no factual rubric and remain `not_assessed`; the managed workflow requires both artifact and member inspection to answer its type and entry-point question. +The closed factual scenarios request complete producer Evidence on the initial +analysis so their source selectors can authenticate all configured facts. +Summary/page/item workflows are exercised separately by `javascript-module-view`; +its heuristic gate does not establish full-producer factual correctness. + All scenarios retain routing, workflow, validation, repetition, process-exit, and token-use gates. Configured factual scenarios additionally require a factual pass. Their answer-text heuristics are diagnostic and do not affect the gate. +Validation counts include complete text-only REA `invalid_request` envelopes +and failed SDK calls whose arguments violate the named current REA contract. +Client approval rejections and provider unavailability remain distinct failures; +truncated JSON previews are not parsed into invented error codes. +Repeated `binary_session` observations separated by a successful `open_binary` +or `close_binary` are treated as lifecycle verification. An unchanged retry, +including one after a failed lifecycle call, still counts as repetition. Scenarios without a rubric retain the legacy text-heuristic gate. `answerTermCoverageMet` checks case-insensitive substrings, `epistemicCuePresent` checks keywords, and `finalCitesEvidence` checks only an ID's @@ -1087,7 +1132,11 @@ presence. These metrics can still accept fabricated prose and must not be read as factual assessments. Set `REA_AGENT_EVAL_TRANSCRIPT_DIR` to retain complete tool results and final answers for review. -Report schema version 3 changes the top-level `factualCorrectness` from a constant +Report schema version 4 adds negative routing scenarios with nullable +`expectedFirstTool` and a `targetClarificationPassed` outcome. Consumers must +allow an expected absence of REA calls; this does not make an analysis scenario +pass without its required tools. Version 3 changed the top-level +`factualCorrectness` from a constant string to an assessment summary with status, scope, and assessed/passed/failed/ not-assessed scenario counts. Scenario records include the factual assessment and configured claim IDs. The evaluation scope is diff --git a/scripts/verify-agent-experience.mjs b/scripts/verify-agent-experience.mjs index 513835860b..99ab060ddf 100644 --- a/scripts/verify-agent-experience.mjs +++ b/scripts/verify-agent-experience.mjs @@ -1,9 +1,18 @@ import { spawn } from "node:child_process"; -import { cp, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { dirname, join, resolve } from "node:path"; +import { + chmod, + copyFile, + mkdir, + mkdtemp, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { homedir } from "node:os"; +import { delimiter, dirname, join, resolve } from "node:path"; import { createInterface } from "node:readline"; import { fileURLToPath } from "node:url"; +import { z } from "zod"; import { evaluateCodexEvents } from "../dist/evaluation/CodexAgentEval.js"; import { @@ -15,32 +24,95 @@ import { agentEvaluationPassed, summarizeFactualCorrectness, } from "../dist/evaluation/AgentEvaluationReport.js"; -import { MCP_STARTUP_POLICY } from "../dist/mcpStartupPolicy.js"; +import { PRODUCT_IDENTITY } from "../dist/identity.js"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const verifierRun = createVerifierRun(); -const evaluationRoot = await mkdtemp(join(tmpdir(), "rea-agent-eval-")); -const fixtureRoot = join(evaluationRoot, "targets"); -const skillDestination = join( - evaluationRoot, - ".agents/skills/reverse-engineer-anything", -); const timeoutMs = Number(process.env.REA_AGENT_EVAL_TIMEOUT_MS ?? 480_000); const codex = process.env.REA_CODEX_CLI ?? "codex"; const optionalModel = process.env.REA_AGENT_EVAL_MODEL; if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10_000) throw new Error("REA_AGENT_EVAL_TIMEOUT_MS must be an integer >= 10000"); -const codexVersion = await readCommandVersion(codex); +// Codex refuses helper aliases when CODEX_HOME is under the OS temp directory. +// Keep the disposable account in a private cache, then remove the owned run. +const evaluationParent = resolve( + process.env.REA_AGENT_EVAL_ROOT ?? + join(homedir(), ".cache", "rea-agent-evaluations"), +); +await mkdir(evaluationParent, { recursive: true, mode: 0o700 }); +const evaluationRoot = await mkdtemp(join(evaluationParent, "run-")); +const fixtureRoot = join(evaluationRoot, "targets"); +const account = join(evaluationRoot, "account"); +const codexHome = join(evaluationRoot, "codex"); +const authPath = join(codexHome, "auth.json"); +const skillDestination = join( + account, + ".agents/skills/reverse-engineer-anything", +); +const evaluationEnvironment = { + ...Object.fromEntries( + Object.entries(process.env).filter( + ([name]) => !/^(?:REA_|CODEX_|GHIDRA_|HOPPER_|IDA_)/u.test(name), + ), + ), + HOME: account, + USERPROFILE: account, + CODEX_HOME: codexHome, + XDG_CONFIG_HOME: join(account, ".config"), + XDG_DATA_HOME: join(account, ".local/share"), + XDG_CACHE_HOME: join(account, ".cache"), + REA_PROCESS_RUN_ID: verifierRun.run_id, + PATH: `${dirname(process.execPath)}${delimiter}${process.env.PATH ?? ""}`, +}; try { - await mkdir(fixtureRoot, { recursive: true }); - await cp( - join(repositoryRoot, "skills/reverse-engineer-anything"), - skillDestination, - { recursive: true }, + const codexVersion = await readCommandVersion(codex); + await Promise.all([ + mkdir(fixtureRoot, { recursive: true }), + mkdir(account), + mkdir(codexHome, { mode: 0o700 }), + ]); + // Reuse authentication only. User config, plugins, rules, history and skills + // must not make a fresh installed-skill trial pass accidentally. + try { + await copyFile( + join(process.env.CODEX_HOME ?? join(homedir(), ".codex"), "auth.json"), + authPath, + ); + await chmod(authPath, 0o600); + } catch (cause) { + if (cause?.code !== "ENOENT") throw cause; + // Environment/API-key authentication can work without a local auth file. + } + await writeFile( + join(codexHome, "config.toml"), + 'approval_policy = "never"\nsandbox_mode = "read-only"\nweb_search = "disabled"\n[features]\nmulti_agent = false\nshell_snapshot = false\n[history]\npersistence = "none"\n', + { mode: 0o600 }, ); + for (const approval of ["--dry-run", "--yes"]) { + await runProcess( + process.execPath, + [ + join(repositoryRoot, "scripts/rea.mjs"), + "setup", + "--client", + "codex", + approval, + "--json", + ], + evaluationRoot, + 120_000, + evaluationEnvironment, + ); + } + const installedSkill = await readFile( + join(skillDestination, "SKILL.md"), + "utf8", + ); + if (!installedSkill.includes(` version: "${PRODUCT_IDENTITY.skillVersion}"`)) + throw new Error("Setup did not install the current packaged REA skill"); const requestedScenarioIds = new Set( (process.env.REA_AGENT_EVAL_SCENARIOS ?? "") .split(",") @@ -57,7 +129,11 @@ try { expectedFirstTool: "open_binary", requiresEvidence: false, requiredAnswerTermGroups: [["unavailable", "could not", "provider"]], - prompt: `Explain what the shipped native program at ${targets.native} does. Base the answer on artifact analysis, and clearly state any facts that remain unavailable.`, + // The verifier authorizes these named session operations, not arbitrary + // writes or runtime execution. Missing-engine behavior stays observable. + approvedTools: ["open_binary", "close_binary"], + requiredToolSubsequence: ["open_binary", "close_binary"], + prompt: `Explain what static artifact evidence establishes about the shipped native program at ${targets.native}. Do not execute the target or install tools. Clearly state unavailable provider evidence, and close any analysis session you opened before finishing.`, }, { id: "asar", @@ -70,6 +146,38 @@ try { ], prompt: `Explain how the desktop application at ${targets.javascript} exposes APIs to its renderer. Base the answer on the shipped application artifact, and state what remains unknown.`, }, + { + id: "asar-zh", + fixtureScenarioId: "asar", + expectedFirstTool: "analyze_javascript_application", + requiresEvidence: true, + requiredAnswerTermGroups: [["profileapi"], ["profile:read"]], + prompt: `請靜態分析已交付的 Electron 程式 ${targets.javascript}，說明 renderer 如何透過 preload 讀取 profile，以及 main handler 的關係。不要執行目標或安裝分析引擎；引用完整的產出 Evidence，保留 runtime 未知事項。`, + }, + { + id: "javascript-module-view", + expectedFirstTool: "analyze_javascript_application", + requiresEvidence: true, + requiredToolSubsequence: [ + "analyze_javascript_application", + "inspect_analysis_view", + "inspect_analysis_view", + ], + requiredAnswerTermGroups: [ + ["main.js"], + ["node", "identity"], + ["observations", "recorded"], + ], + prompt: `Inspect the recorded module identity, exports and observations for main.js in the shipped JavaScript tree ${targets.largeJavascript}. Use REA without directly reading target files or executing the target. Keep results focused on that module, cite Evidence, and state the limits of what its module view establishes.`, + }, + { + id: "missing-target", + expectedFirstTool: null, + requiresEvidence: false, + requiredAnswerTermGroups: [], + prompt: + 'Reverse engineer my desktop app and explain how it saves user settings. Use static analysis; do not install software. End with one JSON object containing status ("needs_target" or "analyzed"), requested_input ("app_name_or_artifact_path" or null), and question (a string or null).', + }, { id: "javascript-export-shape", expectedFirstTool: "analyze_javascript_application", @@ -145,12 +253,14 @@ try { const results = []; for (const scenario of scenarios) { process.stderr.write(`Running Codex agent evaluation: ${scenario.id}\n`); - const fixtureClaims = agentFixtureClaims(scenario.id, targets); - const answerInstructions = agentFixtureAnswerInstructions(scenario.id); + const fixtureId = scenario.fixtureScenarioId ?? scenario.id; + const fixtureClaims = agentFixtureClaims(fixtureId, targets); + const answerInstructions = agentFixtureAnswerInstructions(fixtureId); const execution = await runCodex( answerInstructions.length === 0 ? scenario.prompt - : `${scenario.prompt}\n\n${answerInstructions}`, + : `${scenario.prompt}\n\nThis closed factual rubric needs complete producer Evidence in the transcript. Request complete detail on each initial application analysis when supported; summary or view records alone are not accepted by this rubric.\n\n${answerInstructions}`, + scenario.approvedTools ?? [], ); const transcriptDirectory = process.env.REA_AGENT_EVAL_TRANSCRIPT_DIR; if (transcriptDirectory !== undefined) { @@ -175,6 +285,12 @@ try { ...(fixtureClaims === undefined ? {} : { fixtureClaims }), }, ); + // Fixtures are deliberately visible beside prior investigation artifacts. + // Their existence does not identify the unnamed app in this request. + const targetClarificationPassed = + scenario.expectedFirstTool === null + ? validTargetClarification(metrics.finalMessage) + : undefined; const result = { id: scenario.id, expectedFirstTool: scenario.expectedFirstTool, @@ -185,6 +301,9 @@ try { exitCode: execution.exitCode, stderr: execution.stderr, ...metrics, + ...(targetClarificationPassed === undefined + ? {} + : { targetClarificationPassed }), }; results.push(result); process.stderr.write( @@ -194,13 +313,20 @@ try { const factualCorrectness = summarizeFactualCorrectness(results); const summary = { - schemaVersion: 3, + schemaVersion: 4, evaluationScope: "routing_workflow_and_configured_fixture_claims", factualCorrectness, verifier_run: await completeVerifierRun(verifierRun), codex, codexVersion, model: optionalModel ?? null, + installation: { + method: "rea_setup_codex", + isolatedHome: true, + isolatedCodexHome: true, + skillVersion: PRODUCT_IDENTITY.skillVersion, + authentication: "caller_auth_only_removed_after_run", + }, scenarios: results, totals: { scenarios: results.length, @@ -253,12 +379,32 @@ try { `Agent routing/workflow/answer checks failed: ${failed.map(({ id }) => id).join(", ")}`, ); } finally { + // Even a deliberately retained fixture directory must never retain auth. + await rm(authPath, { force: true }); if (process.env.REA_AGENT_EVAL_KEEP_FIXTURES !== "true") await rm(evaluationRoot, { recursive: true, force: true }); } async function createTargets(root, includeManaged) { const fixtures = await createAgentEvaluationFixtures(root, repositoryRoot); + const largeJavascript = join(root, "large-app"); + await mkdir(join(largeJavascript, "lib"), { recursive: true }); + await Promise.all([ + writeFile( + join(largeJavascript, "package.json"), + '{"name":"large-app","main":"main.js"}\n', + ), + writeFile( + join(largeJavascript, "main.js"), + 'const { ipcMain } = require("electron");\nipcMain.handle("profile:read", (_event, id) => ({ id }));\n', + ), + ...Array.from({ length: 180 }, (_, index) => + writeFile( + join(largeJavascript, "lib", `formatter-${String(index)}.js`), + `export const format = (value) => String(value).trim();\n`, + ), + ), + ]); const managedOutput = join(root, "managed-output"); if (includeManaged) { @@ -292,15 +438,29 @@ async function createTargets(root, includeManaged) { return { native: "/bin/true", ...fixtures, + largeJavascript, managed: join(managedOutput, "AgentEval.dll"), }; } -async function runCodex(prompt) { +function validTargetClarification(message) { + const schema = z.strictObject({ + status: z.literal("needs_target"), + requested_input: z.literal("app_name_or_artifact_path"), + question: z.string().refine((value) => value.trim().length > 0), + }); + try { + return schema.safeParse(JSON.parse(message)).success; + } catch (cause) { + if (cause instanceof SyntaxError) return false; + throw cause; + } +} + +async function runCodex(prompt, approvedTools) { const arguments_ = [ + "--no-daemon", "exec", - "--ignore-user-config", - "--ignore-rules", "--ephemeral", "--skip-git-repo-check", "--sandbox", @@ -312,18 +472,16 @@ async function runCodex(prompt) { evaluationRoot, "-c", 'approval_policy="never"', - "-c", - `mcp_servers.rea.command=${JSON.stringify(process.execPath)}`, - "-c", - `mcp_servers.rea.args=${JSON.stringify([join(repositoryRoot, "scripts/rea.mjs"), "mcp"])}`, - "-c", - `mcp_servers.rea.startup_timeout_sec=${String(MCP_STARTUP_POLICY.codexStartupTimeoutSeconds)}`, + ...approvedTools.flatMap((name) => [ + "-c", + `mcp_servers.rea.tools.${name}.approval_mode="approve"`, + ]), ...(optionalModel === undefined ? [] : ["--model", optionalModel]), prompt, ]; const child = spawn(codex, arguments_, { cwd: evaluationRoot, - env: process.env, + env: evaluationEnvironment, stdio: ["ignore", "pipe", "pipe"], }); const events = []; @@ -362,9 +520,16 @@ async function runCodex(prompt) { return { events, exitCode, stderr: stderr.join("").slice(0, 32_768) }; } -async function runProcess(command, arguments_, cwd, timeout) { +async function runProcess( + command, + arguments_, + cwd, + timeout, + env = process.env, +) { const child = spawn(command, arguments_, { cwd, + env, stdio: ["ignore", "ignore", "pipe"], }); let stderr = ""; diff --git a/src/application/SetupSkill.ts b/src/application/SetupSkill.ts index 25228d14ba..50d0d96f9b 100644 --- a/src/application/SetupSkill.ts +++ b/src/application/SetupSkill.ts @@ -10,6 +10,7 @@ import { clientSkillDirectories } from "./SupportedClients.js"; export const MANAGED_SKILL_FILES = [ "SKILL.md", + "references/connection-and-recovery.md", "references/native-and-artifacts.md", "references/javascript-applications.md", "references/android-applications.md", diff --git a/src/evaluation/AgentEvaluationReport.ts b/src/evaluation/AgentEvaluationReport.ts index 37e6a70eb6..4af002f40c 100644 --- a/src/evaluation/AgentEvaluationReport.ts +++ b/src/evaluation/AgentEvaluationReport.ts @@ -3,6 +3,9 @@ export type FixtureCorrectness = "passed" | "failed" | "not_assessed"; /** Measurements consumed by the evaluation runner's release gate. */ export interface AgentEvaluationGateResult { + /** Explicit null requires a target clarification with no REA calls. */ + readonly expectedFirstTool?: string | null; + readonly targetClarificationPassed?: boolean; readonly exitCode: number; readonly naturalUse: boolean; readonly correctFirstTool: boolean; @@ -20,16 +23,20 @@ export const agentEvaluationPassed = ( result: AgentEvaluationGateResult, ): boolean => result.exitCode === 0 && - result.naturalUse && + (result.expectedFirstTool === null + ? !result.naturalUse + : result.naturalUse) && result.correctFirstTool && result.repeatedCallCount === 0 && result.inputValidationFailureCount === 0 && result.requiredToolSubsequenceMet && result.inputTokens > 0 && - (result.factualCorrectness === "passed" || - (result.factualCorrectness === "not_assessed" && - result.answerHeuristicsMet && - result.epistemicCuePresent)); + (result.expectedFirstTool === null + ? result.targetClarificationPassed === true + : result.factualCorrectness === "passed" || + (result.factualCorrectness === "not_assessed" && + result.answerHeuristicsMet && + result.epistemicCuePresent)); /** Keep assessed fixture outcomes and scenarios without truth oracles distinct. */ export const summarizeFactualCorrectness = ( diff --git a/src/evaluation/CodexAgentEval.ts b/src/evaluation/CodexAgentEval.ts index 2aebc7ca7d..ba5c500d05 100644 --- a/src/evaluation/CodexAgentEval.ts +++ b/src/evaluation/CodexAgentEval.ts @@ -4,6 +4,7 @@ import { type KnownAnswerAssessment, } from "./KnownAnswerEvaluation.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; +import { TOOL_CONTRACTS } from "../contracts/toolContracts.js"; /** One REA MCP invocation observed in a Codex JSONL transcript. */ interface CodexMcpCall { @@ -72,7 +73,35 @@ const evidenceIdsFrom = (value: unknown): readonly string[] => { return [...new Set(encoded.match(/ev_[a-f0-9]{64}/gu) ?? [])]; }; -const toolResultFailed = (item: Record): boolean => { +const toolResultError = ( + item: Record, +): Record | undefined => { + const result = record(item.result ?? item.output); + const structured = record( + result?.structured_content ?? result?.structuredContent, + ); + const direct = record(structured?.error ?? result?.error ?? item.error); + if (direct !== undefined || structured !== undefined) return direct; + // Codex preserves some REA failures only as one complete JSON text block. + // A client-truncated preview or SDK prose is not a structured error record. + const content = result?.content; + if (!Array.isArray(content) || content.length !== 1) return undefined; + const block = record(content[0]); + if (block?.type !== "text" || typeof block.text !== "string") + return undefined; + try { + return record(record(JSON.parse(block.text))?.error); + } catch (cause) { + if (cause instanceof SyntaxError || cause instanceof RangeError) + return undefined; + throw cause; + } +}; + +const toolResultFailed = ( + item: Record, + error: Record | undefined, +): boolean => { if ( item.status === "failed" || (item.error !== undefined && item.error !== null) @@ -82,16 +111,11 @@ const toolResultFailed = (item: Record): boolean => { const structured = record( result?.structured_content ?? result?.structuredContent, ); - return result?.isError === true || structured?.error !== undefined; -}; - -const toolResultErrorCode = (item: Record): string | null => { - const result = record(item.result ?? item.output); - const structured = record( - result?.structured_content ?? result?.structuredContent, + return ( + result?.isError === true || + structured?.error !== undefined || + error !== undefined ); - const error = record(structured?.error ?? result?.error ?? item.error); - return textValue(error?.code) ?? null; }; const callFromItem = (value: unknown): CodexMcpCall | undefined => { @@ -99,14 +123,15 @@ const callFromItem = (value: unknown): CodexMcpCall | undefined => { if (item?.type !== "mcp_tool_call") return undefined; const tool = textValue(item.tool) ?? textValue(item.name); if (tool === undefined) return undefined; + const error = toolResultError(item); return { id: textValue(item.id) ?? null, server: textValue(item.server) ?? textValue(item.server_name) ?? null, tool, arguments: item.arguments ?? item.input ?? {}, evidenceIds: evidenceIdsFrom(item.result ?? item.output), - error: toolResultFailed(item), - errorCode: toolResultErrorCode(item), + error: toolResultFailed(item, error), + errorCode: textValue(error?.code) ?? null, }; }; @@ -171,7 +196,7 @@ const consumeEvent = (state: EvaluationState, value: unknown): void => { /** Measure agent routing, repetition, model usage, and explicitly limited answer heuristics. */ export const evaluateCodexEvents = ( events: readonly unknown[], - expectedFirstTool: string, + expectedFirstTool: string | null, options: { readonly requireEvidence?: boolean; readonly requiredAnswerTermGroups?: readonly (readonly string[])[]; @@ -194,16 +219,31 @@ export const evaluateCodexEvents = ( ({ server }) => server === null || server.toLowerCase() === "rea", ); const signatures = new Map(); + let targetRevision = 0; for (const call of reaCalls) { - const signature = `${call.tool}:${JSON.stringify(canonicalValue(call.arguments))}`; + // A status read before and after a successful target lifecycle change is + // useful verification, not a duplicate analysis or unchanged failed retry. + const context = + call.tool === "binary_session" ? `${String(targetRevision)}:` : ""; + const signature = `${context}${call.tool}:${JSON.stringify(canonicalValue(call.arguments))}`; signatures.set(signature, (signatures.get(signature) ?? 0) + 1); + if ( + !call.error && + (call.tool === "open_binary" || call.tool === "close_binary") + ) + targetRevision += 1; } const repeatedCallCount = [...signatures.values()].reduce( (total, count) => total + Math.max(0, count - 1), 0, ); const inputValidationFailureCount = reaCalls.filter( - ({ errorCode }) => errorCode === "invalid_request", + (call) => + call.errorCode === "invalid_request" || + (call.error && + TOOL_CONTRACTS.find( + ({ name }) => name === call.tool, + )?.inputSchema.safeParse(call.arguments).success === false), ).length; const requiredToolSubsequenceMet = containsOrderedSubsequence( reaCalls.filter(({ error }) => !error).map(({ tool }) => tool), @@ -234,7 +274,7 @@ export const evaluateCodexEvents = ( ); return { naturalUse: reaCalls.length > 0, - correctFirstTool: reaCalls[0]?.tool === expectedFirstTool, + correctFirstTool: (reaCalls[0]?.tool ?? null) === expectedFirstTool, firstTool: reaCalls[0]?.tool ?? null, reaCalls, repeatedCallCount, diff --git a/src/generatedPackageMetadata.ts b/src/generatedPackageMetadata.ts index aaa5b9a805..ab4334d0a3 100644 --- a/src/generatedPackageMetadata.ts +++ b/src/generatedPackageMetadata.ts @@ -5,5 +5,5 @@ export const PACKAGE_METADATA = { serverSdkVersion: "2.3.1", clientSdkVersion: "2.3.1", coreSdkVersion: "2.3.1", - skillVersion: "36", + skillVersion: "37", } as const; diff --git a/tests/boundary/filesystem/setupSkill.test.ts b/tests/boundary/filesystem/setupSkill.test.ts index 40bbcfd825..4da0225c34 100644 --- a/tests/boundary/filesystem/setupSkill.test.ts +++ b/tests/boundary/filesystem/setupSkill.test.ts @@ -187,12 +187,11 @@ describe("canonical skill transaction", () => { expect(installedSkill).toContain( `version: "${PRODUCT_IDENTITY.skillVersion}"`, ); - expect(installedSkill).toContain("call available analysis tools"); - expect(installedSkill).toContain("obtain approval before setup writes"); expect(await readFile(`${nativeGuide}.rea.backup`, "utf8")).toBe( "stale filesystem-write permission grant\n", ); for (const reference of [ + "connection-and-recovery.md", "native-and-artifacts.md", "javascript-applications.md", "android-applications.md", @@ -243,6 +242,7 @@ describe("canonical skill transaction", () => { const paths = [ "SKILL.md", "references/android-applications.md", + "references/connection-and-recovery.md", "references/evidence-workflows.md", "references/javascript-applications.md", "references/native-and-artifacts.md", diff --git a/tests/evaluation/model-evals/agentEvaluationReport.test.ts b/tests/evaluation/model-evals/agentEvaluationReport.test.ts index 1913da97d2..562fd591ea 100644 --- a/tests/evaluation/model-evals/agentEvaluationReport.test.ts +++ b/tests/evaluation/model-evals/agentEvaluationReport.test.ts @@ -77,6 +77,29 @@ async function runnerFailurePredicate(): Promise<(item: unknown) => unknown> { } describe("scoped factual report", () => { + it("requires a validated target clarification and zero REA calls for the negative route", () => { + const clarification = { + ...routingSuccess, + expectedFirstTool: null, + naturalUse: false, + factualCorrectness: "not_assessed" as const, + answerHeuristicsMet: false, + epistemicCuePresent: false, + targetClarificationPassed: true, + }; + expect(agentEvaluationPassed(clarification)).toBe(true); + for (const change of [ + { naturalUse: true }, + { correctFirstTool: false }, + { targetClarificationPassed: false }, + { exitCode: 1 }, + { inputTokens: 0 }, + ]) + expect(agentEvaluationPassed({ ...clarification, ...change })).toBe( + false, + ); + }); + it("accepts correct structured facts without superficial prose cues", () => { expect( agentEvaluationPassed({ diff --git a/tests/evaluation/model-evals/codexAgentEval.test.ts b/tests/evaluation/model-evals/codexAgentEval.test.ts index 61aaa34fbb..f802d68d29 100644 --- a/tests/evaluation/model-evals/codexAgentEval.test.ts +++ b/tests/evaluation/model-evals/codexAgentEval.test.ts @@ -101,6 +101,35 @@ describe("Codex agent release evaluation", () => { ).toBe(1); }); + it("distinguishes session verification after a successful close from an unchanged retry", () => { + const call = (id: string, tool: string, status = "completed") => ({ + type: "item.completed", + item: { + id, + type: "mcp_tool_call", + server: "rea", + tool, + arguments: {}, + status, + }, + }); + for (const [status, repeats] of [ + ["completed", 0], + ["failed", 1], + ] as const) { + expect( + evaluateCodexEvents( + [ + call("before", "binary_session"), + call("close", "close_binary", status), + call("after", "binary_session"), + ], + "binary_session", + ).repeatedCallCount, + ).toBe(repeats); + } + }); + it("counts identical arguments with reversed Unicode key insertion order", () => { const call = (id: string, arguments_: Record) => ({ type: "item.completed", @@ -125,6 +154,119 @@ describe("Codex agent release evaluation", () => { }); }); +describe("Codex agent routing and error evaluation", () => { + it("preserves an unrecognized structured failure without inventing its code", () => { + const metrics = evaluateCodexEvents( + [ + { + type: "item.completed", + item: { + type: "mcp_tool_call", + server: "rea", + tool: "close_binary", + arguments: {}, + result: { structured_content: { error: "unrecognized failure" } }, + }, + }, + ], + "close_binary", + { requiredToolSubsequence: ["close_binary"] }, + ); + expect(metrics.reaCalls[0]).toMatchObject({ error: true, errorCode: null }); + expect(metrics.requiredToolSubsequenceMet).toBe(false); + }); + + it("requires no REA call when target selection is missing", () => { + const events = [ + { + type: "item.completed", + item: { type: "agent_message", text: "Which app should I inspect?" }, + }, + ]; + expect(evaluateCodexEvents(events, null)).toMatchObject({ + naturalUse: false, + correctFirstTool: true, + firstTool: null, + }); + expect( + evaluateCodexEvents( + [ + ...events, + { + type: "item.completed", + item: { + type: "mcp_tool_call", + server: "rea", + tool: "analyze_javascript_application", + arguments: { input_path: "/nearby/test-fixture" }, + }, + }, + ], + null, + ).correctFirstTool, + ).toBe(false); + }); + + it("counts actual text-only REA errors and SDK-rejected schema inputs once per call", () => { + const source = { + kind: "retained-evidence", + evidence_id: `ev_${"a".repeat(64)}`, + }; + const failed = (id: string, arguments_: unknown, text: string) => ({ + type: "item.completed", + item: { + id, + type: "mcp_tool_call", + server: "rea", + tool: "inspect_analysis_view", + arguments: arguments_, + status: "failed", + error: null, + result: { content: [{ type: "text", text }], structured_content: null }, + }, + }); + const metrics = evaluateCodexEvents( + [ + failed( + "sdk", + { source, view: { kind: "module", path: "main.js" } }, + "Input validation error: Invalid arguments for tool inspect_analysis_view: view.kind: Invalid discriminator value.", + ), + failed( + "ambiguous", + { + source, + view: { + kind: "item", + collection: "modules", + selector: { path: "main.js" }, + }, + }, + JSON.stringify({ + error: { + code: "invalid_request", + message: + "Multiple modules have this path; select one by node_id.", + }, + }), + ), + failed( + "incomplete-preview", + { source, view: { kind: "summary" } }, + '{"error":{"code":"invalid_request"', + ), + ], + "inspect_analysis_view", + ); + expect(metrics.inputValidationFailureCount).toBe(2); + expect(metrics.reaCalls.map(({ errorCode }) => errorCode)).toEqual([ + null, + "invalid_request", + null, + ]); + }); +}); + describe("Codex agent completion evaluation", () => { it("does not present matching answer terms as factual correctness or honesty", () => { const evidenceId = `ev_${"a".repeat(64)}`;