diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index f87a4e1103..7e35c0db04 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -2,7 +2,7 @@ name: reverse-engineer-anything description: Reverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. metadata: - version: "36" + version: "37" --- # REA @@ -25,57 +25,18 @@ a skill installed from repository main may describe capabilities absent from an older npm release. Keep complete inline Evidence when that server does not advertise retained references. -When tools are absent or registration is stale: +If tools are absent, a call is blocked by the client, or a required provider is +unavailable, read [connection and recovery](references/connection-and-recovery.md). +Diagnose the observed blocker, continue useful supported work, and resume the +original task after repair. Client approval, shell PATH, registration and engine +availability have different remedies; avoid repeating setup for all of them. -1. Diagnose without changing files. For Codex, run - `npx -y rea-agents@latest doctor --client codex --json`. Substitute the current - supported client: `claude_code`, `claude_desktop`, `codex`, `cursor`, - `gemini_cli`, `windsurf`, `devin`, `opencode`, `antigravity`, `copilot_cli`, - `commandcode`, `qwen_code`, `vscode`, `grok_build`, `omp`, `pi`, `hermes`, or `grok_bot`. `grok_bot` has no local - file registration: doctor reports the Grok Bot chat step, and setup does not - write `mcp.json`. If the client is unknown, use `doctor --json` and - inspect its registration results before choosing a setup scope. -2. Distinguish the reason. Missing, malformed, or stale registration needs a - scoped configuration repair. An aligned registration with no tools in the - active session needs a restart/reconnection; doctor cannot prove that the - current agent has connected. A missing provider affects only tasks requiring - that provider: static JavaScript inspection needs neither Hopper nor Ghidra, - and Android inspection has separate bring-your-own JADX/Java prerequisites. -3. For a configuration repair, prepare the read-only plan: - `npx -y rea-agents@latest setup --client codex --dry-run --json`. - Use the current client's ID, show its exact proposed paths, backups, and - changes, and obtain approval before setup writes configuration or installs - Hopper. Setup normally installs the matching bundled skill too; include that - replacement in the reviewed plan. After approval, apply the same scope with - `npx -y rea-agents@latest setup --client codex --yes`. Add `--install-hopper` - only if that separate installation was needed and explicitly approved. -4. Restart/reconnect the affected agent. Verify that REA tools actually appear - in the session, then resume the original investigation. If they remain - absent, inspect the client's MCP launch error rather than repeating setup. - -REA setup never installs or upgrades Node.js, npm, Homebrew, Java, Ghidra, IDA, -JADX, Binwalk, or Unblob. Use existing prerequisites; do not install unrelated -software to repair -MCP registration. For an unsupported client, use manual stdio registration -with a version-pinned `rea-agents` package or continue through the CLI. - -A concrete CLI fallback for an operator-supplied JavaScript tree or ASAR is: - -```bash -npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json > app-evidence.json -jq -c '{source: {kind: "inline", evidence: .}, view: {kind: "summary"}}' app-evidence.json > app-view.json -npx -y rea-agents@latest inspect-analysis-view app-view.json -``` - -No MCP registration or native engine is required. The CLI returns the complete -Evidence record directly, and a real application can produce hundreds of -megabytes, so save it to a file rather than reading it into context. Start from -the summary view, then request a module page -(`{"kind": "page", "collection": "modules", "offset": 0, "limit": 32}`) or one -module the same way, or pass the saved Evidence inline to -`trace-application-feature`. Read graph, limitations, and unknowns with the -same care as an MCP result. This fallback does not establish that MCP is -configured. Native CLI tasks still require their selected engine. +Prefer an existing `rea` CLI. When none is installed and REA package acquisition +is within the authorized scope, use `npx -y rea-agents@latest` in its place. +Configuration repair starts with a read-only setup plan; show paths, backups and +skill replacement before applying authorized changes. Preserve existing approval +and target-execution boundaries. REA setup never installs or upgrades unrelated +software, including Node.js, npm, Homebrew, Java, Ghidra, IDA, JADX, Binwalk or Unblob. ## Route the target first @@ -140,9 +101,11 @@ after opening; IDA does not supply `binary_overview`. Consult session availabili when composing broader workflows. Modern direct callers and unsupported dossier facets remain unknown; live IDA results are not replayed from snapshots. -If the app is missing, ask which app to inspect. Resolve a human-readable app -name to one clear installed artifact when possible; ask only when matches are -ambiguous. Never choose an example app on the user's behalf. +If the user says only "my app" without a name, path, or target already selected +for this task, ask which app before searching for artifacts. Paths in unrelated +saved reports and nearby test fixtures do not select the user's target. Resolve +an actual human-readable app name to one clear installed artifact when possible; +ask when matches are ambiguous. Never choose an example app on the user's behalf. In a target-free session, use `open_binary` to bind any archive/package or native target whose analysis tool operates on the active target. Do not call an @@ -157,7 +120,11 @@ For a JavaScript application or ELF layout whose size is unknown or large, call `analyze_javascript_application` or `inspect_binary_layout` with `"detail": "summary"`, then inspect the retained `parent_evidence_id` with `inspect_analysis_view`. -Do not repeat an identical tool call. Make a focused follow-up only when the +Page module identities and select the returned `node_id` before requesting an +item; a single file path can represent several graph roles. Do not repeat an +identical analysis or rejected call without a changed input or resolved blocker. +Recheck session state when a lifecycle change makes that status useful. +Make a focused follow-up only when the returned result leaves a specific question unanswered. If MCP reports `resource_constraint` with `details.resource: "transport"`, call @@ -168,6 +135,14 @@ session to a selected path. The analysis remains complete in the ledger; avoid repeating it merely to request the same oversized response. Complete CLI JSON output also streams without a single MCP frame. +A client can also truncate a successful tool result without a REA transport +error. Treat that preview as incomplete; use the retained parent ID for smaller +views or export when the full record is needed. Do not parse the preview as +canonical JSON or repeat the producer. For JavaScript, read the +[module-view and trace recipes](references/javascript-applications.md) before +drilling into a large graph; a broad bidirectional trace can expand unrelated +containment relationships even after summary analysis. + Every conclusion must distinguish observations, inferences, and unknowns. Cite Evidence IDs, preserve limitations and incomplete coverage, and never imply that static analysis observed execution. Runtime requests execute the declared @@ -175,7 +150,9 @@ target and lifecycle; do not broaden the target or action beyond those fields. Within the user's requested investigation, call available analysis tools directly. REA does not require permission grants or per-call approval flags. -Follow the declared request scope and the host's actual access requirements. +The agent client can independently require approval. Follow the declared request +scope and the host's actual access requirements; use the recovery guide for an +observed client-policy rejection. ## Plan broader investigations diff --git a/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md b/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md new file mode 100644 index 0000000000..5e5f6251da --- /dev/null +++ b/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md @@ -0,0 +1,127 @@ +# Connection and task recovery + +Read this guide only for an observed connection, command, approval or provider +blocker. Keep the user's target and question in view. Explain what prevented the +operation and the next useful step; after repair, continue the investigation. + +Commands below use an installed `rea`. If none is available, use the package +invocation in `SKILL.md` when acquiring REA is authorized. Follow the connected +server's schemas and the installed CLI's help rather than assuming repository +main and an older release expose the same arguments. + +## Choose the remedy from the observed failure + +| Observation | Meaning and next step | +| --------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| No REA tools in the active session | Inspect registration with `rea doctor --client codex --json`. A skill installation alone does not register MCP. | +| Missing, malformed or stale registration | Prepare a scoped setup plan; apply only authorized changes with backups. | +| Aligned registration, but no connected tools | Restart/reconnect the client. Doctor checks registration; it cannot establish the current agent's connection. Inspect the client's launch error if reconnection fails. | +| `MCP tool call requires approval, but approval policy is never` | Codex rejected the call before REA executed it. Do not install an engine or repeat setup to solve this client-policy decision. See the approval section below. | +| REA returns `provider_unavailable` or an unavailable tool entry | Read that operation's provider diagnostics or `binary_session.result.tool_availability`. Resolve only the prerequisite needed for the task; a complete tool catalog does not imply every operation is available. | +| `rea` is missing in a child shell, or Node is incompatible | Inspect the actual shell PATH and launcher/runtime. A prefix installation may be absent from a login shell's PATH; this does not establish a broken MCP registration. | + +If the client cannot launch its shell/sandbox helper or read the installed skill, +inspect its launch diagnostic. For example, Codex can refuse helper aliases when +CODEX_HOME is under `/tmp`; use an authorized profile outside the OS temporary +directory. REA setup cannot repair the client's executor. Preserve the sandbox +instead of silently disabling it to make an analysis command work. + +Static JavaScript and ordinary ZIP inventory need neither Hopper nor Ghidra. +HAR inspection is offline and does not require a live browser. Native +decompilation, Android analysis and firmware extraction have their own +bring-your-own prerequisites. Use supported inventory/static evidence when it +answers part of the question, and name the unavailable evidence separately. +Do not substitute another target. Label observations from ordinary shell tools +as such; only actual REA output supplies REA Evidence. + +## Repair registration when needed + +For Codex: + +```bash +rea doctor --client codex --json +rea setup --client codex --dry-run --json +``` + +Use the current supported client's ID: `claude_code`, `claude_desktop`, `codex`, +`cursor`, `gemini_cli`, `windsurf`, `devin`, `opencode`, `antigravity`, +`copilot_cli`, `commandcode`, `qwen_code`, `vscode`, `grok_build`, `omp`, `pi`, +`hermes`, or `grok_bot`. If unknown, inspect `doctor --json` before choosing a +scope. `grok_bot` uses a chat registration step rather than local `mcp.json`. +For unsupported clients, use version-pinned manual stdio registration or CLI. + +Show the plan's exact paths, backups, changes and bundled-skill replacement. +Obtain approval for writes when that concrete scope is not already authorized, +then apply the same scope with `rea setup --client codex --yes`. Add +`--install-hopper` only when separately needed and explicitly approved. Do not +install or upgrade unrelated prerequisites. Restart/reconnect, verify tools +actually appear, and resume the original task. If they remain absent, inspect +the launch failure rather than looping through setup. + +## Codex tool approval is separate from REA readiness + +REA tools have no permission-grant handshake. Codex can still require approval +because an operation creates owned resources, launches a helper or has cleanup +effects. An advertised tool or successful doctor result does not override that +client policy. `workspace-write` alone does not resolve an MCP approval +rejection, and `approval_policy = "never"` does not mean every MCP tool is +preapproved. + +When interactive approval is available, use the client's approval flow for the +requested operation. In an unattended run, continue through an authorized CLI +workflow if the execution environment permits it, or report the blocked +operation and the approval needed to proceed. Do not retry the same rejected +call, silently broaden approvals, or rewrite MCP effects as read-only. + +For an operator who explicitly wants to preapprove named operations in a +disposable Codex profile, current Codex supports per-tool configuration. For +example, opening and closing an analysis session can be scoped independently: + +```toml +[mcp_servers.rea.tools.open_binary] +approval_mode = "approve" + +[mcp_servers.rea.tools.close_binary] +approval_mode = "approve" +``` + +Use the actual server ID and only the operations whose effects were approved; +this example does not approve inspection, extraction or runtime execution. +Show the proposed config edit and preserve unrelated entries and backups before +writing. Managed policy can constrain client settings. Consult the +[Codex configuration reference](https://learn.chatgpt.com/docs/config-file/config-reference) +for the installed client's supported settings. This is an optional recovery +choice, not a prerequisite for every REA task. + +## Continue through an installed CLI + +Check the command environment only if the launcher fails: + +```bash +command -v rea +command -v node +node --version +rea --version +``` + +Use the public launcher at its known installation path with a supported existing +Node on PATH when the child shell loses a prefix installation. Avoid internal +`dist/cli.js` entrypoints or global toolchain changes. MCP may have its own valid +launcher even when a shell cannot resolve `rea`. + +For a user-supplied JavaScript tree or ASAR, select output files in the authorized +workspace and save complete Evidence before reading a view: + +```bash +rea analyze-javascript-application /absolute/path/to/app --json > app-evidence.json +jq -c '{source: {kind: "inline", evidence: .}, view: {kind: "summary"}}' app-evidence.json > app-view.json +rea inspect-analysis-view app-view.json +``` + +These commands do not register MCP or install an analysis engine. CLI JavaScript +JSON is complete and can be very large; redirect it rather than loading it into +context. Use an existing JSON processor if `jq` is absent, without installing +one just for this recipe. Inspect a module page or item using the same inline +source and the [JavaScript recipes](javascript-applications.md). Preserve graph +coverage, limitations and unknowns. Native CLI analysis still requires its +selected engine, and all CLI operations retain the host's execution permissions. diff --git a/.agents/skills/reverse-engineer-anything/references/javascript-applications.md b/.agents/skills/reverse-engineer-anything/references/javascript-applications.md index 0ee112a511..01b49a8887 100644 --- a/.agents/skills/reverse-engineer-anything/references/javascript-applications.md +++ b/.agents/skills/reverse-engineer-anything/references/javascript-applications.md @@ -1,20 +1,90 @@ # JavaScript and Electron application artifacts Use `analyze_javascript_application` directly on the operator-supplied ASAR or -extracted tree. The complete result, graph, and Evidence context are returned -inline when they fit the MCP frame. Source-map contents are part of the static analysis. Start from its -findings, coverage, unknowns, and graph context, then make a focused follow-up -only when a specific question remains unanswered. +extracted tree. When size is unknown or large and the connected schema advertises +`detail`, request `detail: "summary"`. This returns an `inspect_analysis_view` +Evidence record with a distinct ID and a `normalized_result.parent_evidence_id` +for the complete retained application analysis. Source-map contents are part of +that static analysis. Read findings, coverage, limitations and unknowns, then +follow a specific unanswered question. Older servers without `detail` return +the complete application Evidence inline when it fits. + +## Read a module without repeating analysis + +Use the actual returned parent ID, on the same connection, as `source.evidence_id`. +The following IDs are placeholders to replace with returned values. When the +module's node ID is not already known, page identities before selecting an item. +Pages return only node IDs, paths and roles; choose a useful page size and follow +`coverage.next_offset` when another page matters: + +```json +{ + "source": { + "kind": "retained-evidence", + "evidence_id": "ev_ACTUAL_PARENT_ID" + }, + "view": { "kind": "page", "collection": "modules", "offset": 0, "limit": 16 } +} +``` + +Select the relevant page entry by its exact `node_id` and recorded role: + +```json +{ + "source": { + "kind": "retained-evidence", + "evidence_id": "ev_ACTUAL_PARENT_ID" + }, + "view": { + "kind": "item", + "collection": "modules", + "selector": { "node_id": "jag_node_ACTUAL_NODE_ID" } + } +} +``` + +Module pages include asset, bundled and source module roles, so one path can +identify several entries. Use `selector.path` only when uniqueness is established; +the returned node ID avoids needing a complete inventory to prove uniqueness. On ambiguity, +select the matching node ID from the page or the error's candidates. Follow +`next_offset` when another page matters. An item exposes that module node's +recorded observations, identity, hashes, exports and source ranges; it does not +include every child syntax observation, related edge or original source file. +An absent item fact is not evidence of absence from the whole application. + +## Trace only the remaining question + +Use `trace_application_feature` on the parent application Evidence for one +literal node ID, route, string, API, IPC channel, module, or native export. +Select `incoming` for relationships leading to the seed, `outgoing` for those +leaving it, and `both` when the question needs both. Trace traversal includes +containment relationships, so even a narrow seed can expand many nodes; a +bidirectional trace can reach unrelated sibling modules. A module item or page +is preferable when its observations already answer the question. Do not treat +all reached nodes as participants in the feature. + +Reuse retained parent Evidence when the server advertises it, or supply complete +Evidence inline. Include Hopper or Ghidra Evidence only when its artifact digest +matches exactly. + +## Recover an oversized display An oversized response reports `resource_constraint` and `details.resource: "transport"`. Reuse `details.reported_limits.evidence_reference` with `inspect_analysis_view` for a -summary, one module, or a stable page of module identities. Use -`trace_application_feature` once a module, route, or string seed is known. +summary, one module, or a stable page of module identities. `export_evidence_bundle` writes the complete canonical session to a caller-selected file. These workflows keep the original Evidence and coverage; a broad follow-up can also exceed framing. +A client can truncate a successful result inside its text envelope without a +REA `resource_constraint`. Treat a truncation notice or incomplete JSON as a +display limitation; do not claim the full record was examined. Reuse the known +parent ID for smaller views, or export complete Evidence to a selected file +while the session remains open. If the parent ID was lost from the preview, +use the prior analysis summary or explicit ledger entries to recover it; do +not invent an ID or rerun analysis just to reproduce the same display. + BrowserWindow preferences, preload and contextBridge surfaces, IPC registrations, utility processes, and native binding requests are static syntax observations. Only a unique exact literal IPC channel match supports an inferred @@ -22,13 +92,6 @@ pairing. Dynamic or ambiguous channels remain unresolved. A requested `.node` member is not a verified native export. Never claim runtime reachability, registration, defaults, or policy enforcement from static analysis. -Use `trace_application_feature` on existing application Evidence for one -literal node ID, route, string, API, IPC channel, module, or native export. -Choose a direction and reuse the returned application Evidence ID when the -connected server advertises retained references, or supply complete Evidence -inline. Include -Hopper or Ghidra Evidence only when its artifact digest matches exactly. - For version comparison, analyze each version once, then call `compare_application_versions`. Accept only its digest, source-map, structural fingerprint, or non-module semantic matches. Module ordinals and minified names diff --git a/docs/testing.md b/docs/testing.md index ecb2600b6e..b6a4a40364 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1053,6 +1053,39 @@ real local Codex CLI with: npm run verify:agent ``` +The lane runs scenarios sequentially in standalone Codex (`--no-daemon`) with a +disposable HOME and CODEX_HOME under `~/.cache/rea-agent-evaluations/`. Set +`REA_AGENT_EVAL_ROOT` to choose another parent directory outside the OS temporary +directory: current Codex refuses helper aliases under `/tmp`, preventing shell +commands and skill reads even if MCP works. Each owned run directory is removed +after verification unless fixture retention is requested. The lane +installs the packaged skill and MCP registration through public `rea setup`, +rather than copying a skill into a project that also inherits the caller's +skills and configuration. Only authentication is copied, when present, and it +is removed even with `REA_AGENT_EVAL_KEEP_FIXTURES=true`. No user configuration, +rules, plugins or history is copied; shell snapshots and additional agents are +disabled. Existing API-key environment authentication remains available. +The native missing-provider scenario preapproves only `open_binary` and +`close_binary` in the disposable client so it reaches REA's actual provider +boundary instead of stopping at Codex approval. This does not validate a real +native engine or authorize arbitrary runtime execution. + +Select a smaller run with `REA_AGENT_EVAL_SCENARIOS`, a comma-separated list of +scenario IDs. `asar-zh` exercises the packaged desktop rubric from a Chinese +request. `javascript-module-view` exercises summary analysis, module paging and +item inspection in a tree with unrelated modules; its routing and text checks +remain heuristic, so review the retained transcript for selector choice and +coverage claims. Set `REA_CODEX_CLI` to an installed Codex executable and +`REA_AGENT_EVAL_MODEL` to an account-supported model. Use an external resource +limit for the whole process tree when testing on a constrained host; a Node +heap limit alone does not bound client and child-process memory. + +`missing-target` leaves desktop fixtures visible without selecting one in the +request. It requires zero REA calls and a structured request for an app name or +artifact path. Its `expectedFirstTool` is `null`, and +`targetClarificationPassed` grades that limited routing outcome independently +of analysis prose heuristics. Analyzing a nearby example cannot pass this case. + The packaged desktop and parser-comparison scenarios now assess a closed set of known fixture claims. Each final answer must be a strict JSON manifest containing exactly the requested claim IDs, values, Evidence IDs, and Evidence authority and @@ -1077,9 +1110,21 @@ navigation-context, and address-context scenarios have no factual rubric and remain `not_assessed`; the managed workflow requires both artifact and member inspection to answer its type and entry-point question. +The closed factual scenarios request complete producer Evidence on the initial +analysis so their source selectors can authenticate all configured facts. +Summary/page/item workflows are exercised separately by `javascript-module-view`; +its heuristic gate does not establish full-producer factual correctness. + All scenarios retain routing, workflow, validation, repetition, process-exit, and token-use gates. Configured factual scenarios additionally require a factual pass. Their answer-text heuristics are diagnostic and do not affect the gate. +Validation counts include complete text-only REA `invalid_request` envelopes +and failed SDK calls whose arguments violate the named current REA contract. +Client approval rejections and provider unavailability remain distinct failures; +truncated JSON previews are not parsed into invented error codes. +Repeated `binary_session` observations separated by a successful `open_binary` +or `close_binary` are treated as lifecycle verification. An unchanged retry, +including one after a failed lifecycle call, still counts as repetition. Scenarios without a rubric retain the legacy text-heuristic gate. `answerTermCoverageMet` checks case-insensitive substrings, `epistemicCuePresent` checks keywords, and `finalCitesEvidence` checks only an ID's @@ -1087,7 +1132,11 @@ presence. These metrics can still accept fabricated prose and must not be read as factual assessments. Set `REA_AGENT_EVAL_TRANSCRIPT_DIR` to retain complete tool results and final answers for review. -Report schema version 3 changes the top-level `factualCorrectness` from a constant +Report schema version 4 adds negative routing scenarios with nullable +`expectedFirstTool` and a `targetClarificationPassed` outcome. Consumers must +allow an expected absence of REA calls; this does not make an analysis scenario +pass without its required tools. Version 3 changed the top-level +`factualCorrectness` from a constant string to an assessment summary with status, scope, and assessed/passed/failed/ not-assessed scenario counts. Scenario records include the factual assessment and configured claim IDs. The evaluation scope is diff --git a/scripts/verify-agent-experience.mjs b/scripts/verify-agent-experience.mjs index 513835860b..99ab060ddf 100644 --- a/scripts/verify-agent-experience.mjs +++ b/scripts/verify-agent-experience.mjs @@ -1,9 +1,18 @@ import { spawn } from "node:child_process"; -import { cp, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { dirname, join, resolve } from "node:path"; +import { + chmod, + copyFile, + mkdir, + mkdtemp, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { homedir } from "node:os"; +import { delimiter, dirname, join, resolve } from "node:path"; import { createInterface } from "node:readline"; import { fileURLToPath } from "node:url"; +import { z } from "zod"; import { evaluateCodexEvents } from "../dist/evaluation/CodexAgentEval.js"; import { @@ -15,32 +24,95 @@ import { agentEvaluationPassed, summarizeFactualCorrectness, } from "../dist/evaluation/AgentEvaluationReport.js"; -import { MCP_STARTUP_POLICY } from "../dist/mcpStartupPolicy.js"; +import { PRODUCT_IDENTITY } from "../dist/identity.js"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const verifierRun = createVerifierRun(); -const evaluationRoot = await mkdtemp(join(tmpdir(), "rea-agent-eval-")); -const fixtureRoot = join(evaluationRoot, "targets"); -const skillDestination = join( - evaluationRoot, - ".agents/skills/reverse-engineer-anything", -); const timeoutMs = Number(process.env.REA_AGENT_EVAL_TIMEOUT_MS ?? 480_000); const codex = process.env.REA_CODEX_CLI ?? "codex"; const optionalModel = process.env.REA_AGENT_EVAL_MODEL; if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10_000) throw new Error("REA_AGENT_EVAL_TIMEOUT_MS must be an integer >= 10000"); -const codexVersion = await readCommandVersion(codex); +// Codex refuses helper aliases when CODEX_HOME is under the OS temp directory. +// Keep the disposable account in a private cache, then remove the owned run. +const evaluationParent = resolve( + process.env.REA_AGENT_EVAL_ROOT ?? + join(homedir(), ".cache", "rea-agent-evaluations"), +); +await mkdir(evaluationParent, { recursive: true, mode: 0o700 }); +const evaluationRoot = await mkdtemp(join(evaluationParent, "run-")); +const fixtureRoot = join(evaluationRoot, "targets"); +const account = join(evaluationRoot, "account"); +const codexHome = join(evaluationRoot, "codex"); +const authPath = join(codexHome, "auth.json"); +const skillDestination = join( + account, + ".agents/skills/reverse-engineer-anything", +); +const evaluationEnvironment = { + ...Object.fromEntries( + Object.entries(process.env).filter( + ([name]) => !/^(?:REA_|CODEX_|GHIDRA_|HOPPER_|IDA_)/u.test(name), + ), + ), + HOME: account, + USERPROFILE: account, + CODEX_HOME: codexHome, + XDG_CONFIG_HOME: join(account, ".config"), + XDG_DATA_HOME: join(account, ".local/share"), + XDG_CACHE_HOME: join(account, ".cache"), + REA_PROCESS_RUN_ID: verifierRun.run_id, + PATH: `${dirname(process.execPath)}${delimiter}${process.env.PATH ?? ""}`, +}; try { - await mkdir(fixtureRoot, { recursive: true }); - await cp( - join(repositoryRoot, "skills/reverse-engineer-anything"), - skillDestination, - { recursive: true }, + const codexVersion = await readCommandVersion(codex); + await Promise.all([ + mkdir(fixtureRoot, { recursive: true }), + mkdir(account), + mkdir(codexHome, { mode: 0o700 }), + ]); + // Reuse authentication only. User config, plugins, rules, history and skills + // must not make a fresh installed-skill trial pass accidentally. + try { + await copyFile( + join(process.env.CODEX_HOME ?? join(homedir(), ".codex"), "auth.json"), + authPath, + ); + await chmod(authPath, 0o600); + } catch (cause) { + if (cause?.code !== "ENOENT") throw cause; + // Environment/API-key authentication can work without a local auth file. + } + await writeFile( + join(codexHome, "config.toml"), + 'approval_policy = "never"\nsandbox_mode = "read-only"\nweb_search = "disabled"\n[features]\nmulti_agent = false\nshell_snapshot = false\n[history]\npersistence = "none"\n', + { mode: 0o600 }, ); + for (const approval of ["--dry-run", "--yes"]) { + await runProcess( + process.execPath, + [ + join(repositoryRoot, "scripts/rea.mjs"), + "setup", + "--client", + "codex", + approval, + "--json", + ], + evaluationRoot, + 120_000, + evaluationEnvironment, + ); + } + const installedSkill = await readFile( + join(skillDestination, "SKILL.md"), + "utf8", + ); + if (!installedSkill.includes(` version: "${PRODUCT_IDENTITY.skillVersion}"`)) + throw new Error("Setup did not install the current packaged REA skill"); const requestedScenarioIds = new Set( (process.env.REA_AGENT_EVAL_SCENARIOS ?? "") .split(",") @@ -57,7 +129,11 @@ try { expectedFirstTool: "open_binary", requiresEvidence: false, requiredAnswerTermGroups: [["unavailable", "could not", "provider"]], - prompt: `Explain what the shipped native program at ${targets.native} does. Base the answer on artifact analysis, and clearly state any facts that remain unavailable.`, + // The verifier authorizes these named session operations, not arbitrary + // writes or runtime execution. Missing-engine behavior stays observable. + approvedTools: ["open_binary", "close_binary"], + requiredToolSubsequence: ["open_binary", "close_binary"], + prompt: `Explain what static artifact evidence establishes about the shipped native program at ${targets.native}. Do not execute the target or install tools. Clearly state unavailable provider evidence, and close any analysis session you opened before finishing.`, }, { id: "asar", @@ -70,6 +146,38 @@ try { ], prompt: `Explain how the desktop application at ${targets.javascript} exposes APIs to its renderer. Base the answer on the shipped application artifact, and state what remains unknown.`, }, + { + id: "asar-zh", + fixtureScenarioId: "asar", + expectedFirstTool: "analyze_javascript_application", + requiresEvidence: true, + requiredAnswerTermGroups: [["profileapi"], ["profile:read"]], + prompt: `請靜態分析已交付的 Electron 程式 ${targets.javascript}，說明 renderer 如何透過 preload 讀取 profile，以及 main handler 的關係。不要執行目標或安裝分析引擎；引用完整的產出 Evidence，保留 runtime 未知事項。`, + }, + { + id: "javascript-module-view", + expectedFirstTool: "analyze_javascript_application", + requiresEvidence: true, + requiredToolSubsequence: [ + "analyze_javascript_application", + "inspect_analysis_view", + "inspect_analysis_view", + ], + requiredAnswerTermGroups: [ + ["main.js"], + ["node", "identity"], + ["observations", "recorded"], + ], + prompt: `Inspect the recorded module identity, exports and observations for main.js in the shipped JavaScript tree ${targets.largeJavascript}. Use REA without directly reading target files or executing the target. Keep results focused on that module, cite Evidence, and state the limits of what its module view establishes.`, + }, + { + id: "missing-target", + expectedFirstTool: null, + requiresEvidence: false, + requiredAnswerTermGroups: [], + prompt: + 'Reverse engineer my desktop app and explain how it saves user settings. Use static analysis; do not install software. End with one JSON object containing status ("needs_target" or "analyzed"), requested_input ("app_name_or_artifact_path" or null), and question (a string or null).', + }, { id: "javascript-export-shape", expectedFirstTool: "analyze_javascript_application", @@ -145,12 +253,14 @@ try { const results = []; for (const scenario of scenarios) { process.stderr.write(`Running Codex agent evaluation: ${scenario.id}\n`); - const fixtureClaims = agentFixtureClaims(scenario.id, targets); - const answerInstructions = agentFixtureAnswerInstructions(scenario.id); + const fixtureId = scenario.fixtureScenarioId ?? scenario.id; + const fixtureClaims = agentFixtureClaims(fixtureId, targets); + const answerInstructions = agentFixtureAnswerInstructions(fixtureId); const execution = await runCodex( answerInstructions.length === 0 ? scenario.prompt - : `${scenario.prompt}\n\n${answerInstructions}`, + : `${scenario.prompt}\n\nThis closed factual rubric needs complete producer Evidence in the transcript. Request complete detail on each initial application analysis when supported; summary or view records alone are not accepted by this rubric.\n\n${answerInstructions}`, + scenario.approvedTools ?? [], ); const transcriptDirectory = process.env.REA_AGENT_EVAL_TRANSCRIPT_DIR; if (transcriptDirectory !== undefined) { @@ -175,6 +285,12 @@ try { ...(fixtureClaims === undefined ? {} : { fixtureClaims }), }, ); + // Fixtures are deliberately visible beside prior investigation artifacts. + // Their existence does not identify the unnamed app in this request. + const targetClarificationPassed = + scenario.expectedFirstTool === null + ? validTargetClarification(metrics.finalMessage) + : undefined; const result = { id: scenario.id, expectedFirstTool: scenario.expectedFirstTool, @@ -185,6 +301,9 @@ try { exitCode: execution.exitCode, stderr: execution.stderr, ...metrics, + ...(targetClarificationPassed === undefined + ? {} + : { targetClarificationPassed }), }; results.push(result); process.stderr.write( @@ -194,13 +313,20 @@ try { const factualCorrectness = summarizeFactualCorrectness(results); const summary = { - schemaVersion: 3, + schemaVersion: 4, evaluationScope: "routing_workflow_and_configured_fixture_claims", factualCorrectness, verifier_run: await completeVerifierRun(verifierRun), codex, codexVersion, model: optionalModel ?? null, + installation: { + method: "rea_setup_codex", + isolatedHome: true, + isolatedCodexHome: true, + skillVersion: PRODUCT_IDENTITY.skillVersion, + authentication: "caller_auth_only_removed_after_run", + }, scenarios: results, totals: { scenarios: results.length, @@ -253,12 +379,32 @@ try { `Agent routing/workflow/answer checks failed: ${failed.map(({ id }) => id).join(", ")}`, ); } finally { + // Even a deliberately retained fixture directory must never retain auth. + await rm(authPath, { force: true }); if (process.env.REA_AGENT_EVAL_KEEP_FIXTURES !== "true") await rm(evaluationRoot, { recursive: true, force: true }); } async function createTargets(root, includeManaged) { const fixtures = await createAgentEvaluationFixtures(root, repositoryRoot); + const largeJavascript = join(root, "large-app"); + await mkdir(join(largeJavascript, "lib"), { recursive: true }); + await Promise.all([ + writeFile( + join(largeJavascript, "package.json"), + '{"name":"large-app","main":"main.js"}\n', + ), + writeFile( + join(largeJavascript, "main.js"), + 'const { ipcMain } = require("electron");\nipcMain.handle("profile:read", (_event, id) => ({ id }));\n', + ), + ...Array.from({ length: 180 }, (_, index) => + writeFile( + join(largeJavascript, "lib", `formatter-${String(index)}.js`), + `export const format = (value) => String(value).trim();\n`, + ), + ), + ]); const managedOutput = join(root, "managed-output"); if (includeManaged) { @@ -292,15 +438,29 @@ async function createTargets(root, includeManaged) { return { native: "/bin/true", ...fixtures, + largeJavascript, managed: join(managedOutput, "AgentEval.dll"), }; } -async function runCodex(prompt) { +function validTargetClarification(message) { + const schema = z.strictObject({ + status: z.literal("needs_target"), + requested_input: z.literal("app_name_or_artifact_path"), + question: z.string().refine((value) => value.trim().length > 0), + }); + try { + return schema.safeParse(JSON.parse(message)).success; + } catch (cause) { + if (cause instanceof SyntaxError) return false; + throw cause; + } +} + +async function runCodex(prompt, approvedTools) { const arguments_ = [ + "--no-daemon", "exec", - "--ignore-user-config", - "--ignore-rules", "--ephemeral", "--skip-git-repo-check", "--sandbox", @@ -312,18 +472,16 @@ async function runCodex(prompt) { evaluationRoot, "-c", 'approval_policy="never"', - "-c", - `mcp_servers.rea.command=${JSON.stringify(process.execPath)}`, - "-c", - `mcp_servers.rea.args=${JSON.stringify([join(repositoryRoot, "scripts/rea.mjs"), "mcp"])}`, - "-c", - `mcp_servers.rea.startup_timeout_sec=${String(MCP_STARTUP_POLICY.codexStartupTimeoutSeconds)}`, + ...approvedTools.flatMap((name) => [ + "-c", + `mcp_servers.rea.tools.${name}.approval_mode="approve"`, + ]), ...(optionalModel === undefined ? [] : ["--model", optionalModel]), prompt, ]; const child = spawn(codex, arguments_, { cwd: evaluationRoot, - env: process.env, + env: evaluationEnvironment, stdio: ["ignore", "pipe", "pipe"], }); const events = []; @@ -362,9 +520,16 @@ async function runCodex(prompt) { return { events, exitCode, stderr: stderr.join("").slice(0, 32_768) }; } -async function runProcess(command, arguments_, cwd, timeout) { +async function runProcess( + command, + arguments_, + cwd, + timeout, + env = process.env, +) { const child = spawn(command, arguments_, { cwd, + env, stdio: ["ignore", "ignore", "pipe"], }); let stderr = ""; diff --git a/src/application/SetupSkill.ts b/src/application/SetupSkill.ts index 25228d14ba..50d0d96f9b 100644 --- a/src/application/SetupSkill.ts +++ b/src/application/SetupSkill.ts @@ -10,6 +10,7 @@ import { clientSkillDirectories } from "./SupportedClients.js"; export const MANAGED_SKILL_FILES = [ "SKILL.md", + "references/connection-and-recovery.md", "references/native-and-artifacts.md", "references/javascript-applications.md", "references/android-applications.md", diff --git a/src/evaluation/AgentEvaluationReport.ts b/src/evaluation/AgentEvaluationReport.ts index 37e6a70eb6..4af002f40c 100644 --- a/src/evaluation/AgentEvaluationReport.ts +++ b/src/evaluation/AgentEvaluationReport.ts @@ -3,6 +3,9 @@ export type FixtureCorrectness = "passed" | "failed" | "not_assessed"; /** Measurements consumed by the evaluation runner's release gate. */ export interface AgentEvaluationGateResult { + /** Explicit null requires a target clarification with no REA calls. */ + readonly expectedFirstTool?: string | null; + readonly targetClarificationPassed?: boolean; readonly exitCode: number; readonly naturalUse: boolean; readonly correctFirstTool: boolean; @@ -20,16 +23,20 @@ export const agentEvaluationPassed = ( result: AgentEvaluationGateResult, ): boolean => result.exitCode === 0 && - result.naturalUse && + (result.expectedFirstTool === null + ? !result.naturalUse + : result.naturalUse) && result.correctFirstTool && result.repeatedCallCount === 0 && result.inputValidationFailureCount === 0 && result.requiredToolSubsequenceMet && result.inputTokens > 0 && - (result.factualCorrectness === "passed" || - (result.factualCorrectness === "not_assessed" && - result.answerHeuristicsMet && - result.epistemicCuePresent)); + (result.expectedFirstTool === null + ? result.targetClarificationPassed === true + : result.factualCorrectness === "passed" || + (result.factualCorrectness === "not_assessed" && + result.answerHeuristicsMet && + result.epistemicCuePresent)); /** Keep assessed fixture outcomes and scenarios without truth oracles distinct. */ export const summarizeFactualCorrectness = ( diff --git a/src/evaluation/CodexAgentEval.ts b/src/evaluation/CodexAgentEval.ts index 2aebc7ca7d..ba5c500d05 100644 --- a/src/evaluation/CodexAgentEval.ts +++ b/src/evaluation/CodexAgentEval.ts @@ -4,6 +4,7 @@ import { type KnownAnswerAssessment, } from "./KnownAnswerEvaluation.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; +import { TOOL_CONTRACTS } from "../contracts/toolContracts.js"; /** One REA MCP invocation observed in a Codex JSONL transcript. */ interface CodexMcpCall { @@ -72,7 +73,35 @@ const evidenceIdsFrom = (value: unknown): readonly string[] => { return [...new Set(encoded.match(/ev_[a-f0-9]{64}/gu) ?? [])]; }; -const toolResultFailed = (item: Record): boolean => { +const toolResultError = ( + item: Record, +): Record | undefined => { + const result = record(item.result ?? item.output); + const structured = record( + result?.structured_content ?? result?.structuredContent, + ); + const direct = record(structured?.error ?? result?.error ?? item.error); + if (direct !== undefined || structured !== undefined) return direct; + // Codex preserves some REA failures only as one complete JSON text block. + // A client-truncated preview or SDK prose is not a structured error record. + const content = result?.content; + if (!Array.isArray(content) || content.length !== 1) return undefined; + const block = record(content[0]); + if (block?.type !== "text" || typeof block.text !== "string") + return undefined; + try { + return record(record(JSON.parse(block.text))?.error); + } catch (cause) { + if (cause instanceof SyntaxError || cause instanceof RangeError) + return undefined; + throw cause; + } +}; + +const toolResultFailed = ( + item: Record, + error: Record | undefined, +): boolean => { if ( item.status === "failed" || (item.error !== undefined && item.error !== null) @@ -82,16 +111,11 @@ const toolResultFailed = (item: Record): boolean => { const structured = record( result?.structured_content ?? result?.structuredContent, ); - return result?.isError === true || structured?.error !== undefined; -}; - -const toolResultErrorCode = (item: Record): string | null => { - const result = record(item.result ?? item.output); - const structured = record( - result?.structured_content ?? result?.structuredContent, + return ( + result?.isError === true || + structured?.error !== undefined || + error !== undefined ); - const error = record(structured?.error ?? result?.error ?? item.error); - return textValue(error?.code) ?? null; }; const callFromItem = (value: unknown): CodexMcpCall | undefined => { @@ -99,14 +123,15 @@ const callFromItem = (value: unknown): CodexMcpCall | undefined => { if (item?.type !== "mcp_tool_call") return undefined; const tool = textValue(item.tool) ?? textValue(item.name); if (tool === undefined) return undefined; + const error = toolResultError(item); return { id: textValue(item.id) ?? null, server: textValue(item.server) ?? textValue(item.server_name) ?? null, tool, arguments: item.arguments ?? item.input ?? {}, evidenceIds: evidenceIdsFrom(item.result ?? item.output), - error: toolResultFailed(item), - errorCode: toolResultErrorCode(item), + error: toolResultFailed(item, error), + errorCode: textValue(error?.code) ?? null, }; }; @@ -171,7 +196,7 @@ const consumeEvent = (state: EvaluationState, value: unknown): void => { /** Measure agent routing, repetition, model usage, and explicitly limited answer heuristics. */ export const evaluateCodexEvents = ( events: readonly unknown[], - expectedFirstTool: string, + expectedFirstTool: string | null, options: { readonly requireEvidence?: boolean; readonly requiredAnswerTermGroups?: readonly (readonly string[])[]; @@ -194,16 +219,31 @@ export const evaluateCodexEvents = ( ({ server }) => server === null || server.toLowerCase() === "rea", ); const signatures = new Map(); + let targetRevision = 0; for (const call of reaCalls) { - const signature = `${call.tool}:${JSON.stringify(canonicalValue(call.arguments))}`; + // A status read before and after a successful target lifecycle change is + // useful verification, not a duplicate analysis or unchanged failed retry. + const context = + call.tool === "binary_session" ? `${String(targetRevision)}:` : ""; + const signature = `${context}${call.tool}:${JSON.stringify(canonicalValue(call.arguments))}`; signatures.set(signature, (signatures.get(signature) ?? 0) + 1); + if ( + !call.error && + (call.tool === "open_binary" || call.tool === "close_binary") + ) + targetRevision += 1; } const repeatedCallCount = [...signatures.values()].reduce( (total, count) => total + Math.max(0, count - 1), 0, ); const inputValidationFailureCount = reaCalls.filter( - ({ errorCode }) => errorCode === "invalid_request", + (call) => + call.errorCode === "invalid_request" || + (call.error && + TOOL_CONTRACTS.find( + ({ name }) => name === call.tool, + )?.inputSchema.safeParse(call.arguments).success === false), ).length; const requiredToolSubsequenceMet = containsOrderedSubsequence( reaCalls.filter(({ error }) => !error).map(({ tool }) => tool), @@ -234,7 +274,7 @@ export const evaluateCodexEvents = ( ); return { naturalUse: reaCalls.length > 0, - correctFirstTool: reaCalls[0]?.tool === expectedFirstTool, + correctFirstTool: (reaCalls[0]?.tool ?? null) === expectedFirstTool, firstTool: reaCalls[0]?.tool ?? null, reaCalls, repeatedCallCount, diff --git a/src/generatedPackageMetadata.ts b/src/generatedPackageMetadata.ts index aaa5b9a805..ab4334d0a3 100644 --- a/src/generatedPackageMetadata.ts +++ b/src/generatedPackageMetadata.ts @@ -5,5 +5,5 @@ export const PACKAGE_METADATA = { serverSdkVersion: "2.3.1", clientSdkVersion: "2.3.1", coreSdkVersion: "2.3.1", - skillVersion: "36", + skillVersion: "37", } as const; diff --git a/tests/boundary/filesystem/setupSkill.test.ts b/tests/boundary/filesystem/setupSkill.test.ts index 40bbcfd825..4da0225c34 100644 --- a/tests/boundary/filesystem/setupSkill.test.ts +++ b/tests/boundary/filesystem/setupSkill.test.ts @@ -187,12 +187,11 @@ describe("canonical skill transaction", () => { expect(installedSkill).toContain( `version: "${PRODUCT_IDENTITY.skillVersion}"`, ); - expect(installedSkill).toContain("call available analysis tools"); - expect(installedSkill).toContain("obtain approval before setup writes"); expect(await readFile(`${nativeGuide}.rea.backup`, "utf8")).toBe( "stale filesystem-write permission grant\n", ); for (const reference of [ + "connection-and-recovery.md", "native-and-artifacts.md", "javascript-applications.md", "android-applications.md", @@ -243,6 +242,7 @@ describe("canonical skill transaction", () => { const paths = [ "SKILL.md", "references/android-applications.md", + "references/connection-and-recovery.md", "references/evidence-workflows.md", "references/javascript-applications.md", "references/native-and-artifacts.md", diff --git a/tests/evaluation/model-evals/agentEvaluationReport.test.ts b/tests/evaluation/model-evals/agentEvaluationReport.test.ts index 1913da97d2..562fd591ea 100644 --- a/tests/evaluation/model-evals/agentEvaluationReport.test.ts +++ b/tests/evaluation/model-evals/agentEvaluationReport.test.ts @@ -77,6 +77,29 @@ async function runnerFailurePredicate(): Promise<(item: unknown) => unknown> { } describe("scoped factual report", () => { + it("requires a validated target clarification and zero REA calls for the negative route", () => { + const clarification = { + ...routingSuccess, + expectedFirstTool: null, + naturalUse: false, + factualCorrectness: "not_assessed" as const, + answerHeuristicsMet: false, + epistemicCuePresent: false, + targetClarificationPassed: true, + }; + expect(agentEvaluationPassed(clarification)).toBe(true); + for (const change of [ + { naturalUse: true }, + { correctFirstTool: false }, + { targetClarificationPassed: false }, + { exitCode: 1 }, + { inputTokens: 0 }, + ]) + expect(agentEvaluationPassed({ ...clarification, ...change })).toBe( + false, + ); + }); + it("accepts correct structured facts without superficial prose cues", () => { expect( agentEvaluationPassed({ diff --git a/tests/evaluation/model-evals/codexAgentEval.test.ts b/tests/evaluation/model-evals/codexAgentEval.test.ts index 61aaa34fbb..f802d68d29 100644 --- a/tests/evaluation/model-evals/codexAgentEval.test.ts +++ b/tests/evaluation/model-evals/codexAgentEval.test.ts @@ -101,6 +101,35 @@ describe("Codex agent release evaluation", () => { ).toBe(1); }); + it("distinguishes session verification after a successful close from an unchanged retry", () => { + const call = (id: string, tool: string, status = "completed") => ({ + type: "item.completed", + item: { + id, + type: "mcp_tool_call", + server: "rea", + tool, + arguments: {}, + status, + }, + }); + for (const [status, repeats] of [ + ["completed", 0], + ["failed", 1], + ] as const) { + expect( + evaluateCodexEvents( + [ + call("before", "binary_session"), + call("close", "close_binary", status), + call("after", "binary_session"), + ], + "binary_session", + ).repeatedCallCount, + ).toBe(repeats); + } + }); + it("counts identical arguments with reversed Unicode key insertion order", () => { const call = (id: string, arguments_: Record) => ({ type: "item.completed", @@ -125,6 +154,119 @@ describe("Codex agent release evaluation", () => { }); }); +describe("Codex agent routing and error evaluation", () => { + it("preserves an unrecognized structured failure without inventing its code", () => { + const metrics = evaluateCodexEvents( + [ + { + type: "item.completed", + item: { + type: "mcp_tool_call", + server: "rea", + tool: "close_binary", + arguments: {}, + result: { structured_content: { error: "unrecognized failure" } }, + }, + }, + ], + "close_binary", + { requiredToolSubsequence: ["close_binary"] }, + ); + expect(metrics.reaCalls[0]).toMatchObject({ error: true, errorCode: null }); + expect(metrics.requiredToolSubsequenceMet).toBe(false); + }); + + it("requires no REA call when target selection is missing", () => { + const events = [ + { + type: "item.completed", + item: { type: "agent_message", text: "Which app should I inspect?" }, + }, + ]; + expect(evaluateCodexEvents(events, null)).toMatchObject({ + naturalUse: false, + correctFirstTool: true, + firstTool: null, + }); + expect( + evaluateCodexEvents( + [ + ...events, + { + type: "item.completed", + item: { + type: "mcp_tool_call", + server: "rea", + tool: "analyze_javascript_application", + arguments: { input_path: "/nearby/test-fixture" }, + }, + }, + ], + null, + ).correctFirstTool, + ).toBe(false); + }); + + it("counts actual text-only REA errors and SDK-rejected schema inputs once per call", () => { + const source = { + kind: "retained-evidence", + evidence_id: `ev_${"a".repeat(64)}`, + }; + const failed = (id: string, arguments_: unknown, text: string) => ({ + type: "item.completed", + item: { + id, + type: "mcp_tool_call", + server: "rea", + tool: "inspect_analysis_view", + arguments: arguments_, + status: "failed", + error: null, + result: { content: [{ type: "text", text }], structured_content: null }, + }, + }); + const metrics = evaluateCodexEvents( + [ + failed( + "sdk", + { source, view: { kind: "module", path: "main.js" } }, + "Input validation error: Invalid arguments for tool inspect_analysis_view: view.kind: Invalid discriminator value.", + ), + failed( + "ambiguous", + { + source, + view: { + kind: "item", + collection: "modules", + selector: { path: "main.js" }, + }, + }, + JSON.stringify({ + error: { + code: "invalid_request", + message: + "Multiple modules have this path; select one by node_id.", + }, + }), + ), + failed( + "incomplete-preview", + { source, view: { kind: "summary" } }, + '{"error":{"code":"invalid_request"', + ), + ], + "inspect_analysis_view", + ); + expect(metrics.inputValidationFailureCount).toBe(2); + expect(metrics.reaCalls.map(({ errorCode }) => errorCode)).toEqual([ + null, + "invalid_request", + null, + ]); + }); +}); + describe("Codex agent completion evaluation", () => { it("does not present matching answer terms as factual correctness or honesty", () => { const evidenceId = `ev_${"a".repeat(64)}`;