From 2250e67bc271c83f9e200ee298aa04a2457438a6 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:38:57 +0800 Subject: [PATCH] fix(verifiers): consolidate native client command lifetimes Migrate the remaining native verifier commands to shared ownership with explicit stdin contracts, command-before-fixture cleanup, and failure receipts. Preserve observed catalog counts and unknowns, retained artifacts, native assertions, and selected launch inputs. Remove the obsolete raw exec export and share output persistence across native verifiers. Exercise actual entrypoints with authored failed preflights and acknowledged descendants, plus filesystem failures and opaque error preservation. --- scripts/lib/verifier-command-output.d.mts | 7 + scripts/lib/verifier-command-output.mjs | 30 + scripts/lib/verify-package-core.mjs | 16 +- scripts/verify-claude-client.mjs | 812 ++++++++------- scripts/verify-commandcode-client.mjs | 806 ++++++++------- scripts/verify-copilot-client.mjs | 864 ++++++++-------- scripts/verify-deepseek-client.mjs | 684 +++++++------ scripts/verify-gemini-client.mjs | 813 ++++++++------- scripts/verify-grok-client.mjs | 938 +++++++++--------- scripts/verify-native-agent-client.mjs | 49 +- scripts/verify-omp-client.mjs | 811 ++++++++------- scripts/verify-opencode-client.mjs | 681 +++++++------ scripts/verify-qwen-client.mjs | 739 +++++++------- .../filesystem/verifierCommandOutput.test.ts | 198 ++++ .../nativeVerifierPreflightOwnership.test.ts | 270 +++++ 15 files changed, 4292 insertions(+), 3426 deletions(-) create mode 100644 scripts/lib/verifier-command-output.d.mts create mode 100644 scripts/lib/verifier-command-output.mjs create mode 100644 tests/boundary/filesystem/verifierCommandOutput.test.ts create mode 100644 tests/boundary/process/nativeVerifierPreflightOwnership.test.ts diff --git a/scripts/lib/verifier-command-output.d.mts b/scripts/lib/verifier-command-output.d.mts new file mode 100644 index 0000000000..5939a63a32 --- /dev/null +++ b/scripts/lib/verifier-command-output.d.mts @@ -0,0 +1,7 @@ +import type { VerifierCommandOutput } from "./verifier-command-owner.mjs"; + +/** Retain available command output without losing command or filesystem failures. */ +export function saveVerifierCommandOutput( + operation: () => Promise, + paths: { readonly stdout: string; readonly stderr: string }, +): Promise; diff --git a/scripts/lib/verifier-command-output.mjs b/scripts/lib/verifier-command-output.mjs new file mode 100644 index 0000000000..ec4fe6ac60 --- /dev/null +++ b/scripts/lib/verifier-command-output.mjs @@ -0,0 +1,30 @@ +import { writeFile } from "node:fs/promises"; + +import { withVerifierCleanup } from "./verifier-cleanup.mjs"; + +/** Retain available command output without losing command or filesystem failures. */ +export async function saveVerifierCommandOutput(operation, paths) { + let output; + try { + output = await operation(); + } catch (cause) { + const snapshot = cause?.snapshot; + const stdout = snapshot?.stdout?.text; + const stderr = snapshot?.stderr?.text; + return withVerifierCleanup(async () => { + throw cause; + }, [ + async () => { + if (typeof stdout === "string") await writeFile(paths.stdout, stdout); + }, + async () => { + if (typeof stderr === "string") await writeFile(paths.stderr, stderr); + }, + ]); + } + await withVerifierCleanup( + () => writeFile(paths.stdout, output.stdout), + [() => writeFile(paths.stderr, output.stderr)], + ); + return output; +} diff --git a/scripts/lib/verify-package-core.mjs b/scripts/lib/verify-package-core.mjs index ef5179c690..eb15c49085 100644 --- a/scripts/lib/verify-package-core.mjs +++ b/scripts/lib/verify-package-core.mjs @@ -1,22 +1,10 @@ -import { execFile } from "node:child_process"; import { lstat } from "node:fs/promises"; -import { promisify } from "node:util"; import { Ajv2020 } from "ajv/dist/2020.js"; -/** - * `mcp-verifier-results.mjs` reaches into `dist/` at module load, so eager - * importing it propagates that build dependency to every consumer in this - * file. `verifyCompleteToolCatalog` is the only consumer, so we lazy-load - * the module there and keep the remaining helpers (`exec`, `json`, - * `pathExists`) usable from `test:fast` callers that only need `dist/`-free - * utilities — see docs/testing.md § test scripts, which promises `test:fast` - * runs "without build". - */ +// Only catalog verification needs built MCP contracts; the other helpers work +// from source without a runtime build. const loadMcpVerifierResults = () => import("./mcp-verifier-results.mjs"); -/** Native-client commands expose the live child handle for caller-owned stdin. */ -export const exec = promisify(execFile); - /** Parse JSON from a command output string. */ export const json = (text) => JSON.parse(text); diff --git a/scripts/verify-claude-client.mjs b/scripts/verify-claude-client.mjs index 10a17ee3c0..d652400705 100644 --- a/scripts/verify-claude-client.mjs +++ b/scripts/verify-claude-client.mjs @@ -15,7 +15,9 @@ import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; -import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { expandMcpSessionStatus, @@ -38,36 +40,9 @@ const profile = join(lab, "selected profile"); const workspace = join(lab, "workspace"); const configPath = join(profile, ".claude.json"); const skillDirectory = join(profile, "skills", "reverse-engineer-anything"); -await Promise.all([mkdir(profile), mkdir(workspace)]); -await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); -const environment = { - ...process.env, - CLAUDE_CONFIG_DIR: profile, - DISABLE_AUTOUPDATER: "1", - CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", - REA_PROCESS_RUN_ID: verifier.run_id, -}; -let clientVersion; -try { - clientVersion = ( - await exec(command, ["--version"], { env: environment, timeout: 30_000 }) - ).stdout.trim(); -} catch (cause) { - // npm installs put only a claude.cmd shim on PATH, which Node does not spawn without a shell. - const windowsHint = - process.platform === "win32" - ? " On Windows, select claude.exe; npm installs provide it under node_modules/@anthropic-ai/claude-code/bin." - : ""; - throw new Error( - `Claude Code preflight failed for ${command}; install Claude Code or set REA_VERIFY_CLAUDE_COMMAND.${windowsHint} ${String(cause)}`, - ); -} -const target = join(lab, "fixture α测试"); -await mkdir(target); -await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', -); +let clientVersion = null; +let fixture; +let passed = false; let skillRequested = false; let sessionRequested = false; let sessionStatusSeen = false; @@ -75,369 +50,440 @@ let skillLoaded = false; let expectedSkillBody; let evidenceSeen = false; let finalSeen = false; -let catalogSize = 0; +let catalogSize = null; let schemasChecked = 0; let pendingServerWaits = 0; let artifactDigest; -const schemaValidator = new Ajv2020({ strict: false, validateFormats: false }); -const containsText = (value, text) => { - if (typeof value === "string") return value.includes(text); - if (value && typeof value === "object") - return Object.values(value).some((nested) => containsText(nested, text)); - return false; -}; -const extractInlineResult = (value, matches) => { - if (typeof value === "string") { +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([mkdir(profile), mkdir(workspace)]); + await commands.exec("git", ["init", "--quiet", workspace], { + timeout: 30_000, + stdin: "pipe", + }); + const environment = { + ...process.env, + CLAUDE_CONFIG_DIR: profile, + DISABLE_AUTOUPDATER: "1", + CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", + REA_PROCESS_RUN_ID: verifier.run_id, + }; try { - // Native Claude appends this resource hint to tool results. Original bodies remain intact in request artifacts. - const json = value.replace( - /\n\n\n\d+ tokens left<\/total_tokens>\n<\/system-reminder>$/u, - "", + clientVersion = ( + await commands.exec(command, ["--version"], { + env: environment, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + } catch (cause) { + // npm installs put only a claude.cmd shim on PATH, which Node does not spawn without a shell. + const windowsHint = + process.platform === "win32" + ? " On Windows, select claude.exe; npm installs provide it under node_modules/@anthropic-ai/claude-code/bin." + : ""; + throw new Error( + `Claude Code preflight failed for ${command}; install Claude Code or set REA_VERIFY_CLAUDE_COMMAND.${windowsHint} ${String(cause)}`, + { cause }, ); - return extractInlineResult(JSON.parse(json), matches); - } catch { - return undefined; } - } - if (value && typeof value === "object") { - if (matches(value)) return value; - for (const nested of Object.values(value)) { - const result = extractInlineResult(nested, matches); - if (result) return result; - } - } - return undefined; -}; -const fixture = await createAnthropicModelFixture({ - directory: lab, - onRequest: async ({ body, tools, requestIndex }) => { - if (tools.length === 0) - return [{ type: "text", text: "REA fixture title" }]; - assert(requestIndex <= 8, "Local model workflow exceeded its turn budget"); - const call = (name, input) => [ - { type: "tool_use", id: `toolu_rea_${requestIndex}`, name, input }, - ]; - const analysis = tools.find((t) => - t.name.endsWith("analyze_javascript_application"), - ); - if (!analysis) { - // Native automatic discovery can begin the conversation while MCP is still connecting. - const wait = tools.find((t) => t.name === "WaitForMcpServers"); - assert( - wait, - "Claude must expose REA tools or its native pending-server wait", - ); - pendingServerWaits++; - return call(wait.name, { servers: ["rea"] }); - } - const prefix = analysis.name.slice( - 0, - -"analyze_javascript_application".length, - ); - const expected = sortVerifierNames( - TOOL_CONTRACTS.map((t) => prefix + t.name), - ); - const reaTools = tools.filter((t) => t.name.startsWith(prefix)); - assert.deepEqual( - sortVerifierNames(reaTools.map((t) => t.name)), - expected, - "Claude must forward the complete REA catalog after connection", - ); - catalogSize = expected.length; - if (schemasChecked === 0) { - for (const tool of reaTools) { - assert( - tool.input_schema, - `${tool.name}: missing advertised input schema`, - ); - assert( - schemaValidator.validateSchema(tool.input_schema), - `${tool.name}: invalid input schema: ${JSON.stringify(schemaValidator.errors)}`, - ); - schemasChecked++; - } - } - if (mode === "call" && !sessionStatusSeen) { - const observedStatus = extractInlineResult( - body.messages, - (value) => - Boolean(value.result?.server_identity) && - Array.isArray(value.result?.tool_availability?.groups), - ); - if (observedStatus) { - const parsedStatus = - toolContract("binary_session").outputSchema.parse(observedStatus); - const status = { - ...parsedStatus, - result: expandMcpSessionStatus(parsedStatus.result), - }; - const inputProfile = parseMcpInputSchemaProfile( - environment.REA_MCP_INPUT_SCHEMA_PROFILE, - ); - const outputProfile = parseMcpOutputSchemaProfile( - environment.REA_MCP_OUTPUT_SCHEMA_PROFILE, - ); - assert.ok(inputProfile.ok); - assert.ok(outputProfile.ok); - assert.deepEqual( - status.result.server_identity.catalog, - createCatalogIdentity( - compactPresentationForMcpInputSchemaProfile(inputProfile.value), - outputProfile.value ?? "full", - ), - "binary_session must return the selected input/output profile's catalog identity", - ); - assert.equal( - Object.hasOwn(status.result.server_identity.catalog, "tools"), - false, - "binary_session must not inline the duplicated canonical tool catalog", - ); - assert.deepEqual( - sortVerifierNames( - status.result.tool_availability.map((entry) => entry.name), - ), - sortVerifierNames(TOOL_CONTRACTS.map((contract) => contract.name)), - "binary_session must retain complete canonical tool availability", - ); - sessionStatusSeen = true; - } else { - assert( - !sessionRequested, - "Claude must return the inline binary_session result before continuing", - ); - const session = tools.find( - (tool) => tool.name === `${prefix}binary_session`, - ); - assert(session, "Claude must expose the REA binary_session tool"); - sessionRequested = true; - return call(session.name, {}); - } - } - skillLoaded ||= - expectedSkillBody !== undefined && - containsText(body.messages, expectedSkillBody) && - containsText(body.messages, skillDirectory); - const observed = extractInlineResult( - body.messages, - (value) => - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0, - ); - if (observed) { - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(observed); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((n) => n.observations) - .map((o) => o.properties) - .find( - (p) => - p.semantic_role === "export-return-shapes" && - p.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((s) => s.fields); - assert( - fields.some( - (f) => - f.path === "/clientCompatibility" && - f.state === "literal" && - f.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (f) => f.path === "/count" && f.state === "literal" && f.value === 7, - ), - ); - const bytes = JSON.stringify(evidence); - artifactDigest = createHash("sha256").update(bytes).digest("hex"); - await writeFile(join(lab, "claude-evidence.json"), bytes); - evidenceSeen = true; - } - if (mode === "chat" || (sessionStatusSeen && evidenceSeen)) { - finalSeen = true; - return [{ type: "text", text: "REA_CLIENT_COMPATIBILITY_OK" }]; - } - if (!skillRequested) { - const skill = tools.find((t) => t.name === "Skill"); - assert(skill, "Claude must expose its native Skill tool"); - skillRequested = true; - return call(skill.name, { skill: "reverse-engineer-anything" }); - } - assert( - skillLoaded, - "Claude must load the complete installed personal REA skill", - ); - return call(analysis.name, { input_path: target, format: "directory" }); - }, -}); -const original = JSON.stringify({ theme: "light", mcpServers: {} }, null, 2); -await writeFile(configPath, original); -let failure; -try { - for (const args of [ - ["setup", "--client", "claude_code", "--dry-run"], - ["setup", "--client", "claude_code", "--yes"], - ["doctor", "--client", "claude_code", "--skill"], - ]) { - const result = await exec( - process.execPath, - [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); + const target = join(lab, "fixture α测试"); + await mkdir(target); await writeFile( - join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), - result.stdout, + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', ); - const parsed = JSON.parse(result.stdout); - if (args[0] === "doctor") assert.equal(parsed.healthy, true); - else { - assert.equal( - parsed.status, - args.includes("--dry-run") ? "planned" : "ready", - ); - if (args.includes("--dry-run")) { - assert(parsed.plannedActions.some((a) => a.target === configPath)); - assert(parsed.plannedActions.some((a) => a.target === skillDirectory)); - assert( - parsed.plannedActions.every( - (a) => - a.target === configPath || a.target.startsWith(skillDirectory), - ), + const schemaValidator = new Ajv2020({ + strict: false, + validateFormats: false, + }); + const containsText = (value, text) => { + if (typeof value === "string") return value.includes(text); + if (value && typeof value === "object") + return Object.values(value).some((nested) => + containsText(nested, text), ); + return false; + }; + const extractInlineResult = (value, matches) => { + if (typeof value === "string") { + try { + // Native Claude appends this resource hint to tool results. Original bodies remain intact in request artifacts. + const json = value.replace( + /\n\n\n\d+ tokens left<\/total_tokens>\n<\/system-reminder>$/u, + "", + ); + return extractInlineResult(JSON.parse(json), matches); + } catch { + return undefined; + } + } + if (value && typeof value === "object") { + if (matches(value)) return value; + for (const nested of Object.values(value)) { + const result = extractInlineResult(nested, matches); + if (result) return result; + } + } + return undefined; + }; + fixture = await createAnthropicModelFixture({ + directory: lab, + onRequest: async ({ body, tools, requestIndex }) => { + if (tools.length === 0) + return [{ type: "text", text: "REA fixture title" }]; + assert( + requestIndex <= 8, + "Local model workflow exceeded its turn budget", + ); + const call = (name, input) => [ + { type: "tool_use", id: `toolu_rea_${requestIndex}`, name, input }, + ]; + const analysis = tools.find((t) => + t.name.endsWith("analyze_javascript_application"), + ); + if (!analysis) { + // Native automatic discovery can begin the conversation while MCP is still connecting. + const wait = tools.find((t) => t.name === "WaitForMcpServers"); + assert( + wait, + "Claude must expose REA tools or its native pending-server wait", + ); + pendingServerWaits++; + return call(wait.name, { servers: ["rea"] }); + } + const prefix = analysis.name.slice( + 0, + -"analyze_javascript_application".length, + ); + const expected = sortVerifierNames( + TOOL_CONTRACTS.map((t) => prefix + t.name), + ); + const reaTools = tools.filter((t) => t.name.startsWith(prefix)); + catalogSize = reaTools.length; + assert.deepEqual( + sortVerifierNames(reaTools.map((t) => t.name)), + expected, + "Claude must forward the complete REA catalog after connection", + ); + if (schemasChecked === 0) { + for (const tool of reaTools) { + assert( + tool.input_schema, + `${tool.name}: missing advertised input schema`, + ); + assert( + schemaValidator.validateSchema(tool.input_schema), + `${tool.name}: invalid input schema: ${JSON.stringify(schemaValidator.errors)}`, + ); + schemasChecked++; + } + } + if (mode === "call" && !sessionStatusSeen) { + const observedStatus = extractInlineResult( + body.messages, + (value) => + Boolean(value.result?.server_identity) && + Array.isArray(value.result?.tool_availability?.groups), + ); + if (observedStatus) { + const parsedStatus = + toolContract("binary_session").outputSchema.parse(observedStatus); + const status = { + ...parsedStatus, + result: expandMcpSessionStatus(parsedStatus.result), + }; + const inputProfile = parseMcpInputSchemaProfile( + environment.REA_MCP_INPUT_SCHEMA_PROFILE, + ); + const outputProfile = parseMcpOutputSchemaProfile( + environment.REA_MCP_OUTPUT_SCHEMA_PROFILE, + ); + assert.ok(inputProfile.ok); + assert.ok(outputProfile.ok); + assert.deepEqual( + status.result.server_identity.catalog, + createCatalogIdentity( + compactPresentationForMcpInputSchemaProfile(inputProfile.value), + outputProfile.value ?? "full", + ), + "binary_session must return the selected input/output profile's catalog identity", + ); + assert.equal( + Object.hasOwn(status.result.server_identity.catalog, "tools"), + false, + "binary_session must not inline the duplicated canonical tool catalog", + ); + assert.deepEqual( + sortVerifierNames( + status.result.tool_availability.map((entry) => entry.name), + ), + sortVerifierNames( + TOOL_CONTRACTS.map((contract) => contract.name), + ), + "binary_session must retain complete canonical tool availability", + ); + sessionStatusSeen = true; + } else { + assert( + !sessionRequested, + "Claude must return the inline binary_session result before continuing", + ); + const session = tools.find( + (tool) => tool.name === `${prefix}binary_session`, + ); + assert(session, "Claude must expose the REA binary_session tool"); + sessionRequested = true; + return call(session.name, {}); + } + } + skillLoaded ||= + expectedSkillBody !== undefined && + containsText(body.messages, expectedSkillBody) && + containsText(body.messages, skillDirectory); + const observed = extractInlineResult( + body.messages, + (value) => + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes?.length > 0, + ); + if (observed) { + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(observed); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((n) => n.observations) + .map((o) => o.properties) + .find( + (p) => + p.semantic_role === "export-return-shapes" && + p.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap((s) => s.fields); + assert( + fields.some( + (f) => + f.path === "/clientCompatibility" && + f.state === "literal" && + f.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (f) => + f.path === "/count" && f.state === "literal" && f.value === 7, + ), + ); + const bytes = JSON.stringify(evidence); + artifactDigest = createHash("sha256").update(bytes).digest("hex"); + await writeFile(join(lab, "claude-evidence.json"), bytes); + evidenceSeen = true; + } + if (mode === "chat" || (sessionStatusSeen && evidenceSeen)) { + finalSeen = true; + return [{ type: "text", text: "REA_CLIENT_COMPATIBILITY_OK" }]; + } + if (!skillRequested) { + const skill = tools.find((t) => t.name === "Skill"); + assert(skill, "Claude must expose its native Skill tool"); + skillRequested = true; + return call(skill.name, { skill: "reverse-engineer-anything" }); + } + assert( + skillLoaded, + "Claude must load the complete installed personal REA skill", + ); + return call(analysis.name, { input_path: target, format: "directory" }); + }, + }); + const original = JSON.stringify( + { theme: "light", mcpServers: {} }, + null, + 2, + ); + await writeFile(configPath, original); + for (const args of [ + ["setup", "--client", "claude_code", "--dry-run"], + ["setup", "--client", "claude_code", "--yes"], + ["doctor", "--client", "claude_code", "--skill"], + ]) { + const result = await commands.exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + await writeFile( + join( + lab, + `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`, + ), + result.stdout, + ); + const parsed = JSON.parse(result.stdout); + if (args[0] === "doctor") assert.equal(parsed.healthy, true); + else { + assert.equal( + parsed.status, + args.includes("--dry-run") ? "planned" : "ready", + ); + if (args.includes("--dry-run")) { + assert(parsed.plannedActions.some((a) => a.target === configPath)); + assert( + parsed.plannedActions.some((a) => a.target === skillDirectory), + ); + assert( + parsed.plannedActions.every( + (a) => + a.target === configPath || a.target.startsWith(skillDirectory), + ), + ); + } } } - } - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const configured = await readFile(configPath, "utf8"); - assert.equal(JSON.parse(configured).theme, "light"); - const repeated = await exec( - process.execPath, - [ - join(runtimeRoot, "scripts/rea.mjs"), - "setup", - "--client", - "claude_code", - "--yes", - "--json", - ], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); - assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); - assert.equal(await readFile(configPath, "utf8"), configured); - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const installedSkill = await readFile( - join(skillDirectory, "SKILL.md"), - "utf8", - ); - assert(installedSkill.includes("name: reverse-engineer-anything")); - const lines = installedSkill.split("\n"); - const headerEnd = lines.indexOf("---", 1); - assert( - lines[0] === "---" && headerEnd > 0, - "Installed skill must have its authored frontmatter", - ); - expectedSkillBody = lines - .slice(headerEnd + 1) - .join("\n") - .trim(); - assert(expectedSkillBody.length > 0); - const execution = exec( - command, - [ - "--setting-sources", - "user", - "--print", - "--output-format", - "json", - "--no-session-persistence", - "--model", - "claude-sonnet-4-6", - "--tools", - "default", - "--permission-mode", - "dontAsk", - "--settings", - JSON.stringify({ - permissions: { - allow: [ - "Skill", - "WaitForMcpServers", - "mcp__rea__analyze_javascript_application", - ...(mode === "call" ? ["mcp__rea__binary_session"] : []), + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + assert.equal(JSON.parse(configured).theme, "light"); + const repeated = await commands.exec( + process.execPath, + [ + join(runtimeRoot, "scripts/rea.mjs"), + "setup", + "--client", + "claude_code", + "--yes", + "--json", + ], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const installedSkill = await readFile( + join(skillDirectory, "SKILL.md"), + "utf8", + ); + assert(installedSkill.includes("name: reverse-engineer-anything")); + const lines = installedSkill.split("\n"); + const headerEnd = lines.indexOf("---", 1); + assert( + lines[0] === "---" && headerEnd > 0, + "Installed skill must have its authored frontmatter", + ); + expectedSkillBody = lines + .slice(headerEnd + 1) + .join("\n") + .trim(); + assert(expectedSkillBody.length > 0); + const response = await saveVerifierCommandOutput( + () => + commands.exec( + command, + [ + "--setting-sources", + "user", + "--print", + "--output-format", + "json", + "--no-session-persistence", + "--model", + "claude-sonnet-4-6", + "--tools", + "default", + "--permission-mode", + "dontAsk", + "--settings", + JSON.stringify({ + permissions: { + allow: [ + "Skill", + "WaitForMcpServers", + "mcp__rea__analyze_javascript_application", + ...(mode === "call" ? ["mcp__rea__binary_session"] : []), + ], + }, + }), + mode === "chat" + ? "Say hello briefly." + : `Use the REA skill and tools to analyze the JavaScript directory ${target}. Report the result.`, ], - }, - }), - mode === "chat" - ? "Say hello briefly." - : `Use the REA skill and tools to analyze the JavaScript directory ${target}. Report the result.`, - ], - { - env: { - ...environment, - ANTHROPIC_API_KEY: "local-fixture-key", - ANTHROPIC_AUTH_TOKEN: "", - ANTHROPIC_BASE_URL: fixture.baseUrl, + { + env: { + ...environment, + ANTHROPIC_API_KEY: "local-fixture-key", + ANTHROPIC_AUTH_TOKEN: "", + ANTHROPIC_BASE_URL: fixture.baseUrl, + }, + cwd: workspace, + timeout: 120_000, + stdin: "closed-pipe", + maxBuffer: 10 * 1024 * 1024, + }, + ), + { + stdout: join(lab, `${mode}.stdout`), + stderr: join(lab, `${mode}.stderr`), }, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, + ); + assert.equal(fixture.failure, undefined); + const parsed = JSON.parse(response.stdout); + assert.equal(parsed.is_error, false); + assert.equal(parsed.subtype, "success"); + assert.equal(parsed.result, "REA_CLIENT_COMPATIBILITY_OK"); + assert.deepEqual(parsed.permission_denials, []); + assert(finalSeen); + if (mode === "call") + assert(evidenceSeen && skillLoaded && sessionStatusSeen); + }, [ + () => commands.close(), + async () => { + await fixture?.close(); }, - ); - execution.child.stdin.end(); - const response = await execution; - await writeFile(join(lab, `${mode}.stdout`), response.stdout); - await writeFile(join(lab, `${mode}.stderr`), response.stderr); - assert.equal(fixture.failure, undefined); - const parsed = JSON.parse(response.stdout); - assert.equal(parsed.is_error, false); - assert.equal(parsed.subtype, "success"); - assert.equal(parsed.result, "REA_CLIENT_COMPATIBILITY_OK"); - assert.deepEqual(parsed.permission_denials, []); - assert(finalSeen); - if (mode === "call") assert(evidenceSeen && skillLoaded && sessionStatusSeen); -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, `${mode}.stdout`), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, `${mode}.stderr`), cause.stderr); -} finally { - await fixture.close(); - const receipt = { - client: "Claude Code", - clientVersion, - mode, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, - }, - modelFixture: - "loopback Anthropic Messages/SSE; synthetic usage and native resource hints; no live API claim", - discovery: - "native CLAUDE_CONFIG_DIR user configuration and skills; separate Git root, user settings only, default built-in tools", - bareMode: - "not used; native bare print mode did not load the personal skill", - catalogSize, - schemasChecked, - pendingServerWaits, - evidenceSeen, - sessionStatusSeen, - skillLoaded, - artifactDigest, - finalSeen, - requests: fixture.requests, - probes: fixture.probes, - fixtureFailure: fixture.failure, - verifier: await completeVerifierRun(verifier), - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; + ]); + passed = true; +}, [ + async () => { + const receipt = { + client: "Claude Code", + clientVersion, + mode, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback Anthropic Messages/SSE; synthetic usage and native resource hints; no live API claim", + discovery: + "native CLAUDE_CONFIG_DIR user configuration and skills; separate Git root, user settings only, default built-in tools", + bareMode: + "not used; native bare print mode did not load the personal skill", + catalogSize, + schemasChecked, + pendingServerWaits, + evidenceSeen, + sessionStatusSeen, + skillLoaded, + artifactDigest, + finalSeen, + requests: fixture?.requests, + probes: fixture?.probes, + fixtureFailure: fixture?.failure, + verifier: await completeVerifierRun(verifier), + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); + }, + ], + ); + }, +]); diff --git a/scripts/verify-commandcode-client.mjs b/scripts/verify-commandcode-client.mjs index 50ba09eac3..8819f4fc45 100644 --- a/scripts/verify-commandcode-client.mjs +++ b/scripts/verify-commandcode-client.mjs @@ -9,7 +9,9 @@ import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; -import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -47,43 +49,13 @@ const skillDirectory = join( "skills", "reverse-engineer-anything", ); -await Promise.all([ - mkdir(target, { recursive: true }), - mkdir(profile, { recursive: true }), - mkdir(join(lab, "tmp")), -]); -await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); -await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', -); -const environment = { - PATH: process.env.PATH, - NODE_OPTIONS: process.env.NODE_OPTIONS, - USERPROFILE: account, - TMPDIR: join(lab, "tmp"), - CMD_LOCAL_ONLY: "1", - // Native print mode checks for an account key even with a keyless BYOK model. - // This synthetic value is never a real credential; local-only refuses hosted API calls. - COMMAND_CODE_API_KEY: "local-fixture-only", - COMMANDCODE_SKIP_UPDATES: "1", - COMMANDCODE_DISABLE_CRON: "1", - OTEL_SDK_DISABLED: "true", - REA_PROCESS_RUN_ID: verifier.run_id, -}; const expectedNames = TOOL_CONTRACTS.map( (tool) => "mcp__rea__" + tool.name, ).sort(); -const schemaValidator = new Ajv2020({ strict: false, validateFormats: false }); -const textValues = (value) => - typeof value === "string" - ? [value] - : value !== null && typeof value === "object" - ? Object.values(value).flatMap(textValues) - : []; -let clientVersion; +let clientVersion = null; let expectedSkill; let inventoryChecked = false; +let catalogSize = null; let skillLoaded = false; let schemasChecked = 0; let catalogArtifact; @@ -94,360 +66,428 @@ let evidenceSeen = false; let evidenceBytes; let finalSeen = false; let stage = 0; -let failure; -const call = (name, args) => ({ - role: "assistant", - content: null, - tool_calls: [ - { - id: `call_commandcode_${++stage}`, - type: "function", - function: { name, arguments: JSON.stringify(args) }, - }, - ], -}); -const validateCatalog = async (text) => { - const entries = [ - ...text.matchAll( - /^### (mcp__rea__[A-Za-z0-9_]+)\n[\s\S]*?\nParameters:\n```json\n([\s\S]*?)\n```/gm, - ), - ]; - assert.deepEqual( - entries.map((entry) => entry[1]).sort(), - expectedNames, - "Native exact-name search must return every REA input schema", - ); - for (const entry of entries) { - const schema = JSON.parse(entry[2]); - assert( - schemaValidator.validateSchema(schema), - `${entry[1]}: invalid native schema: ${JSON.stringify(schemaValidator.errors)}`, - ); - } - schemasChecked = entries.length; - catalogDigest = createHash("sha256").update(text).digest("hex"); - await writeFile(join(lab, "native-catalog.md"), text); -}; -const fixture = await createOpenAiModelFixture({ - directory: lab, - prefix: "commandcode", - onRequest: async ({ body, tools }) => { - assert(tools.some((tool) => tool.name === "search_tools")); - const strings = textValues(body.messages); - if (!inventoryChecked) { - const names = [ - ...new Set( - strings.flatMap( - (text) => text.match(/mcp__rea__[A-Za-z0-9_]+/g) ?? [], - ), - ), - ].sort(); - assert.deepEqual( - names, - expectedNames, - "Default native deferred-tool prompt must advertise the complete REA inventory", - ); - assert.equal( - tools.filter((tool) => tool.name.startsWith("mcp__rea__")).length, - 0, - "Keep native lazy schema delivery enabled", - ); - assert( - tools - .find((tool) => tool.name === "activate_skill") - ?.parameters.properties.name.enum.includes( - "reverse-engineer-anything", - ), - ); - inventoryChecked = true; - } - if (mode === "chat") { - assert.equal( - body.messages.filter((message) => message.role === "tool").length, - 0, - ); - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - } - if (stage === 0) - return call("activate_skill", { name: "reverse-engineer-anything" }); - assert( - strings.some((text) => text.includes(expectedSkill)), - "The model must receive the complete activated skill body", - ); - skillLoaded = true; - const last = body.messages - .filter((message) => message.role === "tool") - .at(-1)?.content; - assert.equal(typeof last, "string"); - if (stage === 1) - return call("search_tools", { - query: "select:" + expectedNames.join(","), - }); - if (stage === 2) { - const spill = last.match( - /\[full output saved to: (.+?) — read it with read_file/, - ); - if (spill) { - catalogArtifact = spill[1]; - const child = relative(environment.TMPDIR, catalogArtifact); - assert( - isAbsolute(catalogArtifact) && - child && - !child.startsWith("..") && - !isAbsolute(child), - "Native spill must remain inside this run's temporary directory", - ); - const text = await readFile(catalogArtifact, "utf8"); - await validateCatalog(text); - catalogProjection = { schemas: schemasChecked, sha256: catalogDigest }; - const query = - "const fs=require('node:fs'),crypto=require('node:crypto');const text=fs.readFileSync(process.argv[1],'utf8');const schemas=[...text.matchAll(/^### (mcp__rea__[A-Za-z0-9_]+)\\n/gm)].length;console.log(JSON.stringify({rea_catalog:{schemas,sha256:crypto.createHash('sha256').update(text).digest('hex')}}));"; - return call("shell_command", { - command: process.execPath, - args: ["-e", query, catalogArtifact], - }); - } - await validateCatalog(last); - return call("mcp__rea__analyze_javascript_application", { - input_path: target, - format: "directory", - }); - } - if (catalogArtifact && !artifactProjectionSeen) { - const line = last - .split("\n") - .find((item) => item.startsWith('{"rea_catalog":')); - assert(line, "Native shell must return the selected saved-catalog facts"); - assert.deepEqual(JSON.parse(line).rea_catalog, catalogProjection); - artifactProjectionSeen = true; - return call("mcp__rea__analyze_javascript_application", { - input_path: target, - format: "directory", - }); - } - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(JSON.parse(last)); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shapes = evidence.normalized_result.graph.nodes - .flatMap((node) => node.observations) - .map((observation) => observation.properties) - .filter( - (properties) => - properties.semantic_role === "export-return-shapes" && - properties.exported_name === "result", - ); - assert( - shapes.length > 0 && - shapes.every( - (shape) => shape.return_shape_coverage?.projection_complete, - ), - ); - const fields = shapes.flatMap((shape) => - shape.static_return_shapes.flatMap((item) => item.fields), - ); - assert( - fields.some( - (field) => - field.path === "/clientCompatibility" && - field.state === "literal" && - field.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (field) => - field.path === "/count" && - field.state === "literal" && - field.value === 7, - ), - ); - evidenceBytes = Buffer.byteLength(last); - await writeFile(join(lab, "native-evidence.json"), last); - evidenceSeen = true; - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - }, -}); -await writeFile( - join(profile, "providers.json"), - JSON.stringify({ - provider: { - "rea-local": { - baseURL: fixture.baseUrl, - apiKey: false, - models: { - "rea-client-fixture": { contextWindow: 1_000_000, maxOutput: 4096 }, - }, - }, - }, - }), -); -const original = - JSON.stringify( - { - mcpServers: { other: { command: "unrelated-server", enabled: false } }, - preserved: "keep unrelated settings", - }, - null, - 2, - ) + "\n"; -await writeFile(configPath, original); -const rea = (args) => - exec( - process.execPath, - [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); -try { - clientVersion = ( - await exec(command, ["--version"], { - env: environment, - cwd: workspace, +let passed = false; +let fixture; +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([ + mkdir(target, { recursive: true }), + mkdir(profile, { recursive: true }), + mkdir(join(lab, "tmp")), + ]); + await commands.exec("git", ["init", "--quiet", workspace], { timeout: 30_000, - }) - ).stdout.trim(); - for (const args of [ - ["setup", "--client", "commandcode", "--dry-run"], - ["setup", "--client", "commandcode", "--yes"], - ["doctor", "--client", "commandcode", "--skill"], - ]) { - const response = await rea(args); + stdin: "pipe", + }); await writeFile( - join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), - response.stdout, + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', ); - const result = JSON.parse(response.stdout); - if (args[0] === "doctor") assert.equal(result.healthy, true); - else - assert.equal( - result.status, - args.includes("--dry-run") ? "planned" : "ready", - ); - if (args.includes("--dry-run")) { - assert( - result.plannedActions.some((action) => action.target === configPath), - ); - assert( - result.plannedActions.every( - (action) => - action.target === configPath || - action.target === skillDirectory || - action.target.startsWith(skillDirectory + "/"), + const environment = { + PATH: process.env.PATH, + NODE_OPTIONS: process.env.NODE_OPTIONS, + USERPROFILE: account, + TMPDIR: join(lab, "tmp"), + CMD_LOCAL_ONLY: "1", + // Native print mode checks for an account key even with a keyless BYOK model. + // This synthetic value is never a real credential; local-only refuses hosted API calls. + COMMAND_CODE_API_KEY: "local-fixture-only", + COMMANDCODE_SKIP_UPDATES: "1", + COMMANDCODE_DISABLE_CRON: "1", + OTEL_SDK_DISABLED: "true", + REA_PROCESS_RUN_ID: verifier.run_id, + }; + const schemaValidator = new Ajv2020({ + strict: false, + validateFormats: false, + }); + const textValues = (value) => + typeof value === "string" + ? [value] + : value !== null && typeof value === "object" + ? Object.values(value).flatMap(textValues) + : []; + const call = (name, args) => ({ + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_commandcode_${++stage}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }); + const validateCatalog = async (text) => { + const entries = [ + ...text.matchAll( + /^### (mcp__rea__[A-Za-z0-9_]+)\n[\s\S]*?\nParameters:\n```json\n([\s\S]*?)\n```/gm, ), - "Refuse setup writes outside the disposable account's configuration and skill", + ]; + assert.deepEqual( + entries.map((entry) => entry[1]).sort(), + expectedNames, + "Native exact-name search must return every REA input schema", ); - } - } - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const configured = await readFile(configPath, "utf8"); - assert.equal(JSON.parse(configured).preserved, "keep unrelated settings"); - assert.equal( - JSON.parse(configured).mcpServers.other.command, - "unrelated-server", - ); - const repeated = JSON.parse( - (await rea(["setup", "--client", "commandcode", "--yes"])).stdout, - ); - assert.deepEqual(repeated.appliedActions, []); - assert.equal(await readFile(configPath, "utf8"), configured); - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - expectedSkill = (await readFile(join(skillDirectory, "SKILL.md"), "utf8")) - .replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/, "") - .trim(); - const execution = exec( - command, - [ - "-p", - mode === "chat" - ? "Say hello briefly." - : `Use the REA skill to analyze ${target}.`, - "-m", - "rea-local/rea-client-fixture", - "--local-only", - "--skip-onboarding", - "--trust", - "--output-format", - "json", - "--max-turns", - "8", - "--yolo", - ], - { - env: environment, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, - }, - ); - execution.child.stdin.end(); - const response = await execution; - await writeFile(join(lab, `${mode}.stdout`), response.stdout); - await writeFile(join(lab, `${mode}.stderr`), response.stderr); - assert.equal(fixture.failure, undefined); - const events = response.stdout - .trim() - .split("\n") - .map((line) => JSON.parse(line)); - assert( - events.some( - (event) => - event.type === "result" && - event.subtype === "success" && - event.finalText === "REA_CLIENT_COMPATIBILITY_OK", - ), - ); - assert(inventoryChecked && finalSeen); - if (mode === "call") - assert( - skillLoaded && - schemasChecked === expectedNames.length && - evidenceSeen && - (!catalogArtifact || artifactProjectionSeen), + for (const entry of entries) { + const schema = JSON.parse(entry[2]); + assert( + schemaValidator.validateSchema(schema), + `${entry[1]}: invalid native schema: ${JSON.stringify(schemaValidator.errors)}`, + ); + } + schemasChecked = entries.length; + catalogDigest = createHash("sha256").update(text).digest("hex"); + await writeFile(join(lab, "native-catalog.md"), text); + }; + fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: "commandcode", + onRequest: async ({ body, tools }) => { + assert(tools.some((tool) => tool.name === "search_tools")); + const strings = textValues(body.messages); + if (!inventoryChecked) { + const names = [ + ...new Set( + strings.flatMap( + (text) => text.match(/mcp__rea__[A-Za-z0-9_]+/g) ?? [], + ), + ), + ].sort(); + catalogSize = names.length; + assert.deepEqual( + names, + expectedNames, + "Default native deferred-tool prompt must advertise the complete REA inventory", + ); + assert.equal( + tools.filter((tool) => tool.name.startsWith("mcp__rea__")).length, + 0, + "Keep native lazy schema delivery enabled", + ); + assert( + tools + .find((tool) => tool.name === "activate_skill") + ?.parameters.properties.name.enum.includes( + "reverse-engineer-anything", + ), + ); + inventoryChecked = true; + } + if (mode === "chat") { + assert.equal( + body.messages.filter((message) => message.role === "tool").length, + 0, + ); + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + if (stage === 0) + return call("activate_skill", { name: "reverse-engineer-anything" }); + assert( + strings.some((text) => text.includes(expectedSkill)), + "The model must receive the complete activated skill body", + ); + skillLoaded = true; + const last = body.messages + .filter((message) => message.role === "tool") + .at(-1)?.content; + assert.equal(typeof last, "string"); + if (stage === 1) + return call("search_tools", { + query: "select:" + expectedNames.join(","), + }); + if (stage === 2) { + const spill = last.match( + /\[full output saved to: (.+?) — read it with read_file/, + ); + if (spill) { + catalogArtifact = spill[1]; + const child = relative(environment.TMPDIR, catalogArtifact); + assert( + isAbsolute(catalogArtifact) && + child && + !child.startsWith("..") && + !isAbsolute(child), + "Native spill must remain inside this run's temporary directory", + ); + const text = await readFile(catalogArtifact, "utf8"); + await validateCatalog(text); + catalogProjection = { + schemas: schemasChecked, + sha256: catalogDigest, + }; + const query = + "const fs=require('node:fs'),crypto=require('node:crypto');const text=fs.readFileSync(process.argv[1],'utf8');const schemas=[...text.matchAll(/^### (mcp__rea__[A-Za-z0-9_]+)\\n/gm)].length;console.log(JSON.stringify({rea_catalog:{schemas,sha256:crypto.createHash('sha256').update(text).digest('hex')}}));"; + return call("shell_command", { + command: process.execPath, + args: ["-e", query, catalogArtifact], + }); + } + await validateCatalog(last); + return call("mcp__rea__analyze_javascript_application", { + input_path: target, + format: "directory", + }); + } + if (catalogArtifact && !artifactProjectionSeen) { + const line = last + .split("\n") + .find((item) => item.startsWith('{"rea_catalog":')); + assert( + line, + "Native shell must return the selected saved-catalog facts", + ); + assert.deepEqual(JSON.parse(line).rea_catalog, catalogProjection); + artifactProjectionSeen = true; + return call("mcp__rea__analyze_javascript_application", { + input_path: target, + format: "directory", + }); + } + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(JSON.parse(last)); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shapes = evidence.normalized_result.graph.nodes + .flatMap((node) => node.observations) + .map((observation) => observation.properties) + .filter( + (properties) => + properties.semantic_role === "export-return-shapes" && + properties.exported_name === "result", + ); + assert( + shapes.length > 0 && + shapes.every( + (shape) => shape.return_shape_coverage?.projection_complete, + ), + ); + const fields = shapes.flatMap((shape) => + shape.static_return_shapes.flatMap((item) => item.fields), + ); + assert( + fields.some( + (field) => + field.path === "/clientCompatibility" && + field.state === "literal" && + field.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (field) => + field.path === "/count" && + field.state === "literal" && + field.value === 7, + ), + ); + evidenceBytes = Buffer.byteLength(last); + await writeFile(join(lab, "native-evidence.json"), last); + evidenceSeen = true; + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + }, + }); + await writeFile( + join(profile, "providers.json"), + JSON.stringify({ + provider: { + "rea-local": { + baseURL: fixture.baseUrl, + apiKey: false, + models: { + "rea-client-fixture": { + contextWindow: 1_000_000, + maxOutput: 4096, + }, + }, + }, + }, + }), ); -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, `${mode}.stdout`), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, `${mode}.stderr`), cause.stderr); -} finally { - await fixture.close(); - const receipt = { - client: "Command Code", - clientVersion, - mode, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, + const original = + JSON.stringify( + { + mcpServers: { + other: { command: "unrelated-server", enabled: false }, + }, + preserved: "keep unrelated settings", + }, + null, + 2, + ) + "\n"; + await writeFile(configPath, original); + const rea = (args) => + commands.exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + clientVersion = ( + await commands.exec(command, ["--version"], { + env: environment, + cwd: workspace, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + for (const args of [ + ["setup", "--client", "commandcode", "--dry-run"], + ["setup", "--client", "commandcode", "--yes"], + ["doctor", "--client", "commandcode", "--skill"], + ]) { + const response = await rea(args); + await writeFile( + join( + lab, + `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`, + ), + response.stdout, + ); + const result = JSON.parse(response.stdout); + if (args[0] === "doctor") assert.equal(result.healthy, true); + else + assert.equal( + result.status, + args.includes("--dry-run") ? "planned" : "ready", + ); + if (args.includes("--dry-run")) { + assert( + result.plannedActions.some((action) => action.target === configPath), + ); + assert( + result.plannedActions.every( + (action) => + action.target === configPath || + action.target === skillDirectory || + action.target.startsWith(skillDirectory + "/"), + ), + "Refuse setup writes outside the disposable account's configuration and skill", + ); + } + } + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + assert.equal(JSON.parse(configured).preserved, "keep unrelated settings"); + assert.equal( + JSON.parse(configured).mcpServers.other.command, + "unrelated-server", + ); + const repeated = JSON.parse( + (await rea(["setup", "--client", "commandcode", "--yes"])).stdout, + ); + assert.deepEqual(repeated.appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + expectedSkill = (await readFile(join(skillDirectory, "SKILL.md"), "utf8")) + .replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/, "") + .trim(); + const response = await saveVerifierCommandOutput( + () => + commands.exec( + command, + [ + "-p", + mode === "chat" + ? "Say hello briefly." + : `Use the REA skill to analyze ${target}.`, + "-m", + "rea-local/rea-client-fixture", + "--local-only", + "--skip-onboarding", + "--trust", + "--output-format", + "json", + "--max-turns", + "8", + "--yolo", + ], + { + env: environment, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + stdin: "closed-pipe", + }, + ), + { + stdout: join(lab, `${mode}.stdout`), + stderr: join(lab, `${mode}.stderr`), + }, + ); + assert.equal(fixture.failure, undefined); + const events = response.stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + assert( + events.some( + (event) => + event.type === "result" && + event.subtype === "success" && + event.finalText === "REA_CLIENT_COMPATIBILITY_OK", + ), + ); + assert(inventoryChecked && finalSeen); + if (mode === "call") + assert( + skillLoaded && + schemasChecked === expectedNames.length && + evidenceSeen && + (!catalogArtifact || artifactProjectionSeen), + ); + }, [ + () => commands.close(), + async () => { + await fixture?.close(); }, - modelFixture: - "loopback OpenAI completions/SSE, keyless BYOK, synthetic account-key gate and usage; no hosted authentication or live-model claim", - discovery: - "actual disposable OS account home; native user MCP configuration and default shared skill discovery; Windows unverified", - interface: - "default deferred schemas, complete exact-name search, native saved-catalog query when needed, actual JavaScript MCP call", - inventoryChecked, - catalogSize: inventoryChecked ? expectedNames.length : 0, - schemasChecked, - skillLoaded, - catalogArtifact, - catalogDigest, - artifactProjectionSeen, - evidenceSeen, - evidenceBytes, - finalSeen, - requests: fixture.requests, - probes: fixture.probes, - fixtureFailure: fixture.failure, - verifier: await completeVerifierRun(verifier), - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; + ]); + passed = true; +}, [ + async () => { + const receipt = { + client: "Command Code", + clientVersion, + mode, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback OpenAI completions/SSE, keyless BYOK, synthetic account-key gate and usage; no hosted authentication or live-model claim", + discovery: + "actual disposable OS account home; native user MCP configuration and default shared skill discovery; Windows unverified", + interface: + "default deferred schemas, complete exact-name search, native saved-catalog query when needed, actual JavaScript MCP call", + inventoryChecked, + catalogSize, + schemasChecked, + skillLoaded, + catalogArtifact, + catalogDigest, + artifactProjectionSeen, + evidenceSeen, + evidenceBytes, + finalSeen, + requests: fixture?.requests, + probes: fixture?.probes, + fixtureFailure: fixture?.failure, + verifier: await completeVerifierRun(verifier), + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); + }, + ], + ); + }, +]); diff --git a/scripts/verify-copilot-client.mjs b/scripts/verify-copilot-client.mjs index 02b069d694..4225f87561 100644 --- a/scripts/verify-copilot-client.mjs +++ b/scripts/verify-copilot-client.mjs @@ -12,7 +12,9 @@ import { import { advertisedInputJsonSchema } from "../dist/contracts/toolSchemaMetadata.js"; import { presentAdvertisedInputSchema } from "../dist/contracts/compactInputSchema.js"; import { compactPresentationForMcpInputSchemaProfile } from "../dist/config/mcpInputSchemaProfile.js"; -import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -49,435 +51,473 @@ const skillDirectory = join( "skills", "reverse-engineer-anything", ); -await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]); -await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); -// Do not inherit the caller's home or credentials into the isolated native profile. -const environment = { - PATH: process.env.PATH, - NODE_OPTIONS: process.env.NODE_OPTIONS, - HOME: account, - USERPROFILE: account, - COPILOT_HOME: profile, - COPILOT_OFFLINE: "true", - COPILOT_PROVIDER_TYPE: "openai", - COPILOT_PROVIDER_WIRE_API: "completions", - COPILOT_PROVIDER_API_KEY: "local-fixture-key", - COPILOT_PROVIDER_API_KEY_COMMAND: "", - COPILOT_PROVIDER_BEARER_TOKEN: "", - COPILOT_PROVIDER_HEADERS: "", - COPILOT_PROVIDER_MODEL_ID: model, - COPILOT_PROVIDER_WIRE_MODEL: wireModel, - COPILOT_PROVIDER_MAX_PROMPT_TOKENS: "1000000", - COPILOT_PROVIDER_MAX_OUTPUT_TOKENS: "4096", - REA_PROCESS_RUN_ID: verifier.run_id, -}; -let clientVersion; -try { - clientVersion = ( - await exec(command, ["--version"], { env: environment, timeout: 30_000 }) - ).stdout.trim(); -} catch (cause) { - throw new Error( - `Copilot CLI preflight failed for ${command}; install Copilot CLI or set REA_VERIFY_COPILOT_COMMAND. ${String(cause)}`, - ); -} -const target = join(lab, "fixture α测试"); -await mkdir(target); -await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', -); +let clientVersion = null; +let fixture; +let passed = false; let skillRequested = false; let skillLoaded = false; let expectedSkillBody; let evidenceSeen = false; let finalSeen = false; -let catalogSize = 0; +let catalogSize = null; let schemasChecked = 0; let spillReadRequested = false; let artifactDigest; -const schemaValidator = new Ajv2020({ strict: false, validateFormats: false }); -const containsText = (value, text) => { - if (typeof value === "string") return value.includes(text); - if (value && typeof value === "object") - return Object.values(value).some((nested) => containsText(nested, text)); - return false; -}; -const extractEvidence = (value) => { - if (typeof value === "string") { - try { - return extractEvidence(JSON.parse(value)); - } catch { - return undefined; - } - } - if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0 - ) - return value; - for (const nested of Object.values(value)) { - const evidence = extractEvidence(nested); - if (evidence) return evidence; - } - } - return undefined; -}; const wireModels = []; -const fixture = await createOpenAiModelFixture({ - directory: lab, - prefix: "copilot", - onRequest: async ({ body, tools, requestIndex }) => { - assert.equal( - body.model, - wireModel, - "Copilot must send the selected wire model", +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]); + await commands.exec("git", ["init", "--quiet", workspace], { + timeout: 30_000, + stdin: "pipe", + }); + // Do not inherit the caller's home or credentials into the isolated native profile. + const environment = { + PATH: process.env.PATH, + NODE_OPTIONS: process.env.NODE_OPTIONS, + HOME: account, + USERPROFILE: account, + COPILOT_HOME: profile, + COPILOT_OFFLINE: "true", + COPILOT_PROVIDER_TYPE: "openai", + COPILOT_PROVIDER_WIRE_API: "completions", + COPILOT_PROVIDER_API_KEY: "local-fixture-key", + COPILOT_PROVIDER_API_KEY_COMMAND: "", + COPILOT_PROVIDER_BEARER_TOKEN: "", + COPILOT_PROVIDER_HEADERS: "", + COPILOT_PROVIDER_MODEL_ID: model, + COPILOT_PROVIDER_WIRE_MODEL: wireModel, + COPILOT_PROVIDER_MAX_PROMPT_TOKENS: "1000000", + COPILOT_PROVIDER_MAX_OUTPUT_TOKENS: "4096", + REA_PROCESS_RUN_ID: verifier.run_id, + }; + try { + clientVersion = ( + await commands.exec(command, ["--version"], { + env: environment, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + } catch (cause) { + throw new Error( + `Copilot CLI preflight failed for ${command}; install Copilot CLI or set REA_VERIFY_COPILOT_COMMAND. ${String(cause)}`, + { cause }, + ); + } + const target = join(lab, "fixture α测试"); + await mkdir(target); + await writeFile( + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', ); - wireModels.push(body.model); - // Auxiliary requests without tools cannot execute the analyst's workflow. - if (tools.length === 0) - return { role: "assistant", content: "REA fixture title" }; - const analysis = tools.find((t) => - t.name.endsWith("analyze_javascript_application"), - ); - assert(analysis, "Copilot CLI must expose the REA analysis tool"); - const prefix = analysis.name.slice( - 0, - -"analyze_javascript_application".length, - ); - const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort(); - const reaTools = tools.filter((t) => expected.includes(t.name)); - assert.deepEqual( - reaTools.map((t) => t.name).sort(), - expected, - "Copilot CLI must forward the complete REA catalog", - ); - catalogSize = expected.length; - if (schemasChecked === 0) { - for (const tool of reaTools) { - assert( - tool.parameters, - `${tool.name}: missing advertised input schema`, + const schemaValidator = new Ajv2020({ + strict: false, + validateFormats: false, + }); + const containsText = (value, text) => { + if (typeof value === "string") return value.includes(text); + if (value && typeof value === "object") + return Object.values(value).some((nested) => + containsText(nested, text), ); - assert( - schemaValidator.validateSchema(tool.parameters), - `${tool.name}: invalid input schema: ${JSON.stringify(schemaValidator.errors)}`, + return false; + }; + const extractEvidence = (value) => { + if (typeof value === "string") { + try { + return extractEvidence(JSON.parse(value)); + } catch { + return undefined; + } + } + if (value && typeof value === "object") { + if ( + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes?.length > 0 + ) + return value; + for (const nested of Object.values(value)) { + const evidence = extractEvidence(nested); + if (evidence) return evidence; + } + } + return undefined; + }; + fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: "copilot", + onRequest: async ({ body, tools, requestIndex }) => { + assert.equal( + body.model, + wireModel, + "Copilot must send the selected wire model", ); + wireModels.push(body.model); + // Auxiliary requests without tools cannot execute the analyst's workflow. + if (tools.length === 0) + return { role: "assistant", content: "REA fixture title" }; + const analysis = tools.find((t) => + t.name.endsWith("analyze_javascript_application"), + ); + assert(analysis, "Copilot CLI must expose the REA analysis tool"); + const prefix = analysis.name.slice( + 0, + -"analyze_javascript_application".length, + ); + const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort(); + const reaTools = tools.filter((t) => expected.includes(t.name)); + catalogSize = reaTools.length; assert.deepEqual( - tool.parameters, - presentAdvertisedInputSchema( - advertisedInputJsonSchema( - toolContract(tool.name.slice(prefix.length)), - ), - compactPresentationForMcpInputSchemaProfile(inputSchemaProfile), - ), - `${tool.name}: Copilot must forward the selected ${inputSchemaProfile} input schema`, + reaTools.map((t) => t.name).sort(), + expected, + "Copilot CLI must forward the complete REA catalog", ); - schemasChecked++; + if (schemasChecked === 0) { + for (const tool of reaTools) { + assert( + tool.parameters, + `${tool.name}: missing advertised input schema`, + ); + assert( + schemaValidator.validateSchema(tool.parameters), + `${tool.name}: invalid input schema: ${JSON.stringify(schemaValidator.errors)}`, + ); + assert.deepEqual( + tool.parameters, + presentAdvertisedInputSchema( + advertisedInputJsonSchema( + toolContract(tool.name.slice(prefix.length)), + ), + compactPresentationForMcpInputSchemaProfile(inputSchemaProfile), + ), + `${tool.name}: Copilot must forward the selected ${inputSchemaProfile} input schema`, + ); + schemasChecked++; + } + } + const replies = body.messages.filter((m) => m.role === "tool"); + // Native skill activation injects its context as a user message after the acknowledgement. + skillLoaded ||= + skillRequested && + expectedSkillBody !== undefined && + containsText(body.messages, expectedSkillBody) && + containsText(body.messages, skillDirectory); + const observed = extractEvidence(replies); + if (observed) { + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(observed); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((n) => n.observations) + .map((o) => o.properties) + .find( + (p) => + p.semantic_role === "export-return-shapes" && + p.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap((s) => s.fields); + assert( + fields.some( + (f) => + f.path === "/clientCompatibility" && + f.state === "literal" && + f.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (f) => + f.path === "/count" && f.state === "literal" && f.value === 7, + ), + ); + const bytes = JSON.stringify(evidence); + artifactDigest = createHash("sha256").update(bytes).digest("hex"); + await writeFile(join(lab, "copilot-evidence.json"), bytes); + evidenceSeen = true; + } + if (mode === "chat" || evidenceSeen) { + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + assert( + requestIndex <= 8, + "Local model workflow exceeded its turn budget", + ); + const call = (name, args) => ({ + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_rea_fixture_${requestIndex}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }); + if (!skillRequested) { + const skill = tools.find((t) => t.name === "skill"); + assert(skill, "Copilot CLI must expose its native skill tool"); + skillRequested = true; + return call(skill.name, { skill: "reverse-engineer-anything" }); + } + assert( + skillLoaded, + "Copilot CLI must load the installed personal REA skill", + ); + const spilled = replies.find( + (message) => + typeof message.content === "string" && + message.content.startsWith("Output too large to read at once "), + ); + if (spilled) { + assert( + !spillReadRequested, + "Copilot did not return complete Evidence after its native spill read", + ); + const path = /\. Saved to: ([^\n]+)\n/u.exec(spilled.content)?.[1]; + assert( + path && + isAbsolute(path) && + /^\d+-copilot-tool-output-[a-f\d]+\.txt$/u.test(basename(path)), + "Unexpected native spill path", + ); + const view = tools.find((tool) => tool.name === "view"); + assert(view, "Copilot must expose its native file reader"); + spillReadRequested = true; + return call(view.name, { path, forceReadLargeFiles: true }); + } + return call(analysis.name, { input_path: target, format: "directory" }); + }, + }); + const original = JSON.stringify( + { + mcpServers: { + other: { + type: "local", + command: "unrelated-server", + args: [], + enabled: false, + }, + }, + }, + null, + 2, + ); + await writeFile(configPath, original); + for (const args of [ + ["setup", "--client", "copilot_cli", "--dry-run"], + ["setup", "--client", "copilot_cli", "--yes"], + ["doctor", "--client", "copilot_cli", "--skill"], + ]) { + const result = await commands.exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + await writeFile( + join( + lab, + `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`, + ), + result.stdout, + ); + const parsed = JSON.parse(result.stdout); + if (args[0] === "doctor") assert.equal(parsed.healthy, true); + else { + assert.equal( + parsed.status, + args.includes("--dry-run") ? "planned" : "ready", + ); + if (args.includes("--dry-run")) { + assert(parsed.plannedActions.some((a) => a.target === configPath)); + assert( + parsed.plannedActions.some((a) => a.target === skillDirectory), + ); + assert( + parsed.plannedActions.every( + (a) => + a.target === configPath || + a.target === skillDirectory || + a.target.startsWith(skillDirectory + "/"), + ), + ); + } } } - const replies = body.messages.filter((m) => m.role === "tool"); - // Native skill activation injects its context as a user message after the acknowledgement. - skillLoaded ||= - skillRequested && - expectedSkillBody !== undefined && - containsText(body.messages, expectedSkillBody) && - containsText(body.messages, skillDirectory); - const observed = extractEvidence(replies); - if (observed) { - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(observed); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((n) => n.observations) - .map((o) => o.properties) - .find( - (p) => - p.semantic_role === "export-return-shapes" && - p.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((s) => s.fields); - assert( - fields.some( - (f) => - f.path === "/clientCompatibility" && - f.state === "literal" && - f.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (f) => f.path === "/count" && f.state === "literal" && f.value === 7, - ), - ); - const bytes = JSON.stringify(evidence); - artifactDigest = createHash("sha256").update(bytes).digest("hex"); - await writeFile(join(lab, "copilot-evidence.json"), bytes); - evidenceSeen = true; + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + assert(configured.includes("unrelated-server")); + + const repeated = await commands.exec( + process.execPath, + [ + join(runtimeRoot, "scripts/rea.mjs"), + "setup", + "--client", + "copilot_cli", + "--yes", + "--json", + ], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + // Bind both choices explicitly: the server's default can change independently + // of the profile this verifier promises to exercise and records in its receipt. + const customized = JSON.parse(configured); + assert(customized.mcpServers.rea); + customized.mcpServers.rea.env = { + ...customized.mcpServers.rea.env, + REA_MCP_INPUT_SCHEMA_PROFILE: inputSchemaProfile, + }; + await writeFile(configPath, JSON.stringify(customized, null, 2)); + const installedSkill = await readFile( + join(skillDirectory, "SKILL.md"), + "utf8", + ); + assert(installedSkill.includes("name: reverse-engineer-anything")); + const skillLines = installedSkill.split("\n"); + const headerEnd = skillLines.indexOf("---", 1); + assert( + skillLines[0] === "---" && headerEnd > 0, + "Installed skill must have its authored frontmatter", + ); + expectedSkillBody = skillLines + .slice(headerEnd + 1) + .join("\n") + .trim(); + assert(expectedSkillBody.length > 0); + for (const [name, args] of [ + ["mcp-list", ["mcp", "list"]], + ["skill-add", ["skill", "add", skillDirectory]], + ["skill-list", ["skill", "list"]], + ]) { + const result = await commands.exec(command, args, { + env: environment, + cwd: workspace, + timeout: 30_000, + stdin: "pipe", + }); + await writeFile(join(lab, `${name}.stdout`), result.stdout); + await writeFile(join(lab, `${name}.stderr`), result.stderr); + if (name === "mcp-list") assert(result.stdout.includes("rea")); + if (name === "skill-list") + assert(result.stdout.includes("reverse-engineer-anything")); } - if (mode === "chat" || evidenceSeen) { - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - } - assert(requestIndex <= 8, "Local model workflow exceeded its turn budget"); - const call = (name, args) => ({ - role: "assistant", - content: null, - tool_calls: [ - { - id: `call_rea_fixture_${requestIndex}`, - type: "function", - function: { name, arguments: JSON.stringify(args) }, + const response = await saveVerifierCommandOutput( + () => + commands.exec( + command, + [ + "--disable-builtin-mcps", + "--disable-mcp-server", + "other", + "--no-ask-user", + "--output-format", + "json", + "--model", + model, + "--allow-tool=skill", + "--allow-tool=read", + "--allow-tool=rea(analyze_javascript_application)", + "--add-dir", + target, + "--prompt", + mode === "chat" + ? "Say hello briefly." + : `Use the REA skill and tools to analyze the JavaScript directory ${target}. Report the result.`, + ], + { + env: { ...environment, COPILOT_PROVIDER_BASE_URL: fixture.baseUrl }, + cwd: workspace, + timeout: 120_000, + stdin: "closed-pipe", + maxBuffer: 10 * 1024 * 1024, + }, + ), + { + stdout: join(lab, `${mode}.stdout`), + stderr: join(lab, `${mode}.stderr`), + }, + ); + assert.equal(fixture.failure, undefined); + const events = response.stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + assert( + !events.some((event) => event.type === "session.error"), + "Copilot emitted a session error", + ); + assert(finalSeen); + assert( + events.some( + (event) => + event.type === "assistant.message" && + event.data.content === "REA_CLIENT_COMPATIBILITY_OK", + ), + "Copilot must finish with the expected native assistant message", + ); + if (mode === "call") assert(evidenceSeen && skillLoaded); + }, [ + () => commands.close(), + async () => { + await fixture?.close(); + }, + ]); + passed = true; +}, [ + async () => { + const lineage = await completeVerifierRun(verifier); + const receipt = { + client: "Copilot CLI", + clientVersion, + mode, + model, + wireModel, + inputSchemaProfile, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback OpenAI completions/SSE, synthetic usage, requested one-million-token BYOK prompt capacity; effective capacity unknown; no live provider claim", + skillDiscovery: + "native COPILOT_HOME MCP configuration and caller-added isolated skill directory; default OS-home discovery unverified", + capacityBoundary: + inputSchemaProfile === "compact" + ? "Explicit compact profile: complete inventory and canonical server validation; advertised schemas may be reduced. This does not fix the full-profile gpt-4.1 context boundary." + : "Full advertised input schemas, checked at the client request boundary. Client model ID and wire model are recorded separately; effective token capacity and live provider compatibility remain unknown.", + catalogSize, + schemasChecked, + spillReadRequested, + evidenceSeen, + skillLoaded, + artifactDigest, + finalSeen, + wireModels, + requests: fixture?.requests, + probes: fixture?.probes, + fixtureFailure: fixture?.failure, + verifier: lineage, + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); }, ], - }); - if (!skillRequested) { - const skill = tools.find((t) => t.name === "skill"); - assert(skill, "Copilot CLI must expose its native skill tool"); - skillRequested = true; - return call(skill.name, { skill: "reverse-engineer-anything" }); - } - assert( - skillLoaded, - "Copilot CLI must load the installed personal REA skill", ); - const spilled = replies.find( - (message) => - typeof message.content === "string" && - message.content.startsWith("Output too large to read at once "), - ); - if (spilled) { - assert( - !spillReadRequested, - "Copilot did not return complete Evidence after its native spill read", - ); - const path = /\. Saved to: ([^\n]+)\n/u.exec(spilled.content)?.[1]; - assert( - path && - isAbsolute(path) && - /^\d+-copilot-tool-output-[a-f\d]+\.txt$/u.test(basename(path)), - "Unexpected native spill path", - ); - const view = tools.find((tool) => tool.name === "view"); - assert(view, "Copilot must expose its native file reader"); - spillReadRequested = true; - return call(view.name, { path, forceReadLargeFiles: true }); - } - return call(analysis.name, { input_path: target, format: "directory" }); }, -}); -const original = JSON.stringify( - { - mcpServers: { - other: { - type: "local", - command: "unrelated-server", - args: [], - enabled: false, - }, - }, - }, - null, - 2, -); -await writeFile(configPath, original); -let failure; -try { - for (const args of [ - ["setup", "--client", "copilot_cli", "--dry-run"], - ["setup", "--client", "copilot_cli", "--yes"], - ["doctor", "--client", "copilot_cli", "--skill"], - ]) { - const result = await exec( - process.execPath, - [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); - await writeFile( - join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), - result.stdout, - ); - const parsed = JSON.parse(result.stdout); - if (args[0] === "doctor") assert.equal(parsed.healthy, true); - else { - assert.equal( - parsed.status, - args.includes("--dry-run") ? "planned" : "ready", - ); - if (args.includes("--dry-run")) { - assert(parsed.plannedActions.some((a) => a.target === configPath)); - assert(parsed.plannedActions.some((a) => a.target === skillDirectory)); - assert( - parsed.plannedActions.every( - (a) => - a.target === configPath || - a.target === skillDirectory || - a.target.startsWith(skillDirectory + "/"), - ), - ); - } - } - } - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const configured = await readFile(configPath, "utf8"); - assert(configured.includes("unrelated-server")); - - const repeated = await exec( - process.execPath, - [ - join(runtimeRoot, "scripts/rea.mjs"), - "setup", - "--client", - "copilot_cli", - "--yes", - "--json", - ], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); - assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); - assert.equal(await readFile(configPath, "utf8"), configured); - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - // Bind both choices explicitly: the server's default can change independently - // of the profile this verifier promises to exercise and records in its receipt. - const customized = JSON.parse(configured); - assert(customized.mcpServers.rea); - customized.mcpServers.rea.env = { - ...customized.mcpServers.rea.env, - REA_MCP_INPUT_SCHEMA_PROFILE: inputSchemaProfile, - }; - await writeFile(configPath, JSON.stringify(customized, null, 2)); - const installedSkill = await readFile( - join(skillDirectory, "SKILL.md"), - "utf8", - ); - assert(installedSkill.includes("name: reverse-engineer-anything")); - const skillLines = installedSkill.split("\n"); - const headerEnd = skillLines.indexOf("---", 1); - assert( - skillLines[0] === "---" && headerEnd > 0, - "Installed skill must have its authored frontmatter", - ); - expectedSkillBody = skillLines - .slice(headerEnd + 1) - .join("\n") - .trim(); - assert(expectedSkillBody.length > 0); - for (const [name, args] of [ - ["mcp-list", ["mcp", "list"]], - ["skill-add", ["skill", "add", skillDirectory]], - ["skill-list", ["skill", "list"]], - ]) { - const result = await exec(command, args, { - env: environment, - cwd: workspace, - timeout: 30_000, - }); - await writeFile(join(lab, `${name}.stdout`), result.stdout); - await writeFile(join(lab, `${name}.stderr`), result.stderr); - if (name === "mcp-list") assert(result.stdout.includes("rea")); - if (name === "skill-list") - assert(result.stdout.includes("reverse-engineer-anything")); - } - const execution = exec( - command, - [ - "--disable-builtin-mcps", - "--disable-mcp-server", - "other", - "--no-ask-user", - "--output-format", - "json", - "--model", - model, - "--allow-tool=skill", - "--allow-tool=read", - "--allow-tool=rea(analyze_javascript_application)", - "--add-dir", - target, - "--prompt", - mode === "chat" - ? "Say hello briefly." - : `Use the REA skill and tools to analyze the JavaScript directory ${target}. Report the result.`, - ], - { - env: { ...environment, COPILOT_PROVIDER_BASE_URL: fixture.baseUrl }, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, - }, - ); - execution.child.stdin.end(); - const response = await execution; - await writeFile(join(lab, `${mode}.stdout`), response.stdout); - await writeFile(join(lab, `${mode}.stderr`), response.stderr); - assert.equal(fixture.failure, undefined); - const events = response.stdout - .trim() - .split("\n") - .map((line) => JSON.parse(line)); - assert( - !events.some((event) => event.type === "session.error"), - "Copilot emitted a session error", - ); - assert(finalSeen); - assert( - events.some( - (event) => - event.type === "assistant.message" && - event.data.content === "REA_CLIENT_COMPATIBILITY_OK", - ), - "Copilot must finish with the expected native assistant message", - ); - if (mode === "call") assert(evidenceSeen && skillLoaded); -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, `${mode}.stdout`), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, `${mode}.stderr`), cause.stderr); -} finally { - await fixture.close(); - const lineage = await completeVerifierRun(verifier); - const receipt = { - client: "Copilot CLI", - clientVersion, - mode, - model, - wireModel, - inputSchemaProfile, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, - }, - modelFixture: - "loopback OpenAI completions/SSE, synthetic usage, requested one-million-token BYOK prompt capacity; effective capacity unknown; no live provider claim", - skillDiscovery: - "native COPILOT_HOME MCP configuration and caller-added isolated skill directory; default OS-home discovery unverified", - capacityBoundary: - inputSchemaProfile === "compact" - ? "Explicit compact profile: complete inventory and canonical server validation; advertised schemas may be reduced. This does not fix the full-profile gpt-4.1 context boundary." - : "Full advertised input schemas, checked at the client request boundary. Client model ID and wire model are recorded separately; effective token capacity and live provider compatibility remain unknown.", - catalogSize, - schemasChecked, - spillReadRequested, - evidenceSeen, - skillLoaded, - artifactDigest, - finalSeen, - wireModels, - requests: fixture.requests, - probes: fixture.probes, - fixtureFailure: fixture.failure, - verifier: lineage, - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; +]); diff --git a/scripts/verify-deepseek-client.mjs b/scripts/verify-deepseek-client.mjs index b8d8dd7409..9f5a3bbc1e 100644 --- a/scripts/verify-deepseek-client.mjs +++ b/scripts/verify-deepseek-client.mjs @@ -10,7 +10,10 @@ import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; -import { exec, pathExists } from "./lib/verify-package-core.mjs"; +import { pathExists } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -36,66 +39,9 @@ const skillPath = join( const cli = join(runtimeRoot, "scripts", "rea.mjs"); const configPath = join(profile, "cordis.patch.yml"); const verifier = createVerifierRun(); -await Promise.all([ - mkdir(account), - mkdir(profile), - mkdir(target, { recursive: true }), -]); -const environment = { - PATH: process.env.PATH ?? "", - HOME: account, - USERPROFILE: account, - DSH_HOME: profile, - OPENAI_API_KEY: "local-fixture-key", - REA_PROCESS_RUN_ID: verifier.run_id, -}; -// A temporary directory under the checkout must not inherit its project skills or AGENTS.md. -await exec( - "git", - ["-c", "init.defaultBranch=main", "init", "--quiet", workspace], - { - env: environment, - timeout: 30_000, - }, -); -await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', -); -const clientVersion = ( - await exec(command, ["--version"], { - env: environment, - cwd: workspace, - timeout: 30_000, - }) -).stdout.trim(); -const lifecycle = []; -const runRea = async (args, artifact) => { - const result = await exec(process.execPath, [cli, ...args, "--json"], { - env: environment, - cwd: workspace, - timeout: 60_000, - }); - await writeFile(join(lab, `${artifact}.json`), result.stdout); - return JSON.parse(result.stdout); -}; -const patches = (text) => - parseYaml(text, { - customTags: [ - { - tag: "tag:yaml.org,2002:js", - resolve: (source) => ({ __jsExpr: source }), - }, - ], - }); -const withoutRea = (text) => - patches(text).flatMap((op) => { - if (op.id === "rea-mcp") return []; - if (!Array.isArray(op.insert)) return [op]; - const insert = op.insert.filter((entry) => entry.id !== "rea-mcp"); - return insert.length === 0 ? [] : [{ ...op, insert }]; - }); - +let clientVersion = null; +let fixture; +let passed = false; let catalog; let patternCount = 0; let skillRequested = false; @@ -104,155 +50,223 @@ let evidenceSeen = false; let finalSeen = false; let artifactDigest; let mainRequests = 0; -const ajv = new Ajv2020({ strict: false, validateFormats: false }); +const lifecycle = []; const patterns = new Set(); -const validatePatterns = (value) => { - if (!value || typeof value !== "object") return; - for (const [key, child] of Object.entries(value)) { - if (key === "pattern" && typeof child === "string") { - // Check the representation the native client actually forwards to its model adapter. - for (const flags of ["", "u", "v"]) new RegExp(child, flags); - patterns.add(child); - patternCount++; - } else validatePatterns(child); - } -}; -const extractEvidence = (value) => { - if (typeof value === "string") { - try { - return extractEvidence(JSON.parse(value)); - } catch { - return undefined; - } - } - if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes - ) - return value; - for (const child of Object.values(value)) { - const found = extractEvidence(child); - if (found) return found; - } - } -}; -const fixture = await createOpenAiModelFixture({ - directory: lab, - prefix: `deepseek-${mode}`, - onRequest: async ({ body, tools, requestIndex }) => { - // Harness generates a session title in a separate, tool-free model request. - if (tools.length === 0) - return { role: "assistant", content: "REA validation" }; - assert( - ++mainRequests <= 4, - "Harness did not complete the bounded fixture workflow.", +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([ + mkdir(account), + mkdir(profile), + mkdir(target, { recursive: true }), + ]); + const environment = { + PATH: process.env.PATH ?? "", + HOME: account, + USERPROFILE: account, + DSH_HOME: profile, + OPENAI_API_KEY: "local-fixture-key", + REA_PROCESS_RUN_ID: verifier.run_id, + }; + // A temporary directory under the checkout must not inherit its project skills or AGENTS.md. + await commands.exec( + "git", + ["-c", "init.defaultBranch=main", "init", "--quiet", workspace], + { + env: environment, + timeout: 30_000, + stdin: "pipe", + }, ); - const rea = tools.filter((tool) => tool.name.startsWith("mcp__rea__")); - const observedCatalog = rea - .map((tool) => tool.name.slice("mcp__rea__".length)) - .sort(); - assert.deepEqual( - observedCatalog, - TOOL_CONTRACTS.map((tool) => tool.name).sort(), + await writeFile( + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', ); - if (catalog === undefined) { - catalog = observedCatalog; - for (const tool of rea) { - assert( - ajv.validateSchema(tool.parameters), - `${tool.name}: ${ajv.errorsText(ajv.errors)}`, - ); - validatePatterns(tool.parameters); - } - assert( - patternCount > 0, - "The advertised regex constraints must reach the model.", - ); - } - const toolMessages = (body.messages ?? []).filter( - (message) => message.role === "tool", - ); - const serialized = JSON.stringify(toolMessages); - skillLoaded ||= - serialized.includes( - '', - ) && serialized.includes(dirname(skillPath)); - for (const message of toolMessages) { - const found = extractEvidence(message.content); - if (found === undefined) continue; - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(found); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((node) => node.observations) - .map((observation) => observation.properties) - .find( - (properties) => - properties.semantic_role === "export-return-shapes" && - properties.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap( - (entry) => entry.fields, - ); - assert( - fields.some( - (field) => - field.path === "/clientCompatibility" && - field.state === "literal" && - field.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (field) => - field.path === "/count" && - field.state === "literal" && - field.value === 7, - ), - ); - const text = JSON.stringify(evidence); - await writeFile(join(lab, "evidence.json"), text); - artifactDigest = createHash("sha256").update(text).digest("hex"); - evidenceSeen = true; - } - if (mode === "chat" || evidenceSeen) { - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - } - const call = (name, args) => ({ - role: "assistant", - content: null, - tool_calls: [ + clientVersion = ( + await commands.exec(command, ["--version"], { + env: environment, + cwd: workspace, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + const runRea = async (args, artifact) => { + const result = await commands.exec( + process.execPath, + [cli, ...args, "--json"], { - id: `call_rea_${requestIndex}`, - type: "function", - function: { name, arguments: JSON.stringify(args) }, + env: environment, + cwd: workspace, + timeout: 60_000, + stdin: "pipe", }, - ], + ); + await writeFile(join(lab, `${artifact}.json`), result.stdout); + return JSON.parse(result.stdout); + }; + const patches = (text) => + parseYaml(text, { + customTags: [ + { + tag: "tag:yaml.org,2002:js", + resolve: (source) => ({ __jsExpr: source }), + }, + ], + }); + const withoutRea = (text) => + patches(text).flatMap((op) => { + if (op.id === "rea-mcp") return []; + if (!Array.isArray(op.insert)) return [op]; + const insert = op.insert.filter((entry) => entry.id !== "rea-mcp"); + return insert.length === 0 ? [] : [{ ...op, insert }]; + }); + + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const validatePatterns = (value) => { + if (!value || typeof value !== "object") return; + for (const [key, child] of Object.entries(value)) { + if (key === "pattern" && typeof child === "string") { + // Check the representation the native client actually forwards to its model adapter. + for (const flags of ["", "u", "v"]) new RegExp(child, flags); + patterns.add(child); + patternCount++; + } else validatePatterns(child); + } + }; + const extractEvidence = (value) => { + if (typeof value === "string") { + try { + return extractEvidence(JSON.parse(value)); + } catch { + return undefined; + } + } + if (value && typeof value === "object") { + if ( + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes + ) + return value; + for (const child of Object.values(value)) { + const found = extractEvidence(child); + if (found) return found; + } + } + }; + fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: `deepseek-${mode}`, + onRequest: async ({ body, tools, requestIndex }) => { + // Harness generates a session title in a separate, tool-free model request. + if (tools.length === 0) + return { role: "assistant", content: "REA validation" }; + assert( + ++mainRequests <= 4, + "Harness did not complete the bounded fixture workflow.", + ); + const rea = tools.filter((tool) => tool.name.startsWith("mcp__rea__")); + const observedCatalog = rea + .map((tool) => tool.name.slice("mcp__rea__".length)) + .sort(); + assert.deepEqual( + observedCatalog, + TOOL_CONTRACTS.map((tool) => tool.name).sort(), + ); + if (catalog === undefined) { + catalog = observedCatalog; + for (const tool of rea) { + assert( + ajv.validateSchema(tool.parameters), + `${tool.name}: ${ajv.errorsText(ajv.errors)}`, + ); + validatePatterns(tool.parameters); + } + assert( + patternCount > 0, + "The advertised regex constraints must reach the model.", + ); + } + const toolMessages = (body.messages ?? []).filter( + (message) => message.role === "tool", + ); + const serialized = JSON.stringify(toolMessages); + skillLoaded ||= + serialized.includes( + '', + ) && serialized.includes(dirname(skillPath)); + for (const message of toolMessages) { + const found = extractEvidence(message.content); + if (found === undefined) continue; + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(found); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((node) => node.observations) + .map((observation) => observation.properties) + .find( + (properties) => + properties.semantic_role === "export-return-shapes" && + properties.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap( + (entry) => entry.fields, + ); + assert( + fields.some( + (field) => + field.path === "/clientCompatibility" && + field.state === "literal" && + field.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (field) => + field.path === "/count" && + field.state === "literal" && + field.value === 7, + ), + ); + const text = JSON.stringify(evidence); + await writeFile(join(lab, "evidence.json"), text); + artifactDigest = createHash("sha256").update(text).digest("hex"); + evidenceSeen = true; + } + if (mode === "chat" || evidenceSeen) { + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + const call = (name, args) => ({ + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_rea_${requestIndex}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }); + if (!skillRequested) { + assert(tools.some((tool) => tool.name === "skill")); + skillRequested = true; + return call("skill", { name: "reverse-engineer-anything" }); + } + assert( + skillLoaded, + "Harness must load the installed skill before analysis.", + ); + return call("mcp__rea__analyze_javascript_application", { + input_path: target, + format: "directory", + }); + }, }); - if (!skillRequested) { - assert(tools.some((tool) => tool.name === "skill")); - skillRequested = true; - return call("skill", { name: "reverse-engineer-anything" }); - } - assert( - skillLoaded, - "Harness must load the installed skill before analysis.", - ); - return call("mcp__rea__analyze_javascript_application", { - input_path: target, - format: "directory", - }); - }, -}); -let failure; -try { - // Seed model settings and a stale registration; the public setup workflow - // must produce the registration this native client actually boots. - const original = ` + // Seed model settings and a stale registration; the public setup workflow + // must produce the registration this native client actually boots. + const original = ` - id: llm-pi-ai config: providers: @@ -283,127 +297,143 @@ try { env: REA_PROCESS_RUN_ID: ${JSON.stringify(verifier.run_id)} `; - await writeFile(configPath, original); - const planned = await runRea( - ["setup", "--client", "dsh", "--dry-run"], - "setup-plan", - ); - assert.equal(planned.status, "planned"); - assert(planned.plannedActions.some((action) => action.target === configPath)); - assert( - planned.plannedActions.some( - (action) => action.target === dirname(skillPath), - ), - ); - assert( - planned.plannedActions.every( - (action) => - action.target === configPath || action.target === dirname(skillPath), - ), - ); - assert.equal(await readFile(configPath, "utf8"), original); - assert.equal(await pathExists(`${configPath}.rea.backup`), false); - lifecycle.push("dry-run"); - const setup = await runRea(["setup", "--client", "dsh", "--yes"], "setup"); - assert.equal(setup.status, "ready"); - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const configured = await readFile(configPath, "utf8"); - assert.deepEqual(withoutRea(configured), withoutRea(original)); - assert(configured.includes("!!js process.cwd()")); - assert( - (await readFile(skillPath, "utf8")).includes( - "name: reverse-engineer-anything", - ), - ); - lifecycle.push("setup"); - const repeated = await runRea( - ["setup", "--client", "dsh", "--yes"], - "setup-repeat", - ); - assert.deepEqual(repeated.appliedActions, []); - assert.equal(await readFile(configPath, "utf8"), configured); - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - lifecycle.push("idempotent-setup"); - const doctor = await runRea( - ["doctor", "--client", "dsh", "--skill"], - "doctor", - ); - assert.equal(doctor.healthy, true); - lifecycle.push("doctor"); - const execution = exec( - command, - [ - "--profile", - "headless", - "--json", - mode === "chat" ? "Say hello briefly." : `Use REA to analyze ${target}.`, - ], - { - env: environment, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, + await writeFile(configPath, original); + const planned = await runRea( + ["setup", "--client", "dsh", "--dry-run"], + "setup-plan", + ); + assert.equal(planned.status, "planned"); + assert( + planned.plannedActions.some((action) => action.target === configPath), + ); + assert( + planned.plannedActions.some( + (action) => action.target === dirname(skillPath), + ), + ); + assert( + planned.plannedActions.every( + (action) => + action.target === configPath || action.target === dirname(skillPath), + ), + ); + assert.equal(await readFile(configPath, "utf8"), original); + assert.equal(await pathExists(`${configPath}.rea.backup`), false); + lifecycle.push("dry-run"); + const setup = await runRea(["setup", "--client", "dsh", "--yes"], "setup"); + assert.equal(setup.status, "ready"); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + assert.deepEqual(withoutRea(configured), withoutRea(original)); + assert(configured.includes("!!js process.cwd()")); + assert( + (await readFile(skillPath, "utf8")).includes( + "name: reverse-engineer-anything", + ), + ); + lifecycle.push("setup"); + const repeated = await runRea( + ["setup", "--client", "dsh", "--yes"], + "setup-repeat", + ); + assert.deepEqual(repeated.appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + lifecycle.push("idempotent-setup"); + const doctor = await runRea( + ["doctor", "--client", "dsh", "--skill"], + "doctor", + ); + assert.equal(doctor.healthy, true); + lifecycle.push("doctor"); + const result = await saveVerifierCommandOutput( + () => + commands.exec( + command, + [ + "--profile", + "headless", + "--json", + mode === "chat" + ? "Say hello briefly." + : `Use REA to analyze ${target}.`, + ], + { + env: environment, + cwd: workspace, + timeout: 120_000, + stdin: "closed-pipe", + maxBuffer: 10 * 1024 * 1024, + }, + ), + { + stdout: join(lab, "client.stdout"), + stderr: join(lab, "client.stderr"), + }, + ); + assert.equal(fixture.failure, undefined); + assert(finalSeen && result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK")); + if (mode === "call") assert(evidenceSeen && skillLoaded); + lifecycle.push(mode === "call" ? "native-tool-invocation" : "native-chat"); + const uninstall = await runRea(["uninstall"], "uninstall"); + assert.equal(uninstall.status, "complete"); + assert( + uninstall.items.some( + (item) => item.name === "dsh" && item.status === "removed", + ), + ); + assert.deepEqual( + patches(await readFile(configPath, "utf8")), + withoutRea(original), + ); + assert.equal(await pathExists(skillPath), false); + lifecycle.push("uninstall"); + }, [ + () => commands.close(), + async () => { + await fixture?.close(); }, - ); - execution.child.stdin.end(); - const result = await execution; - await writeFile(join(lab, "client.stdout"), result.stdout); - await writeFile(join(lab, "client.stderr"), result.stderr); - assert.equal(fixture.failure, undefined); - assert(finalSeen && result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK")); - if (mode === "call") assert(evidenceSeen && skillLoaded); - lifecycle.push(mode === "call" ? "native-tool-invocation" : "native-chat"); - const uninstall = await runRea(["uninstall"], "uninstall"); - assert.equal(uninstall.status, "complete"); - assert( - uninstall.items.some( - (item) => item.name === "dsh" && item.status === "removed", - ), - ); - assert.deepEqual( - patches(await readFile(configPath, "utf8")), - withoutRea(original), - ); - assert.equal(await pathExists(skillPath), false); - lifecycle.push("uninstall"); -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, "client.stdout"), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, "client.stderr"), cause.stderr); -} finally { - await fixture.close(); - const receipt = { - client: "deepseek-harness", - clientVersion, - mode, - lifecycle, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, - }, - modelFixture: - "loopback custom OpenAI-compatible adapter; no live DeepSeek provider claim", - modelContextWindow: 1000000, - skillDiscovery: - "native HOME shared personal directory; isolated project root", - catalogSize: catalog?.length, - patternCount, - uniquePatterns: patterns.size, - skillLoaded, - evidenceSeen, - artifactDigest, - finalSeen, - requests: fixture.requests, - fixtureFailure: fixture.failure, - verifier: await completeVerifierRun(verifier), - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; + ]); + passed = true; +}, [ + async () => { + const receipt = { + client: "deepseek-harness", + clientVersion, + mode, + lifecycle, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback custom OpenAI-compatible adapter; no live DeepSeek provider claim", + modelContextWindow: 1000000, + skillDiscovery: + "native HOME shared personal directory; isolated project root", + catalogSize: catalog?.length, + patternCount, + uniquePatterns: patterns.size, + skillLoaded, + evidenceSeen, + artifactDigest, + finalSeen, + requests: fixture?.requests, + fixtureFailure: fixture?.failure, + verifier: await completeVerifierRun(verifier), + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); + }, + ], + ); + }, +]); diff --git a/scripts/verify-gemini-client.mjs b/scripts/verify-gemini-client.mjs index ad4d3c00ff..eec9f69ce6 100644 --- a/scripts/verify-gemini-client.mjs +++ b/scripts/verify-gemini-client.mjs @@ -14,7 +14,9 @@ import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; -import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createGeminiModelFixture } from "./verify/clients/gemini-model-fixture.mjs"; @@ -50,402 +52,441 @@ const skillPath = join( "reverse-engineer-anything", "SKILL.md", ); -await Promise.all([ - mkdir(account), - mkdir(workspace), - mkdir(join(profile, ".gemini"), { recursive: true }), -]); -await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); -await writeFile(join(lab, "system-settings.json"), "{}\n"); -await writeFile(join(lab, "system-defaults.json"), "{}\n"); -const environment = { - PATH: process.env.PATH ?? "", - USERPROFILE: account, - GEMINI_CLI_HOME: profile, - GEMINI_CLI_SYSTEM_SETTINGS_PATH: join(lab, "system-settings.json"), - GEMINI_CLI_SYSTEM_DEFAULTS_PATH: join(lab, "system-defaults.json"), - // Preserve the caller's bounded Node heap instead of the launcher's host-memory auto-sizing. - GEMINI_CLI_NO_RELAUNCH: "1", - GEMINI_CLI_TRUST_WORKSPACE: "true", - REA_PROCESS_RUN_ID: verifier.run_id, - REA_MCP_INPUT_SCHEMA_PROFILE: schemaProfile, -}; -let clientVersion; -try { - clientVersion = ( - await exec(...launch(["--version"]), { env: environment, timeout: 30_000 }) - ).stdout.trim(); -} catch (cause) { - const windowsHint = - process.platform === "win32" - ? " On Windows, select the JavaScript entry point; npm installs provide it as node_modules/@google/gemini-cli/bundle/gemini.js." - : ""; - throw new Error( - `Gemini preflight failed for ${command}; install Gemini CLI or set REA_VERIFY_GEMINI_COMMAND.${windowsHint} ${String(cause)}`, - ); -} -const original = JSON.stringify( - { - security: { - auth: { selectedType: "gemini-api-key" }, - folderTrust: { enabled: false }, - }, - telemetry: { enabled: false }, - advanced: { autoConfigureMemory: false }, - model: { name: "gemini-2.5-flash", maxSessionTurns: 8 }, - mcpServers: { other: { command: "unrelated-server" } }, - }, - null, - 2, -); -await writeFile(configPath, original); -for (const args of [ - ["setup", "--client", "gemini_cli", "--dry-run"], - ["setup", "--client", "gemini_cli", "--yes"], - ["doctor", "--client", "gemini_cli", "--skill"], -]) { - const result = await exec( - process.execPath, - [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); - const parsed = JSON.parse(result.stdout); - if (args[0] === "doctor") assert.equal(parsed.healthy, true); - else { - assert.equal( - parsed.status, - args.includes("--dry-run") ? "planned" : "ready", - ); - if (args.includes("--dry-run")) { - assert(parsed.plannedActions.some((a) => a.target === configPath)); - assert( - parsed.plannedActions.some((a) => - a.target.startsWith(join(profile, ".agents", "skills") + sep), - ), - ); - assert( - parsed.plannedActions.every((a) => a.target.startsWith(profile + sep)), - ); - } - } - await writeFile( - join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), - result.stdout, - ); -} -assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); -const configured = await readFile(configPath, "utf8"); -assert.deepEqual(JSON.parse(configured).mcpServers.other, { - command: "unrelated-server", -}); -const repeated = await exec( - process.execPath, - [ - join(runtimeRoot, "scripts/rea.mjs"), - "setup", - "--client", - "gemini_cli", - "--yes", - "--json", - ], - { env: environment, cwd: workspace, timeout: 60_000 }, -); -assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); -assert.equal(await readFile(configPath, "utf8"), configured); -assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); -assert( - (await readFile(skillPath, "utf8")).includes( - "name: reverse-engineer-anything", - ), -); -const target = join(lab, "fixture α测试"); -await mkdir(target); -await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', -); +let clientVersion = null; +let fixture; +let passed = false; let evidenceSeen = false; let skillLoaded = false; let skillRequested = false; let finalSeen = false; -let catalogSize = 0; +let catalogSize = null; let schemasChecked = 0; let examplesChecked = 0; let artifactDigest; -const schemaValidator = new Ajv2020({ strict: false, validateFormats: false }); -const unsupportedLiteralPath = (schema, path = "$", seen = new Set()) => { - if (!schema || typeof schema !== "object" || seen.has(schema)) - return undefined; - seen.add(schema); - for (const [key, child] of Object.entries(schema)) { - const childPath = `${path}.${key}`; - if (key === "const" && (typeof child === "boolean" || child === "")) - return childPath; - if ( - key === "enum" && - Array.isArray(child) && - child.some((item) => typeof item === "boolean" || item === "") - ) - return childPath; - const schemaChildrenAtKey = - schemaMaps.has(key) && - child && - typeof child === "object" && - !Array.isArray(child) - ? Object.entries(child).map(([name, item]) => [ - `${childPath}.${name}`, - item, - ]) - : schemaArrays.has(key) && Array.isArray(child) - ? child.map((item, index) => [`${childPath}[${index}]`, item]) - : schemaChildren.has(key) - ? [[childPath, child]] - : []; - for (const [nestedPath, nested] of schemaChildrenAtKey) { - const found = unsupportedLiteralPath(nested, nestedPath, seen); - if (found !== undefined) return found; - } - } - return undefined; -}; -const extractEvidence = (value) => { - if (typeof value === "string") { - if ( - value.startsWith("\n") && - value.endsWith("\n") - ) { - return extractEvidence( - value.slice( - "\n".length, - -"\n".length, - ), - ); - } +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([ + mkdir(account), + mkdir(workspace), + mkdir(join(profile, ".gemini"), { recursive: true }), + ]); + await commands.exec("git", ["init", "--quiet", workspace], { + timeout: 30_000, + stdin: "pipe", + }); + await writeFile(join(lab, "system-settings.json"), "{}\n"); + await writeFile(join(lab, "system-defaults.json"), "{}\n"); + const environment = { + PATH: process.env.PATH ?? "", + USERPROFILE: account, + GEMINI_CLI_HOME: profile, + GEMINI_CLI_SYSTEM_SETTINGS_PATH: join(lab, "system-settings.json"), + GEMINI_CLI_SYSTEM_DEFAULTS_PATH: join(lab, "system-defaults.json"), + // Preserve the caller's bounded Node heap instead of the launcher's host-memory auto-sizing. + GEMINI_CLI_NO_RELAUNCH: "1", + GEMINI_CLI_TRUST_WORKSPACE: "true", + REA_PROCESS_RUN_ID: verifier.run_id, + REA_MCP_INPUT_SCHEMA_PROFILE: schemaProfile, + }; try { - return extractEvidence(JSON.parse(value)); - } catch { - return undefined; - } - } - if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0 - ) - return value; - for (const nested of Object.values(value)) { - const evidence = extractEvidence(nested); - if (evidence) return evidence; - } - } - return undefined; -}; -const fixture = await createGeminiModelFixture({ - directory: lab, - onRequest: async ({ body, tools }) => { - const analysis = tools.find((t) => - t.name.endsWith("analyze_javascript_application"), - ); - assert(analysis, "Gemini must expose the REA analysis tool"); - const prefix = analysis.name.slice( - 0, - -"analyze_javascript_application".length, - ); - const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort(); - assert.deepEqual( - tools - .filter((t) => expected.includes(t.name)) - .map((t) => t.name) - .sort(), - expected, - "Gemini must forward the complete REA tool catalog", - ); - catalogSize = expected.length; - if (schemasChecked === 0) { - for (const tool of tools.filter((t) => expected.includes(t.name))) { - assert( - tool.parametersJsonSchema, - `${tool.name}: missing advertised JSON Schema`, - ); - assert( - schemaValidator.validateSchema(tool.parametersJsonSchema), - `${tool.name}: invalid advertised JSON Schema: ${JSON.stringify(schemaValidator.errors)}`, - ); - const unsupportedLiteral = unsupportedLiteralPath( - tool.parametersJsonSchema, - ); - assert.equal( - unsupportedLiteral, - undefined, - `${tool.name}: Gemini SDK declaration retains a boolean or empty-string const/enum at ${unsupportedLiteral}`, - ); - const name = tool.name.slice(prefix.length); - const contract = toolContract(name); - const validate = schemaValidator.compile(tool.parametersJsonSchema); - for (const example of contract.examples) { - assert( - validate(example.input), - `${tool.name}: Gemini SDK declaration rejects canonical example ${example.title}: ${JSON.stringify(schemaValidator.errors)}`, - ); - examplesChecked++; - } - schemasChecked++; - } - } - const responses = body.contents - .flatMap((c) => c.parts ?? []) - .filter((p) => p.functionResponse) - .map((p) => p.functionResponse); - skillLoaded ||= responses.some( - (r) => - r.name === "activate_skill" && - r.response?.output?.includes( - ``, - ) && - r.response.output.includes( - join(profile, ".agents", "skills", "reverse-engineer-anything"), - ), - ); - const observed = extractEvidence(responses); - if (observed) { - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(observed); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((n) => n.observations) - .map((o) => o.properties) - .find( - (p) => - p.semantic_role === "export-return-shapes" && - p.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((s) => s.fields); - assert( - fields.some( - (f) => - f.path === "/clientCompatibility" && - f.state === "literal" && - f.value === "REA_CLIENT_FIXTURE", - ), + clientVersion = ( + await commands.exec(...launch(["--version"]), { + env: environment, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + } catch (cause) { + const windowsHint = + process.platform === "win32" + ? " On Windows, select the JavaScript entry point; npm installs provide it as node_modules/@google/gemini-cli/bundle/gemini.js." + : ""; + throw new Error( + `Gemini preflight failed for ${command}; install Gemini CLI or set REA_VERIFY_GEMINI_COMMAND.${windowsHint} ${String(cause)}`, + { cause }, ); - assert( - fields.some( - (f) => f.path === "/count" && f.state === "literal" && f.value === 7, - ), - ); - const bytes = JSON.stringify(evidence); - artifactDigest = createHash("sha256").update(bytes).digest("hex"); - await writeFile(join(lab, "gemini-evidence.json"), bytes); - evidenceSeen = true; } - if (mode === "chat" || evidenceSeen) { - finalSeen = true; - return [{ text: "REA_CLIENT_COMPATIBILITY_OK" }]; - } - if (!skillRequested) { - const skill = tools.find((t) => t.name === "activate_skill"); - assert(skill, "Gemini must expose its native skill activation tool"); - skillRequested = true; - return [ - { - functionCall: { - name: skill.name, - args: { name: "reverse-engineer-anything" }, - }, - }, - ]; - } - assert(skillLoaded, "Gemini must load the installed personal REA skill"); - return [ + const original = JSON.stringify( { - functionCall: { - name: analysis.name, - args: { input_path: target, format: "directory" }, + security: { + auth: { selectedType: "gemini-api-key" }, + folderTrust: { enabled: false }, }, + telemetry: { enabled: false }, + advanced: { autoConfigureMemory: false }, + model: { name: "gemini-2.5-flash", maxSessionTurns: 8 }, + mcpServers: { other: { command: "unrelated-server" } }, }, - ]; + null, + 2, + ); + await writeFile(configPath, original); + for (const args of [ + ["setup", "--client", "gemini_cli", "--dry-run"], + ["setup", "--client", "gemini_cli", "--yes"], + ["doctor", "--client", "gemini_cli", "--skill"], + ]) { + const result = await commands.exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + const parsed = JSON.parse(result.stdout); + if (args[0] === "doctor") assert.equal(parsed.healthy, true); + else { + assert.equal( + parsed.status, + args.includes("--dry-run") ? "planned" : "ready", + ); + if (args.includes("--dry-run")) { + assert(parsed.plannedActions.some((a) => a.target === configPath)); + assert( + parsed.plannedActions.some((a) => + a.target.startsWith(join(profile, ".agents", "skills") + sep), + ), + ); + assert( + parsed.plannedActions.every((a) => + a.target.startsWith(profile + sep), + ), + ); + } + } + await writeFile( + join( + lab, + `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`, + ), + result.stdout, + ); + } + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + assert.deepEqual(JSON.parse(configured).mcpServers.other, { + command: "unrelated-server", + }); + const repeated = await commands.exec( + process.execPath, + [ + join(runtimeRoot, "scripts/rea.mjs"), + "setup", + "--client", + "gemini_cli", + "--yes", + "--json", + ], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + assert( + (await readFile(skillPath, "utf8")).includes( + "name: reverse-engineer-anything", + ), + ); + const target = join(lab, "fixture α测试"); + await mkdir(target); + await writeFile( + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', + ); + const schemaValidator = new Ajv2020({ + strict: false, + validateFormats: false, + }); + const unsupportedLiteralPath = (schema, path = "$", seen = new Set()) => { + if (!schema || typeof schema !== "object" || seen.has(schema)) + return undefined; + seen.add(schema); + for (const [key, child] of Object.entries(schema)) { + const childPath = `${path}.${key}`; + if (key === "const" && (typeof child === "boolean" || child === "")) + return childPath; + if ( + key === "enum" && + Array.isArray(child) && + child.some((item) => typeof item === "boolean" || item === "") + ) + return childPath; + const schemaChildrenAtKey = + schemaMaps.has(key) && + child && + typeof child === "object" && + !Array.isArray(child) + ? Object.entries(child).map(([name, item]) => [ + `${childPath}.${name}`, + item, + ]) + : schemaArrays.has(key) && Array.isArray(child) + ? child.map((item, index) => [`${childPath}[${index}]`, item]) + : schemaChildren.has(key) + ? [[childPath, child]] + : []; + for (const [nestedPath, nested] of schemaChildrenAtKey) { + const found = unsupportedLiteralPath(nested, nestedPath, seen); + if (found !== undefined) return found; + } + } + return undefined; + }; + const extractEvidence = (value) => { + if (typeof value === "string") { + if ( + value.startsWith("\n") && + value.endsWith("\n") + ) { + return extractEvidence( + value.slice( + "\n".length, + -"\n".length, + ), + ); + } + try { + return extractEvidence(JSON.parse(value)); + } catch { + return undefined; + } + } + if (value && typeof value === "object") { + if ( + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes?.length > 0 + ) + return value; + for (const nested of Object.values(value)) { + const evidence = extractEvidence(nested); + if (evidence) return evidence; + } + } + return undefined; + }; + fixture = await createGeminiModelFixture({ + directory: lab, + onRequest: async ({ body, tools }) => { + const analysis = tools.find((t) => + t.name.endsWith("analyze_javascript_application"), + ); + assert(analysis, "Gemini must expose the REA analysis tool"); + const prefix = analysis.name.slice( + 0, + -"analyze_javascript_application".length, + ); + const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort(); + const reaTools = tools.filter((t) => expected.includes(t.name)); + catalogSize = reaTools.length; + assert.deepEqual( + reaTools.map((t) => t.name).sort(), + expected, + "Gemini must forward the complete REA tool catalog", + ); + if (schemasChecked === 0) { + for (const tool of reaTools) { + assert( + tool.parametersJsonSchema, + `${tool.name}: missing advertised JSON Schema`, + ); + assert( + schemaValidator.validateSchema(tool.parametersJsonSchema), + `${tool.name}: invalid advertised JSON Schema: ${JSON.stringify(schemaValidator.errors)}`, + ); + const unsupportedLiteral = unsupportedLiteralPath( + tool.parametersJsonSchema, + ); + assert.equal( + unsupportedLiteral, + undefined, + `${tool.name}: Gemini SDK declaration retains a boolean or empty-string const/enum at ${unsupportedLiteral}`, + ); + const name = tool.name.slice(prefix.length); + const contract = toolContract(name); + const validate = schemaValidator.compile(tool.parametersJsonSchema); + for (const example of contract.examples) { + assert( + validate(example.input), + `${tool.name}: Gemini SDK declaration rejects canonical example ${example.title}: ${JSON.stringify(schemaValidator.errors)}`, + ); + examplesChecked++; + } + schemasChecked++; + } + } + const responses = body.contents + .flatMap((c) => c.parts ?? []) + .filter((p) => p.functionResponse) + .map((p) => p.functionResponse); + skillLoaded ||= responses.some( + (r) => + r.name === "activate_skill" && + r.response?.output?.includes( + ``, + ) && + r.response.output.includes( + join(profile, ".agents", "skills", "reverse-engineer-anything"), + ), + ); + const observed = extractEvidence(responses); + if (observed) { + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(observed); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((n) => n.observations) + .map((o) => o.properties) + .find( + (p) => + p.semantic_role === "export-return-shapes" && + p.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap((s) => s.fields); + assert( + fields.some( + (f) => + f.path === "/clientCompatibility" && + f.state === "literal" && + f.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (f) => + f.path === "/count" && f.state === "literal" && f.value === 7, + ), + ); + const bytes = JSON.stringify(evidence); + artifactDigest = createHash("sha256").update(bytes).digest("hex"); + await writeFile(join(lab, "gemini-evidence.json"), bytes); + evidenceSeen = true; + } + if (mode === "chat" || evidenceSeen) { + finalSeen = true; + return [{ text: "REA_CLIENT_COMPATIBILITY_OK" }]; + } + if (!skillRequested) { + const skill = tools.find((t) => t.name === "activate_skill"); + assert(skill, "Gemini must expose its native skill activation tool"); + skillRequested = true; + return [ + { + functionCall: { + name: skill.name, + args: { name: "reverse-engineer-anything" }, + }, + }, + ]; + } + assert( + skillLoaded, + "Gemini must load the installed personal REA skill", + ); + return [ + { + functionCall: { + name: analysis.name, + args: { input_path: target, format: "directory" }, + }, + }, + ]; + }, + }); + const response = await saveVerifierCommandOutput( + () => + commands.exec( + ...launch([ + "--model", + "gemini-2.5-flash", + "--approval-mode", + mode === "chat" ? "default" : "yolo", + "--allowed-mcp-server-names", + "rea", + "--output-format", + "stream-json", + "-p", + mode === "chat" + ? "Say hello briefly." + : `Use REA to analyze the JavaScript directory ${target}. Report the result.`, + ]), + { + env: { + ...environment, + GEMINI_API_KEY: "local-fixture-key", + GOOGLE_GEMINI_BASE_URL: fixture.baseUrl, + }, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + stdin: "pipe", + }, + ), + { + stdout: join(lab, `${mode}.stdout`), + stderr: join(lab, `${mode}.stderr`), + }, + ); + const events = response.stdout + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); + assert.equal( + events.findLast((e) => e.type === "result")?.status, + "success", + ); + assert.equal(fixture.failure, undefined); + assert( + finalSeen && response.stdout.includes("REA_CLIENT_COMPATIBILITY_OK"), + ); + if (mode === "call") assert(evidenceSeen && skillLoaded); + }, [ + () => commands.close(), + async () => { + await fixture?.close(); + }, + ]); + passed = true; +}, [ + async () => { + const lineage = await completeVerifierRun(verifier); + const receipt = { + client: "Gemini CLI", + clientVersion, + mode, + schemaProfile, + schemasChecked, + examplesChecked, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback native Gemini API adapter; token counts synthetic; no live Google API claim", + skillDiscovery: + "native GEMINI_CLI_HOME personal shared directory; isolated Git project", + catalogSize, + evidenceSeen, + skillLoaded, + artifactDigest, + finalSeen, + requests: fixture?.requests, + probes: fixture?.probes, + fixtureFailure: fixture?.failure, + verifier: lineage, + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); + }, + ], + ); }, -}); -let failure; -try { - const response = await exec( - ...launch([ - "--model", - "gemini-2.5-flash", - "--approval-mode", - mode === "chat" ? "default" : "yolo", - "--allowed-mcp-server-names", - "rea", - "--output-format", - "stream-json", - "-p", - mode === "chat" - ? "Say hello briefly." - : `Use REA to analyze the JavaScript directory ${target}. Report the result.`, - ]), - { - env: { - ...environment, - GEMINI_API_KEY: "local-fixture-key", - GOOGLE_GEMINI_BASE_URL: fixture.baseUrl, - }, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, - }, - ); - await writeFile(join(lab, `${mode}.stdout`), response.stdout); - await writeFile(join(lab, `${mode}.stderr`), response.stderr); - const events = response.stdout - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line)); - assert.equal(events.findLast((e) => e.type === "result")?.status, "success"); - assert.equal(fixture.failure, undefined); - assert(finalSeen && response.stdout.includes("REA_CLIENT_COMPATIBILITY_OK")); - if (mode === "call") assert(evidenceSeen && skillLoaded); -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, `${mode}.stdout`), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, `${mode}.stderr`), cause.stderr); -} finally { - await fixture.close(); - const lineage = await completeVerifierRun(verifier); - const receipt = { - client: "Gemini CLI", - clientVersion, - mode, - schemaProfile, - schemasChecked, - examplesChecked, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, - }, - modelFixture: - "loopback native Gemini API adapter; token counts synthetic; no live Google API claim", - skillDiscovery: - "native GEMINI_CLI_HOME personal shared directory; isolated Git project", - catalogSize, - schemasChecked, - evidenceSeen, - skillLoaded, - artifactDigest, - finalSeen, - requests: fixture.requests, - probes: fixture.probes, - fixtureFailure: fixture.failure, - verifier: lineage, - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; +]); diff --git a/scripts/verify-grok-client.mjs b/scripts/verify-grok-client.mjs index befe2de8e9..67ec83a200 100644 --- a/scripts/verify-grok-client.mjs +++ b/scripts/verify-grok-client.mjs @@ -9,7 +9,9 @@ import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; -import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -38,57 +40,8 @@ const skillDirectory = join( "skills", "reverse-engineer-anything", ); -await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]); -await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); -// Native compatibility discovery is independent of GROK_HOME. Disable those -// sources and ignore the real shared skill root instead of changing HOME. -const environment = { - PATH: process.env.PATH, - NODE_OPTIONS: process.env.NODE_OPTIONS, - USERPROFILE: account, - GROK_HOME: profile, - GROK_DISABLE_AUTOUPDATER: "1", - ...Object.fromEntries( - ["CLAUDE", "CURSOR"].flatMap((vendor) => - ["SKILLS", "RULES", "AGENTS", "MCPS", "HOOKS", "SESSIONS"].map((cell) => [ - `GROK_${vendor}_${cell}_ENABLED`, - "false", - ]), - ), - ), - XDG_DATA_HOME: join(lab, "data"), - XDG_CONFIG_HOME: join(lab, "config"), - XDG_CACHE_HOME: join(lab, "cache"), - REA_PROCESS_RUN_ID: verifier.run_id, -}; -const target = join(lab, "fixture α测试"); -await mkdir(target); -await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n' + - // A real analysis result large enough to exercise native MCP offloading. - Array.from( - { length: 60 }, - (_, index) => - `export function extra${index}() { return { index: ${index}, tag: "GROK_LARGE_FIXTURE" }; }`, - ).join("\n"), -); const expectedNames = TOOL_CONTRACTS.map((tool) => "rea__" + tool.name).sort(); -assert( - expectedNames.length <= 255, - "Native search_tool limit is uint8; this full-catalog query cannot cover more than 255 tools", -); -const schemaValidator = new Ajv2020({ strict: false, validateFormats: false }); -const shellQuote = (value) => "'" + value.replaceAll("'", "'\"'\"'") + "'"; -const containsText = (value, text) => { - if (typeof value === "string") return value.includes(text); - return ( - value !== null && - typeof value === "object" && - Object.values(value).some((item) => containsText(item, text)) - ); -}; -let clientVersion; +let clientVersion = null; let expectedSkill; let turn = 0; let skillRequested = false; @@ -97,6 +50,7 @@ let diagnosticsRequested = false; let diagnosticsVerified = false; let lastTool; let catalogChecked = false; +let catalogSize = null; let schemasChecked = 0; let analysisRequested = false; let evidenceSeen = false; @@ -107,240 +61,312 @@ let artifactBytes; let retainedArtifact; let expectedProjection; let finalSeen = false; -let failure; -const call = (name, args) => { - lastTool = name; - return { - role: "assistant", - content: null, - tool_calls: [ - { - id: `call_grok_${turn}`, - type: "function", - function: { name, arguments: JSON.stringify(args) }, - }, - ], - }; -}; -const verifyEvidence = async (bytes) => { - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(JSON.parse(bytes.toString("utf8"))); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shapes = evidence.normalized_result.graph.nodes - .flatMap((node) => node.observations) - .map((observation) => observation.properties) - .filter( - (properties) => - properties.semantic_role === "export-return-shapes" && - properties.exported_name === "result", - ); - assert( - shapes.length > 0 && - shapes.every((shape) => shape.return_shape_coverage?.projection_complete), - ); - const fields = shapes.flatMap((shape) => - shape.static_return_shapes.flatMap((item) => item.fields), - ); - assert( - fields.some( - (field) => - field.path === "/clientCompatibility" && - field.state === "literal" && - field.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (field) => - field.path === "/count" && - field.state === "literal" && - field.value === 7, - ), - ); - artifactDigest = createHash("sha256").update(bytes).digest("hex"); - artifactBytes = bytes.length; - evidenceSeen = true; - return { - evidence_id: evidence.evidence_id, - subject: evidence.subject, - operation: evidence.operation, - artifact_sha256: artifactDigest, - byte_length: artifactBytes, - shapes, - }; -}; -const diagnosticFile = join(lab, "native-doctor.json"); -const diagnosticCode = - "const fs=require('node:fs');const d=JSON.parse(fs.readFileSync(process.argv[1],'utf8'));console.log(JSON.stringify({rea_client_diagnostic:{healthy:d.healthy,registrations:d.identity.registrations.filter(r=>r.client==='grok_build'),skill:d.identity.skill.state}}));"; -// Query the saved result through a native agent command. Only the facts needed -// for this analyst question enter context; the complete named-schema Evidence -// and its digest remain available in the native artifact. -const recoveryCode = - "const fs=require('node:fs'),crypto=require('node:crypto');const bytes=fs.readFileSync(process.argv[1]),e=JSON.parse(bytes);const shapes=e.normalized_result.graph.nodes.flatMap(n=>n.observations).map(o=>o.properties).filter(p=>p.semantic_role==='export-return-shapes'&&p.exported_name==='result');console.log(JSON.stringify({rea_recovered:{evidence_id:e.evidence_id,subject:e.subject,operation:e.operation,artifact_sha256:crypto.createHash('sha256').update(bytes).digest('hex'),byte_length:bytes.length,shapes}}));"; -const fixture = await createOpenAiModelFixture({ - directory: lab, - prefix: "grok", - onRequest: async ({ body, tools }) => { - // Native title/summary requests cannot execute the analyst workflow. - if (!tools.some((tool) => tool.name === "search_tool")) - return { role: "assistant", content: "REA fixture auxiliary response" }; - turn++; - assert( - containsText(body.messages, skillDirectory), - "Grok must discover the isolated installed skill", +let passed = false; +let fixture; +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]); + await commands.exec("git", ["init", "--quiet", workspace], { + timeout: 30_000, + stdin: "pipe", + }); + // Native compatibility discovery is independent of GROK_HOME. Disable those + // sources and ignore the real shared skill root instead of changing HOME. + const environment = { + PATH: process.env.PATH, + NODE_OPTIONS: process.env.NODE_OPTIONS, + USERPROFILE: account, + GROK_HOME: profile, + GROK_DISABLE_AUTOUPDATER: "1", + ...Object.fromEntries( + ["CLAUDE", "CURSOR"].flatMap((vendor) => + ["SKILLS", "RULES", "AGENTS", "MCPS", "HOOKS", "SESSIONS"].map( + (cell) => [`GROK_${vendor}_${cell}_ENABLED`, "false"], + ), + ), + ), + XDG_DATA_HOME: join(lab, "data"), + XDG_CONFIG_HOME: join(lab, "config"), + XDG_CACHE_HOME: join(lab, "cache"), + REA_PROCESS_RUN_ID: verifier.run_id, + }; + const target = join(lab, "fixture α测试"); + await mkdir(target); + await writeFile( + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n' + + // A real analysis result large enough to exercise native MCP offloading. + Array.from( + { length: 60 }, + (_, index) => + `export function extra${index}() { return { index: ${index}, tag: "GROK_LARGE_FIXTURE" }; }`, + ).join("\n"), ); assert( - !containsText(body.messages, join(homedir(), ".agents", "skills")), - "Do not inherit real personal skills", + expectedNames.length <= 255, + "Native search_tool limit is uint8; this full-catalog query cannot cover more than 255 tools", ); - if (mode === "chat") { - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - } - const replies = body.messages.filter((message) => message.role === "tool"); - if (!skillRequested) { - skillRequested = true; - return call("read_file", { - target_file: join(skillDirectory, "SKILL.md"), - }); - } - assert( - expectedSkill !== undefined && containsText(replies, expectedSkill), - "Native read_file must return the full skill with its documented line-number rendering", - ); - skillLoaded = true; - const last = replies.at(-1)?.content; - assert.equal(typeof last, "string"); - if (!catalogChecked) { - const discovery = - lastTool === "search_tool" ? JSON.parse(last) : undefined; - if (discovery?.status === "partial" && !diagnosticsRequested) { - diagnosticsRequested = true; - const diagnostic = [ - process.execPath, - join(runtimeRoot, "scripts/rea.mjs"), - "doctor", - "--client", - "grok_build", - "--skill", - "--json", - ] - .map(shellQuote) - .join(" "); - const projection = [ - process.execPath, - "-e", - diagnosticCode, - diagnosticFile, - ] - .map(shellQuote) - .join(" "); - return call("run_terminal_command", { - command: `${diagnostic} > ${shellQuote(diagnosticFile)} && ${projection}`, - description: - "Diagnose REA registration while native MCP discovery is pending", - }); - } - if (lastTool === "run_terminal_command") { + const schemaValidator = new Ajv2020({ + strict: false, + validateFormats: false, + }); + const shellQuote = (value) => "'" + value.replaceAll("'", "'\"'\"'") + "'"; + const containsText = (value, text) => { + if (typeof value === "string") return value.includes(text); + return ( + value !== null && + typeof value === "object" && + Object.values(value).some((item) => containsText(item, text)) + ); + }; + const call = (name, args) => { + lastTool = name; + return { + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_grok_${turn}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }; + }; + const verifyEvidence = async (bytes) => { + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(JSON.parse(bytes.toString("utf8"))); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shapes = evidence.normalized_result.graph.nodes + .flatMap((node) => node.observations) + .map((observation) => observation.properties) + .filter( + (properties) => + properties.semantic_role === "export-return-shapes" && + properties.exported_name === "result", + ); + assert( + shapes.length > 0 && + shapes.every( + (shape) => shape.return_shape_coverage?.projection_complete, + ), + ); + const fields = shapes.flatMap((shape) => + shape.static_return_shapes.flatMap((item) => item.fields), + ); + assert( + fields.some( + (field) => + field.path === "/clientCompatibility" && + field.state === "literal" && + field.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (field) => + field.path === "/count" && + field.state === "literal" && + field.value === 7, + ), + ); + artifactDigest = createHash("sha256").update(bytes).digest("hex"); + artifactBytes = bytes.length; + evidenceSeen = true; + return { + evidence_id: evidence.evidence_id, + subject: evidence.subject, + operation: evidence.operation, + artifact_sha256: artifactDigest, + byte_length: artifactBytes, + shapes, + }; + }; + const diagnosticFile = join(lab, "native-doctor.json"); + const diagnosticCode = + "const fs=require('node:fs');const d=JSON.parse(fs.readFileSync(process.argv[1],'utf8'));console.log(JSON.stringify({rea_client_diagnostic:{healthy:d.healthy,registrations:d.identity.registrations.filter(r=>r.client==='grok_build'),skill:d.identity.skill.state}}));"; + // Query the saved result through a native agent command. Only the facts needed + // for this analyst question enter context; the complete named-schema Evidence + // and its digest remain available in the native artifact. + const recoveryCode = + "const fs=require('node:fs'),crypto=require('node:crypto');const bytes=fs.readFileSync(process.argv[1]),e=JSON.parse(bytes);const shapes=e.normalized_result.graph.nodes.flatMap(n=>n.observations).map(o=>o.properties).filter(p=>p.semantic_role==='export-return-shapes'&&p.exported_name==='result');console.log(JSON.stringify({rea_recovered:{evidence_id:e.evidence_id,subject:e.subject,operation:e.operation,artifact_sha256:crypto.createHash('sha256').update(bytes).digest('hex'),byte_length:bytes.length,shapes}}));"; + fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: "grok", + onRequest: async ({ body, tools }) => { + // Native title/summary requests cannot execute the analyst workflow. + if (!tools.some((tool) => tool.name === "search_tool")) + return { + role: "assistant", + content: "REA fixture auxiliary response", + }; + turn++; + assert( + containsText(body.messages, skillDirectory), + "Grok must discover the isolated installed skill", + ); + assert( + !containsText(body.messages, join(homedir(), ".agents", "skills")), + "Do not inherit real personal skills", + ); + if (mode === "chat") { + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + const replies = body.messages.filter( + (message) => message.role === "tool", + ); + if (!skillRequested) { + skillRequested = true; + return call("read_file", { + target_file: join(skillDirectory, "SKILL.md"), + }); + } + assert( + expectedSkill !== undefined && containsText(replies, expectedSkill), + "Native read_file must return the full skill with its documented line-number rendering", + ); + skillLoaded = true; + const last = replies.at(-1)?.content; + assert.equal(typeof last, "string"); + if (!catalogChecked) { + const discovery = + lastTool === "search_tool" ? JSON.parse(last) : undefined; + if (discovery?.status === "partial" && !diagnosticsRequested) { + diagnosticsRequested = true; + const diagnostic = [ + process.execPath, + join(runtimeRoot, "scripts/rea.mjs"), + "doctor", + "--client", + "grok_build", + "--skill", + "--json", + ] + .map(shellQuote) + .join(" "); + const projection = [ + process.execPath, + "-e", + diagnosticCode, + diagnosticFile, + ] + .map(shellQuote) + .join(" "); + return call("run_terminal_command", { + command: `${diagnostic} > ${shellQuote(diagnosticFile)} && ${projection}`, + description: + "Diagnose REA registration while native MCP discovery is pending", + }); + } + if (lastTool === "run_terminal_command") { + const line = last + .split("\n") + .find((item) => item.startsWith('{"rea_client_diagnostic":')); + assert( + line, + "Native REA diagnostics failed or did not return the selected status", + ); + const diagnostic = JSON.parse(line).rea_client_diagnostic; + assert.equal(diagnostic.healthy, true); + assert.equal(diagnostic.skill, "aligned"); + assert( + diagnostic.registrations.some( + (registration) => + registration.state === "aligned" && + registration.config_path === configPath, + ), + ); + diagnosticsVerified = true; + } + if (discovery?.status !== "ready") + return call("search_tool", { query: "rea", limit: 255 }); + const inventory = discovery.results + .filter((result) => result.server === "rea") + .flatMap((result) => result.tools); + catalogSize = inventory.length; + assert.deepEqual( + inventory.map((tool) => tool.tool_name).sort(), + expectedNames, + "Native search_tool must discover the complete REA catalog", + ); + for (const tool of inventory) { + assert( + tool.input_schema, + `${tool.tool_name}: missing input schema`, + ); + assert( + schemaValidator.validateSchema(tool.input_schema), + `${tool.tool_name}: invalid advertised schema: ${JSON.stringify(schemaValidator.errors)}`, + ); + schemasChecked++; + } + await writeFile(join(lab, "native-discovery.json"), last); + catalogChecked = true; + analysisRequested = true; + return call("use_tool", { + tool_name: "rea__analyze_javascript_application", + tool_input: { input_path: target, format: "directory" }, + }); + } + if (!spillQueryRequested) { + const marker = "Full output written to: "; + const start = last.indexOf(marker); + if (start < 0) { + await verifyEvidence(Buffer.from(last, "utf8")); + finalSeen = true; + return { + role: "assistant", + content: "REA_CLIENT_COMPATIBILITY_OK", + }; + } + // The native hint explicitly calls for a query rather than read_file on + // the saved JSON's very long line. Preserve the original hint in requests. + const file = last + .slice(start + marker.length) + .split(". The full output ")[0]; + assert(isAbsolute(file)); + const childPath = relative(join(profile, "sessions"), file); + assert( + childPath !== "" && + !childPath.startsWith("..") && + !isAbsolute(childPath) && + file.endsWith(".json"), + "Unexpected native MCP spill path", + ); + retainedArtifact = file; + expectedProjection = await verifyEvidence(await readFile(file)); + spillQueryRequested = true; + return call("run_terminal_command", { + command: [process.execPath, "-e", recoveryCode, file] + .map(shellQuote) + .join(" "), + description: + "Query full saved REA Evidence while preserving artifact identity", + }); + } const line = last .split("\n") - .find((item) => item.startsWith('{"rea_client_diagnostic":')); + .find((item) => item.startsWith('{"rea_recovered":')); assert( line, - "Native REA diagnostics failed or did not return the selected status", + "The next model request must contain native artifact query output", ); - const diagnostic = JSON.parse(line).rea_client_diagnostic; - assert.equal(diagnostic.healthy, true); - assert.equal(diagnostic.skill, "aligned"); - assert( - diagnostic.registrations.some( - (registration) => - registration.state === "aligned" && - registration.config_path === configPath, - ), + const projection = JSON.parse(line).rea_recovered; + assert.deepEqual( + projection, + expectedProjection, + "Native query must recover the observed facts and complete artifact digest", ); - diagnosticsVerified = true; - } - if (discovery?.status !== "ready") - return call("search_tool", { query: "rea", limit: 255 }); - const inventory = discovery.results - .filter((result) => result.server === "rea") - .flatMap((result) => result.tools); - assert.deepEqual( - inventory.map((tool) => tool.tool_name).sort(), - expectedNames, - "Native search_tool must discover the complete REA catalog", - ); - for (const tool of inventory) { - assert(tool.input_schema, `${tool.tool_name}: missing input schema`); - assert( - schemaValidator.validateSchema(tool.input_schema), - `${tool.tool_name}: invalid advertised schema: ${JSON.stringify(schemaValidator.errors)}`, - ); - schemasChecked++; - } - await writeFile(join(lab, "native-discovery.json"), last); - catalogChecked = true; - analysisRequested = true; - return call("use_tool", { - tool_name: "rea__analyze_javascript_application", - tool_input: { input_path: target, format: "directory" }, - }); - } - if (!spillQueryRequested) { - const marker = "Full output written to: "; - const start = last.indexOf(marker); - if (start < 0) { - await verifyEvidence(Buffer.from(last, "utf8")); + projectionSeen = true; finalSeen = true; return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - } - // The native hint explicitly calls for a query rather than read_file on - // the saved JSON's very long line. Preserve the original hint in requests. - const file = last - .slice(start + marker.length) - .split(". The full output ")[0]; - assert(isAbsolute(file)); - const childPath = relative(join(profile, "sessions"), file); - assert( - childPath !== "" && - !childPath.startsWith("..") && - !isAbsolute(childPath) && - file.endsWith(".json"), - "Unexpected native MCP spill path", - ); - retainedArtifact = file; - expectedProjection = await verifyEvidence(await readFile(file)); - spillQueryRequested = true; - return call("run_terminal_command", { - command: [process.execPath, "-e", recoveryCode, file] - .map(shellQuote) - .join(" "), - description: - "Query full saved REA Evidence while preserving artifact identity", - }); - } - const line = last - .split("\n") - .find((item) => item.startsWith('{"rea_recovered":')); - assert( - line, - "The next model request must contain native artifact query output", - ); - const projection = JSON.parse(line).rea_recovered; - assert.deepEqual( - projection, - expectedProjection, - "Native query must recover the observed facts and complete artifact digest", - ); - projectionSeen = true; - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - }, -}); -const q = JSON.stringify; -const original = `# Keep unrelated settings and server state. + }, + }); + const q = JSON.stringify; + const original = `# Keep unrelated settings and server state. disabled_mcp_servers = ["rea", "other"] [cli] auto_update = false @@ -360,181 +386,203 @@ max_completion_tokens = 4096 command = "unrelated-server" enabled = false `; -await writeFile(configPath, original); -const rea = async (args) => - exec( - process.execPath, - [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); -try { - clientVersion = ( - await exec(command, ["--version"], { env: environment, timeout: 30_000 }) - ).stdout.trim(); - for (const args of [ - ["setup", "--client", "grok_build", "--dry-run"], - ["setup", "--client", "grok_build", "--yes"], - ["doctor", "--client", "grok_build", "--skill"], - ]) { - const result = await rea(args); - await writeFile( - join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), - result.stdout, - ); - const parsed = JSON.parse(result.stdout); - if (args[0] === "doctor") assert.equal(parsed.healthy, true); - else { - assert.equal( - parsed.status, - args.includes("--dry-run") ? "planned" : "ready", + await writeFile(configPath, original); + const rea = async (args) => + commands.exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, ); - if (args.includes("--dry-run")) { - assert( - parsed.plannedActions.some((action) => action.target === configPath), - ); - assert( - parsed.plannedActions.some( - (action) => action.target === skillDirectory, - ), - ); - assert( - parsed.plannedActions.every( - (action) => - action.target === configPath || - action.target === skillDirectory || - action.target.startsWith(skillDirectory + "/"), - ), - "Refuse writes outside the owned profile/skill", + clientVersion = ( + await commands.exec(command, ["--version"], { + env: environment, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + for (const args of [ + ["setup", "--client", "grok_build", "--dry-run"], + ["setup", "--client", "grok_build", "--yes"], + ["doctor", "--client", "grok_build", "--skill"], + ]) { + const result = await rea(args); + await writeFile( + join( + lab, + `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`, + ), + result.stdout, + ); + const parsed = JSON.parse(result.stdout); + if (args[0] === "doctor") assert.equal(parsed.healthy, true); + else { + assert.equal( + parsed.status, + args.includes("--dry-run") ? "planned" : "ready", ); + if (args.includes("--dry-run")) { + assert( + parsed.plannedActions.some( + (action) => action.target === configPath, + ), + ); + assert( + parsed.plannedActions.some( + (action) => action.target === skillDirectory, + ), + ); + assert( + parsed.plannedActions.every( + (action) => + action.target === configPath || + action.target === skillDirectory || + action.target.startsWith(skillDirectory + "/"), + ), + "Refuse writes outside the owned profile/skill", + ); + } } } - } - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const configured = await readFile(configPath, "utf8"); - assert( - configured.includes("unrelated-server") && - configured.includes("# Keep unrelated settings"), - ); - const repeated = JSON.parse( - (await rea(["setup", "--client", "grok_build", "--yes"])).stdout, - ); - assert.deepEqual(repeated.appliedActions, []); - assert.equal(await readFile(configPath, "utf8"), configured); - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const skill = await readFile(join(skillDirectory, "SKILL.md"), "utf8"); - expectedSkill = skill - .trimEnd() - .split("\n") - .map((line, index) => - index === 0 || (index + 1) % 10 === 0 ? `${index + 1}→${line}` : line, - ) - .join("\n"); - const execution = exec( - command, - [ - "-p", - mode === "chat" - ? "Say hello briefly." - : `Use the REA skill to analyze ${target}.`, - "-m", - "rea-client-fixture", - "--output-format", - "streaming-messages-json", - "--always-approve", - "--no-subagents", - ], - { - env: environment, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, - }, - ); - execution.child.stdin.end(); - const response = await execution; - await writeFile(join(lab, `${mode}.stdout`), response.stdout); - await writeFile(join(lab, `${mode}.stderr`), response.stderr); - assert.equal(fixture.failure, undefined); - const events = response.stdout - .trim() - .split("\n") - .map((line) => JSON.parse(line)); - const init = events.find( - (event) => event.type === "system" && event.subtype === "init", - ); - assert(init?.mcp_servers?.some((server) => server.name === "rea")); - assert( - init.mcp_servers.every((server) => ["rea", "other"].includes(server.name)), - "Do not inherit foreign MCP configuration", - ); - assert( - events.some( - (event) => - event.type === "result" && - event.subtype === "success" && - event.is_error === false && - event.result === "REA_CLIENT_COMPATIBILITY_OK", - ), - ); - assert(finalSeen); - if (mode === "call") + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); assert( - skillLoaded && - catalogChecked && - analysisRequested && - evidenceSeen && - (!spillQueryRequested || projectionSeen) && - (!diagnosticsRequested || diagnosticsVerified), + configured.includes("unrelated-server") && + configured.includes("# Keep unrelated settings"), ); -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, `${mode}.stdout`), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, `${mode}.stderr`), cause.stderr); -} finally { - await fixture.close(); - const lineage = await completeVerifierRun(verifier); - const receipt = { - client: "Grok Build", - clientVersion, - mode, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, + const repeated = JSON.parse( + (await rea(["setup", "--client", "grok_build", "--yes"])).stdout, + ); + assert.deepEqual(repeated.appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const skill = await readFile(join(skillDirectory, "SKILL.md"), "utf8"); + expectedSkill = skill + .trimEnd() + .split("\n") + .map((line, index) => + index === 0 || (index + 1) % 10 === 0 ? `${index + 1}→${line}` : line, + ) + .join("\n"); + const response = await saveVerifierCommandOutput( + () => + commands.exec( + command, + [ + "-p", + mode === "chat" + ? "Say hello briefly." + : `Use the REA skill to analyze ${target}.`, + "-m", + "rea-client-fixture", + "--output-format", + "streaming-messages-json", + "--always-approve", + "--no-subagents", + ], + { + env: environment, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + stdin: "closed-pipe", + }, + ), + { + stdout: join(lab, `${mode}.stdout`), + stderr: join(lab, `${mode}.stderr`), + }, + ); + assert.equal(fixture.failure, undefined); + const events = response.stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + const init = events.find( + (event) => event.type === "system" && event.subtype === "init", + ); + assert(init?.mcp_servers?.some((server) => server.name === "rea")); + assert( + init.mcp_servers.every((server) => + ["rea", "other"].includes(server.name), + ), + "Do not inherit foreign MCP configuration", + ); + assert( + events.some( + (event) => + event.type === "result" && + event.subtype === "success" && + event.is_error === false && + event.result === "REA_CLIENT_COMPATIBILITY_OK", + ), + ); + assert(finalSeen); + if (mode === "call") + assert( + skillLoaded && + catalogChecked && + analysisRequested && + evidenceSeen && + (!spillQueryRequested || projectionSeen) && + (!diagnosticsRequested || diagnosticsVerified), + ); + }, [ + () => commands.close(), + async () => { + await fixture?.close(); }, - modelFixture: - "loopback OpenAI completions/SSE, declared one-million-token custom model and synthetic usage; no live model claim", - discovery: - "native GROK_HOME, isolated additional skill directory, disabled foreign compatibility sources; default OS-home skill discovery and Windows unverified", - interface: - "default native search_tool/use_tool with a complete explicit inventory query; asynchronous partial discovery may trigger native REA diagnosis; large MCP outputs are queried via native terminal", - catalogChecked, - catalogSize: catalogChecked ? expectedNames.length : 0, - schemasChecked, - skillLoaded, - diagnosticsRequested, - diagnosticsVerified, - analysisRequested, - evidenceSeen, - spillQueryRequested, - projectionSeen, - retainedArtifact, - artifactBytes, - artifactDigest, - finalSeen, - evidenceScope: - "complete saved Evidence validated with its named schema; the model receives the selected export facts, subject and full artifact digest, not every offloaded graph fact", - requests: fixture.requests, - probes: fixture.probes, - fixtureFailure: fixture.failure, - verifier: lineage, - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; + ]); + passed = true; +}, [ + async () => { + const lineage = await completeVerifierRun(verifier); + const receipt = { + client: "Grok Build", + clientVersion, + mode, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback OpenAI completions/SSE, declared one-million-token custom model and synthetic usage; no live model claim", + discovery: + "native GROK_HOME, isolated additional skill directory, disabled foreign compatibility sources; default OS-home skill discovery and Windows unverified", + interface: + "default native search_tool/use_tool with a complete explicit inventory query; asynchronous partial discovery may trigger native REA diagnosis; large MCP outputs are queried via native terminal", + catalogChecked, + catalogSize, + schemasChecked, + skillLoaded, + diagnosticsRequested, + diagnosticsVerified, + analysisRequested, + evidenceSeen, + spillQueryRequested, + projectionSeen, + retainedArtifact, + artifactBytes, + artifactDigest, + finalSeen, + evidenceScope: + "complete saved Evidence validated with its named schema; the model receives the selected export facts, subject and full artifact digest, not every offloaded graph fact", + requests: fixture?.requests, + probes: fixture?.probes, + fixtureFailure: fixture?.failure, + verifier: lineage, + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); + }, + ], + ); + }, +]); diff --git a/scripts/verify-native-agent-client.mjs b/scripts/verify-native-agent-client.mjs index 9c05492d3b..b99337d651 100644 --- a/scripts/verify-native-agent-client.mjs +++ b/scripts/verify-native-agent-client.mjs @@ -9,6 +9,7 @@ import { toolContract, } from "../dist/contracts/toolContracts.js"; import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; @@ -418,36 +419,24 @@ await withVerifierCleanup(async () => { "--reasoning", "none", ]; - const result = await commands - .exec(command, args, { - env: { - ...environment, - OPENAI_API_KEY: "local-fixture-key", - OPENAI_BASE_URL: fixture.baseUrl, - }, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, - // Pi consumes piped input before its first prompt; an unused open pipe is not EOF. - stdin: "closed-pipe", - }) - .catch(async (cause) => { - const outputWriters = []; - if (typeof cause?.snapshot?.stdout?.text === "string") - outputWriters.push(() => - writeFile(join(lab, "client.stdout"), cause.snapshot.stdout.text), - ); - if (typeof cause?.snapshot?.stderr?.text === "string") - outputWriters.push(() => - writeFile(join(lab, "client.stderr"), cause.snapshot.stderr.text), - ); - return withVerifierCleanup(() => { - throw cause; - }, outputWriters); - }); - await withVerifierCleanup( - () => writeFile(join(lab, "client.stdout"), result.stdout), - [() => writeFile(join(lab, "client.stderr"), result.stderr)], + const result = await saveVerifierCommandOutput( + () => + commands.exec(command, args, { + env: { + ...environment, + OPENAI_API_KEY: "local-fixture-key", + OPENAI_BASE_URL: fixture.baseUrl, + }, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + // Pi consumes piped input before its first prompt; an unused open pipe is not EOF. + stdin: "closed-pipe", + }), + { + stdout: join(lab, "client.stdout"), + stderr: join(lab, "client.stderr"), + }, ); assert.equal(fixture.failure, undefined); assert(finalSeen); diff --git a/scripts/verify-omp-client.mjs b/scripts/verify-omp-client.mjs index f107ce9b06..973e85c47b 100644 --- a/scripts/verify-omp-client.mjs +++ b/scripts/verify-omp-client.mjs @@ -8,7 +8,9 @@ import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; -import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -37,40 +39,10 @@ const skillDirectory = join( "skills", "reverse-engineer-anything", ); -await Promise.all([ - mkdir(account), - mkdir(agent), - mkdir(workspace), - ...["data", "state", "cache"].map((name) => - mkdir(join(lab, name, "omp"), { recursive: true }), - ), -]); -await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); -// OMP's global config root uses native homedir independently of the agent override. -// A documented relative PI_CONFIG_DIR redirects it without changing HOME. -const environment = { - PATH: process.env.PATH, - NODE_OPTIONS: process.env.NODE_OPTIONS, - USERPROFILE: account, - PI_CODING_AGENT_DIR: agent, - PI_CONFIG_DIR: relative(homedir(), join(lab, "native-config")), - OMP_PROFILE: "", - XDG_DATA_HOME: join(lab, "data"), - XDG_STATE_HOME: join(lab, "state"), - XDG_CACHE_HOME: join(lab, "cache"), - REA_PROCESS_RUN_ID: verifier.run_id, -}; -const target = join(lab, "fixture α测试"); -await mkdir(target); -await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', -); const expectedDevices = TOOL_CONTRACTS.map( (tool) => "mcp__rea_" + tool.name, ).sort(); -const analysisDevice = "xd://mcp__rea_analyze_javascript_application"; -let clientVersion; +let clientVersion = null; let expectedSkillBody; let skillLoaded = false; let contractRead = false; @@ -78,365 +50,448 @@ let evidenceSeen = false; let artifactDigest; let finalSeen = false; let catalogMounted = false; +let catalogSize = null; let invalidProfileRejected = false; -let failure; -const containsText = (value, text) => { - if (typeof value === "string") return value.includes(text); - return ( - value !== null && - typeof value === "object" && - Object.values(value).some((item) => containsText(item, text)) - ); -}; -const call = (index, name, args) => ({ - role: "assistant", - content: null, - tool_calls: [ - { - id: `call_omp_${index}`, - type: "function", - function: { name, arguments: JSON.stringify(args) }, - }, - ], -}); -const fixture = await createOpenAiModelFixture({ - directory: lab, - prefix: "omp", - onRequest: async ({ body, tools, requestIndex }) => { - assert(tools.some((tool) => tool.name === "read")); - assert(tools.some((tool) => tool.name === "write")); - const instructions = body.messages.filter( - (message) => message.role === "system", - ); - for (const device of expectedDevices) - assert( - containsText(instructions, device), - `OMP must advertise mounted device ${device}`, - ); - if (mode === "chat") { - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - } - const replies = body.messages.filter((message) => message.role === "tool"); - if (requestIndex === 1) - return call(requestIndex, "read", { - i: "Load REA skill", - path: "skill://reverse-engineer-anything:raw", - }); - assert( - expectedSkillBody !== undefined && - containsText(replies, expectedSkillBody), - "OMP must return the complete installed skill body", - ); - skillLoaded = true; - if (requestIndex === 2) - return call(requestIndex, "read", { - i: "Inspect REA native tool contract", - path: analysisDevice, - }); - assert(containsText(replies, "# mcp__rea_analyze_javascript_application")); - assert( - containsText(replies, "input_path") && - containsText(replies, '"directory"'), - ); - contractRead = true; - if (requestIndex === 3) - return call(requestIndex, "write", { - path: analysisDevice, - content: JSON.stringify({ input_path: target, format: "directory" }), - }); - const evidenceReply = replies.find( - (reply) => - typeof reply.content === "string" && - reply.content.startsWith('{"evidence_id":'), - ); - assert( - evidenceReply, - "OMP must deliver the full analysis result to the next model turn", - ); - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(JSON.parse(evidenceReply.content)); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((node) => node.observations) - .map((observation) => observation.properties) - .find( - (properties) => - properties.semantic_role === "export-return-shapes" && - properties.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((item) => item.fields); - assert( - fields.some( - (field) => - field.path === "/clientCompatibility" && - field.state === "literal" && - field.value === "REA_CLIENT_FIXTURE", +let passed = false; +let fixture; +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([ + mkdir(account), + mkdir(agent), + mkdir(workspace), + ...["data", "state", "cache"].map((name) => + mkdir(join(lab, name, "omp"), { recursive: true }), ), - ); - assert( - fields.some( - (field) => - field.path === "/count" && - field.state === "literal" && - field.value === 7, - ), - ); - const bytes = JSON.stringify(evidence); - artifactDigest = createHash("sha256").update(bytes).digest("hex"); - await writeFile(join(lab, "omp-evidence.json"), bytes); - evidenceSeen = true; - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - }, -}); -const original = JSON.stringify( - { - mcpServers: { other: { command: "unrelated-server", enabled: false } }, - disabledServers: ["rea", "other"], - }, - null, - 2, -); -await writeFile(configPath, original); -await writeFile( - join(agent, "config.yml"), - JSON.stringify({ - skills: { customDirectories: [join(account, ".agents", "skills")] }, - retry: { fallbackChains: { default: [] } }, - }), -); -await writeFile( - join(agent, "models.yml"), - JSON.stringify({ - providers: { - "rea-fixture": { - baseUrl: fixture.baseUrl, - apiKey: "local-fixture-key", - api: "openai-completions", - models: [ - { - id: "rea-client-fixture", - name: "REA local fixture", - reasoning: false, - input: ["text"], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 1000000, - maxTokens: 4096, - }, - ], - }, - }, - }), -); -const rea = async (args, env = environment) => - exec( - process.execPath, - [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], - { env, cwd: workspace, timeout: 60_000 }, - ); -try { - clientVersion = ( - await exec(command, ["--version"], { env: environment, timeout: 30_000 }) - ).stdout.trim(); - const invalidEnvironment = { ...environment, OMP_PROFILE: "Bad Name" }; - const nativeRejected = await exec( - command, - ["--print", "--no-session", "--no-title", "Hello"], - { env: invalidEnvironment, cwd: workspace, timeout: 30_000 }, - ).then( - () => undefined, - (cause) => cause, - ); - assert( - nativeRejected?.code === 1 && - nativeRejected.stderr.includes('Invalid OMP profile "Bad Name"'), - ); - await writeFile( - join(lab, "invalid-profile-native.stderr"), - nativeRejected.stderr, - ); - const setupRejected = await rea( - ["setup", "--client", "omp", "--dry-run"], - invalidEnvironment, - ).then( - () => undefined, - (cause) => cause, - ); - assert(typeof setupRejected?.code === "number" && setupRejected.code !== 0); - const rejected = JSON.parse(setupRejected.stdout); - assert.equal(rejected.status, "needs_human"); - assert.deepEqual(rejected.plannedActions, []); - assert.deepEqual(rejected.appliedActions, []); - assert(rejected.remediation.includes('Invalid OMP profile "Bad Name"')); - assert.equal(await readFile(configPath, "utf8"), original); - await writeFile( - join(lab, "invalid-profile-setup.json"), - setupRejected.stdout, - ); - invalidProfileRejected = true; - for (const args of [ - ["setup", "--client", "omp", "--dry-run"], - ["setup", "--client", "omp", "--yes"], - ["doctor", "--client", "omp", "--skill"], - ]) { - const result = await rea(args); + ]); + await commands.exec("git", ["init", "--quiet", workspace], { + timeout: 30_000, + stdin: "pipe", + }); + // OMP's global config root uses native homedir independently of the agent override. + // A documented relative PI_CONFIG_DIR redirects it without changing HOME. + const environment = { + PATH: process.env.PATH, + NODE_OPTIONS: process.env.NODE_OPTIONS, + USERPROFILE: account, + PI_CODING_AGENT_DIR: agent, + PI_CONFIG_DIR: relative(homedir(), join(lab, "native-config")), + OMP_PROFILE: "", + XDG_DATA_HOME: join(lab, "data"), + XDG_STATE_HOME: join(lab, "state"), + XDG_CACHE_HOME: join(lab, "cache"), + REA_PROCESS_RUN_ID: verifier.run_id, + }; + const target = join(lab, "fixture α测试"); + await mkdir(target); await writeFile( - join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), - result.stdout, + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', ); - const parsed = JSON.parse(result.stdout); - if (args[0] === "doctor") assert.equal(parsed.healthy, true); - else { - assert.equal( - parsed.status, - args.includes("--dry-run") ? "planned" : "ready", + const analysisDevice = "xd://mcp__rea_analyze_javascript_application"; + const containsText = (value, text) => { + if (typeof value === "string") return value.includes(text); + return ( + value !== null && + typeof value === "object" && + Object.values(value).some((item) => containsText(item, text)) ); - if (args.includes("--dry-run")) { + }; + const call = (index, name, args) => ({ + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_omp_${index}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }); + fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: "omp", + onRequest: async ({ body, tools, requestIndex }) => { + assert(tools.some((tool) => tool.name === "read")); + assert(tools.some((tool) => tool.name === "write")); + const instructions = body.messages.filter( + (message) => message.role === "system", + ); + for (const device of expectedDevices) + assert( + containsText(instructions, device), + `OMP must advertise mounted device ${device}`, + ); + if (mode === "chat") { + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + const replies = body.messages.filter( + (message) => message.role === "tool", + ); + if (requestIndex === 1) + return call(requestIndex, "read", { + i: "Load REA skill", + path: "skill://reverse-engineer-anything:raw", + }); assert( - parsed.plannedActions.some((action) => action.target === configPath), + expectedSkillBody !== undefined && + containsText(replies, expectedSkillBody), + "OMP must return the complete installed skill body", + ); + skillLoaded = true; + if (requestIndex === 2) + return call(requestIndex, "read", { + i: "Inspect REA native tool contract", + path: analysisDevice, + }); + assert( + containsText(replies, "# mcp__rea_analyze_javascript_application"), ); assert( - parsed.plannedActions.some( - (action) => action.target === skillDirectory, + containsText(replies, "input_path") && + containsText(replies, '"directory"'), + ); + contractRead = true; + if (requestIndex === 3) + return call(requestIndex, "write", { + path: analysisDevice, + content: JSON.stringify({ + input_path: target, + format: "directory", + }), + }); + const evidenceReply = replies.find( + (reply) => + typeof reply.content === "string" && + reply.content.startsWith('{"evidence_id":'), + ); + assert( + evidenceReply, + "OMP must deliver the full analysis result to the next model turn", + ); + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(JSON.parse(evidenceReply.content)); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((node) => node.observations) + .map((observation) => observation.properties) + .find( + (properties) => + properties.semantic_role === "export-return-shapes" && + properties.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap( + (item) => item.fields, + ); + assert( + fields.some( + (field) => + field.path === "/clientCompatibility" && + field.state === "literal" && + field.value === "REA_CLIENT_FIXTURE", ), ); assert( - parsed.plannedActions.every( - (action) => - action.target === configPath || - action.target === skillDirectory || - action.target.startsWith(skillDirectory + "/"), + fields.some( + (field) => + field.path === "/count" && + field.state === "literal" && + field.value === 7, ), - "Refuse setup writes outside the owned config/skill", ); + const bytes = JSON.stringify(evidence); + artifactDigest = createHash("sha256").update(bytes).digest("hex"); + await writeFile(join(lab, "omp-evidence.json"), bytes); + evidenceSeen = true; + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + }, + }); + const original = JSON.stringify( + { + mcpServers: { other: { command: "unrelated-server", enabled: false } }, + disabledServers: ["rea", "other"], + }, + null, + 2, + ); + await writeFile(configPath, original); + await writeFile( + join(agent, "config.yml"), + JSON.stringify({ + skills: { customDirectories: [join(account, ".agents", "skills")] }, + retry: { fallbackChains: { default: [] } }, + }), + ); + await writeFile( + join(agent, "models.yml"), + JSON.stringify({ + providers: { + "rea-fixture": { + baseUrl: fixture.baseUrl, + apiKey: "local-fixture-key", + api: "openai-completions", + models: [ + { + id: "rea-client-fixture", + name: "REA local fixture", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1000000, + maxTokens: 4096, + }, + ], + }, + }, + }), + ); + const rea = async (args, env = environment) => + commands.exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + clientVersion = ( + await commands.exec(command, ["--version"], { + env: environment, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + const invalidEnvironment = { ...environment, OMP_PROFILE: "Bad Name" }; + const nativeRejected = await commands + .exec(command, ["--print", "--no-session", "--no-title", "Hello"], { + env: invalidEnvironment, + cwd: workspace, + timeout: 30_000, + stdin: "pipe", + }) + .then( + () => undefined, + (cause) => { + if (typeof cause?.code !== "number" || cause.code === 0) throw cause; + return cause; + }, + ); + assert( + nativeRejected?.code === 1 && + nativeRejected.snapshot.stderr.text.includes( + 'Invalid OMP profile "Bad Name"', + ), + ); + await writeFile( + join(lab, "invalid-profile-native.stderr"), + nativeRejected.snapshot.stderr.text, + ); + const setupRejected = await rea( + ["setup", "--client", "omp", "--dry-run"], + invalidEnvironment, + ).then( + () => undefined, + (cause) => { + if (typeof cause?.code !== "number" || cause.code === 0) throw cause; + return cause; + }, + ); + assert(typeof setupRejected?.code === "number" && setupRejected.code !== 0); + const rejected = JSON.parse(setupRejected.snapshot.stdout.text); + assert.equal(rejected.status, "needs_human"); + assert.deepEqual(rejected.plannedActions, []); + assert.deepEqual(rejected.appliedActions, []); + assert(rejected.remediation.includes('Invalid OMP profile "Bad Name"')); + assert.equal(await readFile(configPath, "utf8"), original); + await writeFile( + join(lab, "invalid-profile-setup.json"), + setupRejected.snapshot.stdout.text, + ); + invalidProfileRejected = true; + for (const args of [ + ["setup", "--client", "omp", "--dry-run"], + ["setup", "--client", "omp", "--yes"], + ["doctor", "--client", "omp", "--skill"], + ]) { + const result = await rea(args); + await writeFile( + join( + lab, + `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`, + ), + result.stdout, + ); + const parsed = JSON.parse(result.stdout); + if (args[0] === "doctor") assert.equal(parsed.healthy, true); + else { + assert.equal( + parsed.status, + args.includes("--dry-run") ? "planned" : "ready", + ); + if (args.includes("--dry-run")) { + assert( + parsed.plannedActions.some( + (action) => action.target === configPath, + ), + ); + assert( + parsed.plannedActions.some( + (action) => action.target === skillDirectory, + ), + ); + assert( + parsed.plannedActions.every( + (action) => + action.target === configPath || + action.target === skillDirectory || + action.target.startsWith(skillDirectory + "/"), + ), + "Refuse setup writes outside the owned config/skill", + ); + } } } - } - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const configured = await readFile(configPath, "utf8"); - const registration = JSON.parse(configured); - assert.equal(registration.mcpServers.other.command, "unrelated-server"); - assert.deepEqual(registration.disabledServers, ["other"]); - const repeated = JSON.parse( - (await rea(["setup", "--client", "omp", "--yes"])).stdout, - ); - assert.deepEqual(repeated.appliedActions, []); - assert.equal(await readFile(configPath, "utf8"), configured); - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - expectedSkillBody = await readFile(join(skillDirectory, "SKILL.md"), "utf8"); - const execution = exec( - command, - [ - "--mode", - "json", - "--print", - "--no-session", - "--no-title", - "--no-lsp", - "--no-pty", - "--approval-mode", - "yolo", - "--model", - "rea-fixture/rea-client-fixture", - mode === "chat" - ? "Say hello briefly." - : `Use the REA skill to analyze ${target}.`, - ], - { - env: environment, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + const registration = JSON.parse(configured); + assert.equal(registration.mcpServers.other.command, "unrelated-server"); + assert.deepEqual(registration.disabledServers, ["other"]); + const repeated = JSON.parse( + (await rea(["setup", "--client", "omp", "--yes"])).stdout, + ); + assert.deepEqual(repeated.appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + expectedSkillBody = await readFile( + join(skillDirectory, "SKILL.md"), + "utf8", + ); + const response = await saveVerifierCommandOutput( + () => + commands.exec( + command, + [ + "--mode", + "json", + "--print", + "--no-session", + "--no-title", + "--no-lsp", + "--no-pty", + "--approval-mode", + "yolo", + "--model", + "rea-fixture/rea-client-fixture", + mode === "chat" + ? "Say hello briefly." + : `Use the REA skill to analyze ${target}.`, + ], + { + env: environment, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + stdin: "closed-pipe", + }, + ), + { + stdout: join(lab, `${mode}.stdout`), + stderr: join(lab, `${mode}.stderr`), + }, + ); + assert.equal(fixture.failure, undefined); + const events = response.stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + const mounted = events + .filter( + (event) => + event.type === "notice" && + event.message?.startsWith("xd://: mounted "), + ) + .flatMap((event) => + event.message.slice("xd://: mounted ".length).split(", "), + ) + .filter((name) => name.startsWith("mcp__rea_")); + catalogSize = mounted.length; + assert.deepEqual( + mounted.sort(), + expectedDevices, + "Native OMP must mount the complete REA catalog", + ); + catalogMounted = true; + assert( + !events.some( + (event) => event.type === "tool_execution_end" && event.isError, + ), + ); + assert( + events.some( + (event) => + event.type === "message_end" && + event.message?.role === "assistant" && + event.message.stopReason === "stop" && + containsText(event.message.content, "REA_CLIENT_COMPATIBILITY_OK"), + ), + "OMP must finish with the expected native assistant message", + ); + assert( + events.some((event) => event.type === "agent_end" && event.isTerminal), + ); + assert(finalSeen); + if (mode === "call") assert(skillLoaded && contractRead && evidenceSeen); + }, [ + () => commands.close(), + async () => { + await fixture?.close(); }, - ); - execution.child.stdin.end(); - const response = await execution; - await writeFile(join(lab, `${mode}.stdout`), response.stdout); - await writeFile(join(lab, `${mode}.stderr`), response.stderr); - assert.equal(fixture.failure, undefined); - const events = response.stdout - .trim() - .split("\n") - .map((line) => JSON.parse(line)); - const mounted = events - .filter( - (event) => - event.type === "notice" && event.message?.startsWith("xd://: mounted "), - ) - .flatMap((event) => - event.message.slice("xd://: mounted ".length).split(", "), - ) - .filter((name) => name.startsWith("mcp__rea_")); - assert.deepEqual( - mounted.sort(), - expectedDevices, - "Native OMP must mount the complete REA catalog", - ); - catalogMounted = true; - assert( - !events.some( - (event) => event.type === "tool_execution_end" && event.isError, - ), - ); - assert( - events.some( - (event) => - event.type === "message_end" && - event.message?.role === "assistant" && - event.message.stopReason === "stop" && - containsText(event.message.content, "REA_CLIENT_COMPATIBILITY_OK"), - ), - "OMP must finish with the expected native assistant message", - ); - assert( - events.some((event) => event.type === "agent_end" && event.isTerminal), - ); - assert(finalSeen); - if (mode === "call") assert(skillLoaded && contractRead && evidenceSeen); -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, `${mode}.stdout`), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, `${mode}.stderr`), cause.stderr); -} finally { - await fixture.close(); - const lineage = await completeVerifierRun(verifier); - const receipt = { - client: "OMP", - clientVersion, - mode, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, - }, - modelFixture: - "loopback OpenAI completions/SSE, caller-declared one-million-token model and synthetic usage; no live model claim", - discovery: - "explicit isolated default agent/global config/XDG roots and custom skill directory; named profiles, default OS-home skill discovery and Windows unverified", - interface: - "default native xd:// device metadata with read/write dispatch; complete JSON schemas are not forwarded as model functions", - catalogMounted, - catalogSize: expectedDevices.length, - invalidProfileRejected, - skillLoaded, - contractRead, - evidenceSeen, - artifactDigest, - finalSeen, - requests: fixture.requests, - probes: fixture.probes, - fixtureFailure: fixture.failure, - verifier: lineage, - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; + ]); + passed = true; +}, [ + async () => { + const lineage = await completeVerifierRun(verifier); + const receipt = { + client: "OMP", + clientVersion, + mode, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback OpenAI completions/SSE, caller-declared one-million-token model and synthetic usage; no live model claim", + discovery: + "explicit isolated default agent/global config/XDG roots and custom skill directory; named profiles, default OS-home skill discovery and Windows unverified", + interface: + "default native xd:// device metadata with read/write dispatch; complete JSON schemas are not forwarded as model functions", + catalogMounted, + catalogSize, + invalidProfileRejected, + skillLoaded, + contractRead, + evidenceSeen, + artifactDigest, + finalSeen, + requests: fixture?.requests, + probes: fixture?.probes, + fixtureFailure: fixture?.failure, + verifier: lineage, + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); + }, + ], + ); + }, +]); diff --git a/scripts/verify-opencode-client.mjs b/scripts/verify-opencode-client.mjs index 2a3e4c0501..6ea41ef3bc 100644 --- a/scripts/verify-opencode-client.mjs +++ b/scripts/verify-opencode-client.mjs @@ -9,7 +9,9 @@ import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; -import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -58,335 +60,382 @@ assert.equal( isManagedSkillTarget(join(skillDirectory, "..", "outside", "AGENTS.md")), false, ); -await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]); -await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); -const environment = { - USERPROFILE: account, - OPENCODE_CONFIG_DIR: profile, - XDG_CONFIG_HOME: join(lab, "config"), - XDG_CACHE_HOME: join(lab, "cache"), - XDG_DATA_HOME: join(lab, "data"), - XDG_STATE_HOME: join(lab, "state"), - OPENCODE_DISABLE_AUTOUPDATE: "true", - OPENCODE_DISABLE_MODELS_FETCH: "true", - // Use the native core without external account plugins. - OPENCODE_PURE: "true", - REA_PROCESS_RUN_ID: verifier.run_id, -}; -let clientVersion; -try { - clientVersion = ( - await exec(command, ["--version"], { env: environment, timeout: 30_000 }) - ).stdout.trim(); -} catch (cause) { - throw new Error( - `OpenCode preflight failed for ${command}; install OpenCode or set REA_VERIFY_OPENCODE_COMMAND. ${String(cause)}`, - ); -} -const target = join(lab, "fixture α测试"); -await mkdir(target); -await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', -); +let clientVersion = null; +let fixture; +let passed = false; let skillRequested = false; let skillLoaded = false; let expectedSkillBody; let evidenceSeen = false; let finalSeen = false; -let catalogSize = 0; +let catalogSize = null; let schemasChecked = 0; let artifactDigest; -const schemaValidator = new Ajv2020({ strict: false, validateFormats: false }); -const extractEvidence = (value) => { - if (typeof value === "string") { - try { - return extractEvidence(JSON.parse(value)); - } catch { - return undefined; - } - } - if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0 - ) - return value; - for (const nested of Object.values(value)) { - const evidence = extractEvidence(nested); - if (evidence) return evidence; - } - } - return undefined; -}; -const fixture = await createOpenAiModelFixture({ - directory: lab, - prefix: "opencode", - onRequest: async ({ body, tools, requestIndex }) => { - // Native session-title requests do not execute the analyst's workflow. - if (tools.length === 0) - return { role: "assistant", content: "REA fixture title" }; - const analysis = tools.find((t) => - t.name.endsWith("analyze_javascript_application"), - ); - assert(analysis, "OpenCode must expose the REA analysis tool"); - const prefix = analysis.name.slice( - 0, - -"analyze_javascript_application".length, - ); - const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort(); - const reaTools = tools.filter((t) => expected.includes(t.name)); - assert.deepEqual( - reaTools.map((t) => t.name).sort(), - expected, - "OpenCode must forward the complete REA catalog", - ); - catalogSize = expected.length; - if (schemasChecked === 0) { - for (const tool of reaTools) { - assert( - tool.parameters, - `${tool.name}: missing advertised input schema`, - ); - assert( - schemaValidator.validateSchema(tool.parameters), - `${tool.name}: invalid input schema: ${JSON.stringify(schemaValidator.errors)}`, - ); - schemasChecked++; - } - } - const replies = body.messages.filter((m) => m.role === "tool"); - skillLoaded ||= replies.some( - (m) => - typeof m.content === "string" && - expectedSkillBody !== undefined && - m.content.includes(skillDirectory) && - m.content.includes(expectedSkillBody), - ); - const observed = extractEvidence(replies); - if (observed) { - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(observed); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((n) => n.observations) - .map((o) => o.properties) - .find( - (p) => - p.semantic_role === "export-return-shapes" && - p.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((s) => s.fields); - assert( - fields.some( - (f) => - f.path === "/clientCompatibility" && - f.state === "literal" && - f.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (f) => f.path === "/count" && f.state === "literal" && f.value === 7, - ), - ); - const bytes = JSON.stringify(evidence); - artifactDigest = createHash("sha256").update(bytes).digest("hex"); - await writeFile(join(lab, "opencode-evidence.json"), bytes); - evidenceSeen = true; - } - if (mode === "chat" || evidenceSeen) { - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - } - assert(requestIndex <= 8, "Local model workflow exceeded its turn budget"); - const call = (name, args) => ({ - role: "assistant", - content: null, - tool_calls: [ - { - id: `call_rea_fixture_${requestIndex}`, - type: "function", - function: { name, arguments: JSON.stringify(args) }, - }, - ], +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]); + await commands.exec("git", ["init", "--quiet", workspace], { + timeout: 30_000, + stdin: "pipe", }); - if (!skillRequested) { - const skill = tools.find((t) => t.name === "skill"); - assert(skill, "OpenCode must expose its native skill tool"); - skillRequested = true; - return call(skill.name, { name: "reverse-engineer-anything" }); + const environment = { + USERPROFILE: account, + OPENCODE_CONFIG_DIR: profile, + XDG_CONFIG_HOME: join(lab, "config"), + XDG_CACHE_HOME: join(lab, "cache"), + XDG_DATA_HOME: join(lab, "data"), + XDG_STATE_HOME: join(lab, "state"), + OPENCODE_DISABLE_AUTOUPDATE: "true", + OPENCODE_DISABLE_MODELS_FETCH: "true", + // Use the native core without external account plugins. + OPENCODE_PURE: "true", + REA_PROCESS_RUN_ID: verifier.run_id, + }; + try { + clientVersion = ( + await commands.exec(command, ["--version"], { + env: environment, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + } catch (cause) { + throw new Error( + `OpenCode preflight failed for ${command}; install OpenCode or set REA_VERIFY_OPENCODE_COMMAND. ${String(cause)}`, + { cause }, + ); } - assert(skillLoaded, "OpenCode must load the installed personal REA skill"); - return call(analysis.name, { input_path: target, format: "directory" }); - }, -}); -const original = - (configFormat === "jsonc" ? "// Keep caller configuration.\n" : "") + - JSON.stringify( - { - autoupdate: false, - skills: { paths: [join(account, ".agents", "skills")] }, - permission: { "*": "allow" }, - provider: { - "rea-fixture": { - npm: "@ai-sdk/openai-compatible", - name: "REA local model fixture", - options: { baseURL: fixture.baseUrl, apiKey: "local-fixture-key" }, - models: { - "rea-client-fixture": { - name: "REA fixture", - tool_call: true, - limit: { context: 1_000_000, output: 4096 }, + const target = join(lab, "fixture α测试"); + await mkdir(target); + await writeFile( + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', + ); + const schemaValidator = new Ajv2020({ + strict: false, + validateFormats: false, + }); + const extractEvidence = (value) => { + if (typeof value === "string") { + try { + return extractEvidence(JSON.parse(value)); + } catch { + return undefined; + } + } + if (value && typeof value === "object") { + if ( + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes?.length > 0 + ) + return value; + for (const nested of Object.values(value)) { + const evidence = extractEvidence(nested); + if (evidence) return evidence; + } + } + return undefined; + }; + fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: "opencode", + onRequest: async ({ body, tools, requestIndex }) => { + // Native session-title requests do not execute the analyst's workflow. + if (tools.length === 0) + return { role: "assistant", content: "REA fixture title" }; + const analysis = tools.find((t) => + t.name.endsWith("analyze_javascript_application"), + ); + assert(analysis, "OpenCode must expose the REA analysis tool"); + const prefix = analysis.name.slice( + 0, + -"analyze_javascript_application".length, + ); + const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort(); + const reaTools = tools.filter((t) => expected.includes(t.name)); + catalogSize = reaTools.length; + assert.deepEqual( + reaTools.map((t) => t.name).sort(), + expected, + "OpenCode must forward the complete REA catalog", + ); + if (schemasChecked === 0) { + for (const tool of reaTools) { + assert( + tool.parameters, + `${tool.name}: missing advertised input schema`, + ); + assert( + schemaValidator.validateSchema(tool.parameters), + `${tool.name}: invalid input schema: ${JSON.stringify(schemaValidator.errors)}`, + ); + schemasChecked++; + } + } + const replies = body.messages.filter((m) => m.role === "tool"); + skillLoaded ||= replies.some( + (m) => + typeof m.content === "string" && + expectedSkillBody !== undefined && + m.content.includes(skillDirectory) && + m.content.includes(expectedSkillBody), + ); + const observed = extractEvidence(replies); + if (observed) { + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(observed); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((n) => n.observations) + .map((o) => o.properties) + .find( + (p) => + p.semantic_role === "export-return-shapes" && + p.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap((s) => s.fields); + assert( + fields.some( + (f) => + f.path === "/clientCompatibility" && + f.state === "literal" && + f.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (f) => + f.path === "/count" && f.state === "literal" && f.value === 7, + ), + ); + const bytes = JSON.stringify(evidence); + artifactDigest = createHash("sha256").update(bytes).digest("hex"); + await writeFile(join(lab, "opencode-evidence.json"), bytes); + evidenceSeen = true; + } + if (mode === "chat" || evidenceSeen) { + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + assert( + requestIndex <= 8, + "Local model workflow exceeded its turn budget", + ); + const call = (name, args) => ({ + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_rea_fixture_${requestIndex}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }); + if (!skillRequested) { + const skill = tools.find((t) => t.name === "skill"); + assert(skill, "OpenCode must expose its native skill tool"); + skillRequested = true; + return call(skill.name, { name: "reverse-engineer-anything" }); + } + assert( + skillLoaded, + "OpenCode must load the installed personal REA skill", + ); + return call(analysis.name, { input_path: target, format: "directory" }); + }, + }); + const original = + (configFormat === "jsonc" ? "// Keep caller configuration.\n" : "") + + JSON.stringify( + { + autoupdate: false, + skills: { paths: [join(account, ".agents", "skills")] }, + permission: { "*": "allow" }, + provider: { + "rea-fixture": { + npm: "@ai-sdk/openai-compatible", + name: "REA local model fixture", + options: { + baseURL: fixture.baseUrl, + apiKey: "local-fixture-key", + }, + models: { + "rea-client-fixture": { + name: "REA fixture", + tool_call: true, + limit: { context: 1_000_000, output: 4096 }, + }, + }, + }, + }, + mcp: { + other: { + type: "local", + command: ["unrelated-server"], + enabled: false, }, }, }, - }, - mcp: { - other: { type: "local", command: ["unrelated-server"], enabled: false }, - }, - }, - null, - 2, - ); -await writeFile(configPath, original); -let failure; -try { - for (const args of [ - ["setup", "--client", "opencode", "--dry-run"], - ["setup", "--client", "opencode", "--yes"], - ["doctor", "--client", "opencode", "--skill"], - ]) { - const result = await exec( - process.execPath, - [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); - await writeFile( - join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), - result.stdout, - ); - const parsed = JSON.parse(result.stdout); - if (args[0] === "doctor") assert.equal(parsed.healthy, true); - else { - assert.equal( - parsed.status, - args.includes("--dry-run") ? "planned" : "ready", + null, + 2, ); - if (args.includes("--dry-run")) { - assert(parsed.plannedActions.some((a) => a.target === configPath)); - assert(parsed.plannedActions.some((a) => a.target === skillDirectory)); - assert( - parsed.plannedActions.every( - (a) => a.target === configPath || isManagedSkillTarget(a.target), - ), + await writeFile(configPath, original); + for (const args of [ + ["setup", "--client", "opencode", "--dry-run"], + ["setup", "--client", "opencode", "--yes"], + ["doctor", "--client", "opencode", "--skill"], + ]) { + const result = await commands.exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + await writeFile( + join( + lab, + `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`, + ), + result.stdout, + ); + const parsed = JSON.parse(result.stdout); + if (args[0] === "doctor") assert.equal(parsed.healthy, true); + else { + assert.equal( + parsed.status, + args.includes("--dry-run") ? "planned" : "ready", ); + if (args.includes("--dry-run")) { + assert(parsed.plannedActions.some((a) => a.target === configPath)); + assert( + parsed.plannedActions.some((a) => a.target === skillDirectory), + ); + assert( + parsed.plannedActions.every( + (a) => a.target === configPath || isManagedSkillTarget(a.target), + ), + ); + } } } - } - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const configured = await readFile(configPath, "utf8"); - assert(configured.includes("unrelated-server")); - if (configFormat === "jsonc") - assert(configured.includes("// Keep caller configuration.")); - const repeated = await exec( - process.execPath, - [ - join(runtimeRoot, "scripts/rea.mjs"), - "setup", - "--client", - "opencode", - "--yes", - "--json", - ], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); - assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); - assert.equal(await readFile(configPath, "utf8"), configured); - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const installedSkill = await readFile( - join(skillDirectory, "SKILL.md"), - "utf8", - ); - assert(installedSkill.includes("name: reverse-engineer-anything")); - const skillLines = installedSkill.split("\n"); - const headerEnd = skillLines.indexOf("---", 1); - assert( - skillLines[0] === "---" && headerEnd > 0, - "Installed skill must have its authored frontmatter", - ); - expectedSkillBody = skillLines - .slice(headerEnd + 1) - .join("\n") - .trim(); - assert(expectedSkillBody.length > 0); - const execution = exec( - command, - [ - "run", - "--format", - "json", - "--model", - "rea-fixture/rea-client-fixture", - mode === "chat" - ? "Say hello briefly." - : `Use REA to analyze the JavaScript directory ${target}. Report the result.`, - ], - { - env: environment, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + assert(configured.includes("unrelated-server")); + if (configFormat === "jsonc") + assert(configured.includes("// Keep caller configuration.")); + const repeated = await commands.exec( + process.execPath, + [ + join(runtimeRoot, "scripts/rea.mjs"), + "setup", + "--client", + "opencode", + "--yes", + "--json", + ], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const installedSkill = await readFile( + join(skillDirectory, "SKILL.md"), + "utf8", + ); + assert(installedSkill.includes("name: reverse-engineer-anything")); + const skillLines = installedSkill.split("\n"); + const headerEnd = skillLines.indexOf("---", 1); + assert( + skillLines[0] === "---" && headerEnd > 0, + "Installed skill must have its authored frontmatter", + ); + expectedSkillBody = skillLines + .slice(headerEnd + 1) + .join("\n") + .trim(); + assert(expectedSkillBody.length > 0); + const response = await saveVerifierCommandOutput( + () => + commands.exec( + command, + [ + "run", + "--format", + "json", + "--model", + "rea-fixture/rea-client-fixture", + mode === "chat" + ? "Say hello briefly." + : `Use REA to analyze the JavaScript directory ${target}. Report the result.`, + ], + { + env: environment, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + stdin: "closed-pipe", + }, + ), + { + stdout: join(lab, `${mode}.stdout`), + stderr: join(lab, `${mode}.stderr`), + }, + ); + assert.equal(fixture.failure, undefined); + assert( + finalSeen && response.stdout.includes("REA_CLIENT_COMPATIBILITY_OK"), + ); + if (mode === "call") assert(evidenceSeen && skillLoaded); + }, [ + () => commands.close(), + async () => { + await fixture?.close(); }, - ); - execution.child.stdin.end(); - const response = await execution; - await writeFile(join(lab, `${mode}.stdout`), response.stdout); - await writeFile(join(lab, `${mode}.stderr`), response.stderr); - assert.equal(fixture.failure, undefined); - assert(finalSeen && response.stdout.includes("REA_CLIENT_COMPATIBILITY_OK")); - if (mode === "call") assert(evidenceSeen && skillLoaded); -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, `${mode}.stdout`), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, `${mode}.stderr`), cause.stderr); -} finally { - await fixture.close(); - const lineage = await completeVerifierRun(verifier); - const receipt = { - client: "OpenCode", - clientVersion, - mode, - configFormat, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, - }, - modelFixture: - "loopback custom OpenAI-compatible provider; one-million-token fixture capacity and synthetic usage; no live provider claim", - skillDiscovery: - "caller-configured isolated shared directory; default OS home discovery unverified", - externalPlugins: "disabled through native OPENCODE_PURE", - catalogSize, - schemasChecked, - evidenceSeen, - skillLoaded, - artifactDigest, - finalSeen, - requests: fixture.requests, - probes: fixture.probes, - fixtureFailure: fixture.failure, - verifier: lineage, - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; + ]); + passed = true; +}, [ + async () => { + const lineage = await completeVerifierRun(verifier); + const receipt = { + client: "OpenCode", + clientVersion, + mode, + configFormat, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback custom OpenAI-compatible provider; one-million-token fixture capacity and synthetic usage; no live provider claim", + skillDiscovery: + "caller-configured isolated shared directory; default OS home discovery unverified", + externalPlugins: "disabled through native OPENCODE_PURE", + catalogSize, + schemasChecked, + evidenceSeen, + skillLoaded, + artifactDigest, + finalSeen, + requests: fixture?.requests, + probes: fixture?.probes, + fixtureFailure: fixture?.failure, + verifier: lineage, + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); + }, + ], + ); + }, +]); diff --git a/scripts/verify-qwen-client.mjs b/scripts/verify-qwen-client.mjs index 50001103c8..b92619721d 100644 --- a/scripts/verify-qwen-client.mjs +++ b/scripts/verify-qwen-client.mjs @@ -11,7 +11,9 @@ import { tmpdir } from "node:os"; import { join, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; import { createHash } from "node:crypto"; -import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { @@ -42,93 +44,9 @@ const account = join(lab, "account"); const runtime = join(profile, "runtime"); const workspace = join(lab, "workspace"); const sharedSkills = join(account, ".agents", "skills"); -await Promise.all([ - mkdir(profile, { recursive: true }), - mkdir(account), - mkdir(workspace), -]); -const baseEnvironment = { - PATH: process.env.PATH ?? "", - HOME: account, - USERPROFILE: account, - QWEN_HOME: profile, - QWEN_RUNTIME_DIR: runtime, - REA_PROCESS_RUN_ID: verifier.run_id, -}; -let clientVersion; -try { - clientVersion = ( - await exec(...launch(["--version"]), { - env: baseEnvironment, - timeout: 30_000, - }) - ).stdout.trim(); -} catch (cause) { - const windowsHint = - process.platform === "win32" - ? " On Windows, select node_modules/@qwen-code/qwen-code/cli-entry.js." - : ""; - throw new Error( - `Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND.${windowsHint} ${String(cause)}`, - ); -} -const original = JSON.stringify( - { - security: { - auth: { selectedType: "openai" }, - folderTrust: { enabled: false }, - }, - telemetry: { enabled: false }, - model: { name: "rea-client-fixture" }, - mcpServers: { other: { command: "unrelated-server", disabled: true } }, - }, - null, - 2, -); -await writeFile(join(profile, "settings.json"), original); -for (const step of [ - ["setup", "--client", "qwen_code", "--dry-run"], - ["setup", "--client", "qwen_code", "--yes"], - ["doctor", "--client", "qwen_code", "--skill"], -]) { - const response = await exec( - process.execPath, - [join(runtimeRoot, "scripts/rea.mjs"), ...step, "--json"], - { env: baseEnvironment, cwd: repo, timeout: 60_000 }, - ); - const result = JSON.parse(response.stdout); - if (step.includes("--dry-run")) - assert( - result.plannedActions.every( - (a) => - a.target.startsWith(profile + sep) || - a.target.startsWith(sharedSkills + sep), - ), - ); - await writeFile( - join(lab, `${step[0]}${step.includes("--dry-run") ? "-plan" : ""}.json`), - response.stdout, - ); -} -assert.equal( - await readFile(join(profile, "settings.json.rea.backup"), "utf8"), - original, -); -const configured = JSON.parse( - await readFile(join(profile, "settings.json"), "utf8"), -); -assert.deepEqual(configured.model, { name: "rea-client-fixture" }); -assert.equal(configured.skills, undefined); -assert.deepEqual(configured.mcpServers.other, { - command: "unrelated-server", - disabled: true, -}); -const target = join(lab, "fixture α测试"); -await mkdir(target, { recursive: true }); -await writeFile( - `${target}/app.js`, - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', -); +let clientVersion = null; +let fixture; +let passed = false; let evidenceSeen = false; let skillSeen = false; let finalSeen = false; @@ -138,278 +56,395 @@ let artifactReadByAgent = false; let artifactPath; let selectedSkillLoaded = false; let artifactDigest; -const extractEvidence = (value) => { - if (typeof value === "string") { +let catalogSize = null; +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([ + mkdir(profile, { recursive: true }), + mkdir(account), + mkdir(workspace), + ]); + const baseEnvironment = { + PATH: process.env.PATH ?? "", + HOME: account, + USERPROFILE: account, + QWEN_HOME: profile, + QWEN_RUNTIME_DIR: runtime, + REA_PROCESS_RUN_ID: verifier.run_id, + }; try { - return extractEvidence(JSON.parse(value)); - } catch { - return null; - } - } - if (Array.isArray(value)) { - for (const item of value) { - const result = extractEvidence(item); - if (result) return result; - } - } else if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0 - ) - return value; - for (const nested of Object.values(value)) { - const result = extractEvidence(nested); - if (result) return result; - } - } - return null; -}; -const fixture = await createOpenAiModelFixture({ - directory: lab, - prefix: `${client}-${mode}`, - onRequest: async ({ body, tools, requestIndex }) => { - skillSeen ||= JSON.stringify(body.messages ?? []).includes( - "reverse-engineer-anything", - ); - const messageText = (body.messages ?? []) - .map((message) => - typeof message.content === "string" - ? message.content - : (message.content ?? []).map((part) => part.text ?? "").join("\n"), - ) - .join("\n"); - selectedSkillLoaded ||= messageText.includes( - join(sharedSkills, "reverse-engineer-anything"), - ); - const toolText = JSON.stringify( - (body.messages ?? []).filter((m) => m.role === "tool"), - ); - if ( - toolText.includes("REA_CLIENT_FIXTURE") && - /(?:\\?"REA_CLIENT_FIXTURE\\?"\s*:\s*true)/u.test(toolText) - ) { - artifactReadByAgent = true; - assert(artifactPath); - const artifact = await readFile(artifactPath); - artifactDigest = createHash("sha256").update(artifact).digest("hex"); - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(JSON.parse(artifact.toString("utf8"))); - assert(extractEvidence(evidence)); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((n) => n.observations) - .map((o) => o.properties) - .find( - (p) => - p.semantic_role === "export-return-shapes" && - p.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((s) => s.fields); - assert( - fields.some( - (f) => - f.path === "/clientCompatibility" && - f.state === "literal" && - f.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (f) => f.path === "/count" && f.state === "literal" && f.value === 7, - ), - ); - evidenceSeen = true; - await writeFile( - `${lab}/${client}-${mode}-evidence.json`, - JSON.stringify(evidence), + clientVersion = ( + await commands.exec(...launch(["--version"]), { + env: baseEnvironment, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + } catch (cause) { + const windowsHint = + process.platform === "win32" + ? " On Windows, select node_modules/@qwen-code/qwen-code/cli-entry.js." + : ""; + throw new Error( + `Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND.${windowsHint} ${String(cause)}`, + { cause }, ); } - for (const message of body.messages ?? []) { - if (message.role !== "tool") continue; - const evidence = extractEvidence(message.content); - if (evidence) { + const original = JSON.stringify( + { + security: { + auth: { selectedType: "openai" }, + folderTrust: { enabled: false }, + }, + telemetry: { enabled: false }, + model: { name: "rea-client-fixture" }, + mcpServers: { other: { command: "unrelated-server", disabled: true } }, + }, + null, + 2, + ); + await writeFile(join(profile, "settings.json"), original); + for (const step of [ + ["setup", "--client", "qwen_code", "--dry-run"], + ["setup", "--client", "qwen_code", "--yes"], + ["doctor", "--client", "qwen_code", "--skill"], + ]) { + const response = await commands.exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...step, "--json"], + { env: baseEnvironment, cwd: repo, timeout: 60_000, stdin: "pipe" }, + ); + const result = JSON.parse(response.stdout); + if (step.includes("--dry-run")) assert( - JSON.stringify(evidence.normalized_result).includes( - "REA_CLIENT_FIXTURE", + result.plannedActions.every( + (a) => + a.target.startsWith(profile + sep) || + a.target.startsWith(sharedSkills + sep), ), ); - evidenceSeen = true; - await writeFile( - `${lab}/${client}-${mode}-evidence.json`, - JSON.stringify(evidence), - ); - } + await writeFile( + join( + lab, + `${step[0]}${step.includes("--dry-run") ? "-plan" : ""}.json`, + ), + response.stdout, + ); } - const selected = tools.find((t) => - t.name.includes("analyze_javascript_application"), + assert.equal( + await readFile(join(profile, "settings.json.rea.backup"), "utf8"), + original, ); - const reviewedNames = [ - ...JSON.stringify( - (body.messages ?? []).filter((m) => m.role === "tool"), - ).matchAll( - /(?:\\?"name\\?"\s*:\s*\\?")([^"\\]*analyze_javascript_application)(?:\\?")/g, - ), - ] - .map((m) => m[1]) - .sort((a, b) => b.length - a.length); - const search = tools.find((t) => t.name === "tool_search"); - const invoke = tools.find((t) => t.name === "tool_call"); - const skill = tools.find((t) => t.name === "skill"); - const shell = tools.find((t) => t.name === "run_shell_command"); - const persisted = messageText.match(/Full output saved to: ([^\r\n]+)/u); - const call = (name, args) => ({ - role: "assistant", - content: null, - tool_calls: [ - { - id: `call_rea_fixture_${requestIndex}`, - type: "function", - function: { name, arguments: JSON.stringify(args) }, + const configured = JSON.parse( + await readFile(join(profile, "settings.json"), "utf8"), + ); + assert.deepEqual(configured.model, { name: "rea-client-fixture" }); + assert.equal(configured.skills, undefined); + assert.deepEqual(configured.mcpServers.other, { + command: "unrelated-server", + disabled: true, + }); + const target = join(lab, "fixture α测试"); + await mkdir(target, { recursive: true }); + await writeFile( + `${target}/app.js`, + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', + ); + const extractEvidence = (value) => { + if (typeof value === "string") { + try { + return extractEvidence(JSON.parse(value)); + } catch { + return null; + } + } + if (Array.isArray(value)) { + for (const item of value) { + const result = extractEvidence(item); + if (result) return result; + } + } else if (value && typeof value === "object") { + if ( + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes?.length > 0 + ) + return value; + for (const nested of Object.values(value)) { + const result = extractEvidence(nested); + if (result) return result; + } + } + return null; + }; + fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: `${client}-${mode}`, + onRequest: async ({ body, tools, requestIndex }) => { + skillSeen ||= JSON.stringify(body.messages ?? []).includes( + "reverse-engineer-anything", + ); + const messageText = (body.messages ?? []) + .map((message) => + typeof message.content === "string" + ? message.content + : (message.content ?? []) + .map((part) => part.text ?? "") + .join("\n"), + ) + .join("\n"); + selectedSkillLoaded ||= messageText.includes( + join(sharedSkills, "reverse-engineer-anything"), + ); + const toolText = JSON.stringify( + (body.messages ?? []).filter((m) => m.role === "tool"), + ); + if ( + toolText.includes("REA_CLIENT_FIXTURE") && + /(?:\\?"REA_CLIENT_FIXTURE\\?"\s*:\s*true)/u.test(toolText) + ) { + artifactReadByAgent = true; + assert(artifactPath); + const artifact = await readFile(artifactPath); + artifactDigest = createHash("sha256").update(artifact).digest("hex"); + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(JSON.parse(artifact.toString("utf8"))); + assert(extractEvidence(evidence)); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((n) => n.observations) + .map((o) => o.properties) + .find( + (p) => + p.semantic_role === "export-return-shapes" && + p.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap((s) => s.fields); + assert( + fields.some( + (f) => + f.path === "/clientCompatibility" && + f.state === "literal" && + f.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (f) => + f.path === "/count" && f.state === "literal" && f.value === 7, + ), + ); + evidenceSeen = true; + await writeFile( + `${lab}/${client}-${mode}-evidence.json`, + JSON.stringify(evidence), + ); + } + for (const message of body.messages ?? []) { + if (message.role !== "tool") continue; + const evidence = extractEvidence(message.content); + if (evidence) { + assert( + JSON.stringify(evidence.normalized_result).includes( + "REA_CLIENT_FIXTURE", + ), + ); + evidenceSeen = true; + await writeFile( + `${lab}/${client}-${mode}-evidence.json`, + JSON.stringify(evidence), + ); + } + } + const selected = tools.find((t) => + t.name.includes("analyze_javascript_application"), + ); + const reviewedNames = [ + ...JSON.stringify( + (body.messages ?? []).filter((m) => m.role === "tool"), + ).matchAll( + /(?:\\?"name\\?"\s*:\s*\\?")([^"\\]*analyze_javascript_application)(?:\\?")/g, + ), + ] + .map((m) => m[1]) + .sort((a, b) => b.length - a.length); + const search = tools.find((t) => t.name === "tool_search"); + const invoke = tools.find((t) => t.name === "tool_call"); + const skill = tools.find((t) => t.name === "skill"); + const shell = tools.find((t) => t.name === "run_shell_command"); + const persisted = messageText.match( + /Full output saved to: ([^\r\n]+)/u, + ); + const call = (name, args) => ({ + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_rea_fixture_${requestIndex}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }); + let message; + if (mode === "chat" || evidenceSeen) { + message = { + role: "assistant", + content: "REA_CLIENT_COMPATIBILITY_OK", + }; + finalSeen = true; + } else if (skill && !skillRequested) { + skillRequested = true; + message = call(skill.name, { skill: "reverse-engineer-anything" }); + } else if (persisted && shell) { + artifactPath = persisted[1]; + assert( + (await realpath(artifactPath)).startsWith( + (await realpath(runtime)) + sep, + ), + ); + const script = `const fs=require('node:fs');const e=JSON.parse(fs.readFileSync(${JSON.stringify(artifactPath)},'utf8'));console.log(JSON.stringify({REA_CLIENT_FIXTURE:JSON.stringify(e.normalized_result).includes('REA_CLIENT_FIXTURE'),evidence_id:e.evidence_id,graph_nodes:e.normalized_result.graph.nodes.length}));`; + // Keep paths and JSON quotes out of shell syntax on Windows and POSIX. + const encodedScript = Buffer.from(script).toString("base64"); + message = call(shell.name, { + command: `node -e "eval(Buffer.from('${encodedScript}','base64').toString('utf8'))"`, + description: + "Read the complete saved REA Evidence and verify the fixture facts.", + }); + } else if (selected) + message = call(selected.name, { + input_path: target, + format: "directory", + }); + else if (reviewedNames.length && invoke) + message = call(invoke.name, { + name: reviewedNames[0], + arguments: { input_path: target, format: "directory" }, + }); + else if (search && !searchRequested) { + searchRequested = true; + message = call(search.name, { + query: "analyze_javascript_application", + }); + } else + throw new Error( + `REA analysis tool not exposed; tools: ${tools.map((t) => t.name).join(", ")}`, + ); + return message; + }, + }); + const endpoint = fixture.baseUrl; + const prompt = + mode === "chat" + ? "Say hello briefly." + : `Use REA to analyze the JavaScript directory ${target}. Report the result.`; + const env = { + ...baseEnvironment, + OPENAI_API_KEY: "local-fixture-key", + OPENAI_BASE_URL: endpoint, + }; + const args = [ + "--auth-type", + "openai", + "--openai-base-url", + endpoint, + "--model", + "rea-client-fixture", + "--approval-mode", + mode === "chat" ? "default" : "yolo", + "--allowed-mcp-server-names", + "rea", + "--output-format", + "stream-json", + "--max-session-turns", + "8", + "-p", + prompt, + ]; + const response = await saveVerifierCommandOutput( + () => + commands.exec(...launch(args), { + env, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + stdin: "pipe", + }), + { + stdout: join(lab, `${mode}.stdout`), + stderr: join(lab, `${mode}.stderr`), + }, + ); + const initialized = response.stdout + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)) + .find((v) => v.type === "system" && v.subtype === "init"); + assert(initialized); + const observed = initialized.tools + .filter((n) => n.startsWith("mcp__rea__")) + .sort(); + catalogSize = observed.length; + assert.deepEqual( + observed, + TOOL_CONTRACTS.map((c) => `mcp__rea__${c.name}`).sort(), + "Qwen must discover the complete REA catalog", + ); + assert.equal(fixture.failure, undefined); + assert(finalSeen); + if (mode === "call") { + assert(evidenceSeen); + assert(selectedSkillLoaded); + assert(artifactReadByAgent); + } + }, [ + () => commands.close(), + async () => { + await fixture?.close(); + }, + ]); + passed = true; +}, [ + async () => { + const lineage = await completeVerifierRun(verifier); + const receipt = { + client: "Qwen Code", + clientVersion, + mode, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback deterministic OpenAI-compatible endpoint; no live model-provider claim", + skillDiscovery: + "native HOME shared personal directory; no explicit skill setting", + catalogSize, + requests: fixture?.requests, + probes: fixture?.probes, + evidenceSeen, + selectedSkillLoaded, + artifactReadByAgent, + artifactPath, + artifactDigest, + finalSeen, + fixtureFailure: fixture?.failure, + verifier: lineage, + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); }, ], - }); - let message; - if (mode === "chat" || evidenceSeen) { - message = { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - finalSeen = true; - } else if (skill && !skillRequested) { - skillRequested = true; - message = call(skill.name, { skill: "reverse-engineer-anything" }); - } else if (persisted && shell) { - artifactPath = persisted[1]; - assert( - (await realpath(artifactPath)).startsWith( - (await realpath(runtime)) + sep, - ), - ); - const script = `const fs=require('node:fs');const e=JSON.parse(fs.readFileSync(${JSON.stringify(artifactPath)},'utf8'));console.log(JSON.stringify({REA_CLIENT_FIXTURE:JSON.stringify(e.normalized_result).includes('REA_CLIENT_FIXTURE'),evidence_id:e.evidence_id,graph_nodes:e.normalized_result.graph.nodes.length}));`; - // Keep paths and JSON quotes out of shell syntax on Windows and POSIX. - const encodedScript = Buffer.from(script).toString("base64"); - message = call(shell.name, { - command: `node -e "eval(Buffer.from('${encodedScript}','base64').toString('utf8'))"`, - description: - "Read the complete saved REA Evidence and verify the fixture facts.", - }); - } else if (selected) - message = call(selected.name, { - input_path: target, - format: "directory", - }); - else if (reviewedNames.length && invoke) - message = call(invoke.name, { - name: reviewedNames[0], - arguments: { input_path: target, format: "directory" }, - }); - else if (search && !searchRequested) { - searchRequested = true; - message = call(search.name, { query: "analyze_javascript_application" }); - } else - throw new Error( - `REA analysis tool not exposed; tools: ${tools.map((t) => t.name).join(", ")}`, - ); - return message; + ); }, -}); -const endpoint = fixture.baseUrl; -const prompt = - mode === "chat" - ? "Say hello briefly." - : `Use REA to analyze the JavaScript directory ${target}. Report the result.`; -const env = { - ...baseEnvironment, - OPENAI_API_KEY: "local-fixture-key", - OPENAI_BASE_URL: endpoint, -}; -const args = [ - "--auth-type", - "openai", - "--openai-base-url", - endpoint, - "--model", - "rea-client-fixture", - "--approval-mode", - mode === "chat" ? "default" : "yolo", - "--allowed-mcp-server-names", - "rea", - "--output-format", - "stream-json", - "--max-session-turns", - "8", - "-p", - prompt, -]; -let response; -let failure; -try { - response = await exec(...launch(args), { - env, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, - }); - await writeFile(join(lab, `${mode}.stdout`), response.stdout); - await writeFile(join(lab, `${mode}.stderr`), response.stderr); - const initialized = response.stdout - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line)) - .find((v) => v.type === "system" && v.subtype === "init"); - assert(initialized); - const observed = initialized.tools - .filter((n) => n.startsWith("mcp__rea__")) - .sort(); - assert.deepEqual( - observed, - TOOL_CONTRACTS.map((c) => `mcp__rea__${c.name}`).sort(), - "Qwen must discover the complete REA catalog", - ); - assert.equal(fixture.failure, undefined); - assert(finalSeen); - if (mode === "call") { - assert(evidenceSeen); - assert(selectedSkillLoaded); - assert(artifactReadByAgent); - } -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, `${mode}.stdout`), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, `${mode}.stderr`), cause.stderr); -} finally { - await fixture.close(); - const lineage = await completeVerifierRun(verifier); - const receipt = { - client: "Qwen Code", - clientVersion, - mode, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, - }, - modelFixture: - "loopback deterministic OpenAI-compatible endpoint; no live model-provider claim", - skillDiscovery: - "native HOME shared personal directory; no explicit skill setting", - catalogSize: TOOL_CONTRACTS.length, - requests: fixture.requests, - probes: fixture.probes, - evidenceSeen, - selectedSkillLoaded, - artifactReadByAgent, - artifactPath, - artifactDigest, - finalSeen, - fixtureFailure: fixture.failure, - verifier: lineage, - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; +]); diff --git a/tests/boundary/filesystem/verifierCommandOutput.test.ts b/tests/boundary/filesystem/verifierCommandOutput.test.ts new file mode 100644 index 0000000000..bc85ffcb7c --- /dev/null +++ b/tests/boundary/filesystem/verifierCommandOutput.test.ts @@ -0,0 +1,198 @@ +import { readFile } from "node:fs/promises"; +import { afterEach, expect, it } from "vitest"; + +import { saveVerifierCommandOutput } from "../../../scripts/lib/verifier-command-output.mjs"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; + +const workspaces: string[] = []; +afterEach(async () => { + for (const root of workspaces.splice(0)) await removeTestWorkspace(root); +}); + +const createOutputWorkspace = async () => { + const workspace = await createTestWorkspace("rea-verifier-command-output-"); + workspaces.push(workspace.root); + return workspace; +}; + +const rejection = (operation: Promise): Promise => + operation.then( + () => { + throw new Error("Expected command output persistence to reject"); + }, + (cause: unknown) => cause, + ); + +it("writes exact Unicode and empty output bytes and returns the original output", async () => { + const workspace = await createOutputWorkspace(); + const paths = { + stdout: workspace.path("stdout"), + stderr: workspace.path("stderr"), + }; + const output = { stdout: "observed α测试\0\n", stderr: "" }; + + expect(await saveVerifierCommandOutput(async () => output, paths)).toBe( + output, + ); + expect(await readFile(paths.stdout)).toEqual(Buffer.from(output.stdout)); + expect(await readFile(paths.stderr)).toEqual(Buffer.alloc(0)); +}); + +it("retains both failed-command streams while preserving its exact cause", async () => { + const workspace = await createOutputWorkspace(); + const paths = { + stdout: workspace.path("stdout"), + stderr: workspace.path("stderr"), + }; + const cause = Object.assign(new Error("selected command failed"), { + snapshot: { + stdout: { text: "" }, + stderr: { text: "partial diagnostics α测试\n" }, + }, + }); + + await expect( + saveVerifierCommandOutput(async () => { + throw cause; + }, paths), + ).rejects.toBe(cause); + expect(await readFile(paths.stdout)).toEqual(Buffer.alloc(0)); + expect(await readFile(paths.stderr)).toEqual( + Buffer.from(cause.snapshot.stderr.text), + ); +}); + +it("writes available partial failure output without replacing unavailable evidence", async () => { + const workspace = await createOutputWorkspace(); + const original = Buffer.from("retained caller output\0\n"); + const paths = { + stdout: await workspace.write("stdout", original), + stderr: workspace.path("stderr"), + }; + const cause = { + snapshot: { + stdout: { text: null }, + stderr: { text: "available failure diagnostics\n" }, + }, + }; + + await expect( + saveVerifierCommandOutput(async () => { + throw cause; + }, paths), + ).rejects.toBe(cause); + expect(await readFile(paths.stdout)).toEqual(original); + expect(await readFile(paths.stderr, "utf8")).toBe(cause.snapshot.stderr.text); +}); + +it.each([undefined, null])( + "preserves the opaque %s failure and both existing output artifacts", + async (cause) => { + const workspace = await createOutputWorkspace(); + const paths = { + stdout: await workspace.write("stdout", "retained stdout\n"), + stderr: await workspace.write("stderr", "retained stderr\n"), + }; + + await expect( + saveVerifierCommandOutput(async () => { + throw cause; + }, paths), + ).rejects.toBe(cause); + expect(await readFile(paths.stdout, "utf8")).toBe("retained stdout\n"); + expect(await readFile(paths.stderr, "utf8")).toBe("retained stderr\n"); + }, +); + +it.skipIf(process.platform === "win32")( + "retains the command cause and actual stdout write failure while writing stderr", + async () => { + const workspace = await createOutputWorkspace(); + const paths = { + stdout: await workspace.mkdir("stdout-directory"), + stderr: workspace.path("stderr"), + }; + const cause = Object.assign(new Error("primary command failure"), { + snapshot: { + stdout: { text: "command stdout" }, + stderr: { text: "command stderr\n" }, + }, + }); + + const failure = await rejection( + saveVerifierCommandOutput(async () => { + throw cause; + }, paths), + ); + expect(failure).toBeInstanceOf(AggregateError); + if (!(failure instanceof AggregateError)) throw failure; + expect(failure.cause).toBe(cause); + expect(failure.errors).toHaveLength(2); + expect(failure.errors[0]).toBe(cause); + expect(failure.errors[1]).toMatchObject({ + code: "EISDIR", + path: paths.stdout, + }); + expect(await readFile(paths.stderr, "utf8")).toBe( + cause.snapshot.stderr.text, + ); + }, +); + +it.skipIf(process.platform === "win32")( + "attempts stderr persistence after a successful command's stdout write fails", + async () => { + const workspace = await createOutputWorkspace(); + const paths = { + stdout: await workspace.mkdir("stdout-directory"), + stderr: workspace.path("stderr"), + }; + const output = { + stdout: "successful stdout", + stderr: "successful stderr\n", + }; + + const failure = await rejection( + saveVerifierCommandOutput(async () => output, paths), + ); + expect(failure).not.toBeInstanceOf(AggregateError); + expect(failure).toMatchObject({ + code: "EISDIR", + path: paths.stdout, + }); + expect(await readFile(paths.stderr, "utf8")).toBe(output.stderr); + }, +); + +it.skipIf(process.platform === "win32")( + "retains both independent filesystem errors when neither output can be written", + async () => { + const workspace = await createOutputWorkspace(); + const paths = { + stdout: await workspace.mkdir("stdout-directory"), + stderr: await workspace.mkdir("stderr-directory"), + }; + + const failure = await rejection( + saveVerifierCommandOutput( + async () => ({ stdout: "stdout", stderr: "stderr" }), + paths, + ), + ); + expect(failure).toBeInstanceOf(AggregateError); + if (!(failure instanceof AggregateError)) throw failure; + expect(failure.errors).toHaveLength(2); + expect(failure.errors[0]).toMatchObject({ + code: "EISDIR", + path: paths.stdout, + }); + expect(failure.errors[1]).toMatchObject({ + code: "EISDIR", + path: paths.stderr, + }); + expect(failure.cause).toBe(failure.errors[0]); + }, +); diff --git a/tests/boundary/process/nativeVerifierPreflightOwnership.test.ts b/tests/boundary/process/nativeVerifierPreflightOwnership.test.ts new file mode 100644 index 0000000000..2aaee785df --- /dev/null +++ b/tests/boundary/process/nativeVerifierPreflightOwnership.test.ts @@ -0,0 +1,270 @@ +import { randomUUID } from "node:crypto"; +import { chmod, readFile, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import { expect, it } from "vitest"; +import { z } from "zod"; + +import { cleanupOwnedProcessGroup } from "../../../src/process/ProcessOwnership.js"; +import { systemProcessOwnershipHost } from "../../../src/process/ProcessOwnershipObservation.js"; +import { liveProcesses } from "../../../src/process/ProcessOwnershipProcessTree.js"; +import { + ProviderProcessSupervisor, + spawnOwnedProviderProcess, +} from "../../../src/process/ProviderProcess.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; + +const repository = fileURLToPath(new URL("../../../", import.meta.url)); +// Select the same actual entrypoints from an owned original-source checkout for +// the regression's RED run. This changes no production imports or behavior. +const verifierSourceRoot = resolve( + process.env.REA_NATIVE_VERIFIER_SOURCE_ROOT ?? repository, +); +const retainedBytes = Buffer.from("native preflight retained evidence α\0\n"); +const versionStdout = "authored native version output\n"; +const versionStderr = "authored native version diagnostics\n"; + +const clients = [ + { client: "claude", profile: "CLAUDE_CONFIG_DIR", javascript: false }, + { client: "copilot", profile: "COPILOT_HOME", javascript: false }, + { client: "deepseek", profile: "DSH_HOME", javascript: false }, + { client: "gemini", profile: "GEMINI_CLI_HOME", javascript: true }, + { client: "grok", profile: "GROK_HOME", javascript: false }, + { client: "omp", profile: "PI_CODING_AGENT_DIR", javascript: false }, + { client: "opencode", profile: "OPENCODE_CONFIG_DIR", javascript: false }, + { client: "qwen", profile: "QWEN_HOME", javascript: true }, +] as const; + +const readySchema = z.object({ + pids: z.array(z.number().int().positive()).length(2), + runId: z.string().min(1), + descendantRunId: z.string().min(1), + argv: z.tuple([z.literal("--version")]), + cwd: z.string(), + lab: z.string(), + artifact: z.string(), + stdin: z.object({ + fifo: z.boolean(), + socket: z.boolean(), + character: z.literal(false), + file: z.literal(false), + }), + stdout: z.literal(versionStdout), + stderr: z.literal(versionStderr), +}); + +const receiptSchema = z.object({ + clientVersion: z.null(), + passed: z.literal(false), + artifacts: z.string(), + catalogSize: z.null().optional(), + requests: z.array(z.unknown()).optional(), +}); + +const descendantSource = ` +import { appendFileSync, existsSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +const release = fileURLToPath(new URL("./release-descendant", import.meta.url)); +const events = fileURLToPath(new URL("./descendant-events.jsonl", import.meta.url)); +const finish = (reason) => { + appendFileSync(events, JSON.stringify({ pid: process.pid, reason }) + "\\n"); + process.exit(0); +}; +// The ready message is an acquisition barrier, independent of process-table +// scheduling. Ignored stdio lets the version leader and verifier finish first. +process.send({ pid: process.pid, runId: process.env.REA_PROCESS_RUN_ID }); +process.disconnect(); +setInterval(() => { + if (existsSync(release)) finish("external-release"); +}, 25); +setTimeout(() => finish("bounded-fixture-deadline"), 20000); +`; + +const selectedClientSource = (profile: string) => ` +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { fstatSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +assert.deepEqual(process.argv.slice(2), ["--version"]); +const metadata = fstatSync(0); +const stdin = { + fifo: metadata.isFIFO(), socket: metadata.isSocket(), + character: metadata.isCharacterDevice(), file: metadata.isFile(), +}; +assert.ok(stdin.fifo || stdin.socket, "Native preflight stdin must retain a real pipe or socket"); +assert.equal(stdin.character, false); +assert.equal(stdin.file, false); +const lab = dirname(process.env[${JSON.stringify(profile)}]); +const artifact = join(lab, "authored-preflight-evidence.bin"); +writeFileSync(artifact, Buffer.from(${JSON.stringify(retainedBytes.toString("base64"))}, "base64")); +const child = spawn(process.execPath, [fileURLToPath(new URL("./descendant.mjs", import.meta.url))], { + stdio: ["ignore", "ignore", "ignore", "ipc"], +}); +const acknowledged = await new Promise((resolve, reject) => { + child.once("message", resolve); + child.once("error", reject); +}); +child.unref(); +const stdout = ${JSON.stringify(versionStdout)}; +const stderr = ${JSON.stringify(versionStderr)}; +writeFileSync(fileURLToPath(new URL("./ready.json", import.meta.url)), JSON.stringify({ + pids: [process.pid, acknowledged.pid], + runId: process.env.REA_PROCESS_RUN_ID, descendantRunId: acknowledged.runId, + argv: process.argv.slice(2), cwd: process.cwd(), lab, artifact, stdin, stdout, stderr, +})); +await Promise.all([ + new Promise((resolve) => process.stdout.write(stdout, resolve)), + new Promise((resolve) => process.stderr.write(stderr, resolve)), +]); +process.exit(7); +`; + +const readReady = async (path: string) => + readFile(path, "utf8").then( + (text) => readySchema.parse(JSON.parse(text)), + (cause: unknown) => { + if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") + return undefined; + throw cause; + }, + ); + +const liveAcknowledgedProcesses = async (pids: readonly number[]) => + liveProcesses(await systemProcessOwnershipHost.listProcesses()) + .filter(({ pid }) => pids.includes(pid)) + .map(({ pid }) => pid); + +const awaitReleasedDescendant = async (readyPath: string) => { + const ready = await readReady(readyPath); + if (ready === undefined) return; + const deadline = Date.now() + 7_500; + while ((await liveAcknowledgedProcesses(ready.pids)).length !== 0) { + if (Date.now() >= deadline) + throw new Error( + "Externally released native preflight fixture stayed live", + ); + await delay(25); + } +}; + +const shellQuote = (text: string) => `'${text.replaceAll("'", `'"'"'`)}'`; + +// These are authored selected clients, not installed-provider verification. +// CommandCode remains unverified: its actual disposable OS account is required. +it.skipIf(process.platform === "win32").each(clients)( + "$client actual verifier reclaims a failed preflight descendant and retains its lab", + async ({ client, profile, javascript }) => { + const workspace = await createTestWorkspace( + `rea-native-${client}-preflight-`, + ); + const readyPath = workspace.path("ready.json"); + const releasePath = workspace.path("release-descendant"); + let acquired: + | Awaited> + | undefined; + let supervisor: ProviderProcessSupervisor | undefined; + try { + const temporaryRoot = await workspace.mkdir("native-labs"); + const selectedJavaScript = await workspace.write( + "selected-client.mjs", + selectedClientSource(profile), + ); + await workspace.write("descendant.mjs", descendantSource); + const command = javascript + ? selectedJavaScript + : await workspace.write( + "selected-client", + `#!/bin/sh\nexec ${shellQuote(process.execPath)} ${shellQuote(selectedJavaScript)} "$@"\n`, + ); + if (!javascript) await chmod(command, 0o700); + acquired = await spawnOwnedProviderProcess({ + command: process.execPath, + arguments: [ + join(verifierSourceRoot, "scripts", `verify-${client}-client.mjs`), + "call", + ], + cwd: repository, + runId: `rea-native-preflight-test-${randomUUID()}`, + sweepTokenOwnedProcesses: true, + hostEnvironment: { + ...process.env, + ...workspace.environment, + TMPDIR: temporaryRoot, + TMP: temporaryRoot, + TEMP: temporaryRoot, + REA_VERIFY_RUNTIME_ROOT: repository, + [`REA_VERIFY_${client.toUpperCase()}_COMMAND`]: command, + }, + }); + const owned = acquired; + supervisor = new ProviderProcessSupervisor( + { + ...owned, + ownsProcessLifetime: true, + cleanup: + owned.cleanup ?? (() => cleanupOwnedProcessGroup(owned.ownership)), + }, + { maxDiagnosticBytes: 64 * 1024 }, + ); + expect(await supervisor.waitForOutputClose(15_000)).toBe(true); + const completed = supervisor.snapshot(); + const ready = await readReady(readyPath); + if (ready === undefined) + throw new Error( + `${client} did not reach the authored version preflight: ${completed.stderr.text}`, + ); + // Observe before stop(), external release, receipt parsing or workspace + // removal. A successful test-owned teardown cannot hide verifier leakage. + const liveAtVerifierCompletion = await liveAcknowledgedProcesses( + ready.pids, + ); + expect( + liveAtVerifierCompletion, + `${client} acknowledged processes still alive when its actual verifier completed`, + ).toEqual([]); + expect(completed.exitCode, completed.stderr.text).not.toBe(0); + expect(completed.signal, completed.stderr.text).toBeNull(); + expect(completed.stderr.text).toContain(versionStderr); + expect(ready.stdin.fifo || ready.stdin.socket).toBe(true); + expect(ready.descendantRunId).toBe(ready.runId); + expect(dirname(ready.lab)).toBe(temporaryRoot); + expect(ready.artifact).toBe( + join(ready.lab, "authored-preflight-evidence.bin"), + ); + const receipt = receiptSchema.parse( + JSON.parse(await readFile(join(ready.lab, "receipt.json"), "utf8")), + ); + expect(receipt.artifacts).toBe(ready.lab); + if (receipt.requests !== undefined) expect(receipt.requests).toEqual([]); + // Read only after the verifier's own cleanup, before removing test scratch. + expect(await readFile(ready.artifact)).toEqual(retainedBytes); + } finally { + // Self-release the authored orphan even on RED. PID observations never + // authorize signals; outer cleanup retains its canonical acquisition proof. + try { + await writeFile( + releasePath, + "release this authored preflight descendant", + ); + await awaitReleasedDescendant(readyPath); + } finally { + if (supervisor !== undefined) { + const stopped = await supervisor.stop(); + if (stopped.status === "incomplete") throw new Error(stopped.reason); + } else if (acquired !== undefined) { + const stopped = await (acquired.cleanup === undefined + ? cleanupOwnedProcessGroup(acquired.ownership) + : acquired.cleanup()); + if (!stopped.cleaned) throw new Error(stopped.reason); + } + } + await removeTestWorkspace(workspace.root); + } + }, + 30_000, +);