diff --git a/scripts/lib/verifier-command-output.d.mts b/scripts/lib/verifier-command-output.d.mts new file mode 100644 index 0000000000..5939a63a32 --- /dev/null +++ b/scripts/lib/verifier-command-output.d.mts @@ -0,0 +1,7 @@ +import type { VerifierCommandOutput } from "./verifier-command-owner.mjs"; + +/** Retain available command output without losing command or filesystem failures. */ +export function saveVerifierCommandOutput( + operation: () => Promise, + paths: { readonly stdout: string; readonly stderr: string }, +): Promise; diff --git a/scripts/lib/verifier-command-output.mjs b/scripts/lib/verifier-command-output.mjs new file mode 100644 index 0000000000..ec4fe6ac60 --- /dev/null +++ b/scripts/lib/verifier-command-output.mjs @@ -0,0 +1,30 @@ +import { writeFile } from "node:fs/promises"; + +import { withVerifierCleanup } from "./verifier-cleanup.mjs"; + +/** Retain available command output without losing command or filesystem failures. */ +export async function saveVerifierCommandOutput(operation, paths) { + let output; + try { + output = await operation(); + } catch (cause) { + const snapshot = cause?.snapshot; + const stdout = snapshot?.stdout?.text; + const stderr = snapshot?.stderr?.text; + return withVerifierCleanup(async () => { + throw cause; + }, [ + async () => { + if (typeof stdout === "string") await writeFile(paths.stdout, stdout); + }, + async () => { + if (typeof stderr === "string") await writeFile(paths.stderr, stderr); + }, + ]); + } + await withVerifierCleanup( + () => writeFile(paths.stdout, output.stdout), + [() => writeFile(paths.stderr, output.stderr)], + ); + return output; +} diff --git a/scripts/lib/verify-package-core.mjs b/scripts/lib/verify-package-core.mjs index ef5179c690..eb15c49085 100644 --- a/scripts/lib/verify-package-core.mjs +++ b/scripts/lib/verify-package-core.mjs @@ -1,22 +1,10 @@ -import { execFile } from "node:child_process"; import { lstat } from "node:fs/promises"; -import { promisify } from "node:util"; import { Ajv2020 } from "ajv/dist/2020.js"; -/** - * `mcp-verifier-results.mjs` reaches into `dist/` at module load, so eager - * importing it propagates that build dependency to every consumer in this - * file. `verifyCompleteToolCatalog` is the only consumer, so we lazy-load - * the module there and keep the remaining helpers (`exec`, `json`, - * `pathExists`) usable from `test:fast` callers that only need `dist/`-free - * utilities — see docs/testing.md § test scripts, which promises `test:fast` - * runs "without build". - */ +// Only catalog verification needs built MCP contracts; the other helpers work +// from source without a runtime build. const loadMcpVerifierResults = () => import("./mcp-verifier-results.mjs"); -/** Native-client commands expose the live child handle for caller-owned stdin. */ -export const exec = promisify(execFile); - /** Parse JSON from a command output string. */ export const json = (text) => JSON.parse(text); diff --git a/scripts/verify-claude-client.mjs b/scripts/verify-claude-client.mjs index 10a17ee3c0..d652400705 100644 --- a/scripts/verify-claude-client.mjs +++ b/scripts/verify-claude-client.mjs @@ -15,7 +15,9 @@ import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; -import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { expandMcpSessionStatus, @@ -38,36 +40,9 @@ const profile = join(lab, "selected profile"); const workspace = join(lab, "workspace"); const configPath = join(profile, ".claude.json"); const skillDirectory = join(profile, "skills", "reverse-engineer-anything"); -await Promise.all([mkdir(profile), mkdir(workspace)]); -await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); -const environment = { - ...process.env, - CLAUDE_CONFIG_DIR: profile, - DISABLE_AUTOUPDATER: "1", - CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", - REA_PROCESS_RUN_ID: verifier.run_id, -}; -let clientVersion; -try { - clientVersion = ( - await exec(command, ["--version"], { env: environment, timeout: 30_000 }) - ).stdout.trim(); -} catch (cause) { - // npm installs put only a claude.cmd shim on PATH, which Node does not spawn without a shell. - const windowsHint = - process.platform === "win32" - ? " On Windows, select claude.exe; npm installs provide it under node_modules/@anthropic-ai/claude-code/bin." - : ""; - throw new Error( - `Claude Code preflight failed for ${command}; install Claude Code or set REA_VERIFY_CLAUDE_COMMAND.${windowsHint} ${String(cause)}`, - ); -} -const target = join(lab, "fixture α测试"); -await mkdir(target); -await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', -); +let clientVersion = null; +let fixture; +let passed = false; let skillRequested = false; let sessionRequested = false; let sessionStatusSeen = false; @@ -75,369 +50,440 @@ let skillLoaded = false; let expectedSkillBody; let evidenceSeen = false; let finalSeen = false; -let catalogSize = 0; +let catalogSize = null; let schemasChecked = 0; let pendingServerWaits = 0; let artifactDigest; -const schemaValidator = new Ajv2020({ strict: false, validateFormats: false }); -const containsText = (value, text) => { - if (typeof value === "string") return value.includes(text); - if (value && typeof value === "object") - return Object.values(value).some((nested) => containsText(nested, text)); - return false; -}; -const extractInlineResult = (value, matches) => { - if (typeof value === "string") { +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([mkdir(profile), mkdir(workspace)]); + await commands.exec("git", ["init", "--quiet", workspace], { + timeout: 30_000, + stdin: "pipe", + }); + const environment = { + ...process.env, + CLAUDE_CONFIG_DIR: profile, + DISABLE_AUTOUPDATER: "1", + CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", + REA_PROCESS_RUN_ID: verifier.run_id, + }; try { - // Native Claude appends this resource hint to tool results. Original bodies remain intact in request artifacts. - const json = value.replace( - /\n\n\n\d+ tokens left<\/total_tokens>\n<\/system-reminder>$/u, - "", + clientVersion = ( + await commands.exec(command, ["--version"], { + env: environment, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + } catch (cause) { + // npm installs put only a claude.cmd shim on PATH, which Node does not spawn without a shell. + const windowsHint = + process.platform === "win32" + ? " On Windows, select claude.exe; npm installs provide it under node_modules/@anthropic-ai/claude-code/bin." + : ""; + throw new Error( + `Claude Code preflight failed for ${command}; install Claude Code or set REA_VERIFY_CLAUDE_COMMAND.${windowsHint} ${String(cause)}`, + { cause }, ); - return extractInlineResult(JSON.parse(json), matches); - } catch { - return undefined; } - } - if (value && typeof value === "object") { - if (matches(value)) return value; - for (const nested of Object.values(value)) { - const result = extractInlineResult(nested, matches); - if (result) return result; - } - } - return undefined; -}; -const fixture = await createAnthropicModelFixture({ - directory: lab, - onRequest: async ({ body, tools, requestIndex }) => { - if (tools.length === 0) - return [{ type: "text", text: "REA fixture title" }]; - assert(requestIndex <= 8, "Local model workflow exceeded its turn budget"); - const call = (name, input) => [ - { type: "tool_use", id: `toolu_rea_${requestIndex}`, name, input }, - ]; - const analysis = tools.find((t) => - t.name.endsWith("analyze_javascript_application"), - ); - if (!analysis) { - // Native automatic discovery can begin the conversation while MCP is still connecting. - const wait = tools.find((t) => t.name === "WaitForMcpServers"); - assert( - wait, - "Claude must expose REA tools or its native pending-server wait", - ); - pendingServerWaits++; - return call(wait.name, { servers: ["rea"] }); - } - const prefix = analysis.name.slice( - 0, - -"analyze_javascript_application".length, - ); - const expected = sortVerifierNames( - TOOL_CONTRACTS.map((t) => prefix + t.name), - ); - const reaTools = tools.filter((t) => t.name.startsWith(prefix)); - assert.deepEqual( - sortVerifierNames(reaTools.map((t) => t.name)), - expected, - "Claude must forward the complete REA catalog after connection", - ); - catalogSize = expected.length; - if (schemasChecked === 0) { - for (const tool of reaTools) { - assert( - tool.input_schema, - `${tool.name}: missing advertised input schema`, - ); - assert( - schemaValidator.validateSchema(tool.input_schema), - `${tool.name}: invalid input schema: ${JSON.stringify(schemaValidator.errors)}`, - ); - schemasChecked++; - } - } - if (mode === "call" && !sessionStatusSeen) { - const observedStatus = extractInlineResult( - body.messages, - (value) => - Boolean(value.result?.server_identity) && - Array.isArray(value.result?.tool_availability?.groups), - ); - if (observedStatus) { - const parsedStatus = - toolContract("binary_session").outputSchema.parse(observedStatus); - const status = { - ...parsedStatus, - result: expandMcpSessionStatus(parsedStatus.result), - }; - const inputProfile = parseMcpInputSchemaProfile( - environment.REA_MCP_INPUT_SCHEMA_PROFILE, - ); - const outputProfile = parseMcpOutputSchemaProfile( - environment.REA_MCP_OUTPUT_SCHEMA_PROFILE, - ); - assert.ok(inputProfile.ok); - assert.ok(outputProfile.ok); - assert.deepEqual( - status.result.server_identity.catalog, - createCatalogIdentity( - compactPresentationForMcpInputSchemaProfile(inputProfile.value), - outputProfile.value ?? "full", - ), - "binary_session must return the selected input/output profile's catalog identity", - ); - assert.equal( - Object.hasOwn(status.result.server_identity.catalog, "tools"), - false, - "binary_session must not inline the duplicated canonical tool catalog", - ); - assert.deepEqual( - sortVerifierNames( - status.result.tool_availability.map((entry) => entry.name), - ), - sortVerifierNames(TOOL_CONTRACTS.map((contract) => contract.name)), - "binary_session must retain complete canonical tool availability", - ); - sessionStatusSeen = true; - } else { - assert( - !sessionRequested, - "Claude must return the inline binary_session result before continuing", - ); - const session = tools.find( - (tool) => tool.name === `${prefix}binary_session`, - ); - assert(session, "Claude must expose the REA binary_session tool"); - sessionRequested = true; - return call(session.name, {}); - } - } - skillLoaded ||= - expectedSkillBody !== undefined && - containsText(body.messages, expectedSkillBody) && - containsText(body.messages, skillDirectory); - const observed = extractInlineResult( - body.messages, - (value) => - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0, - ); - if (observed) { - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(observed); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((n) => n.observations) - .map((o) => o.properties) - .find( - (p) => - p.semantic_role === "export-return-shapes" && - p.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((s) => s.fields); - assert( - fields.some( - (f) => - f.path === "/clientCompatibility" && - f.state === "literal" && - f.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (f) => f.path === "/count" && f.state === "literal" && f.value === 7, - ), - ); - const bytes = JSON.stringify(evidence); - artifactDigest = createHash("sha256").update(bytes).digest("hex"); - await writeFile(join(lab, "claude-evidence.json"), bytes); - evidenceSeen = true; - } - if (mode === "chat" || (sessionStatusSeen && evidenceSeen)) { - finalSeen = true; - return [{ type: "text", text: "REA_CLIENT_COMPATIBILITY_OK" }]; - } - if (!skillRequested) { - const skill = tools.find((t) => t.name === "Skill"); - assert(skill, "Claude must expose its native Skill tool"); - skillRequested = true; - return call(skill.name, { skill: "reverse-engineer-anything" }); - } - assert( - skillLoaded, - "Claude must load the complete installed personal REA skill", - ); - return call(analysis.name, { input_path: target, format: "directory" }); - }, -}); -const original = JSON.stringify({ theme: "light", mcpServers: {} }, null, 2); -await writeFile(configPath, original); -let failure; -try { - for (const args of [ - ["setup", "--client", "claude_code", "--dry-run"], - ["setup", "--client", "claude_code", "--yes"], - ["doctor", "--client", "claude_code", "--skill"], - ]) { - const result = await exec( - process.execPath, - [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); + const target = join(lab, "fixture α测试"); + await mkdir(target); await writeFile( - join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), - result.stdout, + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', ); - const parsed = JSON.parse(result.stdout); - if (args[0] === "doctor") assert.equal(parsed.healthy, true); - else { - assert.equal( - parsed.status, - args.includes("--dry-run") ? "planned" : "ready", - ); - if (args.includes("--dry-run")) { - assert(parsed.plannedActions.some((a) => a.target === configPath)); - assert(parsed.plannedActions.some((a) => a.target === skillDirectory)); - assert( - parsed.plannedActions.every( - (a) => - a.target === configPath || a.target.startsWith(skillDirectory), - ), + const schemaValidator = new Ajv2020({ + strict: false, + validateFormats: false, + }); + const containsText = (value, text) => { + if (typeof value === "string") return value.includes(text); + if (value && typeof value === "object") + return Object.values(value).some((nested) => + containsText(nested, text), ); + return false; + }; + const extractInlineResult = (value, matches) => { + if (typeof value === "string") { + try { + // Native Claude appends this resource hint to tool results. Original bodies remain intact in request artifacts. + const json = value.replace( + /\n\n\n\d+ tokens left<\/total_tokens>\n<\/system-reminder>$/u, + "", + ); + return extractInlineResult(JSON.parse(json), matches); + } catch { + return undefined; + } + } + if (value && typeof value === "object") { + if (matches(value)) return value; + for (const nested of Object.values(value)) { + const result = extractInlineResult(nested, matches); + if (result) return result; + } + } + return undefined; + }; + fixture = await createAnthropicModelFixture({ + directory: lab, + onRequest: async ({ body, tools, requestIndex }) => { + if (tools.length === 0) + return [{ type: "text", text: "REA fixture title" }]; + assert( + requestIndex <= 8, + "Local model workflow exceeded its turn budget", + ); + const call = (name, input) => [ + { type: "tool_use", id: `toolu_rea_${requestIndex}`, name, input }, + ]; + const analysis = tools.find((t) => + t.name.endsWith("analyze_javascript_application"), + ); + if (!analysis) { + // Native automatic discovery can begin the conversation while MCP is still connecting. + const wait = tools.find((t) => t.name === "WaitForMcpServers"); + assert( + wait, + "Claude must expose REA tools or its native pending-server wait", + ); + pendingServerWaits++; + return call(wait.name, { servers: ["rea"] }); + } + const prefix = analysis.name.slice( + 0, + -"analyze_javascript_application".length, + ); + const expected = sortVerifierNames( + TOOL_CONTRACTS.map((t) => prefix + t.name), + ); + const reaTools = tools.filter((t) => t.name.startsWith(prefix)); + catalogSize = reaTools.length; + assert.deepEqual( + sortVerifierNames(reaTools.map((t) => t.name)), + expected, + "Claude must forward the complete REA catalog after connection", + ); + if (schemasChecked === 0) { + for (const tool of reaTools) { + assert( + tool.input_schema, + `${tool.name}: missing advertised input schema`, + ); + assert( + schemaValidator.validateSchema(tool.input_schema), + `${tool.name}: invalid input schema: ${JSON.stringify(schemaValidator.errors)}`, + ); + schemasChecked++; + } + } + if (mode === "call" && !sessionStatusSeen) { + const observedStatus = extractInlineResult( + body.messages, + (value) => + Boolean(value.result?.server_identity) && + Array.isArray(value.result?.tool_availability?.groups), + ); + if (observedStatus) { + const parsedStatus = + toolContract("binary_session").outputSchema.parse(observedStatus); + const status = { + ...parsedStatus, + result: expandMcpSessionStatus(parsedStatus.result), + }; + const inputProfile = parseMcpInputSchemaProfile( + environment.REA_MCP_INPUT_SCHEMA_PROFILE, + ); + const outputProfile = parseMcpOutputSchemaProfile( + environment.REA_MCP_OUTPUT_SCHEMA_PROFILE, + ); + assert.ok(inputProfile.ok); + assert.ok(outputProfile.ok); + assert.deepEqual( + status.result.server_identity.catalog, + createCatalogIdentity( + compactPresentationForMcpInputSchemaProfile(inputProfile.value), + outputProfile.value ?? "full", + ), + "binary_session must return the selected input/output profile's catalog identity", + ); + assert.equal( + Object.hasOwn(status.result.server_identity.catalog, "tools"), + false, + "binary_session must not inline the duplicated canonical tool catalog", + ); + assert.deepEqual( + sortVerifierNames( + status.result.tool_availability.map((entry) => entry.name), + ), + sortVerifierNames( + TOOL_CONTRACTS.map((contract) => contract.name), + ), + "binary_session must retain complete canonical tool availability", + ); + sessionStatusSeen = true; + } else { + assert( + !sessionRequested, + "Claude must return the inline binary_session result before continuing", + ); + const session = tools.find( + (tool) => tool.name === `${prefix}binary_session`, + ); + assert(session, "Claude must expose the REA binary_session tool"); + sessionRequested = true; + return call(session.name, {}); + } + } + skillLoaded ||= + expectedSkillBody !== undefined && + containsText(body.messages, expectedSkillBody) && + containsText(body.messages, skillDirectory); + const observed = extractInlineResult( + body.messages, + (value) => + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes?.length > 0, + ); + if (observed) { + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(observed); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((n) => n.observations) + .map((o) => o.properties) + .find( + (p) => + p.semantic_role === "export-return-shapes" && + p.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap((s) => s.fields); + assert( + fields.some( + (f) => + f.path === "/clientCompatibility" && + f.state === "literal" && + f.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (f) => + f.path === "/count" && f.state === "literal" && f.value === 7, + ), + ); + const bytes = JSON.stringify(evidence); + artifactDigest = createHash("sha256").update(bytes).digest("hex"); + await writeFile(join(lab, "claude-evidence.json"), bytes); + evidenceSeen = true; + } + if (mode === "chat" || (sessionStatusSeen && evidenceSeen)) { + finalSeen = true; + return [{ type: "text", text: "REA_CLIENT_COMPATIBILITY_OK" }]; + } + if (!skillRequested) { + const skill = tools.find((t) => t.name === "Skill"); + assert(skill, "Claude must expose its native Skill tool"); + skillRequested = true; + return call(skill.name, { skill: "reverse-engineer-anything" }); + } + assert( + skillLoaded, + "Claude must load the complete installed personal REA skill", + ); + return call(analysis.name, { input_path: target, format: "directory" }); + }, + }); + const original = JSON.stringify( + { theme: "light", mcpServers: {} }, + null, + 2, + ); + await writeFile(configPath, original); + for (const args of [ + ["setup", "--client", "claude_code", "--dry-run"], + ["setup", "--client", "claude_code", "--yes"], + ["doctor", "--client", "claude_code", "--skill"], + ]) { + const result = await commands.exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + await writeFile( + join( + lab, + `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`, + ), + result.stdout, + ); + const parsed = JSON.parse(result.stdout); + if (args[0] === "doctor") assert.equal(parsed.healthy, true); + else { + assert.equal( + parsed.status, + args.includes("--dry-run") ? "planned" : "ready", + ); + if (args.includes("--dry-run")) { + assert(parsed.plannedActions.some((a) => a.target === configPath)); + assert( + parsed.plannedActions.some((a) => a.target === skillDirectory), + ); + assert( + parsed.plannedActions.every( + (a) => + a.target === configPath || a.target.startsWith(skillDirectory), + ), + ); + } } } - } - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const configured = await readFile(configPath, "utf8"); - assert.equal(JSON.parse(configured).theme, "light"); - const repeated = await exec( - process.execPath, - [ - join(runtimeRoot, "scripts/rea.mjs"), - "setup", - "--client", - "claude_code", - "--yes", - "--json", - ], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); - assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); - assert.equal(await readFile(configPath, "utf8"), configured); - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const installedSkill = await readFile( - join(skillDirectory, "SKILL.md"), - "utf8", - ); - assert(installedSkill.includes("name: reverse-engineer-anything")); - const lines = installedSkill.split("\n"); - const headerEnd = lines.indexOf("---", 1); - assert( - lines[0] === "---" && headerEnd > 0, - "Installed skill must have its authored frontmatter", - ); - expectedSkillBody = lines - .slice(headerEnd + 1) - .join("\n") - .trim(); - assert(expectedSkillBody.length > 0); - const execution = exec( - command, - [ - "--setting-sources", - "user", - "--print", - "--output-format", - "json", - "--no-session-persistence", - "--model", - "claude-sonnet-4-6", - "--tools", - "default", - "--permission-mode", - "dontAsk", - "--settings", - JSON.stringify({ - permissions: { - allow: [ - "Skill", - "WaitForMcpServers", - "mcp__rea__analyze_javascript_application", - ...(mode === "call" ? ["mcp__rea__binary_session"] : []), + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + assert.equal(JSON.parse(configured).theme, "light"); + const repeated = await commands.exec( + process.execPath, + [ + join(runtimeRoot, "scripts/rea.mjs"), + "setup", + "--client", + "claude_code", + "--yes", + "--json", + ], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const installedSkill = await readFile( + join(skillDirectory, "SKILL.md"), + "utf8", + ); + assert(installedSkill.includes("name: reverse-engineer-anything")); + const lines = installedSkill.split("\n"); + const headerEnd = lines.indexOf("---", 1); + assert( + lines[0] === "---" && headerEnd > 0, + "Installed skill must have its authored frontmatter", + ); + expectedSkillBody = lines + .slice(headerEnd + 1) + .join("\n") + .trim(); + assert(expectedSkillBody.length > 0); + const response = await saveVerifierCommandOutput( + () => + commands.exec( + command, + [ + "--setting-sources", + "user", + "--print", + "--output-format", + "json", + "--no-session-persistence", + "--model", + "claude-sonnet-4-6", + "--tools", + "default", + "--permission-mode", + "dontAsk", + "--settings", + JSON.stringify({ + permissions: { + allow: [ + "Skill", + "WaitForMcpServers", + "mcp__rea__analyze_javascript_application", + ...(mode === "call" ? ["mcp__rea__binary_session"] : []), + ], + }, + }), + mode === "chat" + ? "Say hello briefly." + : `Use the REA skill and tools to analyze the JavaScript directory ${target}. Report the result.`, ], - }, - }), - mode === "chat" - ? "Say hello briefly." - : `Use the REA skill and tools to analyze the JavaScript directory ${target}. Report the result.`, - ], - { - env: { - ...environment, - ANTHROPIC_API_KEY: "local-fixture-key", - ANTHROPIC_AUTH_TOKEN: "", - ANTHROPIC_BASE_URL: fixture.baseUrl, + { + env: { + ...environment, + ANTHROPIC_API_KEY: "local-fixture-key", + ANTHROPIC_AUTH_TOKEN: "", + ANTHROPIC_BASE_URL: fixture.baseUrl, + }, + cwd: workspace, + timeout: 120_000, + stdin: "closed-pipe", + maxBuffer: 10 * 1024 * 1024, + }, + ), + { + stdout: join(lab, `${mode}.stdout`), + stderr: join(lab, `${mode}.stderr`), }, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, + ); + assert.equal(fixture.failure, undefined); + const parsed = JSON.parse(response.stdout); + assert.equal(parsed.is_error, false); + assert.equal(parsed.subtype, "success"); + assert.equal(parsed.result, "REA_CLIENT_COMPATIBILITY_OK"); + assert.deepEqual(parsed.permission_denials, []); + assert(finalSeen); + if (mode === "call") + assert(evidenceSeen && skillLoaded && sessionStatusSeen); + }, [ + () => commands.close(), + async () => { + await fixture?.close(); }, - ); - execution.child.stdin.end(); - const response = await execution; - await writeFile(join(lab, `${mode}.stdout`), response.stdout); - await writeFile(join(lab, `${mode}.stderr`), response.stderr); - assert.equal(fixture.failure, undefined); - const parsed = JSON.parse(response.stdout); - assert.equal(parsed.is_error, false); - assert.equal(parsed.subtype, "success"); - assert.equal(parsed.result, "REA_CLIENT_COMPATIBILITY_OK"); - assert.deepEqual(parsed.permission_denials, []); - assert(finalSeen); - if (mode === "call") assert(evidenceSeen && skillLoaded && sessionStatusSeen); -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, `${mode}.stdout`), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, `${mode}.stderr`), cause.stderr); -} finally { - await fixture.close(); - const receipt = { - client: "Claude Code", - clientVersion, - mode, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, - }, - modelFixture: - "loopback Anthropic Messages/SSE; synthetic usage and native resource hints; no live API claim", - discovery: - "native CLAUDE_CONFIG_DIR user configuration and skills; separate Git root, user settings only, default built-in tools", - bareMode: - "not used; native bare print mode did not load the personal skill", - catalogSize, - schemasChecked, - pendingServerWaits, - evidenceSeen, - sessionStatusSeen, - skillLoaded, - artifactDigest, - finalSeen, - requests: fixture.requests, - probes: fixture.probes, - fixtureFailure: fixture.failure, - verifier: await completeVerifierRun(verifier), - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; + ]); + passed = true; +}, [ + async () => { + const receipt = { + client: "Claude Code", + clientVersion, + mode, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback Anthropic Messages/SSE; synthetic usage and native resource hints; no live API claim", + discovery: + "native CLAUDE_CONFIG_DIR user configuration and skills; separate Git root, user settings only, default built-in tools", + bareMode: + "not used; native bare print mode did not load the personal skill", + catalogSize, + schemasChecked, + pendingServerWaits, + evidenceSeen, + sessionStatusSeen, + skillLoaded, + artifactDigest, + finalSeen, + requests: fixture?.requests, + probes: fixture?.probes, + fixtureFailure: fixture?.failure, + verifier: await completeVerifierRun(verifier), + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); + }, + ], + ); + }, +]); diff --git a/scripts/verify-commandcode-client.mjs b/scripts/verify-commandcode-client.mjs index 50ba09eac3..8819f4fc45 100644 --- a/scripts/verify-commandcode-client.mjs +++ b/scripts/verify-commandcode-client.mjs @@ -9,7 +9,9 @@ import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; -import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -47,43 +49,13 @@ const skillDirectory = join( "skills", "reverse-engineer-anything", ); -await Promise.all([ - mkdir(target, { recursive: true }), - mkdir(profile, { recursive: true }), - mkdir(join(lab, "tmp")), -]); -await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); -await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', -); -const environment = { - PATH: process.env.PATH, - NODE_OPTIONS: process.env.NODE_OPTIONS, - USERPROFILE: account, - TMPDIR: join(lab, "tmp"), - CMD_LOCAL_ONLY: "1", - // Native print mode checks for an account key even with a keyless BYOK model. - // This synthetic value is never a real credential; local-only refuses hosted API calls. - COMMAND_CODE_API_KEY: "local-fixture-only", - COMMANDCODE_SKIP_UPDATES: "1", - COMMANDCODE_DISABLE_CRON: "1", - OTEL_SDK_DISABLED: "true", - REA_PROCESS_RUN_ID: verifier.run_id, -}; const expectedNames = TOOL_CONTRACTS.map( (tool) => "mcp__rea__" + tool.name, ).sort(); -const schemaValidator = new Ajv2020({ strict: false, validateFormats: false }); -const textValues = (value) => - typeof value === "string" - ? [value] - : value !== null && typeof value === "object" - ? Object.values(value).flatMap(textValues) - : []; -let clientVersion; +let clientVersion = null; let expectedSkill; let inventoryChecked = false; +let catalogSize = null; let skillLoaded = false; let schemasChecked = 0; let catalogArtifact; @@ -94,360 +66,428 @@ let evidenceSeen = false; let evidenceBytes; let finalSeen = false; let stage = 0; -let failure; -const call = (name, args) => ({ - role: "assistant", - content: null, - tool_calls: [ - { - id: `call_commandcode_${++stage}`, - type: "function", - function: { name, arguments: JSON.stringify(args) }, - }, - ], -}); -const validateCatalog = async (text) => { - const entries = [ - ...text.matchAll( - /^### (mcp__rea__[A-Za-z0-9_]+)\n[\s\S]*?\nParameters:\n```json\n([\s\S]*?)\n```/gm, - ), - ]; - assert.deepEqual( - entries.map((entry) => entry[1]).sort(), - expectedNames, - "Native exact-name search must return every REA input schema", - ); - for (const entry of entries) { - const schema = JSON.parse(entry[2]); - assert( - schemaValidator.validateSchema(schema), - `${entry[1]}: invalid native schema: ${JSON.stringify(schemaValidator.errors)}`, - ); - } - schemasChecked = entries.length; - catalogDigest = createHash("sha256").update(text).digest("hex"); - await writeFile(join(lab, "native-catalog.md"), text); -}; -const fixture = await createOpenAiModelFixture({ - directory: lab, - prefix: "commandcode", - onRequest: async ({ body, tools }) => { - assert(tools.some((tool) => tool.name === "search_tools")); - const strings = textValues(body.messages); - if (!inventoryChecked) { - const names = [ - ...new Set( - strings.flatMap( - (text) => text.match(/mcp__rea__[A-Za-z0-9_]+/g) ?? [], - ), - ), - ].sort(); - assert.deepEqual( - names, - expectedNames, - "Default native deferred-tool prompt must advertise the complete REA inventory", - ); - assert.equal( - tools.filter((tool) => tool.name.startsWith("mcp__rea__")).length, - 0, - "Keep native lazy schema delivery enabled", - ); - assert( - tools - .find((tool) => tool.name === "activate_skill") - ?.parameters.properties.name.enum.includes( - "reverse-engineer-anything", - ), - ); - inventoryChecked = true; - } - if (mode === "chat") { - assert.equal( - body.messages.filter((message) => message.role === "tool").length, - 0, - ); - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - } - if (stage === 0) - return call("activate_skill", { name: "reverse-engineer-anything" }); - assert( - strings.some((text) => text.includes(expectedSkill)), - "The model must receive the complete activated skill body", - ); - skillLoaded = true; - const last = body.messages - .filter((message) => message.role === "tool") - .at(-1)?.content; - assert.equal(typeof last, "string"); - if (stage === 1) - return call("search_tools", { - query: "select:" + expectedNames.join(","), - }); - if (stage === 2) { - const spill = last.match( - /\[full output saved to: (.+?) — read it with read_file/, - ); - if (spill) { - catalogArtifact = spill[1]; - const child = relative(environment.TMPDIR, catalogArtifact); - assert( - isAbsolute(catalogArtifact) && - child && - !child.startsWith("..") && - !isAbsolute(child), - "Native spill must remain inside this run's temporary directory", - ); - const text = await readFile(catalogArtifact, "utf8"); - await validateCatalog(text); - catalogProjection = { schemas: schemasChecked, sha256: catalogDigest }; - const query = - "const fs=require('node:fs'),crypto=require('node:crypto');const text=fs.readFileSync(process.argv[1],'utf8');const schemas=[...text.matchAll(/^### (mcp__rea__[A-Za-z0-9_]+)\\n/gm)].length;console.log(JSON.stringify({rea_catalog:{schemas,sha256:crypto.createHash('sha256').update(text).digest('hex')}}));"; - return call("shell_command", { - command: process.execPath, - args: ["-e", query, catalogArtifact], - }); - } - await validateCatalog(last); - return call("mcp__rea__analyze_javascript_application", { - input_path: target, - format: "directory", - }); - } - if (catalogArtifact && !artifactProjectionSeen) { - const line = last - .split("\n") - .find((item) => item.startsWith('{"rea_catalog":')); - assert(line, "Native shell must return the selected saved-catalog facts"); - assert.deepEqual(JSON.parse(line).rea_catalog, catalogProjection); - artifactProjectionSeen = true; - return call("mcp__rea__analyze_javascript_application", { - input_path: target, - format: "directory", - }); - } - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(JSON.parse(last)); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shapes = evidence.normalized_result.graph.nodes - .flatMap((node) => node.observations) - .map((observation) => observation.properties) - .filter( - (properties) => - properties.semantic_role === "export-return-shapes" && - properties.exported_name === "result", - ); - assert( - shapes.length > 0 && - shapes.every( - (shape) => shape.return_shape_coverage?.projection_complete, - ), - ); - const fields = shapes.flatMap((shape) => - shape.static_return_shapes.flatMap((item) => item.fields), - ); - assert( - fields.some( - (field) => - field.path === "/clientCompatibility" && - field.state === "literal" && - field.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (field) => - field.path === "/count" && - field.state === "literal" && - field.value === 7, - ), - ); - evidenceBytes = Buffer.byteLength(last); - await writeFile(join(lab, "native-evidence.json"), last); - evidenceSeen = true; - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - }, -}); -await writeFile( - join(profile, "providers.json"), - JSON.stringify({ - provider: { - "rea-local": { - baseURL: fixture.baseUrl, - apiKey: false, - models: { - "rea-client-fixture": { contextWindow: 1_000_000, maxOutput: 4096 }, - }, - }, - }, - }), -); -const original = - JSON.stringify( - { - mcpServers: { other: { command: "unrelated-server", enabled: false } }, - preserved: "keep unrelated settings", - }, - null, - 2, - ) + "\n"; -await writeFile(configPath, original); -const rea = (args) => - exec( - process.execPath, - [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); -try { - clientVersion = ( - await exec(command, ["--version"], { - env: environment, - cwd: workspace, +let passed = false; +let fixture; +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([ + mkdir(target, { recursive: true }), + mkdir(profile, { recursive: true }), + mkdir(join(lab, "tmp")), + ]); + await commands.exec("git", ["init", "--quiet", workspace], { timeout: 30_000, - }) - ).stdout.trim(); - for (const args of [ - ["setup", "--client", "commandcode", "--dry-run"], - ["setup", "--client", "commandcode", "--yes"], - ["doctor", "--client", "commandcode", "--skill"], - ]) { - const response = await rea(args); + stdin: "pipe", + }); await writeFile( - join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), - response.stdout, + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', ); - const result = JSON.parse(response.stdout); - if (args[0] === "doctor") assert.equal(result.healthy, true); - else - assert.equal( - result.status, - args.includes("--dry-run") ? "planned" : "ready", - ); - if (args.includes("--dry-run")) { - assert( - result.plannedActions.some((action) => action.target === configPath), - ); - assert( - result.plannedActions.every( - (action) => - action.target === configPath || - action.target === skillDirectory || - action.target.startsWith(skillDirectory + "/"), + const environment = { + PATH: process.env.PATH, + NODE_OPTIONS: process.env.NODE_OPTIONS, + USERPROFILE: account, + TMPDIR: join(lab, "tmp"), + CMD_LOCAL_ONLY: "1", + // Native print mode checks for an account key even with a keyless BYOK model. + // This synthetic value is never a real credential; local-only refuses hosted API calls. + COMMAND_CODE_API_KEY: "local-fixture-only", + COMMANDCODE_SKIP_UPDATES: "1", + COMMANDCODE_DISABLE_CRON: "1", + OTEL_SDK_DISABLED: "true", + REA_PROCESS_RUN_ID: verifier.run_id, + }; + const schemaValidator = new Ajv2020({ + strict: false, + validateFormats: false, + }); + const textValues = (value) => + typeof value === "string" + ? [value] + : value !== null && typeof value === "object" + ? Object.values(value).flatMap(textValues) + : []; + const call = (name, args) => ({ + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_commandcode_${++stage}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }); + const validateCatalog = async (text) => { + const entries = [ + ...text.matchAll( + /^### (mcp__rea__[A-Za-z0-9_]+)\n[\s\S]*?\nParameters:\n```json\n([\s\S]*?)\n```/gm, ), - "Refuse setup writes outside the disposable account's configuration and skill", + ]; + assert.deepEqual( + entries.map((entry) => entry[1]).sort(), + expectedNames, + "Native exact-name search must return every REA input schema", ); - } - } - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const configured = await readFile(configPath, "utf8"); - assert.equal(JSON.parse(configured).preserved, "keep unrelated settings"); - assert.equal( - JSON.parse(configured).mcpServers.other.command, - "unrelated-server", - ); - const repeated = JSON.parse( - (await rea(["setup", "--client", "commandcode", "--yes"])).stdout, - ); - assert.deepEqual(repeated.appliedActions, []); - assert.equal(await readFile(configPath, "utf8"), configured); - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - expectedSkill = (await readFile(join(skillDirectory, "SKILL.md"), "utf8")) - .replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/, "") - .trim(); - const execution = exec( - command, - [ - "-p", - mode === "chat" - ? "Say hello briefly." - : `Use the REA skill to analyze ${target}.`, - "-m", - "rea-local/rea-client-fixture", - "--local-only", - "--skip-onboarding", - "--trust", - "--output-format", - "json", - "--max-turns", - "8", - "--yolo", - ], - { - env: environment, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, - }, - ); - execution.child.stdin.end(); - const response = await execution; - await writeFile(join(lab, `${mode}.stdout`), response.stdout); - await writeFile(join(lab, `${mode}.stderr`), response.stderr); - assert.equal(fixture.failure, undefined); - const events = response.stdout - .trim() - .split("\n") - .map((line) => JSON.parse(line)); - assert( - events.some( - (event) => - event.type === "result" && - event.subtype === "success" && - event.finalText === "REA_CLIENT_COMPATIBILITY_OK", - ), - ); - assert(inventoryChecked && finalSeen); - if (mode === "call") - assert( - skillLoaded && - schemasChecked === expectedNames.length && - evidenceSeen && - (!catalogArtifact || artifactProjectionSeen), + for (const entry of entries) { + const schema = JSON.parse(entry[2]); + assert( + schemaValidator.validateSchema(schema), + `${entry[1]}: invalid native schema: ${JSON.stringify(schemaValidator.errors)}`, + ); + } + schemasChecked = entries.length; + catalogDigest = createHash("sha256").update(text).digest("hex"); + await writeFile(join(lab, "native-catalog.md"), text); + }; + fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: "commandcode", + onRequest: async ({ body, tools }) => { + assert(tools.some((tool) => tool.name === "search_tools")); + const strings = textValues(body.messages); + if (!inventoryChecked) { + const names = [ + ...new Set( + strings.flatMap( + (text) => text.match(/mcp__rea__[A-Za-z0-9_]+/g) ?? [], + ), + ), + ].sort(); + catalogSize = names.length; + assert.deepEqual( + names, + expectedNames, + "Default native deferred-tool prompt must advertise the complete REA inventory", + ); + assert.equal( + tools.filter((tool) => tool.name.startsWith("mcp__rea__")).length, + 0, + "Keep native lazy schema delivery enabled", + ); + assert( + tools + .find((tool) => tool.name === "activate_skill") + ?.parameters.properties.name.enum.includes( + "reverse-engineer-anything", + ), + ); + inventoryChecked = true; + } + if (mode === "chat") { + assert.equal( + body.messages.filter((message) => message.role === "tool").length, + 0, + ); + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + if (stage === 0) + return call("activate_skill", { name: "reverse-engineer-anything" }); + assert( + strings.some((text) => text.includes(expectedSkill)), + "The model must receive the complete activated skill body", + ); + skillLoaded = true; + const last = body.messages + .filter((message) => message.role === "tool") + .at(-1)?.content; + assert.equal(typeof last, "string"); + if (stage === 1) + return call("search_tools", { + query: "select:" + expectedNames.join(","), + }); + if (stage === 2) { + const spill = last.match( + /\[full output saved to: (.+?) — read it with read_file/, + ); + if (spill) { + catalogArtifact = spill[1]; + const child = relative(environment.TMPDIR, catalogArtifact); + assert( + isAbsolute(catalogArtifact) && + child && + !child.startsWith("..") && + !isAbsolute(child), + "Native spill must remain inside this run's temporary directory", + ); + const text = await readFile(catalogArtifact, "utf8"); + await validateCatalog(text); + catalogProjection = { + schemas: schemasChecked, + sha256: catalogDigest, + }; + const query = + "const fs=require('node:fs'),crypto=require('node:crypto');const text=fs.readFileSync(process.argv[1],'utf8');const schemas=[...text.matchAll(/^### (mcp__rea__[A-Za-z0-9_]+)\\n/gm)].length;console.log(JSON.stringify({rea_catalog:{schemas,sha256:crypto.createHash('sha256').update(text).digest('hex')}}));"; + return call("shell_command", { + command: process.execPath, + args: ["-e", query, catalogArtifact], + }); + } + await validateCatalog(last); + return call("mcp__rea__analyze_javascript_application", { + input_path: target, + format: "directory", + }); + } + if (catalogArtifact && !artifactProjectionSeen) { + const line = last + .split("\n") + .find((item) => item.startsWith('{"rea_catalog":')); + assert( + line, + "Native shell must return the selected saved-catalog facts", + ); + assert.deepEqual(JSON.parse(line).rea_catalog, catalogProjection); + artifactProjectionSeen = true; + return call("mcp__rea__analyze_javascript_application", { + input_path: target, + format: "directory", + }); + } + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(JSON.parse(last)); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shapes = evidence.normalized_result.graph.nodes + .flatMap((node) => node.observations) + .map((observation) => observation.properties) + .filter( + (properties) => + properties.semantic_role === "export-return-shapes" && + properties.exported_name === "result", + ); + assert( + shapes.length > 0 && + shapes.every( + (shape) => shape.return_shape_coverage?.projection_complete, + ), + ); + const fields = shapes.flatMap((shape) => + shape.static_return_shapes.flatMap((item) => item.fields), + ); + assert( + fields.some( + (field) => + field.path === "/clientCompatibility" && + field.state === "literal" && + field.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (field) => + field.path === "/count" && + field.state === "literal" && + field.value === 7, + ), + ); + evidenceBytes = Buffer.byteLength(last); + await writeFile(join(lab, "native-evidence.json"), last); + evidenceSeen = true; + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + }, + }); + await writeFile( + join(profile, "providers.json"), + JSON.stringify({ + provider: { + "rea-local": { + baseURL: fixture.baseUrl, + apiKey: false, + models: { + "rea-client-fixture": { + contextWindow: 1_000_000, + maxOutput: 4096, + }, + }, + }, + }, + }), ); -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, `${mode}.stdout`), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, `${mode}.stderr`), cause.stderr); -} finally { - await fixture.close(); - const receipt = { - client: "Command Code", - clientVersion, - mode, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, + const original = + JSON.stringify( + { + mcpServers: { + other: { command: "unrelated-server", enabled: false }, + }, + preserved: "keep unrelated settings", + }, + null, + 2, + ) + "\n"; + await writeFile(configPath, original); + const rea = (args) => + commands.exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + clientVersion = ( + await commands.exec(command, ["--version"], { + env: environment, + cwd: workspace, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + for (const args of [ + ["setup", "--client", "commandcode", "--dry-run"], + ["setup", "--client", "commandcode", "--yes"], + ["doctor", "--client", "commandcode", "--skill"], + ]) { + const response = await rea(args); + await writeFile( + join( + lab, + `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`, + ), + response.stdout, + ); + const result = JSON.parse(response.stdout); + if (args[0] === "doctor") assert.equal(result.healthy, true); + else + assert.equal( + result.status, + args.includes("--dry-run") ? "planned" : "ready", + ); + if (args.includes("--dry-run")) { + assert( + result.plannedActions.some((action) => action.target === configPath), + ); + assert( + result.plannedActions.every( + (action) => + action.target === configPath || + action.target === skillDirectory || + action.target.startsWith(skillDirectory + "/"), + ), + "Refuse setup writes outside the disposable account's configuration and skill", + ); + } + } + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + assert.equal(JSON.parse(configured).preserved, "keep unrelated settings"); + assert.equal( + JSON.parse(configured).mcpServers.other.command, + "unrelated-server", + ); + const repeated = JSON.parse( + (await rea(["setup", "--client", "commandcode", "--yes"])).stdout, + ); + assert.deepEqual(repeated.appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + expectedSkill = (await readFile(join(skillDirectory, "SKILL.md"), "utf8")) + .replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/, "") + .trim(); + const response = await saveVerifierCommandOutput( + () => + commands.exec( + command, + [ + "-p", + mode === "chat" + ? "Say hello briefly." + : `Use the REA skill to analyze ${target}.`, + "-m", + "rea-local/rea-client-fixture", + "--local-only", + "--skip-onboarding", + "--trust", + "--output-format", + "json", + "--max-turns", + "8", + "--yolo", + ], + { + env: environment, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + stdin: "closed-pipe", + }, + ), + { + stdout: join(lab, `${mode}.stdout`), + stderr: join(lab, `${mode}.stderr`), + }, + ); + assert.equal(fixture.failure, undefined); + const events = response.stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + assert( + events.some( + (event) => + event.type === "result" && + event.subtype === "success" && + event.finalText === "REA_CLIENT_COMPATIBILITY_OK", + ), + ); + assert(inventoryChecked && finalSeen); + if (mode === "call") + assert( + skillLoaded && + schemasChecked === expectedNames.length && + evidenceSeen && + (!catalogArtifact || artifactProjectionSeen), + ); + }, [ + () => commands.close(), + async () => { + await fixture?.close(); }, - modelFixture: - "loopback OpenAI completions/SSE, keyless BYOK, synthetic account-key gate and usage; no hosted authentication or live-model claim", - discovery: - "actual disposable OS account home; native user MCP configuration and default shared skill discovery; Windows unverified", - interface: - "default deferred schemas, complete exact-name search, native saved-catalog query when needed, actual JavaScript MCP call", - inventoryChecked, - catalogSize: inventoryChecked ? expectedNames.length : 0, - schemasChecked, - skillLoaded, - catalogArtifact, - catalogDigest, - artifactProjectionSeen, - evidenceSeen, - evidenceBytes, - finalSeen, - requests: fixture.requests, - probes: fixture.probes, - fixtureFailure: fixture.failure, - verifier: await completeVerifierRun(verifier), - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; + ]); + passed = true; +}, [ + async () => { + const receipt = { + client: "Command Code", + clientVersion, + mode, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback OpenAI completions/SSE, keyless BYOK, synthetic account-key gate and usage; no hosted authentication or live-model claim", + discovery: + "actual disposable OS account home; native user MCP configuration and default shared skill discovery; Windows unverified", + interface: + "default deferred schemas, complete exact-name search, native saved-catalog query when needed, actual JavaScript MCP call", + inventoryChecked, + catalogSize, + schemasChecked, + skillLoaded, + catalogArtifact, + catalogDigest, + artifactProjectionSeen, + evidenceSeen, + evidenceBytes, + finalSeen, + requests: fixture?.requests, + probes: fixture?.probes, + fixtureFailure: fixture?.failure, + verifier: await completeVerifierRun(verifier), + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); + }, + ], + ); + }, +]); diff --git a/scripts/verify-copilot-client.mjs b/scripts/verify-copilot-client.mjs index 02b069d694..4225f87561 100644 --- a/scripts/verify-copilot-client.mjs +++ b/scripts/verify-copilot-client.mjs @@ -12,7 +12,9 @@ import { import { advertisedInputJsonSchema } from "../dist/contracts/toolSchemaMetadata.js"; import { presentAdvertisedInputSchema } from "../dist/contracts/compactInputSchema.js"; import { compactPresentationForMcpInputSchemaProfile } from "../dist/config/mcpInputSchemaProfile.js"; -import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -49,435 +51,473 @@ const skillDirectory = join( "skills", "reverse-engineer-anything", ); -await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]); -await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); -// Do not inherit the caller's home or credentials into the isolated native profile. -const environment = { - PATH: process.env.PATH, - NODE_OPTIONS: process.env.NODE_OPTIONS, - HOME: account, - USERPROFILE: account, - COPILOT_HOME: profile, - COPILOT_OFFLINE: "true", - COPILOT_PROVIDER_TYPE: "openai", - COPILOT_PROVIDER_WIRE_API: "completions", - COPILOT_PROVIDER_API_KEY: "local-fixture-key", - COPILOT_PROVIDER_API_KEY_COMMAND: "", - COPILOT_PROVIDER_BEARER_TOKEN: "", - COPILOT_PROVIDER_HEADERS: "", - COPILOT_PROVIDER_MODEL_ID: model, - COPILOT_PROVIDER_WIRE_MODEL: wireModel, - COPILOT_PROVIDER_MAX_PROMPT_TOKENS: "1000000", - COPILOT_PROVIDER_MAX_OUTPUT_TOKENS: "4096", - REA_PROCESS_RUN_ID: verifier.run_id, -}; -let clientVersion; -try { - clientVersion = ( - await exec(command, ["--version"], { env: environment, timeout: 30_000 }) - ).stdout.trim(); -} catch (cause) { - throw new Error( - `Copilot CLI preflight failed for ${command}; install Copilot CLI or set REA_VERIFY_COPILOT_COMMAND. ${String(cause)}`, - ); -} -const target = join(lab, "fixture α测试"); -await mkdir(target); -await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', -); +let clientVersion = null; +let fixture; +let passed = false; let skillRequested = false; let skillLoaded = false; let expectedSkillBody; let evidenceSeen = false; let finalSeen = false; -let catalogSize = 0; +let catalogSize = null; let schemasChecked = 0; let spillReadRequested = false; let artifactDigest; -const schemaValidator = new Ajv2020({ strict: false, validateFormats: false }); -const containsText = (value, text) => { - if (typeof value === "string") return value.includes(text); - if (value && typeof value === "object") - return Object.values(value).some((nested) => containsText(nested, text)); - return false; -}; -const extractEvidence = (value) => { - if (typeof value === "string") { - try { - return extractEvidence(JSON.parse(value)); - } catch { - return undefined; - } - } - if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0 - ) - return value; - for (const nested of Object.values(value)) { - const evidence = extractEvidence(nested); - if (evidence) return evidence; - } - } - return undefined; -}; const wireModels = []; -const fixture = await createOpenAiModelFixture({ - directory: lab, - prefix: "copilot", - onRequest: async ({ body, tools, requestIndex }) => { - assert.equal( - body.model, - wireModel, - "Copilot must send the selected wire model", +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]); + await commands.exec("git", ["init", "--quiet", workspace], { + timeout: 30_000, + stdin: "pipe", + }); + // Do not inherit the caller's home or credentials into the isolated native profile. + const environment = { + PATH: process.env.PATH, + NODE_OPTIONS: process.env.NODE_OPTIONS, + HOME: account, + USERPROFILE: account, + COPILOT_HOME: profile, + COPILOT_OFFLINE: "true", + COPILOT_PROVIDER_TYPE: "openai", + COPILOT_PROVIDER_WIRE_API: "completions", + COPILOT_PROVIDER_API_KEY: "local-fixture-key", + COPILOT_PROVIDER_API_KEY_COMMAND: "", + COPILOT_PROVIDER_BEARER_TOKEN: "", + COPILOT_PROVIDER_HEADERS: "", + COPILOT_PROVIDER_MODEL_ID: model, + COPILOT_PROVIDER_WIRE_MODEL: wireModel, + COPILOT_PROVIDER_MAX_PROMPT_TOKENS: "1000000", + COPILOT_PROVIDER_MAX_OUTPUT_TOKENS: "4096", + REA_PROCESS_RUN_ID: verifier.run_id, + }; + try { + clientVersion = ( + await commands.exec(command, ["--version"], { + env: environment, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + } catch (cause) { + throw new Error( + `Copilot CLI preflight failed for ${command}; install Copilot CLI or set REA_VERIFY_COPILOT_COMMAND. ${String(cause)}`, + { cause }, + ); + } + const target = join(lab, "fixture α测试"); + await mkdir(target); + await writeFile( + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', ); - wireModels.push(body.model); - // Auxiliary requests without tools cannot execute the analyst's workflow. - if (tools.length === 0) - return { role: "assistant", content: "REA fixture title" }; - const analysis = tools.find((t) => - t.name.endsWith("analyze_javascript_application"), - ); - assert(analysis, "Copilot CLI must expose the REA analysis tool"); - const prefix = analysis.name.slice( - 0, - -"analyze_javascript_application".length, - ); - const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort(); - const reaTools = tools.filter((t) => expected.includes(t.name)); - assert.deepEqual( - reaTools.map((t) => t.name).sort(), - expected, - "Copilot CLI must forward the complete REA catalog", - ); - catalogSize = expected.length; - if (schemasChecked === 0) { - for (const tool of reaTools) { - assert( - tool.parameters, - `${tool.name}: missing advertised input schema`, + const schemaValidator = new Ajv2020({ + strict: false, + validateFormats: false, + }); + const containsText = (value, text) => { + if (typeof value === "string") return value.includes(text); + if (value && typeof value === "object") + return Object.values(value).some((nested) => + containsText(nested, text), ); - assert( - schemaValidator.validateSchema(tool.parameters), - `${tool.name}: invalid input schema: ${JSON.stringify(schemaValidator.errors)}`, + return false; + }; + const extractEvidence = (value) => { + if (typeof value === "string") { + try { + return extractEvidence(JSON.parse(value)); + } catch { + return undefined; + } + } + if (value && typeof value === "object") { + if ( + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes?.length > 0 + ) + return value; + for (const nested of Object.values(value)) { + const evidence = extractEvidence(nested); + if (evidence) return evidence; + } + } + return undefined; + }; + fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: "copilot", + onRequest: async ({ body, tools, requestIndex }) => { + assert.equal( + body.model, + wireModel, + "Copilot must send the selected wire model", ); + wireModels.push(body.model); + // Auxiliary requests without tools cannot execute the analyst's workflow. + if (tools.length === 0) + return { role: "assistant", content: "REA fixture title" }; + const analysis = tools.find((t) => + t.name.endsWith("analyze_javascript_application"), + ); + assert(analysis, "Copilot CLI must expose the REA analysis tool"); + const prefix = analysis.name.slice( + 0, + -"analyze_javascript_application".length, + ); + const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort(); + const reaTools = tools.filter((t) => expected.includes(t.name)); + catalogSize = reaTools.length; assert.deepEqual( - tool.parameters, - presentAdvertisedInputSchema( - advertisedInputJsonSchema( - toolContract(tool.name.slice(prefix.length)), - ), - compactPresentationForMcpInputSchemaProfile(inputSchemaProfile), - ), - `${tool.name}: Copilot must forward the selected ${inputSchemaProfile} input schema`, + reaTools.map((t) => t.name).sort(), + expected, + "Copilot CLI must forward the complete REA catalog", ); - schemasChecked++; + if (schemasChecked === 0) { + for (const tool of reaTools) { + assert( + tool.parameters, + `${tool.name}: missing advertised input schema`, + ); + assert( + schemaValidator.validateSchema(tool.parameters), + `${tool.name}: invalid input schema: ${JSON.stringify(schemaValidator.errors)}`, + ); + assert.deepEqual( + tool.parameters, + presentAdvertisedInputSchema( + advertisedInputJsonSchema( + toolContract(tool.name.slice(prefix.length)), + ), + compactPresentationForMcpInputSchemaProfile(inputSchemaProfile), + ), + `${tool.name}: Copilot must forward the selected ${inputSchemaProfile} input schema`, + ); + schemasChecked++; + } + } + const replies = body.messages.filter((m) => m.role === "tool"); + // Native skill activation injects its context as a user message after the acknowledgement. + skillLoaded ||= + skillRequested && + expectedSkillBody !== undefined && + containsText(body.messages, expectedSkillBody) && + containsText(body.messages, skillDirectory); + const observed = extractEvidence(replies); + if (observed) { + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(observed); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((n) => n.observations) + .map((o) => o.properties) + .find( + (p) => + p.semantic_role === "export-return-shapes" && + p.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap((s) => s.fields); + assert( + fields.some( + (f) => + f.path === "/clientCompatibility" && + f.state === "literal" && + f.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (f) => + f.path === "/count" && f.state === "literal" && f.value === 7, + ), + ); + const bytes = JSON.stringify(evidence); + artifactDigest = createHash("sha256").update(bytes).digest("hex"); + await writeFile(join(lab, "copilot-evidence.json"), bytes); + evidenceSeen = true; + } + if (mode === "chat" || evidenceSeen) { + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + assert( + requestIndex <= 8, + "Local model workflow exceeded its turn budget", + ); + const call = (name, args) => ({ + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_rea_fixture_${requestIndex}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }); + if (!skillRequested) { + const skill = tools.find((t) => t.name === "skill"); + assert(skill, "Copilot CLI must expose its native skill tool"); + skillRequested = true; + return call(skill.name, { skill: "reverse-engineer-anything" }); + } + assert( + skillLoaded, + "Copilot CLI must load the installed personal REA skill", + ); + const spilled = replies.find( + (message) => + typeof message.content === "string" && + message.content.startsWith("Output too large to read at once "), + ); + if (spilled) { + assert( + !spillReadRequested, + "Copilot did not return complete Evidence after its native spill read", + ); + const path = /\. Saved to: ([^\n]+)\n/u.exec(spilled.content)?.[1]; + assert( + path && + isAbsolute(path) && + /^\d+-copilot-tool-output-[a-f\d]+\.txt$/u.test(basename(path)), + "Unexpected native spill path", + ); + const view = tools.find((tool) => tool.name === "view"); + assert(view, "Copilot must expose its native file reader"); + spillReadRequested = true; + return call(view.name, { path, forceReadLargeFiles: true }); + } + return call(analysis.name, { input_path: target, format: "directory" }); + }, + }); + const original = JSON.stringify( + { + mcpServers: { + other: { + type: "local", + command: "unrelated-server", + args: [], + enabled: false, + }, + }, + }, + null, + 2, + ); + await writeFile(configPath, original); + for (const args of [ + ["setup", "--client", "copilot_cli", "--dry-run"], + ["setup", "--client", "copilot_cli", "--yes"], + ["doctor", "--client", "copilot_cli", "--skill"], + ]) { + const result = await commands.exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + await writeFile( + join( + lab, + `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`, + ), + result.stdout, + ); + const parsed = JSON.parse(result.stdout); + if (args[0] === "doctor") assert.equal(parsed.healthy, true); + else { + assert.equal( + parsed.status, + args.includes("--dry-run") ? "planned" : "ready", + ); + if (args.includes("--dry-run")) { + assert(parsed.plannedActions.some((a) => a.target === configPath)); + assert( + parsed.plannedActions.some((a) => a.target === skillDirectory), + ); + assert( + parsed.plannedActions.every( + (a) => + a.target === configPath || + a.target === skillDirectory || + a.target.startsWith(skillDirectory + "/"), + ), + ); + } } } - const replies = body.messages.filter((m) => m.role === "tool"); - // Native skill activation injects its context as a user message after the acknowledgement. - skillLoaded ||= - skillRequested && - expectedSkillBody !== undefined && - containsText(body.messages, expectedSkillBody) && - containsText(body.messages, skillDirectory); - const observed = extractEvidence(replies); - if (observed) { - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(observed); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((n) => n.observations) - .map((o) => o.properties) - .find( - (p) => - p.semantic_role === "export-return-shapes" && - p.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((s) => s.fields); - assert( - fields.some( - (f) => - f.path === "/clientCompatibility" && - f.state === "literal" && - f.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (f) => f.path === "/count" && f.state === "literal" && f.value === 7, - ), - ); - const bytes = JSON.stringify(evidence); - artifactDigest = createHash("sha256").update(bytes).digest("hex"); - await writeFile(join(lab, "copilot-evidence.json"), bytes); - evidenceSeen = true; + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + assert(configured.includes("unrelated-server")); + + const repeated = await commands.exec( + process.execPath, + [ + join(runtimeRoot, "scripts/rea.mjs"), + "setup", + "--client", + "copilot_cli", + "--yes", + "--json", + ], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + // Bind both choices explicitly: the server's default can change independently + // of the profile this verifier promises to exercise and records in its receipt. + const customized = JSON.parse(configured); + assert(customized.mcpServers.rea); + customized.mcpServers.rea.env = { + ...customized.mcpServers.rea.env, + REA_MCP_INPUT_SCHEMA_PROFILE: inputSchemaProfile, + }; + await writeFile(configPath, JSON.stringify(customized, null, 2)); + const installedSkill = await readFile( + join(skillDirectory, "SKILL.md"), + "utf8", + ); + assert(installedSkill.includes("name: reverse-engineer-anything")); + const skillLines = installedSkill.split("\n"); + const headerEnd = skillLines.indexOf("---", 1); + assert( + skillLines[0] === "---" && headerEnd > 0, + "Installed skill must have its authored frontmatter", + ); + expectedSkillBody = skillLines + .slice(headerEnd + 1) + .join("\n") + .trim(); + assert(expectedSkillBody.length > 0); + for (const [name, args] of [ + ["mcp-list", ["mcp", "list"]], + ["skill-add", ["skill", "add", skillDirectory]], + ["skill-list", ["skill", "list"]], + ]) { + const result = await commands.exec(command, args, { + env: environment, + cwd: workspace, + timeout: 30_000, + stdin: "pipe", + }); + await writeFile(join(lab, `${name}.stdout`), result.stdout); + await writeFile(join(lab, `${name}.stderr`), result.stderr); + if (name === "mcp-list") assert(result.stdout.includes("rea")); + if (name === "skill-list") + assert(result.stdout.includes("reverse-engineer-anything")); } - if (mode === "chat" || evidenceSeen) { - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - } - assert(requestIndex <= 8, "Local model workflow exceeded its turn budget"); - const call = (name, args) => ({ - role: "assistant", - content: null, - tool_calls: [ - { - id: `call_rea_fixture_${requestIndex}`, - type: "function", - function: { name, arguments: JSON.stringify(args) }, + const response = await saveVerifierCommandOutput( + () => + commands.exec( + command, + [ + "--disable-builtin-mcps", + "--disable-mcp-server", + "other", + "--no-ask-user", + "--output-format", + "json", + "--model", + model, + "--allow-tool=skill", + "--allow-tool=read", + "--allow-tool=rea(analyze_javascript_application)", + "--add-dir", + target, + "--prompt", + mode === "chat" + ? "Say hello briefly." + : `Use the REA skill and tools to analyze the JavaScript directory ${target}. Report the result.`, + ], + { + env: { ...environment, COPILOT_PROVIDER_BASE_URL: fixture.baseUrl }, + cwd: workspace, + timeout: 120_000, + stdin: "closed-pipe", + maxBuffer: 10 * 1024 * 1024, + }, + ), + { + stdout: join(lab, `${mode}.stdout`), + stderr: join(lab, `${mode}.stderr`), + }, + ); + assert.equal(fixture.failure, undefined); + const events = response.stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + assert( + !events.some((event) => event.type === "session.error"), + "Copilot emitted a session error", + ); + assert(finalSeen); + assert( + events.some( + (event) => + event.type === "assistant.message" && + event.data.content === "REA_CLIENT_COMPATIBILITY_OK", + ), + "Copilot must finish with the expected native assistant message", + ); + if (mode === "call") assert(evidenceSeen && skillLoaded); + }, [ + () => commands.close(), + async () => { + await fixture?.close(); + }, + ]); + passed = true; +}, [ + async () => { + const lineage = await completeVerifierRun(verifier); + const receipt = { + client: "Copilot CLI", + clientVersion, + mode, + model, + wireModel, + inputSchemaProfile, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback OpenAI completions/SSE, synthetic usage, requested one-million-token BYOK prompt capacity; effective capacity unknown; no live provider claim", + skillDiscovery: + "native COPILOT_HOME MCP configuration and caller-added isolated skill directory; default OS-home discovery unverified", + capacityBoundary: + inputSchemaProfile === "compact" + ? "Explicit compact profile: complete inventory and canonical server validation; advertised schemas may be reduced. This does not fix the full-profile gpt-4.1 context boundary." + : "Full advertised input schemas, checked at the client request boundary. Client model ID and wire model are recorded separately; effective token capacity and live provider compatibility remain unknown.", + catalogSize, + schemasChecked, + spillReadRequested, + evidenceSeen, + skillLoaded, + artifactDigest, + finalSeen, + wireModels, + requests: fixture?.requests, + probes: fixture?.probes, + fixtureFailure: fixture?.failure, + verifier: lineage, + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); }, ], - }); - if (!skillRequested) { - const skill = tools.find((t) => t.name === "skill"); - assert(skill, "Copilot CLI must expose its native skill tool"); - skillRequested = true; - return call(skill.name, { skill: "reverse-engineer-anything" }); - } - assert( - skillLoaded, - "Copilot CLI must load the installed personal REA skill", ); - const spilled = replies.find( - (message) => - typeof message.content === "string" && - message.content.startsWith("Output too large to read at once "), - ); - if (spilled) { - assert( - !spillReadRequested, - "Copilot did not return complete Evidence after its native spill read", - ); - const path = /\. Saved to: ([^\n]+)\n/u.exec(spilled.content)?.[1]; - assert( - path && - isAbsolute(path) && - /^\d+-copilot-tool-output-[a-f\d]+\.txt$/u.test(basename(path)), - "Unexpected native spill path", - ); - const view = tools.find((tool) => tool.name === "view"); - assert(view, "Copilot must expose its native file reader"); - spillReadRequested = true; - return call(view.name, { path, forceReadLargeFiles: true }); - } - return call(analysis.name, { input_path: target, format: "directory" }); }, -}); -const original = JSON.stringify( - { - mcpServers: { - other: { - type: "local", - command: "unrelated-server", - args: [], - enabled: false, - }, - }, - }, - null, - 2, -); -await writeFile(configPath, original); -let failure; -try { - for (const args of [ - ["setup", "--client", "copilot_cli", "--dry-run"], - ["setup", "--client", "copilot_cli", "--yes"], - ["doctor", "--client", "copilot_cli", "--skill"], - ]) { - const result = await exec( - process.execPath, - [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); - await writeFile( - join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), - result.stdout, - ); - const parsed = JSON.parse(result.stdout); - if (args[0] === "doctor") assert.equal(parsed.healthy, true); - else { - assert.equal( - parsed.status, - args.includes("--dry-run") ? "planned" : "ready", - ); - if (args.includes("--dry-run")) { - assert(parsed.plannedActions.some((a) => a.target === configPath)); - assert(parsed.plannedActions.some((a) => a.target === skillDirectory)); - assert( - parsed.plannedActions.every( - (a) => - a.target === configPath || - a.target === skillDirectory || - a.target.startsWith(skillDirectory + "/"), - ), - ); - } - } - } - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const configured = await readFile(configPath, "utf8"); - assert(configured.includes("unrelated-server")); - - const repeated = await exec( - process.execPath, - [ - join(runtimeRoot, "scripts/rea.mjs"), - "setup", - "--client", - "copilot_cli", - "--yes", - "--json", - ], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); - assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); - assert.equal(await readFile(configPath, "utf8"), configured); - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - // Bind both choices explicitly: the server's default can change independently - // of the profile this verifier promises to exercise and records in its receipt. - const customized = JSON.parse(configured); - assert(customized.mcpServers.rea); - customized.mcpServers.rea.env = { - ...customized.mcpServers.rea.env, - REA_MCP_INPUT_SCHEMA_PROFILE: inputSchemaProfile, - }; - await writeFile(configPath, JSON.stringify(customized, null, 2)); - const installedSkill = await readFile( - join(skillDirectory, "SKILL.md"), - "utf8", - ); - assert(installedSkill.includes("name: reverse-engineer-anything")); - const skillLines = installedSkill.split("\n"); - const headerEnd = skillLines.indexOf("---", 1); - assert( - skillLines[0] === "---" && headerEnd > 0, - "Installed skill must have its authored frontmatter", - ); - expectedSkillBody = skillLines - .slice(headerEnd + 1) - .join("\n") - .trim(); - assert(expectedSkillBody.length > 0); - for (const [name, args] of [ - ["mcp-list", ["mcp", "list"]], - ["skill-add", ["skill", "add", skillDirectory]], - ["skill-list", ["skill", "list"]], - ]) { - const result = await exec(command, args, { - env: environment, - cwd: workspace, - timeout: 30_000, - }); - await writeFile(join(lab, `${name}.stdout`), result.stdout); - await writeFile(join(lab, `${name}.stderr`), result.stderr); - if (name === "mcp-list") assert(result.stdout.includes("rea")); - if (name === "skill-list") - assert(result.stdout.includes("reverse-engineer-anything")); - } - const execution = exec( - command, - [ - "--disable-builtin-mcps", - "--disable-mcp-server", - "other", - "--no-ask-user", - "--output-format", - "json", - "--model", - model, - "--allow-tool=skill", - "--allow-tool=read", - "--allow-tool=rea(analyze_javascript_application)", - "--add-dir", - target, - "--prompt", - mode === "chat" - ? "Say hello briefly." - : `Use the REA skill and tools to analyze the JavaScript directory ${target}. Report the result.`, - ], - { - env: { ...environment, COPILOT_PROVIDER_BASE_URL: fixture.baseUrl }, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, - }, - ); - execution.child.stdin.end(); - const response = await execution; - await writeFile(join(lab, `${mode}.stdout`), response.stdout); - await writeFile(join(lab, `${mode}.stderr`), response.stderr); - assert.equal(fixture.failure, undefined); - const events = response.stdout - .trim() - .split("\n") - .map((line) => JSON.parse(line)); - assert( - !events.some((event) => event.type === "session.error"), - "Copilot emitted a session error", - ); - assert(finalSeen); - assert( - events.some( - (event) => - event.type === "assistant.message" && - event.data.content === "REA_CLIENT_COMPATIBILITY_OK", - ), - "Copilot must finish with the expected native assistant message", - ); - if (mode === "call") assert(evidenceSeen && skillLoaded); -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, `${mode}.stdout`), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, `${mode}.stderr`), cause.stderr); -} finally { - await fixture.close(); - const lineage = await completeVerifierRun(verifier); - const receipt = { - client: "Copilot CLI", - clientVersion, - mode, - model, - wireModel, - inputSchemaProfile, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, - }, - modelFixture: - "loopback OpenAI completions/SSE, synthetic usage, requested one-million-token BYOK prompt capacity; effective capacity unknown; no live provider claim", - skillDiscovery: - "native COPILOT_HOME MCP configuration and caller-added isolated skill directory; default OS-home discovery unverified", - capacityBoundary: - inputSchemaProfile === "compact" - ? "Explicit compact profile: complete inventory and canonical server validation; advertised schemas may be reduced. This does not fix the full-profile gpt-4.1 context boundary." - : "Full advertised input schemas, checked at the client request boundary. Client model ID and wire model are recorded separately; effective token capacity and live provider compatibility remain unknown.", - catalogSize, - schemasChecked, - spillReadRequested, - evidenceSeen, - skillLoaded, - artifactDigest, - finalSeen, - wireModels, - requests: fixture.requests, - probes: fixture.probes, - fixtureFailure: fixture.failure, - verifier: lineage, - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; +]); diff --git a/scripts/verify-deepseek-client.mjs b/scripts/verify-deepseek-client.mjs index b8d8dd7409..9f5a3bbc1e 100644 --- a/scripts/verify-deepseek-client.mjs +++ b/scripts/verify-deepseek-client.mjs @@ -10,7 +10,10 @@ import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; -import { exec, pathExists } from "./lib/verify-package-core.mjs"; +import { pathExists } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -36,66 +39,9 @@ const skillPath = join( const cli = join(runtimeRoot, "scripts", "rea.mjs"); const configPath = join(profile, "cordis.patch.yml"); const verifier = createVerifierRun(); -await Promise.all([ - mkdir(account), - mkdir(profile), - mkdir(target, { recursive: true }), -]); -const environment = { - PATH: process.env.PATH ?? "", - HOME: account, - USERPROFILE: account, - DSH_HOME: profile, - OPENAI_API_KEY: "local-fixture-key", - REA_PROCESS_RUN_ID: verifier.run_id, -}; -// A temporary directory under the checkout must not inherit its project skills or AGENTS.md. -await exec( - "git", - ["-c", "init.defaultBranch=main", "init", "--quiet", workspace], - { - env: environment, - timeout: 30_000, - }, -); -await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', -); -const clientVersion = ( - await exec(command, ["--version"], { - env: environment, - cwd: workspace, - timeout: 30_000, - }) -).stdout.trim(); -const lifecycle = []; -const runRea = async (args, artifact) => { - const result = await exec(process.execPath, [cli, ...args, "--json"], { - env: environment, - cwd: workspace, - timeout: 60_000, - }); - await writeFile(join(lab, `${artifact}.json`), result.stdout); - return JSON.parse(result.stdout); -}; -const patches = (text) => - parseYaml(text, { - customTags: [ - { - tag: "tag:yaml.org,2002:js", - resolve: (source) => ({ __jsExpr: source }), - }, - ], - }); -const withoutRea = (text) => - patches(text).flatMap((op) => { - if (op.id === "rea-mcp") return []; - if (!Array.isArray(op.insert)) return [op]; - const insert = op.insert.filter((entry) => entry.id !== "rea-mcp"); - return insert.length === 0 ? [] : [{ ...op, insert }]; - }); - +let clientVersion = null; +let fixture; +let passed = false; let catalog; let patternCount = 0; let skillRequested = false; @@ -104,155 +50,223 @@ let evidenceSeen = false; let finalSeen = false; let artifactDigest; let mainRequests = 0; -const ajv = new Ajv2020({ strict: false, validateFormats: false }); +const lifecycle = []; const patterns = new Set(); -const validatePatterns = (value) => { - if (!value || typeof value !== "object") return; - for (const [key, child] of Object.entries(value)) { - if (key === "pattern" && typeof child === "string") { - // Check the representation the native client actually forwards to its model adapter. - for (const flags of ["", "u", "v"]) new RegExp(child, flags); - patterns.add(child); - patternCount++; - } else validatePatterns(child); - } -}; -const extractEvidence = (value) => { - if (typeof value === "string") { - try { - return extractEvidence(JSON.parse(value)); - } catch { - return undefined; - } - } - if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes - ) - return value; - for (const child of Object.values(value)) { - const found = extractEvidence(child); - if (found) return found; - } - } -}; -const fixture = await createOpenAiModelFixture({ - directory: lab, - prefix: `deepseek-${mode}`, - onRequest: async ({ body, tools, requestIndex }) => { - // Harness generates a session title in a separate, tool-free model request. - if (tools.length === 0) - return { role: "assistant", content: "REA validation" }; - assert( - ++mainRequests <= 4, - "Harness did not complete the bounded fixture workflow.", +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([ + mkdir(account), + mkdir(profile), + mkdir(target, { recursive: true }), + ]); + const environment = { + PATH: process.env.PATH ?? "", + HOME: account, + USERPROFILE: account, + DSH_HOME: profile, + OPENAI_API_KEY: "local-fixture-key", + REA_PROCESS_RUN_ID: verifier.run_id, + }; + // A temporary directory under the checkout must not inherit its project skills or AGENTS.md. + await commands.exec( + "git", + ["-c", "init.defaultBranch=main", "init", "--quiet", workspace], + { + env: environment, + timeout: 30_000, + stdin: "pipe", + }, ); - const rea = tools.filter((tool) => tool.name.startsWith("mcp__rea__")); - const observedCatalog = rea - .map((tool) => tool.name.slice("mcp__rea__".length)) - .sort(); - assert.deepEqual( - observedCatalog, - TOOL_CONTRACTS.map((tool) => tool.name).sort(), + await writeFile( + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', ); - if (catalog === undefined) { - catalog = observedCatalog; - for (const tool of rea) { - assert( - ajv.validateSchema(tool.parameters), - `${tool.name}: ${ajv.errorsText(ajv.errors)}`, - ); - validatePatterns(tool.parameters); - } - assert( - patternCount > 0, - "The advertised regex constraints must reach the model.", - ); - } - const toolMessages = (body.messages ?? []).filter( - (message) => message.role === "tool", - ); - const serialized = JSON.stringify(toolMessages); - skillLoaded ||= - serialized.includes( - '', - ) && serialized.includes(dirname(skillPath)); - for (const message of toolMessages) { - const found = extractEvidence(message.content); - if (found === undefined) continue; - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(found); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((node) => node.observations) - .map((observation) => observation.properties) - .find( - (properties) => - properties.semantic_role === "export-return-shapes" && - properties.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap( - (entry) => entry.fields, - ); - assert( - fields.some( - (field) => - field.path === "/clientCompatibility" && - field.state === "literal" && - field.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (field) => - field.path === "/count" && - field.state === "literal" && - field.value === 7, - ), - ); - const text = JSON.stringify(evidence); - await writeFile(join(lab, "evidence.json"), text); - artifactDigest = createHash("sha256").update(text).digest("hex"); - evidenceSeen = true; - } - if (mode === "chat" || evidenceSeen) { - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - } - const call = (name, args) => ({ - role: "assistant", - content: null, - tool_calls: [ + clientVersion = ( + await commands.exec(command, ["--version"], { + env: environment, + cwd: workspace, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + const runRea = async (args, artifact) => { + const result = await commands.exec( + process.execPath, + [cli, ...args, "--json"], { - id: `call_rea_${requestIndex}`, - type: "function", - function: { name, arguments: JSON.stringify(args) }, + env: environment, + cwd: workspace, + timeout: 60_000, + stdin: "pipe", }, - ], + ); + await writeFile(join(lab, `${artifact}.json`), result.stdout); + return JSON.parse(result.stdout); + }; + const patches = (text) => + parseYaml(text, { + customTags: [ + { + tag: "tag:yaml.org,2002:js", + resolve: (source) => ({ __jsExpr: source }), + }, + ], + }); + const withoutRea = (text) => + patches(text).flatMap((op) => { + if (op.id === "rea-mcp") return []; + if (!Array.isArray(op.insert)) return [op]; + const insert = op.insert.filter((entry) => entry.id !== "rea-mcp"); + return insert.length === 0 ? [] : [{ ...op, insert }]; + }); + + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const validatePatterns = (value) => { + if (!value || typeof value !== "object") return; + for (const [key, child] of Object.entries(value)) { + if (key === "pattern" && typeof child === "string") { + // Check the representation the native client actually forwards to its model adapter. + for (const flags of ["", "u", "v"]) new RegExp(child, flags); + patterns.add(child); + patternCount++; + } else validatePatterns(child); + } + }; + const extractEvidence = (value) => { + if (typeof value === "string") { + try { + return extractEvidence(JSON.parse(value)); + } catch { + return undefined; + } + } + if (value && typeof value === "object") { + if ( + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes + ) + return value; + for (const child of Object.values(value)) { + const found = extractEvidence(child); + if (found) return found; + } + } + }; + fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: `deepseek-${mode}`, + onRequest: async ({ body, tools, requestIndex }) => { + // Harness generates a session title in a separate, tool-free model request. + if (tools.length === 0) + return { role: "assistant", content: "REA validation" }; + assert( + ++mainRequests <= 4, + "Harness did not complete the bounded fixture workflow.", + ); + const rea = tools.filter((tool) => tool.name.startsWith("mcp__rea__")); + const observedCatalog = rea + .map((tool) => tool.name.slice("mcp__rea__".length)) + .sort(); + assert.deepEqual( + observedCatalog, + TOOL_CONTRACTS.map((tool) => tool.name).sort(), + ); + if (catalog === undefined) { + catalog = observedCatalog; + for (const tool of rea) { + assert( + ajv.validateSchema(tool.parameters), + `${tool.name}: ${ajv.errorsText(ajv.errors)}`, + ); + validatePatterns(tool.parameters); + } + assert( + patternCount > 0, + "The advertised regex constraints must reach the model.", + ); + } + const toolMessages = (body.messages ?? []).filter( + (message) => message.role === "tool", + ); + const serialized = JSON.stringify(toolMessages); + skillLoaded ||= + serialized.includes( + '', + ) && serialized.includes(dirname(skillPath)); + for (const message of toolMessages) { + const found = extractEvidence(message.content); + if (found === undefined) continue; + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(found); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((node) => node.observations) + .map((observation) => observation.properties) + .find( + (properties) => + properties.semantic_role === "export-return-shapes" && + properties.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap( + (entry) => entry.fields, + ); + assert( + fields.some( + (field) => + field.path === "/clientCompatibility" && + field.state === "literal" && + field.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (field) => + field.path === "/count" && + field.state === "literal" && + field.value === 7, + ), + ); + const text = JSON.stringify(evidence); + await writeFile(join(lab, "evidence.json"), text); + artifactDigest = createHash("sha256").update(text).digest("hex"); + evidenceSeen = true; + } + if (mode === "chat" || evidenceSeen) { + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + const call = (name, args) => ({ + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_rea_${requestIndex}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }); + if (!skillRequested) { + assert(tools.some((tool) => tool.name === "skill")); + skillRequested = true; + return call("skill", { name: "reverse-engineer-anything" }); + } + assert( + skillLoaded, + "Harness must load the installed skill before analysis.", + ); + return call("mcp__rea__analyze_javascript_application", { + input_path: target, + format: "directory", + }); + }, }); - if (!skillRequested) { - assert(tools.some((tool) => tool.name === "skill")); - skillRequested = true; - return call("skill", { name: "reverse-engineer-anything" }); - } - assert( - skillLoaded, - "Harness must load the installed skill before analysis.", - ); - return call("mcp__rea__analyze_javascript_application", { - input_path: target, - format: "directory", - }); - }, -}); -let failure; -try { - // Seed model settings and a stale registration; the public setup workflow - // must produce the registration this native client actually boots. - const original = ` + // Seed model settings and a stale registration; the public setup workflow + // must produce the registration this native client actually boots. + const original = ` - id: llm-pi-ai config: providers: @@ -283,127 +297,143 @@ try { env: REA_PROCESS_RUN_ID: ${JSON.stringify(verifier.run_id)} `; - await writeFile(configPath, original); - const planned = await runRea( - ["setup", "--client", "dsh", "--dry-run"], - "setup-plan", - ); - assert.equal(planned.status, "planned"); - assert(planned.plannedActions.some((action) => action.target === configPath)); - assert( - planned.plannedActions.some( - (action) => action.target === dirname(skillPath), - ), - ); - assert( - planned.plannedActions.every( - (action) => - action.target === configPath || action.target === dirname(skillPath), - ), - ); - assert.equal(await readFile(configPath, "utf8"), original); - assert.equal(await pathExists(`${configPath}.rea.backup`), false); - lifecycle.push("dry-run"); - const setup = await runRea(["setup", "--client", "dsh", "--yes"], "setup"); - assert.equal(setup.status, "ready"); - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const configured = await readFile(configPath, "utf8"); - assert.deepEqual(withoutRea(configured), withoutRea(original)); - assert(configured.includes("!!js process.cwd()")); - assert( - (await readFile(skillPath, "utf8")).includes( - "name: reverse-engineer-anything", - ), - ); - lifecycle.push("setup"); - const repeated = await runRea( - ["setup", "--client", "dsh", "--yes"], - "setup-repeat", - ); - assert.deepEqual(repeated.appliedActions, []); - assert.equal(await readFile(configPath, "utf8"), configured); - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - lifecycle.push("idempotent-setup"); - const doctor = await runRea( - ["doctor", "--client", "dsh", "--skill"], - "doctor", - ); - assert.equal(doctor.healthy, true); - lifecycle.push("doctor"); - const execution = exec( - command, - [ - "--profile", - "headless", - "--json", - mode === "chat" ? "Say hello briefly." : `Use REA to analyze ${target}.`, - ], - { - env: environment, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, + await writeFile(configPath, original); + const planned = await runRea( + ["setup", "--client", "dsh", "--dry-run"], + "setup-plan", + ); + assert.equal(planned.status, "planned"); + assert( + planned.plannedActions.some((action) => action.target === configPath), + ); + assert( + planned.plannedActions.some( + (action) => action.target === dirname(skillPath), + ), + ); + assert( + planned.plannedActions.every( + (action) => + action.target === configPath || action.target === dirname(skillPath), + ), + ); + assert.equal(await readFile(configPath, "utf8"), original); + assert.equal(await pathExists(`${configPath}.rea.backup`), false); + lifecycle.push("dry-run"); + const setup = await runRea(["setup", "--client", "dsh", "--yes"], "setup"); + assert.equal(setup.status, "ready"); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + assert.deepEqual(withoutRea(configured), withoutRea(original)); + assert(configured.includes("!!js process.cwd()")); + assert( + (await readFile(skillPath, "utf8")).includes( + "name: reverse-engineer-anything", + ), + ); + lifecycle.push("setup"); + const repeated = await runRea( + ["setup", "--client", "dsh", "--yes"], + "setup-repeat", + ); + assert.deepEqual(repeated.appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + lifecycle.push("idempotent-setup"); + const doctor = await runRea( + ["doctor", "--client", "dsh", "--skill"], + "doctor", + ); + assert.equal(doctor.healthy, true); + lifecycle.push("doctor"); + const result = await saveVerifierCommandOutput( + () => + commands.exec( + command, + [ + "--profile", + "headless", + "--json", + mode === "chat" + ? "Say hello briefly." + : `Use REA to analyze ${target}.`, + ], + { + env: environment, + cwd: workspace, + timeout: 120_000, + stdin: "closed-pipe", + maxBuffer: 10 * 1024 * 1024, + }, + ), + { + stdout: join(lab, "client.stdout"), + stderr: join(lab, "client.stderr"), + }, + ); + assert.equal(fixture.failure, undefined); + assert(finalSeen && result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK")); + if (mode === "call") assert(evidenceSeen && skillLoaded); + lifecycle.push(mode === "call" ? "native-tool-invocation" : "native-chat"); + const uninstall = await runRea(["uninstall"], "uninstall"); + assert.equal(uninstall.status, "complete"); + assert( + uninstall.items.some( + (item) => item.name === "dsh" && item.status === "removed", + ), + ); + assert.deepEqual( + patches(await readFile(configPath, "utf8")), + withoutRea(original), + ); + assert.equal(await pathExists(skillPath), false); + lifecycle.push("uninstall"); + }, [ + () => commands.close(), + async () => { + await fixture?.close(); }, - ); - execution.child.stdin.end(); - const result = await execution; - await writeFile(join(lab, "client.stdout"), result.stdout); - await writeFile(join(lab, "client.stderr"), result.stderr); - assert.equal(fixture.failure, undefined); - assert(finalSeen && result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK")); - if (mode === "call") assert(evidenceSeen && skillLoaded); - lifecycle.push(mode === "call" ? "native-tool-invocation" : "native-chat"); - const uninstall = await runRea(["uninstall"], "uninstall"); - assert.equal(uninstall.status, "complete"); - assert( - uninstall.items.some( - (item) => item.name === "dsh" && item.status === "removed", - ), - ); - assert.deepEqual( - patches(await readFile(configPath, "utf8")), - withoutRea(original), - ); - assert.equal(await pathExists(skillPath), false); - lifecycle.push("uninstall"); -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, "client.stdout"), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, "client.stderr"), cause.stderr); -} finally { - await fixture.close(); - const receipt = { - client: "deepseek-harness", - clientVersion, - mode, - lifecycle, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, - }, - modelFixture: - "loopback custom OpenAI-compatible adapter; no live DeepSeek provider claim", - modelContextWindow: 1000000, - skillDiscovery: - "native HOME shared personal directory; isolated project root", - catalogSize: catalog?.length, - patternCount, - uniquePatterns: patterns.size, - skillLoaded, - evidenceSeen, - artifactDigest, - finalSeen, - requests: fixture.requests, - fixtureFailure: fixture.failure, - verifier: await completeVerifierRun(verifier), - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; + ]); + passed = true; +}, [ + async () => { + const receipt = { + client: "deepseek-harness", + clientVersion, + mode, + lifecycle, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback custom OpenAI-compatible adapter; no live DeepSeek provider claim", + modelContextWindow: 1000000, + skillDiscovery: + "native HOME shared personal directory; isolated project root", + catalogSize: catalog?.length, + patternCount, + uniquePatterns: patterns.size, + skillLoaded, + evidenceSeen, + artifactDigest, + finalSeen, + requests: fixture?.requests, + fixtureFailure: fixture?.failure, + verifier: await completeVerifierRun(verifier), + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); + }, + ], + ); + }, +]); diff --git a/scripts/verify-gemini-client.mjs b/scripts/verify-gemini-client.mjs index ad4d3c00ff..eec9f69ce6 100644 --- a/scripts/verify-gemini-client.mjs +++ b/scripts/verify-gemini-client.mjs @@ -14,7 +14,9 @@ import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; -import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createGeminiModelFixture } from "./verify/clients/gemini-model-fixture.mjs"; @@ -50,402 +52,441 @@ const skillPath = join( "reverse-engineer-anything", "SKILL.md", ); -await Promise.all([ - mkdir(account), - mkdir(workspace), - mkdir(join(profile, ".gemini"), { recursive: true }), -]); -await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); -await writeFile(join(lab, "system-settings.json"), "{}\n"); -await writeFile(join(lab, "system-defaults.json"), "{}\n"); -const environment = { - PATH: process.env.PATH ?? "", - USERPROFILE: account, - GEMINI_CLI_HOME: profile, - GEMINI_CLI_SYSTEM_SETTINGS_PATH: join(lab, "system-settings.json"), - GEMINI_CLI_SYSTEM_DEFAULTS_PATH: join(lab, "system-defaults.json"), - // Preserve the caller's bounded Node heap instead of the launcher's host-memory auto-sizing. - GEMINI_CLI_NO_RELAUNCH: "1", - GEMINI_CLI_TRUST_WORKSPACE: "true", - REA_PROCESS_RUN_ID: verifier.run_id, - REA_MCP_INPUT_SCHEMA_PROFILE: schemaProfile, -}; -let clientVersion; -try { - clientVersion = ( - await exec(...launch(["--version"]), { env: environment, timeout: 30_000 }) - ).stdout.trim(); -} catch (cause) { - const windowsHint = - process.platform === "win32" - ? " On Windows, select the JavaScript entry point; npm installs provide it as node_modules/@google/gemini-cli/bundle/gemini.js." - : ""; - throw new Error( - `Gemini preflight failed for ${command}; install Gemini CLI or set REA_VERIFY_GEMINI_COMMAND.${windowsHint} ${String(cause)}`, - ); -} -const original = JSON.stringify( - { - security: { - auth: { selectedType: "gemini-api-key" }, - folderTrust: { enabled: false }, - }, - telemetry: { enabled: false }, - advanced: { autoConfigureMemory: false }, - model: { name: "gemini-2.5-flash", maxSessionTurns: 8 }, - mcpServers: { other: { command: "unrelated-server" } }, - }, - null, - 2, -); -await writeFile(configPath, original); -for (const args of [ - ["setup", "--client", "gemini_cli", "--dry-run"], - ["setup", "--client", "gemini_cli", "--yes"], - ["doctor", "--client", "gemini_cli", "--skill"], -]) { - const result = await exec( - process.execPath, - [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); - const parsed = JSON.parse(result.stdout); - if (args[0] === "doctor") assert.equal(parsed.healthy, true); - else { - assert.equal( - parsed.status, - args.includes("--dry-run") ? "planned" : "ready", - ); - if (args.includes("--dry-run")) { - assert(parsed.plannedActions.some((a) => a.target === configPath)); - assert( - parsed.plannedActions.some((a) => - a.target.startsWith(join(profile, ".agents", "skills") + sep), - ), - ); - assert( - parsed.plannedActions.every((a) => a.target.startsWith(profile + sep)), - ); - } - } - await writeFile( - join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), - result.stdout, - ); -} -assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); -const configured = await readFile(configPath, "utf8"); -assert.deepEqual(JSON.parse(configured).mcpServers.other, { - command: "unrelated-server", -}); -const repeated = await exec( - process.execPath, - [ - join(runtimeRoot, "scripts/rea.mjs"), - "setup", - "--client", - "gemini_cli", - "--yes", - "--json", - ], - { env: environment, cwd: workspace, timeout: 60_000 }, -); -assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); -assert.equal(await readFile(configPath, "utf8"), configured); -assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); -assert( - (await readFile(skillPath, "utf8")).includes( - "name: reverse-engineer-anything", - ), -); -const target = join(lab, "fixture α测试"); -await mkdir(target); -await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', -); +let clientVersion = null; +let fixture; +let passed = false; let evidenceSeen = false; let skillLoaded = false; let skillRequested = false; let finalSeen = false; -let catalogSize = 0; +let catalogSize = null; let schemasChecked = 0; let examplesChecked = 0; let artifactDigest; -const schemaValidator = new Ajv2020({ strict: false, validateFormats: false }); -const unsupportedLiteralPath = (schema, path = "$", seen = new Set()) => { - if (!schema || typeof schema !== "object" || seen.has(schema)) - return undefined; - seen.add(schema); - for (const [key, child] of Object.entries(schema)) { - const childPath = `${path}.${key}`; - if (key === "const" && (typeof child === "boolean" || child === "")) - return childPath; - if ( - key === "enum" && - Array.isArray(child) && - child.some((item) => typeof item === "boolean" || item === "") - ) - return childPath; - const schemaChildrenAtKey = - schemaMaps.has(key) && - child && - typeof child === "object" && - !Array.isArray(child) - ? Object.entries(child).map(([name, item]) => [ - `${childPath}.${name}`, - item, - ]) - : schemaArrays.has(key) && Array.isArray(child) - ? child.map((item, index) => [`${childPath}[${index}]`, item]) - : schemaChildren.has(key) - ? [[childPath, child]] - : []; - for (const [nestedPath, nested] of schemaChildrenAtKey) { - const found = unsupportedLiteralPath(nested, nestedPath, seen); - if (found !== undefined) return found; - } - } - return undefined; -}; -const extractEvidence = (value) => { - if (typeof value === "string") { - if ( - value.startsWith("\n") && - value.endsWith("\n") - ) { - return extractEvidence( - value.slice( - "\n".length, - -"\n".length, - ), - ); - } +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([ + mkdir(account), + mkdir(workspace), + mkdir(join(profile, ".gemini"), { recursive: true }), + ]); + await commands.exec("git", ["init", "--quiet", workspace], { + timeout: 30_000, + stdin: "pipe", + }); + await writeFile(join(lab, "system-settings.json"), "{}\n"); + await writeFile(join(lab, "system-defaults.json"), "{}\n"); + const environment = { + PATH: process.env.PATH ?? "", + USERPROFILE: account, + GEMINI_CLI_HOME: profile, + GEMINI_CLI_SYSTEM_SETTINGS_PATH: join(lab, "system-settings.json"), + GEMINI_CLI_SYSTEM_DEFAULTS_PATH: join(lab, "system-defaults.json"), + // Preserve the caller's bounded Node heap instead of the launcher's host-memory auto-sizing. + GEMINI_CLI_NO_RELAUNCH: "1", + GEMINI_CLI_TRUST_WORKSPACE: "true", + REA_PROCESS_RUN_ID: verifier.run_id, + REA_MCP_INPUT_SCHEMA_PROFILE: schemaProfile, + }; try { - return extractEvidence(JSON.parse(value)); - } catch { - return undefined; - } - } - if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0 - ) - return value; - for (const nested of Object.values(value)) { - const evidence = extractEvidence(nested); - if (evidence) return evidence; - } - } - return undefined; -}; -const fixture = await createGeminiModelFixture({ - directory: lab, - onRequest: async ({ body, tools }) => { - const analysis = tools.find((t) => - t.name.endsWith("analyze_javascript_application"), - ); - assert(analysis, "Gemini must expose the REA analysis tool"); - const prefix = analysis.name.slice( - 0, - -"analyze_javascript_application".length, - ); - const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort(); - assert.deepEqual( - tools - .filter((t) => expected.includes(t.name)) - .map((t) => t.name) - .sort(), - expected, - "Gemini must forward the complete REA tool catalog", - ); - catalogSize = expected.length; - if (schemasChecked === 0) { - for (const tool of tools.filter((t) => expected.includes(t.name))) { - assert( - tool.parametersJsonSchema, - `${tool.name}: missing advertised JSON Schema`, - ); - assert( - schemaValidator.validateSchema(tool.parametersJsonSchema), - `${tool.name}: invalid advertised JSON Schema: ${JSON.stringify(schemaValidator.errors)}`, - ); - const unsupportedLiteral = unsupportedLiteralPath( - tool.parametersJsonSchema, - ); - assert.equal( - unsupportedLiteral, - undefined, - `${tool.name}: Gemini SDK declaration retains a boolean or empty-string const/enum at ${unsupportedLiteral}`, - ); - const name = tool.name.slice(prefix.length); - const contract = toolContract(name); - const validate = schemaValidator.compile(tool.parametersJsonSchema); - for (const example of contract.examples) { - assert( - validate(example.input), - `${tool.name}: Gemini SDK declaration rejects canonical example ${example.title}: ${JSON.stringify(schemaValidator.errors)}`, - ); - examplesChecked++; - } - schemasChecked++; - } - } - const responses = body.contents - .flatMap((c) => c.parts ?? []) - .filter((p) => p.functionResponse) - .map((p) => p.functionResponse); - skillLoaded ||= responses.some( - (r) => - r.name === "activate_skill" && - r.response?.output?.includes( - ``, - ) && - r.response.output.includes( - join(profile, ".agents", "skills", "reverse-engineer-anything"), - ), - ); - const observed = extractEvidence(responses); - if (observed) { - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(observed); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((n) => n.observations) - .map((o) => o.properties) - .find( - (p) => - p.semantic_role === "export-return-shapes" && - p.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((s) => s.fields); - assert( - fields.some( - (f) => - f.path === "/clientCompatibility" && - f.state === "literal" && - f.value === "REA_CLIENT_FIXTURE", - ), + clientVersion = ( + await commands.exec(...launch(["--version"]), { + env: environment, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + } catch (cause) { + const windowsHint = + process.platform === "win32" + ? " On Windows, select the JavaScript entry point; npm installs provide it as node_modules/@google/gemini-cli/bundle/gemini.js." + : ""; + throw new Error( + `Gemini preflight failed for ${command}; install Gemini CLI or set REA_VERIFY_GEMINI_COMMAND.${windowsHint} ${String(cause)}`, + { cause }, ); - assert( - fields.some( - (f) => f.path === "/count" && f.state === "literal" && f.value === 7, - ), - ); - const bytes = JSON.stringify(evidence); - artifactDigest = createHash("sha256").update(bytes).digest("hex"); - await writeFile(join(lab, "gemini-evidence.json"), bytes); - evidenceSeen = true; } - if (mode === "chat" || evidenceSeen) { - finalSeen = true; - return [{ text: "REA_CLIENT_COMPATIBILITY_OK" }]; - } - if (!skillRequested) { - const skill = tools.find((t) => t.name === "activate_skill"); - assert(skill, "Gemini must expose its native skill activation tool"); - skillRequested = true; - return [ - { - functionCall: { - name: skill.name, - args: { name: "reverse-engineer-anything" }, - }, - }, - ]; - } - assert(skillLoaded, "Gemini must load the installed personal REA skill"); - return [ + const original = JSON.stringify( { - functionCall: { - name: analysis.name, - args: { input_path: target, format: "directory" }, + security: { + auth: { selectedType: "gemini-api-key" }, + folderTrust: { enabled: false }, }, + telemetry: { enabled: false }, + advanced: { autoConfigureMemory: false }, + model: { name: "gemini-2.5-flash", maxSessionTurns: 8 }, + mcpServers: { other: { command: "unrelated-server" } }, }, - ]; + null, + 2, + ); + await writeFile(configPath, original); + for (const args of [ + ["setup", "--client", "gemini_cli", "--dry-run"], + ["setup", "--client", "gemini_cli", "--yes"], + ["doctor", "--client", "gemini_cli", "--skill"], + ]) { + const result = await commands.exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + const parsed = JSON.parse(result.stdout); + if (args[0] === "doctor") assert.equal(parsed.healthy, true); + else { + assert.equal( + parsed.status, + args.includes("--dry-run") ? "planned" : "ready", + ); + if (args.includes("--dry-run")) { + assert(parsed.plannedActions.some((a) => a.target === configPath)); + assert( + parsed.plannedActions.some((a) => + a.target.startsWith(join(profile, ".agents", "skills") + sep), + ), + ); + assert( + parsed.plannedActions.every((a) => + a.target.startsWith(profile + sep), + ), + ); + } + } + await writeFile( + join( + lab, + `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`, + ), + result.stdout, + ); + } + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + assert.deepEqual(JSON.parse(configured).mcpServers.other, { + command: "unrelated-server", + }); + const repeated = await commands.exec( + process.execPath, + [ + join(runtimeRoot, "scripts/rea.mjs"), + "setup", + "--client", + "gemini_cli", + "--yes", + "--json", + ], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + assert( + (await readFile(skillPath, "utf8")).includes( + "name: reverse-engineer-anything", + ), + ); + const target = join(lab, "fixture α测试"); + await mkdir(target); + await writeFile( + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', + ); + const schemaValidator = new Ajv2020({ + strict: false, + validateFormats: false, + }); + const unsupportedLiteralPath = (schema, path = "$", seen = new Set()) => { + if (!schema || typeof schema !== "object" || seen.has(schema)) + return undefined; + seen.add(schema); + for (const [key, child] of Object.entries(schema)) { + const childPath = `${path}.${key}`; + if (key === "const" && (typeof child === "boolean" || child === "")) + return childPath; + if ( + key === "enum" && + Array.isArray(child) && + child.some((item) => typeof item === "boolean" || item === "") + ) + return childPath; + const schemaChildrenAtKey = + schemaMaps.has(key) && + child && + typeof child === "object" && + !Array.isArray(child) + ? Object.entries(child).map(([name, item]) => [ + `${childPath}.${name}`, + item, + ]) + : schemaArrays.has(key) && Array.isArray(child) + ? child.map((item, index) => [`${childPath}[${index}]`, item]) + : schemaChildren.has(key) + ? [[childPath, child]] + : []; + for (const [nestedPath, nested] of schemaChildrenAtKey) { + const found = unsupportedLiteralPath(nested, nestedPath, seen); + if (found !== undefined) return found; + } + } + return undefined; + }; + const extractEvidence = (value) => { + if (typeof value === "string") { + if ( + value.startsWith("\n") && + value.endsWith("\n") + ) { + return extractEvidence( + value.slice( + "\n".length, + -"\n".length, + ), + ); + } + try { + return extractEvidence(JSON.parse(value)); + } catch { + return undefined; + } + } + if (value && typeof value === "object") { + if ( + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes?.length > 0 + ) + return value; + for (const nested of Object.values(value)) { + const evidence = extractEvidence(nested); + if (evidence) return evidence; + } + } + return undefined; + }; + fixture = await createGeminiModelFixture({ + directory: lab, + onRequest: async ({ body, tools }) => { + const analysis = tools.find((t) => + t.name.endsWith("analyze_javascript_application"), + ); + assert(analysis, "Gemini must expose the REA analysis tool"); + const prefix = analysis.name.slice( + 0, + -"analyze_javascript_application".length, + ); + const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort(); + const reaTools = tools.filter((t) => expected.includes(t.name)); + catalogSize = reaTools.length; + assert.deepEqual( + reaTools.map((t) => t.name).sort(), + expected, + "Gemini must forward the complete REA tool catalog", + ); + if (schemasChecked === 0) { + for (const tool of reaTools) { + assert( + tool.parametersJsonSchema, + `${tool.name}: missing advertised JSON Schema`, + ); + assert( + schemaValidator.validateSchema(tool.parametersJsonSchema), + `${tool.name}: invalid advertised JSON Schema: ${JSON.stringify(schemaValidator.errors)}`, + ); + const unsupportedLiteral = unsupportedLiteralPath( + tool.parametersJsonSchema, + ); + assert.equal( + unsupportedLiteral, + undefined, + `${tool.name}: Gemini SDK declaration retains a boolean or empty-string const/enum at ${unsupportedLiteral}`, + ); + const name = tool.name.slice(prefix.length); + const contract = toolContract(name); + const validate = schemaValidator.compile(tool.parametersJsonSchema); + for (const example of contract.examples) { + assert( + validate(example.input), + `${tool.name}: Gemini SDK declaration rejects canonical example ${example.title}: ${JSON.stringify(schemaValidator.errors)}`, + ); + examplesChecked++; + } + schemasChecked++; + } + } + const responses = body.contents + .flatMap((c) => c.parts ?? []) + .filter((p) => p.functionResponse) + .map((p) => p.functionResponse); + skillLoaded ||= responses.some( + (r) => + r.name === "activate_skill" && + r.response?.output?.includes( + ``, + ) && + r.response.output.includes( + join(profile, ".agents", "skills", "reverse-engineer-anything"), + ), + ); + const observed = extractEvidence(responses); + if (observed) { + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(observed); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((n) => n.observations) + .map((o) => o.properties) + .find( + (p) => + p.semantic_role === "export-return-shapes" && + p.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap((s) => s.fields); + assert( + fields.some( + (f) => + f.path === "/clientCompatibility" && + f.state === "literal" && + f.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (f) => + f.path === "/count" && f.state === "literal" && f.value === 7, + ), + ); + const bytes = JSON.stringify(evidence); + artifactDigest = createHash("sha256").update(bytes).digest("hex"); + await writeFile(join(lab, "gemini-evidence.json"), bytes); + evidenceSeen = true; + } + if (mode === "chat" || evidenceSeen) { + finalSeen = true; + return [{ text: "REA_CLIENT_COMPATIBILITY_OK" }]; + } + if (!skillRequested) { + const skill = tools.find((t) => t.name === "activate_skill"); + assert(skill, "Gemini must expose its native skill activation tool"); + skillRequested = true; + return [ + { + functionCall: { + name: skill.name, + args: { name: "reverse-engineer-anything" }, + }, + }, + ]; + } + assert( + skillLoaded, + "Gemini must load the installed personal REA skill", + ); + return [ + { + functionCall: { + name: analysis.name, + args: { input_path: target, format: "directory" }, + }, + }, + ]; + }, + }); + const response = await saveVerifierCommandOutput( + () => + commands.exec( + ...launch([ + "--model", + "gemini-2.5-flash", + "--approval-mode", + mode === "chat" ? "default" : "yolo", + "--allowed-mcp-server-names", + "rea", + "--output-format", + "stream-json", + "-p", + mode === "chat" + ? "Say hello briefly." + : `Use REA to analyze the JavaScript directory ${target}. Report the result.`, + ]), + { + env: { + ...environment, + GEMINI_API_KEY: "local-fixture-key", + GOOGLE_GEMINI_BASE_URL: fixture.baseUrl, + }, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + stdin: "pipe", + }, + ), + { + stdout: join(lab, `${mode}.stdout`), + stderr: join(lab, `${mode}.stderr`), + }, + ); + const events = response.stdout + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); + assert.equal( + events.findLast((e) => e.type === "result")?.status, + "success", + ); + assert.equal(fixture.failure, undefined); + assert( + finalSeen && response.stdout.includes("REA_CLIENT_COMPATIBILITY_OK"), + ); + if (mode === "call") assert(evidenceSeen && skillLoaded); + }, [ + () => commands.close(), + async () => { + await fixture?.close(); + }, + ]); + passed = true; +}, [ + async () => { + const lineage = await completeVerifierRun(verifier); + const receipt = { + client: "Gemini CLI", + clientVersion, + mode, + schemaProfile, + schemasChecked, + examplesChecked, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback native Gemini API adapter; token counts synthetic; no live Google API claim", + skillDiscovery: + "native GEMINI_CLI_HOME personal shared directory; isolated Git project", + catalogSize, + evidenceSeen, + skillLoaded, + artifactDigest, + finalSeen, + requests: fixture?.requests, + probes: fixture?.probes, + fixtureFailure: fixture?.failure, + verifier: lineage, + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); + }, + ], + ); }, -}); -let failure; -try { - const response = await exec( - ...launch([ - "--model", - "gemini-2.5-flash", - "--approval-mode", - mode === "chat" ? "default" : "yolo", - "--allowed-mcp-server-names", - "rea", - "--output-format", - "stream-json", - "-p", - mode === "chat" - ? "Say hello briefly." - : `Use REA to analyze the JavaScript directory ${target}. Report the result.`, - ]), - { - env: { - ...environment, - GEMINI_API_KEY: "local-fixture-key", - GOOGLE_GEMINI_BASE_URL: fixture.baseUrl, - }, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, - }, - ); - await writeFile(join(lab, `${mode}.stdout`), response.stdout); - await writeFile(join(lab, `${mode}.stderr`), response.stderr); - const events = response.stdout - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line)); - assert.equal(events.findLast((e) => e.type === "result")?.status, "success"); - assert.equal(fixture.failure, undefined); - assert(finalSeen && response.stdout.includes("REA_CLIENT_COMPATIBILITY_OK")); - if (mode === "call") assert(evidenceSeen && skillLoaded); -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, `${mode}.stdout`), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, `${mode}.stderr`), cause.stderr); -} finally { - await fixture.close(); - const lineage = await completeVerifierRun(verifier); - const receipt = { - client: "Gemini CLI", - clientVersion, - mode, - schemaProfile, - schemasChecked, - examplesChecked, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, - }, - modelFixture: - "loopback native Gemini API adapter; token counts synthetic; no live Google API claim", - skillDiscovery: - "native GEMINI_CLI_HOME personal shared directory; isolated Git project", - catalogSize, - schemasChecked, - evidenceSeen, - skillLoaded, - artifactDigest, - finalSeen, - requests: fixture.requests, - probes: fixture.probes, - fixtureFailure: fixture.failure, - verifier: lineage, - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; +]); diff --git a/scripts/verify-grok-client.mjs b/scripts/verify-grok-client.mjs index befe2de8e9..67ec83a200 100644 --- a/scripts/verify-grok-client.mjs +++ b/scripts/verify-grok-client.mjs @@ -9,7 +9,9 @@ import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; -import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -38,57 +40,8 @@ const skillDirectory = join( "skills", "reverse-engineer-anything", ); -await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]); -await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); -// Native compatibility discovery is independent of GROK_HOME. Disable those -// sources and ignore the real shared skill root instead of changing HOME. -const environment = { - PATH: process.env.PATH, - NODE_OPTIONS: process.env.NODE_OPTIONS, - USERPROFILE: account, - GROK_HOME: profile, - GROK_DISABLE_AUTOUPDATER: "1", - ...Object.fromEntries( - ["CLAUDE", "CURSOR"].flatMap((vendor) => - ["SKILLS", "RULES", "AGENTS", "MCPS", "HOOKS", "SESSIONS"].map((cell) => [ - `GROK_${vendor}_${cell}_ENABLED`, - "false", - ]), - ), - ), - XDG_DATA_HOME: join(lab, "data"), - XDG_CONFIG_HOME: join(lab, "config"), - XDG_CACHE_HOME: join(lab, "cache"), - REA_PROCESS_RUN_ID: verifier.run_id, -}; -const target = join(lab, "fixture α测试"); -await mkdir(target); -await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n' + - // A real analysis result large enough to exercise native MCP offloading. - Array.from( - { length: 60 }, - (_, index) => - `export function extra${index}() { return { index: ${index}, tag: "GROK_LARGE_FIXTURE" }; }`, - ).join("\n"), -); const expectedNames = TOOL_CONTRACTS.map((tool) => "rea__" + tool.name).sort(); -assert( - expectedNames.length <= 255, - "Native search_tool limit is uint8; this full-catalog query cannot cover more than 255 tools", -); -const schemaValidator = new Ajv2020({ strict: false, validateFormats: false }); -const shellQuote = (value) => "'" + value.replaceAll("'", "'\"'\"'") + "'"; -const containsText = (value, text) => { - if (typeof value === "string") return value.includes(text); - return ( - value !== null && - typeof value === "object" && - Object.values(value).some((item) => containsText(item, text)) - ); -}; -let clientVersion; +let clientVersion = null; let expectedSkill; let turn = 0; let skillRequested = false; @@ -97,6 +50,7 @@ let diagnosticsRequested = false; let diagnosticsVerified = false; let lastTool; let catalogChecked = false; +let catalogSize = null; let schemasChecked = 0; let analysisRequested = false; let evidenceSeen = false; @@ -107,240 +61,312 @@ let artifactBytes; let retainedArtifact; let expectedProjection; let finalSeen = false; -let failure; -const call = (name, args) => { - lastTool = name; - return { - role: "assistant", - content: null, - tool_calls: [ - { - id: `call_grok_${turn}`, - type: "function", - function: { name, arguments: JSON.stringify(args) }, - }, - ], - }; -}; -const verifyEvidence = async (bytes) => { - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(JSON.parse(bytes.toString("utf8"))); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shapes = evidence.normalized_result.graph.nodes - .flatMap((node) => node.observations) - .map((observation) => observation.properties) - .filter( - (properties) => - properties.semantic_role === "export-return-shapes" && - properties.exported_name === "result", - ); - assert( - shapes.length > 0 && - shapes.every((shape) => shape.return_shape_coverage?.projection_complete), - ); - const fields = shapes.flatMap((shape) => - shape.static_return_shapes.flatMap((item) => item.fields), - ); - assert( - fields.some( - (field) => - field.path === "/clientCompatibility" && - field.state === "literal" && - field.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (field) => - field.path === "/count" && - field.state === "literal" && - field.value === 7, - ), - ); - artifactDigest = createHash("sha256").update(bytes).digest("hex"); - artifactBytes = bytes.length; - evidenceSeen = true; - return { - evidence_id: evidence.evidence_id, - subject: evidence.subject, - operation: evidence.operation, - artifact_sha256: artifactDigest, - byte_length: artifactBytes, - shapes, - }; -}; -const diagnosticFile = join(lab, "native-doctor.json"); -const diagnosticCode = - "const fs=require('node:fs');const d=JSON.parse(fs.readFileSync(process.argv[1],'utf8'));console.log(JSON.stringify({rea_client_diagnostic:{healthy:d.healthy,registrations:d.identity.registrations.filter(r=>r.client==='grok_build'),skill:d.identity.skill.state}}));"; -// Query the saved result through a native agent command. Only the facts needed -// for this analyst question enter context; the complete named-schema Evidence -// and its digest remain available in the native artifact. -const recoveryCode = - "const fs=require('node:fs'),crypto=require('node:crypto');const bytes=fs.readFileSync(process.argv[1]),e=JSON.parse(bytes);const shapes=e.normalized_result.graph.nodes.flatMap(n=>n.observations).map(o=>o.properties).filter(p=>p.semantic_role==='export-return-shapes'&&p.exported_name==='result');console.log(JSON.stringify({rea_recovered:{evidence_id:e.evidence_id,subject:e.subject,operation:e.operation,artifact_sha256:crypto.createHash('sha256').update(bytes).digest('hex'),byte_length:bytes.length,shapes}}));"; -const fixture = await createOpenAiModelFixture({ - directory: lab, - prefix: "grok", - onRequest: async ({ body, tools }) => { - // Native title/summary requests cannot execute the analyst workflow. - if (!tools.some((tool) => tool.name === "search_tool")) - return { role: "assistant", content: "REA fixture auxiliary response" }; - turn++; - assert( - containsText(body.messages, skillDirectory), - "Grok must discover the isolated installed skill", +let passed = false; +let fixture; +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]); + await commands.exec("git", ["init", "--quiet", workspace], { + timeout: 30_000, + stdin: "pipe", + }); + // Native compatibility discovery is independent of GROK_HOME. Disable those + // sources and ignore the real shared skill root instead of changing HOME. + const environment = { + PATH: process.env.PATH, + NODE_OPTIONS: process.env.NODE_OPTIONS, + USERPROFILE: account, + GROK_HOME: profile, + GROK_DISABLE_AUTOUPDATER: "1", + ...Object.fromEntries( + ["CLAUDE", "CURSOR"].flatMap((vendor) => + ["SKILLS", "RULES", "AGENTS", "MCPS", "HOOKS", "SESSIONS"].map( + (cell) => [`GROK_${vendor}_${cell}_ENABLED`, "false"], + ), + ), + ), + XDG_DATA_HOME: join(lab, "data"), + XDG_CONFIG_HOME: join(lab, "config"), + XDG_CACHE_HOME: join(lab, "cache"), + REA_PROCESS_RUN_ID: verifier.run_id, + }; + const target = join(lab, "fixture α测试"); + await mkdir(target); + await writeFile( + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n' + + // A real analysis result large enough to exercise native MCP offloading. + Array.from( + { length: 60 }, + (_, index) => + `export function extra${index}() { return { index: ${index}, tag: "GROK_LARGE_FIXTURE" }; }`, + ).join("\n"), ); assert( - !containsText(body.messages, join(homedir(), ".agents", "skills")), - "Do not inherit real personal skills", + expectedNames.length <= 255, + "Native search_tool limit is uint8; this full-catalog query cannot cover more than 255 tools", ); - if (mode === "chat") { - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - } - const replies = body.messages.filter((message) => message.role === "tool"); - if (!skillRequested) { - skillRequested = true; - return call("read_file", { - target_file: join(skillDirectory, "SKILL.md"), - }); - } - assert( - expectedSkill !== undefined && containsText(replies, expectedSkill), - "Native read_file must return the full skill with its documented line-number rendering", - ); - skillLoaded = true; - const last = replies.at(-1)?.content; - assert.equal(typeof last, "string"); - if (!catalogChecked) { - const discovery = - lastTool === "search_tool" ? JSON.parse(last) : undefined; - if (discovery?.status === "partial" && !diagnosticsRequested) { - diagnosticsRequested = true; - const diagnostic = [ - process.execPath, - join(runtimeRoot, "scripts/rea.mjs"), - "doctor", - "--client", - "grok_build", - "--skill", - "--json", - ] - .map(shellQuote) - .join(" "); - const projection = [ - process.execPath, - "-e", - diagnosticCode, - diagnosticFile, - ] - .map(shellQuote) - .join(" "); - return call("run_terminal_command", { - command: `${diagnostic} > ${shellQuote(diagnosticFile)} && ${projection}`, - description: - "Diagnose REA registration while native MCP discovery is pending", - }); - } - if (lastTool === "run_terminal_command") { + const schemaValidator = new Ajv2020({ + strict: false, + validateFormats: false, + }); + const shellQuote = (value) => "'" + value.replaceAll("'", "'\"'\"'") + "'"; + const containsText = (value, text) => { + if (typeof value === "string") return value.includes(text); + return ( + value !== null && + typeof value === "object" && + Object.values(value).some((item) => containsText(item, text)) + ); + }; + const call = (name, args) => { + lastTool = name; + return { + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_grok_${turn}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }; + }; + const verifyEvidence = async (bytes) => { + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(JSON.parse(bytes.toString("utf8"))); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shapes = evidence.normalized_result.graph.nodes + .flatMap((node) => node.observations) + .map((observation) => observation.properties) + .filter( + (properties) => + properties.semantic_role === "export-return-shapes" && + properties.exported_name === "result", + ); + assert( + shapes.length > 0 && + shapes.every( + (shape) => shape.return_shape_coverage?.projection_complete, + ), + ); + const fields = shapes.flatMap((shape) => + shape.static_return_shapes.flatMap((item) => item.fields), + ); + assert( + fields.some( + (field) => + field.path === "/clientCompatibility" && + field.state === "literal" && + field.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (field) => + field.path === "/count" && + field.state === "literal" && + field.value === 7, + ), + ); + artifactDigest = createHash("sha256").update(bytes).digest("hex"); + artifactBytes = bytes.length; + evidenceSeen = true; + return { + evidence_id: evidence.evidence_id, + subject: evidence.subject, + operation: evidence.operation, + artifact_sha256: artifactDigest, + byte_length: artifactBytes, + shapes, + }; + }; + const diagnosticFile = join(lab, "native-doctor.json"); + const diagnosticCode = + "const fs=require('node:fs');const d=JSON.parse(fs.readFileSync(process.argv[1],'utf8'));console.log(JSON.stringify({rea_client_diagnostic:{healthy:d.healthy,registrations:d.identity.registrations.filter(r=>r.client==='grok_build'),skill:d.identity.skill.state}}));"; + // Query the saved result through a native agent command. Only the facts needed + // for this analyst question enter context; the complete named-schema Evidence + // and its digest remain available in the native artifact. + const recoveryCode = + "const fs=require('node:fs'),crypto=require('node:crypto');const bytes=fs.readFileSync(process.argv[1]),e=JSON.parse(bytes);const shapes=e.normalized_result.graph.nodes.flatMap(n=>n.observations).map(o=>o.properties).filter(p=>p.semantic_role==='export-return-shapes'&&p.exported_name==='result');console.log(JSON.stringify({rea_recovered:{evidence_id:e.evidence_id,subject:e.subject,operation:e.operation,artifact_sha256:crypto.createHash('sha256').update(bytes).digest('hex'),byte_length:bytes.length,shapes}}));"; + fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: "grok", + onRequest: async ({ body, tools }) => { + // Native title/summary requests cannot execute the analyst workflow. + if (!tools.some((tool) => tool.name === "search_tool")) + return { + role: "assistant", + content: "REA fixture auxiliary response", + }; + turn++; + assert( + containsText(body.messages, skillDirectory), + "Grok must discover the isolated installed skill", + ); + assert( + !containsText(body.messages, join(homedir(), ".agents", "skills")), + "Do not inherit real personal skills", + ); + if (mode === "chat") { + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + const replies = body.messages.filter( + (message) => message.role === "tool", + ); + if (!skillRequested) { + skillRequested = true; + return call("read_file", { + target_file: join(skillDirectory, "SKILL.md"), + }); + } + assert( + expectedSkill !== undefined && containsText(replies, expectedSkill), + "Native read_file must return the full skill with its documented line-number rendering", + ); + skillLoaded = true; + const last = replies.at(-1)?.content; + assert.equal(typeof last, "string"); + if (!catalogChecked) { + const discovery = + lastTool === "search_tool" ? JSON.parse(last) : undefined; + if (discovery?.status === "partial" && !diagnosticsRequested) { + diagnosticsRequested = true; + const diagnostic = [ + process.execPath, + join(runtimeRoot, "scripts/rea.mjs"), + "doctor", + "--client", + "grok_build", + "--skill", + "--json", + ] + .map(shellQuote) + .join(" "); + const projection = [ + process.execPath, + "-e", + diagnosticCode, + diagnosticFile, + ] + .map(shellQuote) + .join(" "); + return call("run_terminal_command", { + command: `${diagnostic} > ${shellQuote(diagnosticFile)} && ${projection}`, + description: + "Diagnose REA registration while native MCP discovery is pending", + }); + } + if (lastTool === "run_terminal_command") { + const line = last + .split("\n") + .find((item) => item.startsWith('{"rea_client_diagnostic":')); + assert( + line, + "Native REA diagnostics failed or did not return the selected status", + ); + const diagnostic = JSON.parse(line).rea_client_diagnostic; + assert.equal(diagnostic.healthy, true); + assert.equal(diagnostic.skill, "aligned"); + assert( + diagnostic.registrations.some( + (registration) => + registration.state === "aligned" && + registration.config_path === configPath, + ), + ); + diagnosticsVerified = true; + } + if (discovery?.status !== "ready") + return call("search_tool", { query: "rea", limit: 255 }); + const inventory = discovery.results + .filter((result) => result.server === "rea") + .flatMap((result) => result.tools); + catalogSize = inventory.length; + assert.deepEqual( + inventory.map((tool) => tool.tool_name).sort(), + expectedNames, + "Native search_tool must discover the complete REA catalog", + ); + for (const tool of inventory) { + assert( + tool.input_schema, + `${tool.tool_name}: missing input schema`, + ); + assert( + schemaValidator.validateSchema(tool.input_schema), + `${tool.tool_name}: invalid advertised schema: ${JSON.stringify(schemaValidator.errors)}`, + ); + schemasChecked++; + } + await writeFile(join(lab, "native-discovery.json"), last); + catalogChecked = true; + analysisRequested = true; + return call("use_tool", { + tool_name: "rea__analyze_javascript_application", + tool_input: { input_path: target, format: "directory" }, + }); + } + if (!spillQueryRequested) { + const marker = "Full output written to: "; + const start = last.indexOf(marker); + if (start < 0) { + await verifyEvidence(Buffer.from(last, "utf8")); + finalSeen = true; + return { + role: "assistant", + content: "REA_CLIENT_COMPATIBILITY_OK", + }; + } + // The native hint explicitly calls for a query rather than read_file on + // the saved JSON's very long line. Preserve the original hint in requests. + const file = last + .slice(start + marker.length) + .split(". The full output ")[0]; + assert(isAbsolute(file)); + const childPath = relative(join(profile, "sessions"), file); + assert( + childPath !== "" && + !childPath.startsWith("..") && + !isAbsolute(childPath) && + file.endsWith(".json"), + "Unexpected native MCP spill path", + ); + retainedArtifact = file; + expectedProjection = await verifyEvidence(await readFile(file)); + spillQueryRequested = true; + return call("run_terminal_command", { + command: [process.execPath, "-e", recoveryCode, file] + .map(shellQuote) + .join(" "), + description: + "Query full saved REA Evidence while preserving artifact identity", + }); + } const line = last .split("\n") - .find((item) => item.startsWith('{"rea_client_diagnostic":')); + .find((item) => item.startsWith('{"rea_recovered":')); assert( line, - "Native REA diagnostics failed or did not return the selected status", + "The next model request must contain native artifact query output", ); - const diagnostic = JSON.parse(line).rea_client_diagnostic; - assert.equal(diagnostic.healthy, true); - assert.equal(diagnostic.skill, "aligned"); - assert( - diagnostic.registrations.some( - (registration) => - registration.state === "aligned" && - registration.config_path === configPath, - ), + const projection = JSON.parse(line).rea_recovered; + assert.deepEqual( + projection, + expectedProjection, + "Native query must recover the observed facts and complete artifact digest", ); - diagnosticsVerified = true; - } - if (discovery?.status !== "ready") - return call("search_tool", { query: "rea", limit: 255 }); - const inventory = discovery.results - .filter((result) => result.server === "rea") - .flatMap((result) => result.tools); - assert.deepEqual( - inventory.map((tool) => tool.tool_name).sort(), - expectedNames, - "Native search_tool must discover the complete REA catalog", - ); - for (const tool of inventory) { - assert(tool.input_schema, `${tool.tool_name}: missing input schema`); - assert( - schemaValidator.validateSchema(tool.input_schema), - `${tool.tool_name}: invalid advertised schema: ${JSON.stringify(schemaValidator.errors)}`, - ); - schemasChecked++; - } - await writeFile(join(lab, "native-discovery.json"), last); - catalogChecked = true; - analysisRequested = true; - return call("use_tool", { - tool_name: "rea__analyze_javascript_application", - tool_input: { input_path: target, format: "directory" }, - }); - } - if (!spillQueryRequested) { - const marker = "Full output written to: "; - const start = last.indexOf(marker); - if (start < 0) { - await verifyEvidence(Buffer.from(last, "utf8")); + projectionSeen = true; finalSeen = true; return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - } - // The native hint explicitly calls for a query rather than read_file on - // the saved JSON's very long line. Preserve the original hint in requests. - const file = last - .slice(start + marker.length) - .split(". The full output ")[0]; - assert(isAbsolute(file)); - const childPath = relative(join(profile, "sessions"), file); - assert( - childPath !== "" && - !childPath.startsWith("..") && - !isAbsolute(childPath) && - file.endsWith(".json"), - "Unexpected native MCP spill path", - ); - retainedArtifact = file; - expectedProjection = await verifyEvidence(await readFile(file)); - spillQueryRequested = true; - return call("run_terminal_command", { - command: [process.execPath, "-e", recoveryCode, file] - .map(shellQuote) - .join(" "), - description: - "Query full saved REA Evidence while preserving artifact identity", - }); - } - const line = last - .split("\n") - .find((item) => item.startsWith('{"rea_recovered":')); - assert( - line, - "The next model request must contain native artifact query output", - ); - const projection = JSON.parse(line).rea_recovered; - assert.deepEqual( - projection, - expectedProjection, - "Native query must recover the observed facts and complete artifact digest", - ); - projectionSeen = true; - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - }, -}); -const q = JSON.stringify; -const original = `# Keep unrelated settings and server state. + }, + }); + const q = JSON.stringify; + const original = `# Keep unrelated settings and server state. disabled_mcp_servers = ["rea", "other"] [cli] auto_update = false @@ -360,181 +386,203 @@ max_completion_tokens = 4096 command = "unrelated-server" enabled = false `; -await writeFile(configPath, original); -const rea = async (args) => - exec( - process.execPath, - [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); -try { - clientVersion = ( - await exec(command, ["--version"], { env: environment, timeout: 30_000 }) - ).stdout.trim(); - for (const args of [ - ["setup", "--client", "grok_build", "--dry-run"], - ["setup", "--client", "grok_build", "--yes"], - ["doctor", "--client", "grok_build", "--skill"], - ]) { - const result = await rea(args); - await writeFile( - join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), - result.stdout, - ); - const parsed = JSON.parse(result.stdout); - if (args[0] === "doctor") assert.equal(parsed.healthy, true); - else { - assert.equal( - parsed.status, - args.includes("--dry-run") ? "planned" : "ready", + await writeFile(configPath, original); + const rea = async (args) => + commands.exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, ); - if (args.includes("--dry-run")) { - assert( - parsed.plannedActions.some((action) => action.target === configPath), - ); - assert( - parsed.plannedActions.some( - (action) => action.target === skillDirectory, - ), - ); - assert( - parsed.plannedActions.every( - (action) => - action.target === configPath || - action.target === skillDirectory || - action.target.startsWith(skillDirectory + "/"), - ), - "Refuse writes outside the owned profile/skill", + clientVersion = ( + await commands.exec(command, ["--version"], { + env: environment, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + for (const args of [ + ["setup", "--client", "grok_build", "--dry-run"], + ["setup", "--client", "grok_build", "--yes"], + ["doctor", "--client", "grok_build", "--skill"], + ]) { + const result = await rea(args); + await writeFile( + join( + lab, + `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`, + ), + result.stdout, + ); + const parsed = JSON.parse(result.stdout); + if (args[0] === "doctor") assert.equal(parsed.healthy, true); + else { + assert.equal( + parsed.status, + args.includes("--dry-run") ? "planned" : "ready", ); + if (args.includes("--dry-run")) { + assert( + parsed.plannedActions.some( + (action) => action.target === configPath, + ), + ); + assert( + parsed.plannedActions.some( + (action) => action.target === skillDirectory, + ), + ); + assert( + parsed.plannedActions.every( + (action) => + action.target === configPath || + action.target === skillDirectory || + action.target.startsWith(skillDirectory + "/"), + ), + "Refuse writes outside the owned profile/skill", + ); + } } } - } - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const configured = await readFile(configPath, "utf8"); - assert( - configured.includes("unrelated-server") && - configured.includes("# Keep unrelated settings"), - ); - const repeated = JSON.parse( - (await rea(["setup", "--client", "grok_build", "--yes"])).stdout, - ); - assert.deepEqual(repeated.appliedActions, []); - assert.equal(await readFile(configPath, "utf8"), configured); - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const skill = await readFile(join(skillDirectory, "SKILL.md"), "utf8"); - expectedSkill = skill - .trimEnd() - .split("\n") - .map((line, index) => - index === 0 || (index + 1) % 10 === 0 ? `${index + 1}→${line}` : line, - ) - .join("\n"); - const execution = exec( - command, - [ - "-p", - mode === "chat" - ? "Say hello briefly." - : `Use the REA skill to analyze ${target}.`, - "-m", - "rea-client-fixture", - "--output-format", - "streaming-messages-json", - "--always-approve", - "--no-subagents", - ], - { - env: environment, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, - }, - ); - execution.child.stdin.end(); - const response = await execution; - await writeFile(join(lab, `${mode}.stdout`), response.stdout); - await writeFile(join(lab, `${mode}.stderr`), response.stderr); - assert.equal(fixture.failure, undefined); - const events = response.stdout - .trim() - .split("\n") - .map((line) => JSON.parse(line)); - const init = events.find( - (event) => event.type === "system" && event.subtype === "init", - ); - assert(init?.mcp_servers?.some((server) => server.name === "rea")); - assert( - init.mcp_servers.every((server) => ["rea", "other"].includes(server.name)), - "Do not inherit foreign MCP configuration", - ); - assert( - events.some( - (event) => - event.type === "result" && - event.subtype === "success" && - event.is_error === false && - event.result === "REA_CLIENT_COMPATIBILITY_OK", - ), - ); - assert(finalSeen); - if (mode === "call") + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); assert( - skillLoaded && - catalogChecked && - analysisRequested && - evidenceSeen && - (!spillQueryRequested || projectionSeen) && - (!diagnosticsRequested || diagnosticsVerified), + configured.includes("unrelated-server") && + configured.includes("# Keep unrelated settings"), ); -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, `${mode}.stdout`), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, `${mode}.stderr`), cause.stderr); -} finally { - await fixture.close(); - const lineage = await completeVerifierRun(verifier); - const receipt = { - client: "Grok Build", - clientVersion, - mode, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, + const repeated = JSON.parse( + (await rea(["setup", "--client", "grok_build", "--yes"])).stdout, + ); + assert.deepEqual(repeated.appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const skill = await readFile(join(skillDirectory, "SKILL.md"), "utf8"); + expectedSkill = skill + .trimEnd() + .split("\n") + .map((line, index) => + index === 0 || (index + 1) % 10 === 0 ? `${index + 1}→${line}` : line, + ) + .join("\n"); + const response = await saveVerifierCommandOutput( + () => + commands.exec( + command, + [ + "-p", + mode === "chat" + ? "Say hello briefly." + : `Use the REA skill to analyze ${target}.`, + "-m", + "rea-client-fixture", + "--output-format", + "streaming-messages-json", + "--always-approve", + "--no-subagents", + ], + { + env: environment, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + stdin: "closed-pipe", + }, + ), + { + stdout: join(lab, `${mode}.stdout`), + stderr: join(lab, `${mode}.stderr`), + }, + ); + assert.equal(fixture.failure, undefined); + const events = response.stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + const init = events.find( + (event) => event.type === "system" && event.subtype === "init", + ); + assert(init?.mcp_servers?.some((server) => server.name === "rea")); + assert( + init.mcp_servers.every((server) => + ["rea", "other"].includes(server.name), + ), + "Do not inherit foreign MCP configuration", + ); + assert( + events.some( + (event) => + event.type === "result" && + event.subtype === "success" && + event.is_error === false && + event.result === "REA_CLIENT_COMPATIBILITY_OK", + ), + ); + assert(finalSeen); + if (mode === "call") + assert( + skillLoaded && + catalogChecked && + analysisRequested && + evidenceSeen && + (!spillQueryRequested || projectionSeen) && + (!diagnosticsRequested || diagnosticsVerified), + ); + }, [ + () => commands.close(), + async () => { + await fixture?.close(); }, - modelFixture: - "loopback OpenAI completions/SSE, declared one-million-token custom model and synthetic usage; no live model claim", - discovery: - "native GROK_HOME, isolated additional skill directory, disabled foreign compatibility sources; default OS-home skill discovery and Windows unverified", - interface: - "default native search_tool/use_tool with a complete explicit inventory query; asynchronous partial discovery may trigger native REA diagnosis; large MCP outputs are queried via native terminal", - catalogChecked, - catalogSize: catalogChecked ? expectedNames.length : 0, - schemasChecked, - skillLoaded, - diagnosticsRequested, - diagnosticsVerified, - analysisRequested, - evidenceSeen, - spillQueryRequested, - projectionSeen, - retainedArtifact, - artifactBytes, - artifactDigest, - finalSeen, - evidenceScope: - "complete saved Evidence validated with its named schema; the model receives the selected export facts, subject and full artifact digest, not every offloaded graph fact", - requests: fixture.requests, - probes: fixture.probes, - fixtureFailure: fixture.failure, - verifier: lineage, - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; + ]); + passed = true; +}, [ + async () => { + const lineage = await completeVerifierRun(verifier); + const receipt = { + client: "Grok Build", + clientVersion, + mode, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback OpenAI completions/SSE, declared one-million-token custom model and synthetic usage; no live model claim", + discovery: + "native GROK_HOME, isolated additional skill directory, disabled foreign compatibility sources; default OS-home skill discovery and Windows unverified", + interface: + "default native search_tool/use_tool with a complete explicit inventory query; asynchronous partial discovery may trigger native REA diagnosis; large MCP outputs are queried via native terminal", + catalogChecked, + catalogSize, + schemasChecked, + skillLoaded, + diagnosticsRequested, + diagnosticsVerified, + analysisRequested, + evidenceSeen, + spillQueryRequested, + projectionSeen, + retainedArtifact, + artifactBytes, + artifactDigest, + finalSeen, + evidenceScope: + "complete saved Evidence validated with its named schema; the model receives the selected export facts, subject and full artifact digest, not every offloaded graph fact", + requests: fixture?.requests, + probes: fixture?.probes, + fixtureFailure: fixture?.failure, + verifier: lineage, + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); + }, + ], + ); + }, +]); diff --git a/scripts/verify-native-agent-client.mjs b/scripts/verify-native-agent-client.mjs index 9c05492d3b..b99337d651 100644 --- a/scripts/verify-native-agent-client.mjs +++ b/scripts/verify-native-agent-client.mjs @@ -9,6 +9,7 @@ import { toolContract, } from "../dist/contracts/toolContracts.js"; import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; @@ -418,36 +419,24 @@ await withVerifierCleanup(async () => { "--reasoning", "none", ]; - const result = await commands - .exec(command, args, { - env: { - ...environment, - OPENAI_API_KEY: "local-fixture-key", - OPENAI_BASE_URL: fixture.baseUrl, - }, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, - // Pi consumes piped input before its first prompt; an unused open pipe is not EOF. - stdin: "closed-pipe", - }) - .catch(async (cause) => { - const outputWriters = []; - if (typeof cause?.snapshot?.stdout?.text === "string") - outputWriters.push(() => - writeFile(join(lab, "client.stdout"), cause.snapshot.stdout.text), - ); - if (typeof cause?.snapshot?.stderr?.text === "string") - outputWriters.push(() => - writeFile(join(lab, "client.stderr"), cause.snapshot.stderr.text), - ); - return withVerifierCleanup(() => { - throw cause; - }, outputWriters); - }); - await withVerifierCleanup( - () => writeFile(join(lab, "client.stdout"), result.stdout), - [() => writeFile(join(lab, "client.stderr"), result.stderr)], + const result = await saveVerifierCommandOutput( + () => + commands.exec(command, args, { + env: { + ...environment, + OPENAI_API_KEY: "local-fixture-key", + OPENAI_BASE_URL: fixture.baseUrl, + }, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + // Pi consumes piped input before its first prompt; an unused open pipe is not EOF. + stdin: "closed-pipe", + }), + { + stdout: join(lab, "client.stdout"), + stderr: join(lab, "client.stderr"), + }, ); assert.equal(fixture.failure, undefined); assert(finalSeen); diff --git a/scripts/verify-omp-client.mjs b/scripts/verify-omp-client.mjs index f107ce9b06..973e85c47b 100644 --- a/scripts/verify-omp-client.mjs +++ b/scripts/verify-omp-client.mjs @@ -8,7 +8,9 @@ import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; -import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -37,40 +39,10 @@ const skillDirectory = join( "skills", "reverse-engineer-anything", ); -await Promise.all([ - mkdir(account), - mkdir(agent), - mkdir(workspace), - ...["data", "state", "cache"].map((name) => - mkdir(join(lab, name, "omp"), { recursive: true }), - ), -]); -await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); -// OMP's global config root uses native homedir independently of the agent override. -// A documented relative PI_CONFIG_DIR redirects it without changing HOME. -const environment = { - PATH: process.env.PATH, - NODE_OPTIONS: process.env.NODE_OPTIONS, - USERPROFILE: account, - PI_CODING_AGENT_DIR: agent, - PI_CONFIG_DIR: relative(homedir(), join(lab, "native-config")), - OMP_PROFILE: "", - XDG_DATA_HOME: join(lab, "data"), - XDG_STATE_HOME: join(lab, "state"), - XDG_CACHE_HOME: join(lab, "cache"), - REA_PROCESS_RUN_ID: verifier.run_id, -}; -const target = join(lab, "fixture α测试"); -await mkdir(target); -await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', -); const expectedDevices = TOOL_CONTRACTS.map( (tool) => "mcp__rea_" + tool.name, ).sort(); -const analysisDevice = "xd://mcp__rea_analyze_javascript_application"; -let clientVersion; +let clientVersion = null; let expectedSkillBody; let skillLoaded = false; let contractRead = false; @@ -78,365 +50,448 @@ let evidenceSeen = false; let artifactDigest; let finalSeen = false; let catalogMounted = false; +let catalogSize = null; let invalidProfileRejected = false; -let failure; -const containsText = (value, text) => { - if (typeof value === "string") return value.includes(text); - return ( - value !== null && - typeof value === "object" && - Object.values(value).some((item) => containsText(item, text)) - ); -}; -const call = (index, name, args) => ({ - role: "assistant", - content: null, - tool_calls: [ - { - id: `call_omp_${index}`, - type: "function", - function: { name, arguments: JSON.stringify(args) }, - }, - ], -}); -const fixture = await createOpenAiModelFixture({ - directory: lab, - prefix: "omp", - onRequest: async ({ body, tools, requestIndex }) => { - assert(tools.some((tool) => tool.name === "read")); - assert(tools.some((tool) => tool.name === "write")); - const instructions = body.messages.filter( - (message) => message.role === "system", - ); - for (const device of expectedDevices) - assert( - containsText(instructions, device), - `OMP must advertise mounted device ${device}`, - ); - if (mode === "chat") { - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - } - const replies = body.messages.filter((message) => message.role === "tool"); - if (requestIndex === 1) - return call(requestIndex, "read", { - i: "Load REA skill", - path: "skill://reverse-engineer-anything:raw", - }); - assert( - expectedSkillBody !== undefined && - containsText(replies, expectedSkillBody), - "OMP must return the complete installed skill body", - ); - skillLoaded = true; - if (requestIndex === 2) - return call(requestIndex, "read", { - i: "Inspect REA native tool contract", - path: analysisDevice, - }); - assert(containsText(replies, "# mcp__rea_analyze_javascript_application")); - assert( - containsText(replies, "input_path") && - containsText(replies, '"directory"'), - ); - contractRead = true; - if (requestIndex === 3) - return call(requestIndex, "write", { - path: analysisDevice, - content: JSON.stringify({ input_path: target, format: "directory" }), - }); - const evidenceReply = replies.find( - (reply) => - typeof reply.content === "string" && - reply.content.startsWith('{"evidence_id":'), - ); - assert( - evidenceReply, - "OMP must deliver the full analysis result to the next model turn", - ); - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(JSON.parse(evidenceReply.content)); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((node) => node.observations) - .map((observation) => observation.properties) - .find( - (properties) => - properties.semantic_role === "export-return-shapes" && - properties.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((item) => item.fields); - assert( - fields.some( - (field) => - field.path === "/clientCompatibility" && - field.state === "literal" && - field.value === "REA_CLIENT_FIXTURE", +let passed = false; +let fixture; +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([ + mkdir(account), + mkdir(agent), + mkdir(workspace), + ...["data", "state", "cache"].map((name) => + mkdir(join(lab, name, "omp"), { recursive: true }), ), - ); - assert( - fields.some( - (field) => - field.path === "/count" && - field.state === "literal" && - field.value === 7, - ), - ); - const bytes = JSON.stringify(evidence); - artifactDigest = createHash("sha256").update(bytes).digest("hex"); - await writeFile(join(lab, "omp-evidence.json"), bytes); - evidenceSeen = true; - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - }, -}); -const original = JSON.stringify( - { - mcpServers: { other: { command: "unrelated-server", enabled: false } }, - disabledServers: ["rea", "other"], - }, - null, - 2, -); -await writeFile(configPath, original); -await writeFile( - join(agent, "config.yml"), - JSON.stringify({ - skills: { customDirectories: [join(account, ".agents", "skills")] }, - retry: { fallbackChains: { default: [] } }, - }), -); -await writeFile( - join(agent, "models.yml"), - JSON.stringify({ - providers: { - "rea-fixture": { - baseUrl: fixture.baseUrl, - apiKey: "local-fixture-key", - api: "openai-completions", - models: [ - { - id: "rea-client-fixture", - name: "REA local fixture", - reasoning: false, - input: ["text"], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 1000000, - maxTokens: 4096, - }, - ], - }, - }, - }), -); -const rea = async (args, env = environment) => - exec( - process.execPath, - [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], - { env, cwd: workspace, timeout: 60_000 }, - ); -try { - clientVersion = ( - await exec(command, ["--version"], { env: environment, timeout: 30_000 }) - ).stdout.trim(); - const invalidEnvironment = { ...environment, OMP_PROFILE: "Bad Name" }; - const nativeRejected = await exec( - command, - ["--print", "--no-session", "--no-title", "Hello"], - { env: invalidEnvironment, cwd: workspace, timeout: 30_000 }, - ).then( - () => undefined, - (cause) => cause, - ); - assert( - nativeRejected?.code === 1 && - nativeRejected.stderr.includes('Invalid OMP profile "Bad Name"'), - ); - await writeFile( - join(lab, "invalid-profile-native.stderr"), - nativeRejected.stderr, - ); - const setupRejected = await rea( - ["setup", "--client", "omp", "--dry-run"], - invalidEnvironment, - ).then( - () => undefined, - (cause) => cause, - ); - assert(typeof setupRejected?.code === "number" && setupRejected.code !== 0); - const rejected = JSON.parse(setupRejected.stdout); - assert.equal(rejected.status, "needs_human"); - assert.deepEqual(rejected.plannedActions, []); - assert.deepEqual(rejected.appliedActions, []); - assert(rejected.remediation.includes('Invalid OMP profile "Bad Name"')); - assert.equal(await readFile(configPath, "utf8"), original); - await writeFile( - join(lab, "invalid-profile-setup.json"), - setupRejected.stdout, - ); - invalidProfileRejected = true; - for (const args of [ - ["setup", "--client", "omp", "--dry-run"], - ["setup", "--client", "omp", "--yes"], - ["doctor", "--client", "omp", "--skill"], - ]) { - const result = await rea(args); + ]); + await commands.exec("git", ["init", "--quiet", workspace], { + timeout: 30_000, + stdin: "pipe", + }); + // OMP's global config root uses native homedir independently of the agent override. + // A documented relative PI_CONFIG_DIR redirects it without changing HOME. + const environment = { + PATH: process.env.PATH, + NODE_OPTIONS: process.env.NODE_OPTIONS, + USERPROFILE: account, + PI_CODING_AGENT_DIR: agent, + PI_CONFIG_DIR: relative(homedir(), join(lab, "native-config")), + OMP_PROFILE: "", + XDG_DATA_HOME: join(lab, "data"), + XDG_STATE_HOME: join(lab, "state"), + XDG_CACHE_HOME: join(lab, "cache"), + REA_PROCESS_RUN_ID: verifier.run_id, + }; + const target = join(lab, "fixture α测试"); + await mkdir(target); await writeFile( - join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), - result.stdout, + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', ); - const parsed = JSON.parse(result.stdout); - if (args[0] === "doctor") assert.equal(parsed.healthy, true); - else { - assert.equal( - parsed.status, - args.includes("--dry-run") ? "planned" : "ready", + const analysisDevice = "xd://mcp__rea_analyze_javascript_application"; + const containsText = (value, text) => { + if (typeof value === "string") return value.includes(text); + return ( + value !== null && + typeof value === "object" && + Object.values(value).some((item) => containsText(item, text)) ); - if (args.includes("--dry-run")) { + }; + const call = (index, name, args) => ({ + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_omp_${index}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }); + fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: "omp", + onRequest: async ({ body, tools, requestIndex }) => { + assert(tools.some((tool) => tool.name === "read")); + assert(tools.some((tool) => tool.name === "write")); + const instructions = body.messages.filter( + (message) => message.role === "system", + ); + for (const device of expectedDevices) + assert( + containsText(instructions, device), + `OMP must advertise mounted device ${device}`, + ); + if (mode === "chat") { + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + const replies = body.messages.filter( + (message) => message.role === "tool", + ); + if (requestIndex === 1) + return call(requestIndex, "read", { + i: "Load REA skill", + path: "skill://reverse-engineer-anything:raw", + }); assert( - parsed.plannedActions.some((action) => action.target === configPath), + expectedSkillBody !== undefined && + containsText(replies, expectedSkillBody), + "OMP must return the complete installed skill body", + ); + skillLoaded = true; + if (requestIndex === 2) + return call(requestIndex, "read", { + i: "Inspect REA native tool contract", + path: analysisDevice, + }); + assert( + containsText(replies, "# mcp__rea_analyze_javascript_application"), ); assert( - parsed.plannedActions.some( - (action) => action.target === skillDirectory, + containsText(replies, "input_path") && + containsText(replies, '"directory"'), + ); + contractRead = true; + if (requestIndex === 3) + return call(requestIndex, "write", { + path: analysisDevice, + content: JSON.stringify({ + input_path: target, + format: "directory", + }), + }); + const evidenceReply = replies.find( + (reply) => + typeof reply.content === "string" && + reply.content.startsWith('{"evidence_id":'), + ); + assert( + evidenceReply, + "OMP must deliver the full analysis result to the next model turn", + ); + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(JSON.parse(evidenceReply.content)); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((node) => node.observations) + .map((observation) => observation.properties) + .find( + (properties) => + properties.semantic_role === "export-return-shapes" && + properties.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap( + (item) => item.fields, + ); + assert( + fields.some( + (field) => + field.path === "/clientCompatibility" && + field.state === "literal" && + field.value === "REA_CLIENT_FIXTURE", ), ); assert( - parsed.plannedActions.every( - (action) => - action.target === configPath || - action.target === skillDirectory || - action.target.startsWith(skillDirectory + "/"), + fields.some( + (field) => + field.path === "/count" && + field.state === "literal" && + field.value === 7, ), - "Refuse setup writes outside the owned config/skill", ); + const bytes = JSON.stringify(evidence); + artifactDigest = createHash("sha256").update(bytes).digest("hex"); + await writeFile(join(lab, "omp-evidence.json"), bytes); + evidenceSeen = true; + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + }, + }); + const original = JSON.stringify( + { + mcpServers: { other: { command: "unrelated-server", enabled: false } }, + disabledServers: ["rea", "other"], + }, + null, + 2, + ); + await writeFile(configPath, original); + await writeFile( + join(agent, "config.yml"), + JSON.stringify({ + skills: { customDirectories: [join(account, ".agents", "skills")] }, + retry: { fallbackChains: { default: [] } }, + }), + ); + await writeFile( + join(agent, "models.yml"), + JSON.stringify({ + providers: { + "rea-fixture": { + baseUrl: fixture.baseUrl, + apiKey: "local-fixture-key", + api: "openai-completions", + models: [ + { + id: "rea-client-fixture", + name: "REA local fixture", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1000000, + maxTokens: 4096, + }, + ], + }, + }, + }), + ); + const rea = async (args, env = environment) => + commands.exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + clientVersion = ( + await commands.exec(command, ["--version"], { + env: environment, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + const invalidEnvironment = { ...environment, OMP_PROFILE: "Bad Name" }; + const nativeRejected = await commands + .exec(command, ["--print", "--no-session", "--no-title", "Hello"], { + env: invalidEnvironment, + cwd: workspace, + timeout: 30_000, + stdin: "pipe", + }) + .then( + () => undefined, + (cause) => { + if (typeof cause?.code !== "number" || cause.code === 0) throw cause; + return cause; + }, + ); + assert( + nativeRejected?.code === 1 && + nativeRejected.snapshot.stderr.text.includes( + 'Invalid OMP profile "Bad Name"', + ), + ); + await writeFile( + join(lab, "invalid-profile-native.stderr"), + nativeRejected.snapshot.stderr.text, + ); + const setupRejected = await rea( + ["setup", "--client", "omp", "--dry-run"], + invalidEnvironment, + ).then( + () => undefined, + (cause) => { + if (typeof cause?.code !== "number" || cause.code === 0) throw cause; + return cause; + }, + ); + assert(typeof setupRejected?.code === "number" && setupRejected.code !== 0); + const rejected = JSON.parse(setupRejected.snapshot.stdout.text); + assert.equal(rejected.status, "needs_human"); + assert.deepEqual(rejected.plannedActions, []); + assert.deepEqual(rejected.appliedActions, []); + assert(rejected.remediation.includes('Invalid OMP profile "Bad Name"')); + assert.equal(await readFile(configPath, "utf8"), original); + await writeFile( + join(lab, "invalid-profile-setup.json"), + setupRejected.snapshot.stdout.text, + ); + invalidProfileRejected = true; + for (const args of [ + ["setup", "--client", "omp", "--dry-run"], + ["setup", "--client", "omp", "--yes"], + ["doctor", "--client", "omp", "--skill"], + ]) { + const result = await rea(args); + await writeFile( + join( + lab, + `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`, + ), + result.stdout, + ); + const parsed = JSON.parse(result.stdout); + if (args[0] === "doctor") assert.equal(parsed.healthy, true); + else { + assert.equal( + parsed.status, + args.includes("--dry-run") ? "planned" : "ready", + ); + if (args.includes("--dry-run")) { + assert( + parsed.plannedActions.some( + (action) => action.target === configPath, + ), + ); + assert( + parsed.plannedActions.some( + (action) => action.target === skillDirectory, + ), + ); + assert( + parsed.plannedActions.every( + (action) => + action.target === configPath || + action.target === skillDirectory || + action.target.startsWith(skillDirectory + "/"), + ), + "Refuse setup writes outside the owned config/skill", + ); + } } } - } - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const configured = await readFile(configPath, "utf8"); - const registration = JSON.parse(configured); - assert.equal(registration.mcpServers.other.command, "unrelated-server"); - assert.deepEqual(registration.disabledServers, ["other"]); - const repeated = JSON.parse( - (await rea(["setup", "--client", "omp", "--yes"])).stdout, - ); - assert.deepEqual(repeated.appliedActions, []); - assert.equal(await readFile(configPath, "utf8"), configured); - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - expectedSkillBody = await readFile(join(skillDirectory, "SKILL.md"), "utf8"); - const execution = exec( - command, - [ - "--mode", - "json", - "--print", - "--no-session", - "--no-title", - "--no-lsp", - "--no-pty", - "--approval-mode", - "yolo", - "--model", - "rea-fixture/rea-client-fixture", - mode === "chat" - ? "Say hello briefly." - : `Use the REA skill to analyze ${target}.`, - ], - { - env: environment, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + const registration = JSON.parse(configured); + assert.equal(registration.mcpServers.other.command, "unrelated-server"); + assert.deepEqual(registration.disabledServers, ["other"]); + const repeated = JSON.parse( + (await rea(["setup", "--client", "omp", "--yes"])).stdout, + ); + assert.deepEqual(repeated.appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + expectedSkillBody = await readFile( + join(skillDirectory, "SKILL.md"), + "utf8", + ); + const response = await saveVerifierCommandOutput( + () => + commands.exec( + command, + [ + "--mode", + "json", + "--print", + "--no-session", + "--no-title", + "--no-lsp", + "--no-pty", + "--approval-mode", + "yolo", + "--model", + "rea-fixture/rea-client-fixture", + mode === "chat" + ? "Say hello briefly." + : `Use the REA skill to analyze ${target}.`, + ], + { + env: environment, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + stdin: "closed-pipe", + }, + ), + { + stdout: join(lab, `${mode}.stdout`), + stderr: join(lab, `${mode}.stderr`), + }, + ); + assert.equal(fixture.failure, undefined); + const events = response.stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + const mounted = events + .filter( + (event) => + event.type === "notice" && + event.message?.startsWith("xd://: mounted "), + ) + .flatMap((event) => + event.message.slice("xd://: mounted ".length).split(", "), + ) + .filter((name) => name.startsWith("mcp__rea_")); + catalogSize = mounted.length; + assert.deepEqual( + mounted.sort(), + expectedDevices, + "Native OMP must mount the complete REA catalog", + ); + catalogMounted = true; + assert( + !events.some( + (event) => event.type === "tool_execution_end" && event.isError, + ), + ); + assert( + events.some( + (event) => + event.type === "message_end" && + event.message?.role === "assistant" && + event.message.stopReason === "stop" && + containsText(event.message.content, "REA_CLIENT_COMPATIBILITY_OK"), + ), + "OMP must finish with the expected native assistant message", + ); + assert( + events.some((event) => event.type === "agent_end" && event.isTerminal), + ); + assert(finalSeen); + if (mode === "call") assert(skillLoaded && contractRead && evidenceSeen); + }, [ + () => commands.close(), + async () => { + await fixture?.close(); }, - ); - execution.child.stdin.end(); - const response = await execution; - await writeFile(join(lab, `${mode}.stdout`), response.stdout); - await writeFile(join(lab, `${mode}.stderr`), response.stderr); - assert.equal(fixture.failure, undefined); - const events = response.stdout - .trim() - .split("\n") - .map((line) => JSON.parse(line)); - const mounted = events - .filter( - (event) => - event.type === "notice" && event.message?.startsWith("xd://: mounted "), - ) - .flatMap((event) => - event.message.slice("xd://: mounted ".length).split(", "), - ) - .filter((name) => name.startsWith("mcp__rea_")); - assert.deepEqual( - mounted.sort(), - expectedDevices, - "Native OMP must mount the complete REA catalog", - ); - catalogMounted = true; - assert( - !events.some( - (event) => event.type === "tool_execution_end" && event.isError, - ), - ); - assert( - events.some( - (event) => - event.type === "message_end" && - event.message?.role === "assistant" && - event.message.stopReason === "stop" && - containsText(event.message.content, "REA_CLIENT_COMPATIBILITY_OK"), - ), - "OMP must finish with the expected native assistant message", - ); - assert( - events.some((event) => event.type === "agent_end" && event.isTerminal), - ); - assert(finalSeen); - if (mode === "call") assert(skillLoaded && contractRead && evidenceSeen); -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, `${mode}.stdout`), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, `${mode}.stderr`), cause.stderr); -} finally { - await fixture.close(); - const lineage = await completeVerifierRun(verifier); - const receipt = { - client: "OMP", - clientVersion, - mode, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, - }, - modelFixture: - "loopback OpenAI completions/SSE, caller-declared one-million-token model and synthetic usage; no live model claim", - discovery: - "explicit isolated default agent/global config/XDG roots and custom skill directory; named profiles, default OS-home skill discovery and Windows unverified", - interface: - "default native xd:// device metadata with read/write dispatch; complete JSON schemas are not forwarded as model functions", - catalogMounted, - catalogSize: expectedDevices.length, - invalidProfileRejected, - skillLoaded, - contractRead, - evidenceSeen, - artifactDigest, - finalSeen, - requests: fixture.requests, - probes: fixture.probes, - fixtureFailure: fixture.failure, - verifier: lineage, - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; + ]); + passed = true; +}, [ + async () => { + const lineage = await completeVerifierRun(verifier); + const receipt = { + client: "OMP", + clientVersion, + mode, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback OpenAI completions/SSE, caller-declared one-million-token model and synthetic usage; no live model claim", + discovery: + "explicit isolated default agent/global config/XDG roots and custom skill directory; named profiles, default OS-home skill discovery and Windows unverified", + interface: + "default native xd:// device metadata with read/write dispatch; complete JSON schemas are not forwarded as model functions", + catalogMounted, + catalogSize, + invalidProfileRejected, + skillLoaded, + contractRead, + evidenceSeen, + artifactDigest, + finalSeen, + requests: fixture?.requests, + probes: fixture?.probes, + fixtureFailure: fixture?.failure, + verifier: lineage, + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); + }, + ], + ); + }, +]); diff --git a/scripts/verify-opencode-client.mjs b/scripts/verify-opencode-client.mjs index 2a3e4c0501..6ea41ef3bc 100644 --- a/scripts/verify-opencode-client.mjs +++ b/scripts/verify-opencode-client.mjs @@ -9,7 +9,9 @@ import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; -import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -58,335 +60,382 @@ assert.equal( isManagedSkillTarget(join(skillDirectory, "..", "outside", "AGENTS.md")), false, ); -await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]); -await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 }); -const environment = { - USERPROFILE: account, - OPENCODE_CONFIG_DIR: profile, - XDG_CONFIG_HOME: join(lab, "config"), - XDG_CACHE_HOME: join(lab, "cache"), - XDG_DATA_HOME: join(lab, "data"), - XDG_STATE_HOME: join(lab, "state"), - OPENCODE_DISABLE_AUTOUPDATE: "true", - OPENCODE_DISABLE_MODELS_FETCH: "true", - // Use the native core without external account plugins. - OPENCODE_PURE: "true", - REA_PROCESS_RUN_ID: verifier.run_id, -}; -let clientVersion; -try { - clientVersion = ( - await exec(command, ["--version"], { env: environment, timeout: 30_000 }) - ).stdout.trim(); -} catch (cause) { - throw new Error( - `OpenCode preflight failed for ${command}; install OpenCode or set REA_VERIFY_OPENCODE_COMMAND. ${String(cause)}`, - ); -} -const target = join(lab, "fixture α测试"); -await mkdir(target); -await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', -); +let clientVersion = null; +let fixture; +let passed = false; let skillRequested = false; let skillLoaded = false; let expectedSkillBody; let evidenceSeen = false; let finalSeen = false; -let catalogSize = 0; +let catalogSize = null; let schemasChecked = 0; let artifactDigest; -const schemaValidator = new Ajv2020({ strict: false, validateFormats: false }); -const extractEvidence = (value) => { - if (typeof value === "string") { - try { - return extractEvidence(JSON.parse(value)); - } catch { - return undefined; - } - } - if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0 - ) - return value; - for (const nested of Object.values(value)) { - const evidence = extractEvidence(nested); - if (evidence) return evidence; - } - } - return undefined; -}; -const fixture = await createOpenAiModelFixture({ - directory: lab, - prefix: "opencode", - onRequest: async ({ body, tools, requestIndex }) => { - // Native session-title requests do not execute the analyst's workflow. - if (tools.length === 0) - return { role: "assistant", content: "REA fixture title" }; - const analysis = tools.find((t) => - t.name.endsWith("analyze_javascript_application"), - ); - assert(analysis, "OpenCode must expose the REA analysis tool"); - const prefix = analysis.name.slice( - 0, - -"analyze_javascript_application".length, - ); - const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort(); - const reaTools = tools.filter((t) => expected.includes(t.name)); - assert.deepEqual( - reaTools.map((t) => t.name).sort(), - expected, - "OpenCode must forward the complete REA catalog", - ); - catalogSize = expected.length; - if (schemasChecked === 0) { - for (const tool of reaTools) { - assert( - tool.parameters, - `${tool.name}: missing advertised input schema`, - ); - assert( - schemaValidator.validateSchema(tool.parameters), - `${tool.name}: invalid input schema: ${JSON.stringify(schemaValidator.errors)}`, - ); - schemasChecked++; - } - } - const replies = body.messages.filter((m) => m.role === "tool"); - skillLoaded ||= replies.some( - (m) => - typeof m.content === "string" && - expectedSkillBody !== undefined && - m.content.includes(skillDirectory) && - m.content.includes(expectedSkillBody), - ); - const observed = extractEvidence(replies); - if (observed) { - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(observed); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((n) => n.observations) - .map((o) => o.properties) - .find( - (p) => - p.semantic_role === "export-return-shapes" && - p.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((s) => s.fields); - assert( - fields.some( - (f) => - f.path === "/clientCompatibility" && - f.state === "literal" && - f.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (f) => f.path === "/count" && f.state === "literal" && f.value === 7, - ), - ); - const bytes = JSON.stringify(evidence); - artifactDigest = createHash("sha256").update(bytes).digest("hex"); - await writeFile(join(lab, "opencode-evidence.json"), bytes); - evidenceSeen = true; - } - if (mode === "chat" || evidenceSeen) { - finalSeen = true; - return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - } - assert(requestIndex <= 8, "Local model workflow exceeded its turn budget"); - const call = (name, args) => ({ - role: "assistant", - content: null, - tool_calls: [ - { - id: `call_rea_fixture_${requestIndex}`, - type: "function", - function: { name, arguments: JSON.stringify(args) }, - }, - ], +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]); + await commands.exec("git", ["init", "--quiet", workspace], { + timeout: 30_000, + stdin: "pipe", }); - if (!skillRequested) { - const skill = tools.find((t) => t.name === "skill"); - assert(skill, "OpenCode must expose its native skill tool"); - skillRequested = true; - return call(skill.name, { name: "reverse-engineer-anything" }); + const environment = { + USERPROFILE: account, + OPENCODE_CONFIG_DIR: profile, + XDG_CONFIG_HOME: join(lab, "config"), + XDG_CACHE_HOME: join(lab, "cache"), + XDG_DATA_HOME: join(lab, "data"), + XDG_STATE_HOME: join(lab, "state"), + OPENCODE_DISABLE_AUTOUPDATE: "true", + OPENCODE_DISABLE_MODELS_FETCH: "true", + // Use the native core without external account plugins. + OPENCODE_PURE: "true", + REA_PROCESS_RUN_ID: verifier.run_id, + }; + try { + clientVersion = ( + await commands.exec(command, ["--version"], { + env: environment, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + } catch (cause) { + throw new Error( + `OpenCode preflight failed for ${command}; install OpenCode or set REA_VERIFY_OPENCODE_COMMAND. ${String(cause)}`, + { cause }, + ); } - assert(skillLoaded, "OpenCode must load the installed personal REA skill"); - return call(analysis.name, { input_path: target, format: "directory" }); - }, -}); -const original = - (configFormat === "jsonc" ? "// Keep caller configuration.\n" : "") + - JSON.stringify( - { - autoupdate: false, - skills: { paths: [join(account, ".agents", "skills")] }, - permission: { "*": "allow" }, - provider: { - "rea-fixture": { - npm: "@ai-sdk/openai-compatible", - name: "REA local model fixture", - options: { baseURL: fixture.baseUrl, apiKey: "local-fixture-key" }, - models: { - "rea-client-fixture": { - name: "REA fixture", - tool_call: true, - limit: { context: 1_000_000, output: 4096 }, + const target = join(lab, "fixture α测试"); + await mkdir(target); + await writeFile( + join(target, "app.js"), + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', + ); + const schemaValidator = new Ajv2020({ + strict: false, + validateFormats: false, + }); + const extractEvidence = (value) => { + if (typeof value === "string") { + try { + return extractEvidence(JSON.parse(value)); + } catch { + return undefined; + } + } + if (value && typeof value === "object") { + if ( + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes?.length > 0 + ) + return value; + for (const nested of Object.values(value)) { + const evidence = extractEvidence(nested); + if (evidence) return evidence; + } + } + return undefined; + }; + fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: "opencode", + onRequest: async ({ body, tools, requestIndex }) => { + // Native session-title requests do not execute the analyst's workflow. + if (tools.length === 0) + return { role: "assistant", content: "REA fixture title" }; + const analysis = tools.find((t) => + t.name.endsWith("analyze_javascript_application"), + ); + assert(analysis, "OpenCode must expose the REA analysis tool"); + const prefix = analysis.name.slice( + 0, + -"analyze_javascript_application".length, + ); + const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort(); + const reaTools = tools.filter((t) => expected.includes(t.name)); + catalogSize = reaTools.length; + assert.deepEqual( + reaTools.map((t) => t.name).sort(), + expected, + "OpenCode must forward the complete REA catalog", + ); + if (schemasChecked === 0) { + for (const tool of reaTools) { + assert( + tool.parameters, + `${tool.name}: missing advertised input schema`, + ); + assert( + schemaValidator.validateSchema(tool.parameters), + `${tool.name}: invalid input schema: ${JSON.stringify(schemaValidator.errors)}`, + ); + schemasChecked++; + } + } + const replies = body.messages.filter((m) => m.role === "tool"); + skillLoaded ||= replies.some( + (m) => + typeof m.content === "string" && + expectedSkillBody !== undefined && + m.content.includes(skillDirectory) && + m.content.includes(expectedSkillBody), + ); + const observed = extractEvidence(replies); + if (observed) { + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(observed); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((n) => n.observations) + .map((o) => o.properties) + .find( + (p) => + p.semantic_role === "export-return-shapes" && + p.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap((s) => s.fields); + assert( + fields.some( + (f) => + f.path === "/clientCompatibility" && + f.state === "literal" && + f.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (f) => + f.path === "/count" && f.state === "literal" && f.value === 7, + ), + ); + const bytes = JSON.stringify(evidence); + artifactDigest = createHash("sha256").update(bytes).digest("hex"); + await writeFile(join(lab, "opencode-evidence.json"), bytes); + evidenceSeen = true; + } + if (mode === "chat" || evidenceSeen) { + finalSeen = true; + return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; + } + assert( + requestIndex <= 8, + "Local model workflow exceeded its turn budget", + ); + const call = (name, args) => ({ + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_rea_fixture_${requestIndex}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }); + if (!skillRequested) { + const skill = tools.find((t) => t.name === "skill"); + assert(skill, "OpenCode must expose its native skill tool"); + skillRequested = true; + return call(skill.name, { name: "reverse-engineer-anything" }); + } + assert( + skillLoaded, + "OpenCode must load the installed personal REA skill", + ); + return call(analysis.name, { input_path: target, format: "directory" }); + }, + }); + const original = + (configFormat === "jsonc" ? "// Keep caller configuration.\n" : "") + + JSON.stringify( + { + autoupdate: false, + skills: { paths: [join(account, ".agents", "skills")] }, + permission: { "*": "allow" }, + provider: { + "rea-fixture": { + npm: "@ai-sdk/openai-compatible", + name: "REA local model fixture", + options: { + baseURL: fixture.baseUrl, + apiKey: "local-fixture-key", + }, + models: { + "rea-client-fixture": { + name: "REA fixture", + tool_call: true, + limit: { context: 1_000_000, output: 4096 }, + }, + }, + }, + }, + mcp: { + other: { + type: "local", + command: ["unrelated-server"], + enabled: false, }, }, }, - }, - mcp: { - other: { type: "local", command: ["unrelated-server"], enabled: false }, - }, - }, - null, - 2, - ); -await writeFile(configPath, original); -let failure; -try { - for (const args of [ - ["setup", "--client", "opencode", "--dry-run"], - ["setup", "--client", "opencode", "--yes"], - ["doctor", "--client", "opencode", "--skill"], - ]) { - const result = await exec( - process.execPath, - [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); - await writeFile( - join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`), - result.stdout, - ); - const parsed = JSON.parse(result.stdout); - if (args[0] === "doctor") assert.equal(parsed.healthy, true); - else { - assert.equal( - parsed.status, - args.includes("--dry-run") ? "planned" : "ready", + null, + 2, ); - if (args.includes("--dry-run")) { - assert(parsed.plannedActions.some((a) => a.target === configPath)); - assert(parsed.plannedActions.some((a) => a.target === skillDirectory)); - assert( - parsed.plannedActions.every( - (a) => a.target === configPath || isManagedSkillTarget(a.target), - ), + await writeFile(configPath, original); + for (const args of [ + ["setup", "--client", "opencode", "--dry-run"], + ["setup", "--client", "opencode", "--yes"], + ["doctor", "--client", "opencode", "--skill"], + ]) { + const result = await commands.exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + await writeFile( + join( + lab, + `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`, + ), + result.stdout, + ); + const parsed = JSON.parse(result.stdout); + if (args[0] === "doctor") assert.equal(parsed.healthy, true); + else { + assert.equal( + parsed.status, + args.includes("--dry-run") ? "planned" : "ready", ); + if (args.includes("--dry-run")) { + assert(parsed.plannedActions.some((a) => a.target === configPath)); + assert( + parsed.plannedActions.some((a) => a.target === skillDirectory), + ); + assert( + parsed.plannedActions.every( + (a) => a.target === configPath || isManagedSkillTarget(a.target), + ), + ); + } } } - } - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const configured = await readFile(configPath, "utf8"); - assert(configured.includes("unrelated-server")); - if (configFormat === "jsonc") - assert(configured.includes("// Keep caller configuration.")); - const repeated = await exec( - process.execPath, - [ - join(runtimeRoot, "scripts/rea.mjs"), - "setup", - "--client", - "opencode", - "--yes", - "--json", - ], - { env: environment, cwd: workspace, timeout: 60_000 }, - ); - assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); - assert.equal(await readFile(configPath, "utf8"), configured); - assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); - const installedSkill = await readFile( - join(skillDirectory, "SKILL.md"), - "utf8", - ); - assert(installedSkill.includes("name: reverse-engineer-anything")); - const skillLines = installedSkill.split("\n"); - const headerEnd = skillLines.indexOf("---", 1); - assert( - skillLines[0] === "---" && headerEnd > 0, - "Installed skill must have its authored frontmatter", - ); - expectedSkillBody = skillLines - .slice(headerEnd + 1) - .join("\n") - .trim(); - assert(expectedSkillBody.length > 0); - const execution = exec( - command, - [ - "run", - "--format", - "json", - "--model", - "rea-fixture/rea-client-fixture", - mode === "chat" - ? "Say hello briefly." - : `Use REA to analyze the JavaScript directory ${target}. Report the result.`, - ], - { - env: environment, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + assert(configured.includes("unrelated-server")); + if (configFormat === "jsonc") + assert(configured.includes("// Keep caller configuration.")); + const repeated = await commands.exec( + process.execPath, + [ + join(runtimeRoot, "scripts/rea.mjs"), + "setup", + "--client", + "opencode", + "--yes", + "--json", + ], + { env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" }, + ); + assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const installedSkill = await readFile( + join(skillDirectory, "SKILL.md"), + "utf8", + ); + assert(installedSkill.includes("name: reverse-engineer-anything")); + const skillLines = installedSkill.split("\n"); + const headerEnd = skillLines.indexOf("---", 1); + assert( + skillLines[0] === "---" && headerEnd > 0, + "Installed skill must have its authored frontmatter", + ); + expectedSkillBody = skillLines + .slice(headerEnd + 1) + .join("\n") + .trim(); + assert(expectedSkillBody.length > 0); + const response = await saveVerifierCommandOutput( + () => + commands.exec( + command, + [ + "run", + "--format", + "json", + "--model", + "rea-fixture/rea-client-fixture", + mode === "chat" + ? "Say hello briefly." + : `Use REA to analyze the JavaScript directory ${target}. Report the result.`, + ], + { + env: environment, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + stdin: "closed-pipe", + }, + ), + { + stdout: join(lab, `${mode}.stdout`), + stderr: join(lab, `${mode}.stderr`), + }, + ); + assert.equal(fixture.failure, undefined); + assert( + finalSeen && response.stdout.includes("REA_CLIENT_COMPATIBILITY_OK"), + ); + if (mode === "call") assert(evidenceSeen && skillLoaded); + }, [ + () => commands.close(), + async () => { + await fixture?.close(); }, - ); - execution.child.stdin.end(); - const response = await execution; - await writeFile(join(lab, `${mode}.stdout`), response.stdout); - await writeFile(join(lab, `${mode}.stderr`), response.stderr); - assert.equal(fixture.failure, undefined); - assert(finalSeen && response.stdout.includes("REA_CLIENT_COMPATIBILITY_OK")); - if (mode === "call") assert(evidenceSeen && skillLoaded); -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, `${mode}.stdout`), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, `${mode}.stderr`), cause.stderr); -} finally { - await fixture.close(); - const lineage = await completeVerifierRun(verifier); - const receipt = { - client: "OpenCode", - clientVersion, - mode, - configFormat, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, - }, - modelFixture: - "loopback custom OpenAI-compatible provider; one-million-token fixture capacity and synthetic usage; no live provider claim", - skillDiscovery: - "caller-configured isolated shared directory; default OS home discovery unverified", - externalPlugins: "disabled through native OPENCODE_PURE", - catalogSize, - schemasChecked, - evidenceSeen, - skillLoaded, - artifactDigest, - finalSeen, - requests: fixture.requests, - probes: fixture.probes, - fixtureFailure: fixture.failure, - verifier: lineage, - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; + ]); + passed = true; +}, [ + async () => { + const lineage = await completeVerifierRun(verifier); + const receipt = { + client: "OpenCode", + clientVersion, + mode, + configFormat, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback custom OpenAI-compatible provider; one-million-token fixture capacity and synthetic usage; no live provider claim", + skillDiscovery: + "caller-configured isolated shared directory; default OS home discovery unverified", + externalPlugins: "disabled through native OPENCODE_PURE", + catalogSize, + schemasChecked, + evidenceSeen, + skillLoaded, + artifactDigest, + finalSeen, + requests: fixture?.requests, + probes: fixture?.probes, + fixtureFailure: fixture?.failure, + verifier: lineage, + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); + }, + ], + ); + }, +]); diff --git a/scripts/verify-qwen-client.mjs b/scripts/verify-qwen-client.mjs index 50001103c8..b92619721d 100644 --- a/scripts/verify-qwen-client.mjs +++ b/scripts/verify-qwen-client.mjs @@ -11,7 +11,9 @@ import { tmpdir } from "node:os"; import { join, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; import { createHash } from "node:crypto"; -import { exec } from "./lib/verify-package-core.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { @@ -42,93 +44,9 @@ const account = join(lab, "account"); const runtime = join(profile, "runtime"); const workspace = join(lab, "workspace"); const sharedSkills = join(account, ".agents", "skills"); -await Promise.all([ - mkdir(profile, { recursive: true }), - mkdir(account), - mkdir(workspace), -]); -const baseEnvironment = { - PATH: process.env.PATH ?? "", - HOME: account, - USERPROFILE: account, - QWEN_HOME: profile, - QWEN_RUNTIME_DIR: runtime, - REA_PROCESS_RUN_ID: verifier.run_id, -}; -let clientVersion; -try { - clientVersion = ( - await exec(...launch(["--version"]), { - env: baseEnvironment, - timeout: 30_000, - }) - ).stdout.trim(); -} catch (cause) { - const windowsHint = - process.platform === "win32" - ? " On Windows, select node_modules/@qwen-code/qwen-code/cli-entry.js." - : ""; - throw new Error( - `Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND.${windowsHint} ${String(cause)}`, - ); -} -const original = JSON.stringify( - { - security: { - auth: { selectedType: "openai" }, - folderTrust: { enabled: false }, - }, - telemetry: { enabled: false }, - model: { name: "rea-client-fixture" }, - mcpServers: { other: { command: "unrelated-server", disabled: true } }, - }, - null, - 2, -); -await writeFile(join(profile, "settings.json"), original); -for (const step of [ - ["setup", "--client", "qwen_code", "--dry-run"], - ["setup", "--client", "qwen_code", "--yes"], - ["doctor", "--client", "qwen_code", "--skill"], -]) { - const response = await exec( - process.execPath, - [join(runtimeRoot, "scripts/rea.mjs"), ...step, "--json"], - { env: baseEnvironment, cwd: repo, timeout: 60_000 }, - ); - const result = JSON.parse(response.stdout); - if (step.includes("--dry-run")) - assert( - result.plannedActions.every( - (a) => - a.target.startsWith(profile + sep) || - a.target.startsWith(sharedSkills + sep), - ), - ); - await writeFile( - join(lab, `${step[0]}${step.includes("--dry-run") ? "-plan" : ""}.json`), - response.stdout, - ); -} -assert.equal( - await readFile(join(profile, "settings.json.rea.backup"), "utf8"), - original, -); -const configured = JSON.parse( - await readFile(join(profile, "settings.json"), "utf8"), -); -assert.deepEqual(configured.model, { name: "rea-client-fixture" }); -assert.equal(configured.skills, undefined); -assert.deepEqual(configured.mcpServers.other, { - command: "unrelated-server", - disabled: true, -}); -const target = join(lab, "fixture α测试"); -await mkdir(target, { recursive: true }); -await writeFile( - `${target}/app.js`, - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', -); +let clientVersion = null; +let fixture; +let passed = false; let evidenceSeen = false; let skillSeen = false; let finalSeen = false; @@ -138,278 +56,395 @@ let artifactReadByAgent = false; let artifactPath; let selectedSkillLoaded = false; let artifactDigest; -const extractEvidence = (value) => { - if (typeof value === "string") { +let catalogSize = null; +const commands = await createVerifierCommands(); +await withVerifierCleanup(async () => { + await withVerifierCleanup(async () => { + await Promise.all([ + mkdir(profile, { recursive: true }), + mkdir(account), + mkdir(workspace), + ]); + const baseEnvironment = { + PATH: process.env.PATH ?? "", + HOME: account, + USERPROFILE: account, + QWEN_HOME: profile, + QWEN_RUNTIME_DIR: runtime, + REA_PROCESS_RUN_ID: verifier.run_id, + }; try { - return extractEvidence(JSON.parse(value)); - } catch { - return null; - } - } - if (Array.isArray(value)) { - for (const item of value) { - const result = extractEvidence(item); - if (result) return result; - } - } else if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0 - ) - return value; - for (const nested of Object.values(value)) { - const result = extractEvidence(nested); - if (result) return result; - } - } - return null; -}; -const fixture = await createOpenAiModelFixture({ - directory: lab, - prefix: `${client}-${mode}`, - onRequest: async ({ body, tools, requestIndex }) => { - skillSeen ||= JSON.stringify(body.messages ?? []).includes( - "reverse-engineer-anything", - ); - const messageText = (body.messages ?? []) - .map((message) => - typeof message.content === "string" - ? message.content - : (message.content ?? []).map((part) => part.text ?? "").join("\n"), - ) - .join("\n"); - selectedSkillLoaded ||= messageText.includes( - join(sharedSkills, "reverse-engineer-anything"), - ); - const toolText = JSON.stringify( - (body.messages ?? []).filter((m) => m.role === "tool"), - ); - if ( - toolText.includes("REA_CLIENT_FIXTURE") && - /(?:\\?"REA_CLIENT_FIXTURE\\?"\s*:\s*true)/u.test(toolText) - ) { - artifactReadByAgent = true; - assert(artifactPath); - const artifact = await readFile(artifactPath); - artifactDigest = createHash("sha256").update(artifact).digest("hex"); - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(JSON.parse(artifact.toString("utf8"))); - assert(extractEvidence(evidence)); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((n) => n.observations) - .map((o) => o.properties) - .find( - (p) => - p.semantic_role === "export-return-shapes" && - p.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((s) => s.fields); - assert( - fields.some( - (f) => - f.path === "/clientCompatibility" && - f.state === "literal" && - f.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (f) => f.path === "/count" && f.state === "literal" && f.value === 7, - ), - ); - evidenceSeen = true; - await writeFile( - `${lab}/${client}-${mode}-evidence.json`, - JSON.stringify(evidence), + clientVersion = ( + await commands.exec(...launch(["--version"]), { + env: baseEnvironment, + timeout: 30_000, + stdin: "pipe", + }) + ).stdout.trim(); + } catch (cause) { + const windowsHint = + process.platform === "win32" + ? " On Windows, select node_modules/@qwen-code/qwen-code/cli-entry.js." + : ""; + throw new Error( + `Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND.${windowsHint} ${String(cause)}`, + { cause }, ); } - for (const message of body.messages ?? []) { - if (message.role !== "tool") continue; - const evidence = extractEvidence(message.content); - if (evidence) { + const original = JSON.stringify( + { + security: { + auth: { selectedType: "openai" }, + folderTrust: { enabled: false }, + }, + telemetry: { enabled: false }, + model: { name: "rea-client-fixture" }, + mcpServers: { other: { command: "unrelated-server", disabled: true } }, + }, + null, + 2, + ); + await writeFile(join(profile, "settings.json"), original); + for (const step of [ + ["setup", "--client", "qwen_code", "--dry-run"], + ["setup", "--client", "qwen_code", "--yes"], + ["doctor", "--client", "qwen_code", "--skill"], + ]) { + const response = await commands.exec( + process.execPath, + [join(runtimeRoot, "scripts/rea.mjs"), ...step, "--json"], + { env: baseEnvironment, cwd: repo, timeout: 60_000, stdin: "pipe" }, + ); + const result = JSON.parse(response.stdout); + if (step.includes("--dry-run")) assert( - JSON.stringify(evidence.normalized_result).includes( - "REA_CLIENT_FIXTURE", + result.plannedActions.every( + (a) => + a.target.startsWith(profile + sep) || + a.target.startsWith(sharedSkills + sep), ), ); - evidenceSeen = true; - await writeFile( - `${lab}/${client}-${mode}-evidence.json`, - JSON.stringify(evidence), - ); - } + await writeFile( + join( + lab, + `${step[0]}${step.includes("--dry-run") ? "-plan" : ""}.json`, + ), + response.stdout, + ); } - const selected = tools.find((t) => - t.name.includes("analyze_javascript_application"), + assert.equal( + await readFile(join(profile, "settings.json.rea.backup"), "utf8"), + original, ); - const reviewedNames = [ - ...JSON.stringify( - (body.messages ?? []).filter((m) => m.role === "tool"), - ).matchAll( - /(?:\\?"name\\?"\s*:\s*\\?")([^"\\]*analyze_javascript_application)(?:\\?")/g, - ), - ] - .map((m) => m[1]) - .sort((a, b) => b.length - a.length); - const search = tools.find((t) => t.name === "tool_search"); - const invoke = tools.find((t) => t.name === "tool_call"); - const skill = tools.find((t) => t.name === "skill"); - const shell = tools.find((t) => t.name === "run_shell_command"); - const persisted = messageText.match(/Full output saved to: ([^\r\n]+)/u); - const call = (name, args) => ({ - role: "assistant", - content: null, - tool_calls: [ - { - id: `call_rea_fixture_${requestIndex}`, - type: "function", - function: { name, arguments: JSON.stringify(args) }, + const configured = JSON.parse( + await readFile(join(profile, "settings.json"), "utf8"), + ); + assert.deepEqual(configured.model, { name: "rea-client-fixture" }); + assert.equal(configured.skills, undefined); + assert.deepEqual(configured.mcpServers.other, { + command: "unrelated-server", + disabled: true, + }); + const target = join(lab, "fixture α测试"); + await mkdir(target, { recursive: true }); + await writeFile( + `${target}/app.js`, + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', + ); + const extractEvidence = (value) => { + if (typeof value === "string") { + try { + return extractEvidence(JSON.parse(value)); + } catch { + return null; + } + } + if (Array.isArray(value)) { + for (const item of value) { + const result = extractEvidence(item); + if (result) return result; + } + } else if (value && typeof value === "object") { + if ( + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes?.length > 0 + ) + return value; + for (const nested of Object.values(value)) { + const result = extractEvidence(nested); + if (result) return result; + } + } + return null; + }; + fixture = await createOpenAiModelFixture({ + directory: lab, + prefix: `${client}-${mode}`, + onRequest: async ({ body, tools, requestIndex }) => { + skillSeen ||= JSON.stringify(body.messages ?? []).includes( + "reverse-engineer-anything", + ); + const messageText = (body.messages ?? []) + .map((message) => + typeof message.content === "string" + ? message.content + : (message.content ?? []) + .map((part) => part.text ?? "") + .join("\n"), + ) + .join("\n"); + selectedSkillLoaded ||= messageText.includes( + join(sharedSkills, "reverse-engineer-anything"), + ); + const toolText = JSON.stringify( + (body.messages ?? []).filter((m) => m.role === "tool"), + ); + if ( + toolText.includes("REA_CLIENT_FIXTURE") && + /(?:\\?"REA_CLIENT_FIXTURE\\?"\s*:\s*true)/u.test(toolText) + ) { + artifactReadByAgent = true; + assert(artifactPath); + const artifact = await readFile(artifactPath); + artifactDigest = createHash("sha256").update(artifact).digest("hex"); + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(JSON.parse(artifact.toString("utf8"))); + assert(extractEvidence(evidence)); + await assertJavaScriptApplicationInputPaths(evidence, target); + const shape = evidence.normalized_result.graph.nodes + .flatMap((n) => n.observations) + .map((o) => o.properties) + .find( + (p) => + p.semantic_role === "export-return-shapes" && + p.exported_name === "result", + ); + assert(shape?.return_shape_coverage?.projection_complete); + const fields = shape.static_return_shapes.flatMap((s) => s.fields); + assert( + fields.some( + (f) => + f.path === "/clientCompatibility" && + f.state === "literal" && + f.value === "REA_CLIENT_FIXTURE", + ), + ); + assert( + fields.some( + (f) => + f.path === "/count" && f.state === "literal" && f.value === 7, + ), + ); + evidenceSeen = true; + await writeFile( + `${lab}/${client}-${mode}-evidence.json`, + JSON.stringify(evidence), + ); + } + for (const message of body.messages ?? []) { + if (message.role !== "tool") continue; + const evidence = extractEvidence(message.content); + if (evidence) { + assert( + JSON.stringify(evidence.normalized_result).includes( + "REA_CLIENT_FIXTURE", + ), + ); + evidenceSeen = true; + await writeFile( + `${lab}/${client}-${mode}-evidence.json`, + JSON.stringify(evidence), + ); + } + } + const selected = tools.find((t) => + t.name.includes("analyze_javascript_application"), + ); + const reviewedNames = [ + ...JSON.stringify( + (body.messages ?? []).filter((m) => m.role === "tool"), + ).matchAll( + /(?:\\?"name\\?"\s*:\s*\\?")([^"\\]*analyze_javascript_application)(?:\\?")/g, + ), + ] + .map((m) => m[1]) + .sort((a, b) => b.length - a.length); + const search = tools.find((t) => t.name === "tool_search"); + const invoke = tools.find((t) => t.name === "tool_call"); + const skill = tools.find((t) => t.name === "skill"); + const shell = tools.find((t) => t.name === "run_shell_command"); + const persisted = messageText.match( + /Full output saved to: ([^\r\n]+)/u, + ); + const call = (name, args) => ({ + role: "assistant", + content: null, + tool_calls: [ + { + id: `call_rea_fixture_${requestIndex}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }); + let message; + if (mode === "chat" || evidenceSeen) { + message = { + role: "assistant", + content: "REA_CLIENT_COMPATIBILITY_OK", + }; + finalSeen = true; + } else if (skill && !skillRequested) { + skillRequested = true; + message = call(skill.name, { skill: "reverse-engineer-anything" }); + } else if (persisted && shell) { + artifactPath = persisted[1]; + assert( + (await realpath(artifactPath)).startsWith( + (await realpath(runtime)) + sep, + ), + ); + const script = `const fs=require('node:fs');const e=JSON.parse(fs.readFileSync(${JSON.stringify(artifactPath)},'utf8'));console.log(JSON.stringify({REA_CLIENT_FIXTURE:JSON.stringify(e.normalized_result).includes('REA_CLIENT_FIXTURE'),evidence_id:e.evidence_id,graph_nodes:e.normalized_result.graph.nodes.length}));`; + // Keep paths and JSON quotes out of shell syntax on Windows and POSIX. + const encodedScript = Buffer.from(script).toString("base64"); + message = call(shell.name, { + command: `node -e "eval(Buffer.from('${encodedScript}','base64').toString('utf8'))"`, + description: + "Read the complete saved REA Evidence and verify the fixture facts.", + }); + } else if (selected) + message = call(selected.name, { + input_path: target, + format: "directory", + }); + else if (reviewedNames.length && invoke) + message = call(invoke.name, { + name: reviewedNames[0], + arguments: { input_path: target, format: "directory" }, + }); + else if (search && !searchRequested) { + searchRequested = true; + message = call(search.name, { + query: "analyze_javascript_application", + }); + } else + throw new Error( + `REA analysis tool not exposed; tools: ${tools.map((t) => t.name).join(", ")}`, + ); + return message; + }, + }); + const endpoint = fixture.baseUrl; + const prompt = + mode === "chat" + ? "Say hello briefly." + : `Use REA to analyze the JavaScript directory ${target}. Report the result.`; + const env = { + ...baseEnvironment, + OPENAI_API_KEY: "local-fixture-key", + OPENAI_BASE_URL: endpoint, + }; + const args = [ + "--auth-type", + "openai", + "--openai-base-url", + endpoint, + "--model", + "rea-client-fixture", + "--approval-mode", + mode === "chat" ? "default" : "yolo", + "--allowed-mcp-server-names", + "rea", + "--output-format", + "stream-json", + "--max-session-turns", + "8", + "-p", + prompt, + ]; + const response = await saveVerifierCommandOutput( + () => + commands.exec(...launch(args), { + env, + cwd: workspace, + timeout: 120_000, + maxBuffer: 10 * 1024 * 1024, + stdin: "pipe", + }), + { + stdout: join(lab, `${mode}.stdout`), + stderr: join(lab, `${mode}.stderr`), + }, + ); + const initialized = response.stdout + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)) + .find((v) => v.type === "system" && v.subtype === "init"); + assert(initialized); + const observed = initialized.tools + .filter((n) => n.startsWith("mcp__rea__")) + .sort(); + catalogSize = observed.length; + assert.deepEqual( + observed, + TOOL_CONTRACTS.map((c) => `mcp__rea__${c.name}`).sort(), + "Qwen must discover the complete REA catalog", + ); + assert.equal(fixture.failure, undefined); + assert(finalSeen); + if (mode === "call") { + assert(evidenceSeen); + assert(selectedSkillLoaded); + assert(artifactReadByAgent); + } + }, [ + () => commands.close(), + async () => { + await fixture?.close(); + }, + ]); + passed = true; +}, [ + async () => { + const lineage = await completeVerifierRun(verifier); + const receipt = { + client: "Qwen Code", + clientVersion, + mode, + passed, + artifacts: lab, + runtimeRoot, + host: { + platform: process.platform, + architecture: process.arch, + node: process.versions.node, + }, + modelFixture: + "loopback deterministic OpenAI-compatible endpoint; no live model-provider claim", + skillDiscovery: + "native HOME shared personal directory; no explicit skill setting", + catalogSize, + requests: fixture?.requests, + probes: fixture?.probes, + evidenceSeen, + selectedSkillLoaded, + artifactReadByAgent, + artifactPath, + artifactDigest, + finalSeen, + fixtureFailure: fixture?.failure, + verifier: lineage, + }; + await withVerifierCleanup( + () => + writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)), + [ + () => { + console.log(JSON.stringify(receipt)); }, ], - }); - let message; - if (mode === "chat" || evidenceSeen) { - message = { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" }; - finalSeen = true; - } else if (skill && !skillRequested) { - skillRequested = true; - message = call(skill.name, { skill: "reverse-engineer-anything" }); - } else if (persisted && shell) { - artifactPath = persisted[1]; - assert( - (await realpath(artifactPath)).startsWith( - (await realpath(runtime)) + sep, - ), - ); - const script = `const fs=require('node:fs');const e=JSON.parse(fs.readFileSync(${JSON.stringify(artifactPath)},'utf8'));console.log(JSON.stringify({REA_CLIENT_FIXTURE:JSON.stringify(e.normalized_result).includes('REA_CLIENT_FIXTURE'),evidence_id:e.evidence_id,graph_nodes:e.normalized_result.graph.nodes.length}));`; - // Keep paths and JSON quotes out of shell syntax on Windows and POSIX. - const encodedScript = Buffer.from(script).toString("base64"); - message = call(shell.name, { - command: `node -e "eval(Buffer.from('${encodedScript}','base64').toString('utf8'))"`, - description: - "Read the complete saved REA Evidence and verify the fixture facts.", - }); - } else if (selected) - message = call(selected.name, { - input_path: target, - format: "directory", - }); - else if (reviewedNames.length && invoke) - message = call(invoke.name, { - name: reviewedNames[0], - arguments: { input_path: target, format: "directory" }, - }); - else if (search && !searchRequested) { - searchRequested = true; - message = call(search.name, { query: "analyze_javascript_application" }); - } else - throw new Error( - `REA analysis tool not exposed; tools: ${tools.map((t) => t.name).join(", ")}`, - ); - return message; + ); }, -}); -const endpoint = fixture.baseUrl; -const prompt = - mode === "chat" - ? "Say hello briefly." - : `Use REA to analyze the JavaScript directory ${target}. Report the result.`; -const env = { - ...baseEnvironment, - OPENAI_API_KEY: "local-fixture-key", - OPENAI_BASE_URL: endpoint, -}; -const args = [ - "--auth-type", - "openai", - "--openai-base-url", - endpoint, - "--model", - "rea-client-fixture", - "--approval-mode", - mode === "chat" ? "default" : "yolo", - "--allowed-mcp-server-names", - "rea", - "--output-format", - "stream-json", - "--max-session-turns", - "8", - "-p", - prompt, -]; -let response; -let failure; -try { - response = await exec(...launch(args), { - env, - cwd: workspace, - timeout: 120_000, - maxBuffer: 10 * 1024 * 1024, - }); - await writeFile(join(lab, `${mode}.stdout`), response.stdout); - await writeFile(join(lab, `${mode}.stderr`), response.stderr); - const initialized = response.stdout - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line)) - .find((v) => v.type === "system" && v.subtype === "init"); - assert(initialized); - const observed = initialized.tools - .filter((n) => n.startsWith("mcp__rea__")) - .sort(); - assert.deepEqual( - observed, - TOOL_CONTRACTS.map((c) => `mcp__rea__${c.name}`).sort(), - "Qwen must discover the complete REA catalog", - ); - assert.equal(fixture.failure, undefined); - assert(finalSeen); - if (mode === "call") { - assert(evidenceSeen); - assert(selectedSkillLoaded); - assert(artifactReadByAgent); - } -} catch (cause) { - failure = cause; - if (typeof cause?.stdout === "string") - await writeFile(join(lab, `${mode}.stdout`), cause.stdout); - if (typeof cause?.stderr === "string") - await writeFile(join(lab, `${mode}.stderr`), cause.stderr); -} finally { - await fixture.close(); - const lineage = await completeVerifierRun(verifier); - const receipt = { - client: "Qwen Code", - clientVersion, - mode, - passed: failure === undefined, - artifacts: lab, - runtimeRoot, - host: { - platform: process.platform, - architecture: process.arch, - node: process.versions.node, - }, - modelFixture: - "loopback deterministic OpenAI-compatible endpoint; no live model-provider claim", - skillDiscovery: - "native HOME shared personal directory; no explicit skill setting", - catalogSize: TOOL_CONTRACTS.length, - requests: fixture.requests, - probes: fixture.probes, - evidenceSeen, - selectedSkillLoaded, - artifactReadByAgent, - artifactPath, - artifactDigest, - finalSeen, - fixtureFailure: fixture.failure, - verifier: lineage, - }; - await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)); - console.log(JSON.stringify(receipt)); -} -if (failure !== undefined) throw failure; +]); diff --git a/tests/boundary/filesystem/verifierCommandOutput.test.ts b/tests/boundary/filesystem/verifierCommandOutput.test.ts new file mode 100644 index 0000000000..bc85ffcb7c --- /dev/null +++ b/tests/boundary/filesystem/verifierCommandOutput.test.ts @@ -0,0 +1,198 @@ +import { readFile } from "node:fs/promises"; +import { afterEach, expect, it } from "vitest"; + +import { saveVerifierCommandOutput } from "../../../scripts/lib/verifier-command-output.mjs"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; + +const workspaces: string[] = []; +afterEach(async () => { + for (const root of workspaces.splice(0)) await removeTestWorkspace(root); +}); + +const createOutputWorkspace = async () => { + const workspace = await createTestWorkspace("rea-verifier-command-output-"); + workspaces.push(workspace.root); + return workspace; +}; + +const rejection = (operation: Promise): Promise => + operation.then( + () => { + throw new Error("Expected command output persistence to reject"); + }, + (cause: unknown) => cause, + ); + +it("writes exact Unicode and empty output bytes and returns the original output", async () => { + const workspace = await createOutputWorkspace(); + const paths = { + stdout: workspace.path("stdout"), + stderr: workspace.path("stderr"), + }; + const output = { stdout: "observed α测试\0\n", stderr: "" }; + + expect(await saveVerifierCommandOutput(async () => output, paths)).toBe( + output, + ); + expect(await readFile(paths.stdout)).toEqual(Buffer.from(output.stdout)); + expect(await readFile(paths.stderr)).toEqual(Buffer.alloc(0)); +}); + +it("retains both failed-command streams while preserving its exact cause", async () => { + const workspace = await createOutputWorkspace(); + const paths = { + stdout: workspace.path("stdout"), + stderr: workspace.path("stderr"), + }; + const cause = Object.assign(new Error("selected command failed"), { + snapshot: { + stdout: { text: "" }, + stderr: { text: "partial diagnostics α测试\n" }, + }, + }); + + await expect( + saveVerifierCommandOutput(async () => { + throw cause; + }, paths), + ).rejects.toBe(cause); + expect(await readFile(paths.stdout)).toEqual(Buffer.alloc(0)); + expect(await readFile(paths.stderr)).toEqual( + Buffer.from(cause.snapshot.stderr.text), + ); +}); + +it("writes available partial failure output without replacing unavailable evidence", async () => { + const workspace = await createOutputWorkspace(); + const original = Buffer.from("retained caller output\0\n"); + const paths = { + stdout: await workspace.write("stdout", original), + stderr: workspace.path("stderr"), + }; + const cause = { + snapshot: { + stdout: { text: null }, + stderr: { text: "available failure diagnostics\n" }, + }, + }; + + await expect( + saveVerifierCommandOutput(async () => { + throw cause; + }, paths), + ).rejects.toBe(cause); + expect(await readFile(paths.stdout)).toEqual(original); + expect(await readFile(paths.stderr, "utf8")).toBe(cause.snapshot.stderr.text); +}); + +it.each([undefined, null])( + "preserves the opaque %s failure and both existing output artifacts", + async (cause) => { + const workspace = await createOutputWorkspace(); + const paths = { + stdout: await workspace.write("stdout", "retained stdout\n"), + stderr: await workspace.write("stderr", "retained stderr\n"), + }; + + await expect( + saveVerifierCommandOutput(async () => { + throw cause; + }, paths), + ).rejects.toBe(cause); + expect(await readFile(paths.stdout, "utf8")).toBe("retained stdout\n"); + expect(await readFile(paths.stderr, "utf8")).toBe("retained stderr\n"); + }, +); + +it.skipIf(process.platform === "win32")( + "retains the command cause and actual stdout write failure while writing stderr", + async () => { + const workspace = await createOutputWorkspace(); + const paths = { + stdout: await workspace.mkdir("stdout-directory"), + stderr: workspace.path("stderr"), + }; + const cause = Object.assign(new Error("primary command failure"), { + snapshot: { + stdout: { text: "command stdout" }, + stderr: { text: "command stderr\n" }, + }, + }); + + const failure = await rejection( + saveVerifierCommandOutput(async () => { + throw cause; + }, paths), + ); + expect(failure).toBeInstanceOf(AggregateError); + if (!(failure instanceof AggregateError)) throw failure; + expect(failure.cause).toBe(cause); + expect(failure.errors).toHaveLength(2); + expect(failure.errors[0]).toBe(cause); + expect(failure.errors[1]).toMatchObject({ + code: "EISDIR", + path: paths.stdout, + }); + expect(await readFile(paths.stderr, "utf8")).toBe( + cause.snapshot.stderr.text, + ); + }, +); + +it.skipIf(process.platform === "win32")( + "attempts stderr persistence after a successful command's stdout write fails", + async () => { + const workspace = await createOutputWorkspace(); + const paths = { + stdout: await workspace.mkdir("stdout-directory"), + stderr: workspace.path("stderr"), + }; + const output = { + stdout: "successful stdout", + stderr: "successful stderr\n", + }; + + const failure = await rejection( + saveVerifierCommandOutput(async () => output, paths), + ); + expect(failure).not.toBeInstanceOf(AggregateError); + expect(failure).toMatchObject({ + code: "EISDIR", + path: paths.stdout, + }); + expect(await readFile(paths.stderr, "utf8")).toBe(output.stderr); + }, +); + +it.skipIf(process.platform === "win32")( + "retains both independent filesystem errors when neither output can be written", + async () => { + const workspace = await createOutputWorkspace(); + const paths = { + stdout: await workspace.mkdir("stdout-directory"), + stderr: await workspace.mkdir("stderr-directory"), + }; + + const failure = await rejection( + saveVerifierCommandOutput( + async () => ({ stdout: "stdout", stderr: "stderr" }), + paths, + ), + ); + expect(failure).toBeInstanceOf(AggregateError); + if (!(failure instanceof AggregateError)) throw failure; + expect(failure.errors).toHaveLength(2); + expect(failure.errors[0]).toMatchObject({ + code: "EISDIR", + path: paths.stdout, + }); + expect(failure.errors[1]).toMatchObject({ + code: "EISDIR", + path: paths.stderr, + }); + expect(failure.cause).toBe(failure.errors[0]); + }, +); diff --git a/tests/boundary/process/nativeVerifierPreflightOwnership.test.ts b/tests/boundary/process/nativeVerifierPreflightOwnership.test.ts new file mode 100644 index 0000000000..2aaee785df --- /dev/null +++ b/tests/boundary/process/nativeVerifierPreflightOwnership.test.ts @@ -0,0 +1,270 @@ +import { randomUUID } from "node:crypto"; +import { chmod, readFile, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import { expect, it } from "vitest"; +import { z } from "zod"; + +import { cleanupOwnedProcessGroup } from "../../../src/process/ProcessOwnership.js"; +import { systemProcessOwnershipHost } from "../../../src/process/ProcessOwnershipObservation.js"; +import { liveProcesses } from "../../../src/process/ProcessOwnershipProcessTree.js"; +import { + ProviderProcessSupervisor, + spawnOwnedProviderProcess, +} from "../../../src/process/ProviderProcess.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; + +const repository = fileURLToPath(new URL("../../../", import.meta.url)); +// Select the same actual entrypoints from an owned original-source checkout for +// the regression's RED run. This changes no production imports or behavior. +const verifierSourceRoot = resolve( + process.env.REA_NATIVE_VERIFIER_SOURCE_ROOT ?? repository, +); +const retainedBytes = Buffer.from("native preflight retained evidence α\0\n"); +const versionStdout = "authored native version output\n"; +const versionStderr = "authored native version diagnostics\n"; + +const clients = [ + { client: "claude", profile: "CLAUDE_CONFIG_DIR", javascript: false }, + { client: "copilot", profile: "COPILOT_HOME", javascript: false }, + { client: "deepseek", profile: "DSH_HOME", javascript: false }, + { client: "gemini", profile: "GEMINI_CLI_HOME", javascript: true }, + { client: "grok", profile: "GROK_HOME", javascript: false }, + { client: "omp", profile: "PI_CODING_AGENT_DIR", javascript: false }, + { client: "opencode", profile: "OPENCODE_CONFIG_DIR", javascript: false }, + { client: "qwen", profile: "QWEN_HOME", javascript: true }, +] as const; + +const readySchema = z.object({ + pids: z.array(z.number().int().positive()).length(2), + runId: z.string().min(1), + descendantRunId: z.string().min(1), + argv: z.tuple([z.literal("--version")]), + cwd: z.string(), + lab: z.string(), + artifact: z.string(), + stdin: z.object({ + fifo: z.boolean(), + socket: z.boolean(), + character: z.literal(false), + file: z.literal(false), + }), + stdout: z.literal(versionStdout), + stderr: z.literal(versionStderr), +}); + +const receiptSchema = z.object({ + clientVersion: z.null(), + passed: z.literal(false), + artifacts: z.string(), + catalogSize: z.null().optional(), + requests: z.array(z.unknown()).optional(), +}); + +const descendantSource = ` +import { appendFileSync, existsSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +const release = fileURLToPath(new URL("./release-descendant", import.meta.url)); +const events = fileURLToPath(new URL("./descendant-events.jsonl", import.meta.url)); +const finish = (reason) => { + appendFileSync(events, JSON.stringify({ pid: process.pid, reason }) + "\\n"); + process.exit(0); +}; +// The ready message is an acquisition barrier, independent of process-table +// scheduling. Ignored stdio lets the version leader and verifier finish first. +process.send({ pid: process.pid, runId: process.env.REA_PROCESS_RUN_ID }); +process.disconnect(); +setInterval(() => { + if (existsSync(release)) finish("external-release"); +}, 25); +setTimeout(() => finish("bounded-fixture-deadline"), 20000); +`; + +const selectedClientSource = (profile: string) => ` +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { fstatSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +assert.deepEqual(process.argv.slice(2), ["--version"]); +const metadata = fstatSync(0); +const stdin = { + fifo: metadata.isFIFO(), socket: metadata.isSocket(), + character: metadata.isCharacterDevice(), file: metadata.isFile(), +}; +assert.ok(stdin.fifo || stdin.socket, "Native preflight stdin must retain a real pipe or socket"); +assert.equal(stdin.character, false); +assert.equal(stdin.file, false); +const lab = dirname(process.env[${JSON.stringify(profile)}]); +const artifact = join(lab, "authored-preflight-evidence.bin"); +writeFileSync(artifact, Buffer.from(${JSON.stringify(retainedBytes.toString("base64"))}, "base64")); +const child = spawn(process.execPath, [fileURLToPath(new URL("./descendant.mjs", import.meta.url))], { + stdio: ["ignore", "ignore", "ignore", "ipc"], +}); +const acknowledged = await new Promise((resolve, reject) => { + child.once("message", resolve); + child.once("error", reject); +}); +child.unref(); +const stdout = ${JSON.stringify(versionStdout)}; +const stderr = ${JSON.stringify(versionStderr)}; +writeFileSync(fileURLToPath(new URL("./ready.json", import.meta.url)), JSON.stringify({ + pids: [process.pid, acknowledged.pid], + runId: process.env.REA_PROCESS_RUN_ID, descendantRunId: acknowledged.runId, + argv: process.argv.slice(2), cwd: process.cwd(), lab, artifact, stdin, stdout, stderr, +})); +await Promise.all([ + new Promise((resolve) => process.stdout.write(stdout, resolve)), + new Promise((resolve) => process.stderr.write(stderr, resolve)), +]); +process.exit(7); +`; + +const readReady = async (path: string) => + readFile(path, "utf8").then( + (text) => readySchema.parse(JSON.parse(text)), + (cause: unknown) => { + if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") + return undefined; + throw cause; + }, + ); + +const liveAcknowledgedProcesses = async (pids: readonly number[]) => + liveProcesses(await systemProcessOwnershipHost.listProcesses()) + .filter(({ pid }) => pids.includes(pid)) + .map(({ pid }) => pid); + +const awaitReleasedDescendant = async (readyPath: string) => { + const ready = await readReady(readyPath); + if (ready === undefined) return; + const deadline = Date.now() + 7_500; + while ((await liveAcknowledgedProcesses(ready.pids)).length !== 0) { + if (Date.now() >= deadline) + throw new Error( + "Externally released native preflight fixture stayed live", + ); + await delay(25); + } +}; + +const shellQuote = (text: string) => `'${text.replaceAll("'", `'"'"'`)}'`; + +// These are authored selected clients, not installed-provider verification. +// CommandCode remains unverified: its actual disposable OS account is required. +it.skipIf(process.platform === "win32").each(clients)( + "$client actual verifier reclaims a failed preflight descendant and retains its lab", + async ({ client, profile, javascript }) => { + const workspace = await createTestWorkspace( + `rea-native-${client}-preflight-`, + ); + const readyPath = workspace.path("ready.json"); + const releasePath = workspace.path("release-descendant"); + let acquired: + | Awaited> + | undefined; + let supervisor: ProviderProcessSupervisor | undefined; + try { + const temporaryRoot = await workspace.mkdir("native-labs"); + const selectedJavaScript = await workspace.write( + "selected-client.mjs", + selectedClientSource(profile), + ); + await workspace.write("descendant.mjs", descendantSource); + const command = javascript + ? selectedJavaScript + : await workspace.write( + "selected-client", + `#!/bin/sh\nexec ${shellQuote(process.execPath)} ${shellQuote(selectedJavaScript)} "$@"\n`, + ); + if (!javascript) await chmod(command, 0o700); + acquired = await spawnOwnedProviderProcess({ + command: process.execPath, + arguments: [ + join(verifierSourceRoot, "scripts", `verify-${client}-client.mjs`), + "call", + ], + cwd: repository, + runId: `rea-native-preflight-test-${randomUUID()}`, + sweepTokenOwnedProcesses: true, + hostEnvironment: { + ...process.env, + ...workspace.environment, + TMPDIR: temporaryRoot, + TMP: temporaryRoot, + TEMP: temporaryRoot, + REA_VERIFY_RUNTIME_ROOT: repository, + [`REA_VERIFY_${client.toUpperCase()}_COMMAND`]: command, + }, + }); + const owned = acquired; + supervisor = new ProviderProcessSupervisor( + { + ...owned, + ownsProcessLifetime: true, + cleanup: + owned.cleanup ?? (() => cleanupOwnedProcessGroup(owned.ownership)), + }, + { maxDiagnosticBytes: 64 * 1024 }, + ); + expect(await supervisor.waitForOutputClose(15_000)).toBe(true); + const completed = supervisor.snapshot(); + const ready = await readReady(readyPath); + if (ready === undefined) + throw new Error( + `${client} did not reach the authored version preflight: ${completed.stderr.text}`, + ); + // Observe before stop(), external release, receipt parsing or workspace + // removal. A successful test-owned teardown cannot hide verifier leakage. + const liveAtVerifierCompletion = await liveAcknowledgedProcesses( + ready.pids, + ); + expect( + liveAtVerifierCompletion, + `${client} acknowledged processes still alive when its actual verifier completed`, + ).toEqual([]); + expect(completed.exitCode, completed.stderr.text).not.toBe(0); + expect(completed.signal, completed.stderr.text).toBeNull(); + expect(completed.stderr.text).toContain(versionStderr); + expect(ready.stdin.fifo || ready.stdin.socket).toBe(true); + expect(ready.descendantRunId).toBe(ready.runId); + expect(dirname(ready.lab)).toBe(temporaryRoot); + expect(ready.artifact).toBe( + join(ready.lab, "authored-preflight-evidence.bin"), + ); + const receipt = receiptSchema.parse( + JSON.parse(await readFile(join(ready.lab, "receipt.json"), "utf8")), + ); + expect(receipt.artifacts).toBe(ready.lab); + if (receipt.requests !== undefined) expect(receipt.requests).toEqual([]); + // Read only after the verifier's own cleanup, before removing test scratch. + expect(await readFile(ready.artifact)).toEqual(retainedBytes); + } finally { + // Self-release the authored orphan even on RED. PID observations never + // authorize signals; outer cleanup retains its canonical acquisition proof. + try { + await writeFile( + releasePath, + "release this authored preflight descendant", + ); + await awaitReleasedDescendant(readyPath); + } finally { + if (supervisor !== undefined) { + const stopped = await supervisor.stop(); + if (stopped.status === "incomplete") throw new Error(stopped.reason); + } else if (acquired !== undefined) { + const stopped = await (acquired.cleanup === undefined + ? cleanupOwnedProcessGroup(acquired.ownership) + : acquired.cleanup()); + if (!stopped.cleaned) throw new Error(stopped.reason); + } + } + await removeTestWorkspace(workspace.root); + } + }, + 30_000, +);