mirror of
https://github.com/morluto/rea
synced 2026-10-26 06:16:17 -04:00
fix(verifiers): consolidate native client command lifetimes
Migrate the remaining native verifier commands to shared ownership with explicit stdin contracts, command-before-fixture cleanup, and failure receipts. Preserve observed catalog counts and unknowns, retained artifacts, native assertions, and selected launch inputs. Remove the obsolete raw exec export and share output persistence across native verifiers. Exercise actual entrypoints with authored failed preflights and acknowledged descendants, plus filesystem failures and opaque error preservation.
This commit is contained in:
morluto
@@ -9,7 +9,9 @@ import {
|
||||
TOOL_CONTRACTS,
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
import { exec } from "./lib/verify-package-core.mjs";
|
||||
import { createVerifierCommands } from "./lib/verify-commands.mjs";
|
||||
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
||||
import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs";
|
||||
import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
|
||||
@@ -47,43 +49,13 @@ const skillDirectory = join(
|
||||
"skills",
|
||||
"reverse-engineer-anything",
|
||||
);
|
||||
await Promise.all([
|
||||
mkdir(target, { recursive: true }),
|
||||
mkdir(profile, { recursive: true }),
|
||||
mkdir(join(lab, "tmp")),
|
||||
]);
|
||||
await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 });
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
const environment = {
|
||||
PATH: process.env.PATH,
|
||||
NODE_OPTIONS: process.env.NODE_OPTIONS,
|
||||
USERPROFILE: account,
|
||||
TMPDIR: join(lab, "tmp"),
|
||||
CMD_LOCAL_ONLY: "1",
|
||||
// Native print mode checks for an account key even with a keyless BYOK model.
|
||||
// This synthetic value is never a real credential; local-only refuses hosted API calls.
|
||||
COMMAND_CODE_API_KEY: "local-fixture-only",
|
||||
COMMANDCODE_SKIP_UPDATES: "1",
|
||||
COMMANDCODE_DISABLE_CRON: "1",
|
||||
OTEL_SDK_DISABLED: "true",
|
||||
REA_PROCESS_RUN_ID: verifier.run_id,
|
||||
};
|
||||
const expectedNames = TOOL_CONTRACTS.map(
|
||||
(tool) => "mcp__rea__" + tool.name,
|
||||
).sort();
|
||||
const schemaValidator = new Ajv2020({ strict: false, validateFormats: false });
|
||||
const textValues = (value) =>
|
||||
typeof value === "string"
|
||||
? [value]
|
||||
: value !== null && typeof value === "object"
|
||||
? Object.values(value).flatMap(textValues)
|
||||
: [];
|
||||
let clientVersion;
|
||||
let clientVersion = null;
|
||||
let expectedSkill;
|
||||
let inventoryChecked = false;
|
||||
let catalogSize = null;
|
||||
let skillLoaded = false;
|
||||
let schemasChecked = 0;
|
||||
let catalogArtifact;
|
||||
@@ -94,360 +66,428 @@ let evidenceSeen = false;
|
||||
let evidenceBytes;
|
||||
let finalSeen = false;
|
||||
let stage = 0;
|
||||
let failure;
|
||||
const call = (name, args) => ({
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: `call_commandcode_${++stage}`,
|
||||
type: "function",
|
||||
function: { name, arguments: JSON.stringify(args) },
|
||||
},
|
||||
],
|
||||
});
|
||||
const validateCatalog = async (text) => {
|
||||
const entries = [
|
||||
...text.matchAll(
|
||||
/^### (mcp__rea__[A-Za-z0-9_]+)\n[\s\S]*?\nParameters:\n```json\n([\s\S]*?)\n```/gm,
|
||||
),
|
||||
];
|
||||
assert.deepEqual(
|
||||
entries.map((entry) => entry[1]).sort(),
|
||||
expectedNames,
|
||||
"Native exact-name search must return every REA input schema",
|
||||
);
|
||||
for (const entry of entries) {
|
||||
const schema = JSON.parse(entry[2]);
|
||||
assert(
|
||||
schemaValidator.validateSchema(schema),
|
||||
`${entry[1]}: invalid native schema: ${JSON.stringify(schemaValidator.errors)}`,
|
||||
);
|
||||
}
|
||||
schemasChecked = entries.length;
|
||||
catalogDigest = createHash("sha256").update(text).digest("hex");
|
||||
await writeFile(join(lab, "native-catalog.md"), text);
|
||||
};
|
||||
const fixture = await createOpenAiModelFixture({
|
||||
directory: lab,
|
||||
prefix: "commandcode",
|
||||
onRequest: async ({ body, tools }) => {
|
||||
assert(tools.some((tool) => tool.name === "search_tools"));
|
||||
const strings = textValues(body.messages);
|
||||
if (!inventoryChecked) {
|
||||
const names = [
|
||||
...new Set(
|
||||
strings.flatMap(
|
||||
(text) => text.match(/mcp__rea__[A-Za-z0-9_]+/g) ?? [],
|
||||
),
|
||||
),
|
||||
].sort();
|
||||
assert.deepEqual(
|
||||
names,
|
||||
expectedNames,
|
||||
"Default native deferred-tool prompt must advertise the complete REA inventory",
|
||||
);
|
||||
assert.equal(
|
||||
tools.filter((tool) => tool.name.startsWith("mcp__rea__")).length,
|
||||
0,
|
||||
"Keep native lazy schema delivery enabled",
|
||||
);
|
||||
assert(
|
||||
tools
|
||||
.find((tool) => tool.name === "activate_skill")
|
||||
?.parameters.properties.name.enum.includes(
|
||||
"reverse-engineer-anything",
|
||||
),
|
||||
);
|
||||
inventoryChecked = true;
|
||||
}
|
||||
if (mode === "chat") {
|
||||
assert.equal(
|
||||
body.messages.filter((message) => message.role === "tool").length,
|
||||
0,
|
||||
);
|
||||
finalSeen = true;
|
||||
return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
|
||||
}
|
||||
if (stage === 0)
|
||||
return call("activate_skill", { name: "reverse-engineer-anything" });
|
||||
assert(
|
||||
strings.some((text) => text.includes(expectedSkill)),
|
||||
"The model must receive the complete activated skill body",
|
||||
);
|
||||
skillLoaded = true;
|
||||
const last = body.messages
|
||||
.filter((message) => message.role === "tool")
|
||||
.at(-1)?.content;
|
||||
assert.equal(typeof last, "string");
|
||||
if (stage === 1)
|
||||
return call("search_tools", {
|
||||
query: "select:" + expectedNames.join(","),
|
||||
});
|
||||
if (stage === 2) {
|
||||
const spill = last.match(
|
||||
/\[full output saved to: (.+?) — read it with read_file/,
|
||||
);
|
||||
if (spill) {
|
||||
catalogArtifact = spill[1];
|
||||
const child = relative(environment.TMPDIR, catalogArtifact);
|
||||
assert(
|
||||
isAbsolute(catalogArtifact) &&
|
||||
child &&
|
||||
!child.startsWith("..") &&
|
||||
!isAbsolute(child),
|
||||
"Native spill must remain inside this run's temporary directory",
|
||||
);
|
||||
const text = await readFile(catalogArtifact, "utf8");
|
||||
await validateCatalog(text);
|
||||
catalogProjection = { schemas: schemasChecked, sha256: catalogDigest };
|
||||
const query =
|
||||
"const fs=require('node:fs'),crypto=require('node:crypto');const text=fs.readFileSync(process.argv[1],'utf8');const schemas=[...text.matchAll(/^### (mcp__rea__[A-Za-z0-9_]+)\\n/gm)].length;console.log(JSON.stringify({rea_catalog:{schemas,sha256:crypto.createHash('sha256').update(text).digest('hex')}}));";
|
||||
return call("shell_command", {
|
||||
command: process.execPath,
|
||||
args: ["-e", query, catalogArtifact],
|
||||
});
|
||||
}
|
||||
await validateCatalog(last);
|
||||
return call("mcp__rea__analyze_javascript_application", {
|
||||
input_path: target,
|
||||
format: "directory",
|
||||
});
|
||||
}
|
||||
if (catalogArtifact && !artifactProjectionSeen) {
|
||||
const line = last
|
||||
.split("\n")
|
||||
.find((item) => item.startsWith('{"rea_catalog":'));
|
||||
assert(line, "Native shell must return the selected saved-catalog facts");
|
||||
assert.deepEqual(JSON.parse(line).rea_catalog, catalogProjection);
|
||||
artifactProjectionSeen = true;
|
||||
return call("mcp__rea__analyze_javascript_application", {
|
||||
input_path: target,
|
||||
format: "directory",
|
||||
});
|
||||
}
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(JSON.parse(last));
|
||||
await assertJavaScriptApplicationInputPaths(evidence, target);
|
||||
const shapes = evidence.normalized_result.graph.nodes
|
||||
.flatMap((node) => node.observations)
|
||||
.map((observation) => observation.properties)
|
||||
.filter(
|
||||
(properties) =>
|
||||
properties.semantic_role === "export-return-shapes" &&
|
||||
properties.exported_name === "result",
|
||||
);
|
||||
assert(
|
||||
shapes.length > 0 &&
|
||||
shapes.every(
|
||||
(shape) => shape.return_shape_coverage?.projection_complete,
|
||||
),
|
||||
);
|
||||
const fields = shapes.flatMap((shape) =>
|
||||
shape.static_return_shapes.flatMap((item) => item.fields),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/clientCompatibility" &&
|
||||
field.state === "literal" &&
|
||||
field.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/count" &&
|
||||
field.state === "literal" &&
|
||||
field.value === 7,
|
||||
),
|
||||
);
|
||||
evidenceBytes = Buffer.byteLength(last);
|
||||
await writeFile(join(lab, "native-evidence.json"), last);
|
||||
evidenceSeen = true;
|
||||
finalSeen = true;
|
||||
return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
|
||||
},
|
||||
});
|
||||
await writeFile(
|
||||
join(profile, "providers.json"),
|
||||
JSON.stringify({
|
||||
provider: {
|
||||
"rea-local": {
|
||||
baseURL: fixture.baseUrl,
|
||||
apiKey: false,
|
||||
models: {
|
||||
"rea-client-fixture": { contextWindow: 1_000_000, maxOutput: 4096 },
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
const original =
|
||||
JSON.stringify(
|
||||
{
|
||||
mcpServers: { other: { command: "unrelated-server", enabled: false } },
|
||||
preserved: "keep unrelated settings",
|
||||
},
|
||||
null,
|
||||
2,
|
||||
) + "\n";
|
||||
await writeFile(configPath, original);
|
||||
const rea = (args) =>
|
||||
exec(
|
||||
process.execPath,
|
||||
[join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],
|
||||
{ env: environment, cwd: workspace, timeout: 60_000 },
|
||||
);
|
||||
try {
|
||||
clientVersion = (
|
||||
await exec(command, ["--version"], {
|
||||
env: environment,
|
||||
cwd: workspace,
|
||||
let passed = false;
|
||||
let fixture;
|
||||
const commands = await createVerifierCommands();
|
||||
await withVerifierCleanup(async () => {
|
||||
await withVerifierCleanup(async () => {
|
||||
await Promise.all([
|
||||
mkdir(target, { recursive: true }),
|
||||
mkdir(profile, { recursive: true }),
|
||||
mkdir(join(lab, "tmp")),
|
||||
]);
|
||||
await commands.exec("git", ["init", "--quiet", workspace], {
|
||||
timeout: 30_000,
|
||||
})
|
||||
).stdout.trim();
|
||||
for (const args of [
|
||||
["setup", "--client", "commandcode", "--dry-run"],
|
||||
["setup", "--client", "commandcode", "--yes"],
|
||||
["doctor", "--client", "commandcode", "--skill"],
|
||||
]) {
|
||||
const response = await rea(args);
|
||||
stdin: "pipe",
|
||||
});
|
||||
await writeFile(
|
||||
join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`),
|
||||
response.stdout,
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
const result = JSON.parse(response.stdout);
|
||||
if (args[0] === "doctor") assert.equal(result.healthy, true);
|
||||
else
|
||||
assert.equal(
|
||||
result.status,
|
||||
args.includes("--dry-run") ? "planned" : "ready",
|
||||
);
|
||||
if (args.includes("--dry-run")) {
|
||||
assert(
|
||||
result.plannedActions.some((action) => action.target === configPath),
|
||||
);
|
||||
assert(
|
||||
result.plannedActions.every(
|
||||
(action) =>
|
||||
action.target === configPath ||
|
||||
action.target === skillDirectory ||
|
||||
action.target.startsWith(skillDirectory + "/"),
|
||||
const environment = {
|
||||
PATH: process.env.PATH,
|
||||
NODE_OPTIONS: process.env.NODE_OPTIONS,
|
||||
USERPROFILE: account,
|
||||
TMPDIR: join(lab, "tmp"),
|
||||
CMD_LOCAL_ONLY: "1",
|
||||
// Native print mode checks for an account key even with a keyless BYOK model.
|
||||
// This synthetic value is never a real credential; local-only refuses hosted API calls.
|
||||
COMMAND_CODE_API_KEY: "local-fixture-only",
|
||||
COMMANDCODE_SKIP_UPDATES: "1",
|
||||
COMMANDCODE_DISABLE_CRON: "1",
|
||||
OTEL_SDK_DISABLED: "true",
|
||||
REA_PROCESS_RUN_ID: verifier.run_id,
|
||||
};
|
||||
const schemaValidator = new Ajv2020({
|
||||
strict: false,
|
||||
validateFormats: false,
|
||||
});
|
||||
const textValues = (value) =>
|
||||
typeof value === "string"
|
||||
? [value]
|
||||
: value !== null && typeof value === "object"
|
||||
? Object.values(value).flatMap(textValues)
|
||||
: [];
|
||||
const call = (name, args) => ({
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: `call_commandcode_${++stage}`,
|
||||
type: "function",
|
||||
function: { name, arguments: JSON.stringify(args) },
|
||||
},
|
||||
],
|
||||
});
|
||||
const validateCatalog = async (text) => {
|
||||
const entries = [
|
||||
...text.matchAll(
|
||||
/^### (mcp__rea__[A-Za-z0-9_]+)\n[\s\S]*?\nParameters:\n```json\n([\s\S]*?)\n```/gm,
|
||||
),
|
||||
"Refuse setup writes outside the disposable account's configuration and skill",
|
||||
];
|
||||
assert.deepEqual(
|
||||
entries.map((entry) => entry[1]).sort(),
|
||||
expectedNames,
|
||||
"Native exact-name search must return every REA input schema",
|
||||
);
|
||||
}
|
||||
}
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
const configured = await readFile(configPath, "utf8");
|
||||
assert.equal(JSON.parse(configured).preserved, "keep unrelated settings");
|
||||
assert.equal(
|
||||
JSON.parse(configured).mcpServers.other.command,
|
||||
"unrelated-server",
|
||||
);
|
||||
const repeated = JSON.parse(
|
||||
(await rea(["setup", "--client", "commandcode", "--yes"])).stdout,
|
||||
);
|
||||
assert.deepEqual(repeated.appliedActions, []);
|
||||
assert.equal(await readFile(configPath, "utf8"), configured);
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
expectedSkill = (await readFile(join(skillDirectory, "SKILL.md"), "utf8"))
|
||||
.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/, "")
|
||||
.trim();
|
||||
const execution = exec(
|
||||
command,
|
||||
[
|
||||
"-p",
|
||||
mode === "chat"
|
||||
? "Say hello briefly."
|
||||
: `Use the REA skill to analyze ${target}.`,
|
||||
"-m",
|
||||
"rea-local/rea-client-fixture",
|
||||
"--local-only",
|
||||
"--skip-onboarding",
|
||||
"--trust",
|
||||
"--output-format",
|
||||
"json",
|
||||
"--max-turns",
|
||||
"8",
|
||||
"--yolo",
|
||||
],
|
||||
{
|
||||
env: environment,
|
||||
cwd: workspace,
|
||||
timeout: 120_000,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
},
|
||||
);
|
||||
execution.child.stdin.end();
|
||||
const response = await execution;
|
||||
await writeFile(join(lab, `${mode}.stdout`), response.stdout);
|
||||
await writeFile(join(lab, `${mode}.stderr`), response.stderr);
|
||||
assert.equal(fixture.failure, undefined);
|
||||
const events = response.stdout
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
assert(
|
||||
events.some(
|
||||
(event) =>
|
||||
event.type === "result" &&
|
||||
event.subtype === "success" &&
|
||||
event.finalText === "REA_CLIENT_COMPATIBILITY_OK",
|
||||
),
|
||||
);
|
||||
assert(inventoryChecked && finalSeen);
|
||||
if (mode === "call")
|
||||
assert(
|
||||
skillLoaded &&
|
||||
schemasChecked === expectedNames.length &&
|
||||
evidenceSeen &&
|
||||
(!catalogArtifact || artifactProjectionSeen),
|
||||
for (const entry of entries) {
|
||||
const schema = JSON.parse(entry[2]);
|
||||
assert(
|
||||
schemaValidator.validateSchema(schema),
|
||||
`${entry[1]}: invalid native schema: ${JSON.stringify(schemaValidator.errors)}`,
|
||||
);
|
||||
}
|
||||
schemasChecked = entries.length;
|
||||
catalogDigest = createHash("sha256").update(text).digest("hex");
|
||||
await writeFile(join(lab, "native-catalog.md"), text);
|
||||
};
|
||||
fixture = await createOpenAiModelFixture({
|
||||
directory: lab,
|
||||
prefix: "commandcode",
|
||||
onRequest: async ({ body, tools }) => {
|
||||
assert(tools.some((tool) => tool.name === "search_tools"));
|
||||
const strings = textValues(body.messages);
|
||||
if (!inventoryChecked) {
|
||||
const names = [
|
||||
...new Set(
|
||||
strings.flatMap(
|
||||
(text) => text.match(/mcp__rea__[A-Za-z0-9_]+/g) ?? [],
|
||||
),
|
||||
),
|
||||
].sort();
|
||||
catalogSize = names.length;
|
||||
assert.deepEqual(
|
||||
names,
|
||||
expectedNames,
|
||||
"Default native deferred-tool prompt must advertise the complete REA inventory",
|
||||
);
|
||||
assert.equal(
|
||||
tools.filter((tool) => tool.name.startsWith("mcp__rea__")).length,
|
||||
0,
|
||||
"Keep native lazy schema delivery enabled",
|
||||
);
|
||||
assert(
|
||||
tools
|
||||
.find((tool) => tool.name === "activate_skill")
|
||||
?.parameters.properties.name.enum.includes(
|
||||
"reverse-engineer-anything",
|
||||
),
|
||||
);
|
||||
inventoryChecked = true;
|
||||
}
|
||||
if (mode === "chat") {
|
||||
assert.equal(
|
||||
body.messages.filter((message) => message.role === "tool").length,
|
||||
0,
|
||||
);
|
||||
finalSeen = true;
|
||||
return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
|
||||
}
|
||||
if (stage === 0)
|
||||
return call("activate_skill", { name: "reverse-engineer-anything" });
|
||||
assert(
|
||||
strings.some((text) => text.includes(expectedSkill)),
|
||||
"The model must receive the complete activated skill body",
|
||||
);
|
||||
skillLoaded = true;
|
||||
const last = body.messages
|
||||
.filter((message) => message.role === "tool")
|
||||
.at(-1)?.content;
|
||||
assert.equal(typeof last, "string");
|
||||
if (stage === 1)
|
||||
return call("search_tools", {
|
||||
query: "select:" + expectedNames.join(","),
|
||||
});
|
||||
if (stage === 2) {
|
||||
const spill = last.match(
|
||||
/\[full output saved to: (.+?) — read it with read_file/,
|
||||
);
|
||||
if (spill) {
|
||||
catalogArtifact = spill[1];
|
||||
const child = relative(environment.TMPDIR, catalogArtifact);
|
||||
assert(
|
||||
isAbsolute(catalogArtifact) &&
|
||||
child &&
|
||||
!child.startsWith("..") &&
|
||||
!isAbsolute(child),
|
||||
"Native spill must remain inside this run's temporary directory",
|
||||
);
|
||||
const text = await readFile(catalogArtifact, "utf8");
|
||||
await validateCatalog(text);
|
||||
catalogProjection = {
|
||||
schemas: schemasChecked,
|
||||
sha256: catalogDigest,
|
||||
};
|
||||
const query =
|
||||
"const fs=require('node:fs'),crypto=require('node:crypto');const text=fs.readFileSync(process.argv[1],'utf8');const schemas=[...text.matchAll(/^### (mcp__rea__[A-Za-z0-9_]+)\\n/gm)].length;console.log(JSON.stringify({rea_catalog:{schemas,sha256:crypto.createHash('sha256').update(text).digest('hex')}}));";
|
||||
return call("shell_command", {
|
||||
command: process.execPath,
|
||||
args: ["-e", query, catalogArtifact],
|
||||
});
|
||||
}
|
||||
await validateCatalog(last);
|
||||
return call("mcp__rea__analyze_javascript_application", {
|
||||
input_path: target,
|
||||
format: "directory",
|
||||
});
|
||||
}
|
||||
if (catalogArtifact && !artifactProjectionSeen) {
|
||||
const line = last
|
||||
.split("\n")
|
||||
.find((item) => item.startsWith('{"rea_catalog":'));
|
||||
assert(
|
||||
line,
|
||||
"Native shell must return the selected saved-catalog facts",
|
||||
);
|
||||
assert.deepEqual(JSON.parse(line).rea_catalog, catalogProjection);
|
||||
artifactProjectionSeen = true;
|
||||
return call("mcp__rea__analyze_javascript_application", {
|
||||
input_path: target,
|
||||
format: "directory",
|
||||
});
|
||||
}
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(JSON.parse(last));
|
||||
await assertJavaScriptApplicationInputPaths(evidence, target);
|
||||
const shapes = evidence.normalized_result.graph.nodes
|
||||
.flatMap((node) => node.observations)
|
||||
.map((observation) => observation.properties)
|
||||
.filter(
|
||||
(properties) =>
|
||||
properties.semantic_role === "export-return-shapes" &&
|
||||
properties.exported_name === "result",
|
||||
);
|
||||
assert(
|
||||
shapes.length > 0 &&
|
||||
shapes.every(
|
||||
(shape) => shape.return_shape_coverage?.projection_complete,
|
||||
),
|
||||
);
|
||||
const fields = shapes.flatMap((shape) =>
|
||||
shape.static_return_shapes.flatMap((item) => item.fields),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/clientCompatibility" &&
|
||||
field.state === "literal" &&
|
||||
field.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/count" &&
|
||||
field.state === "literal" &&
|
||||
field.value === 7,
|
||||
),
|
||||
);
|
||||
evidenceBytes = Buffer.byteLength(last);
|
||||
await writeFile(join(lab, "native-evidence.json"), last);
|
||||
evidenceSeen = true;
|
||||
finalSeen = true;
|
||||
return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
|
||||
},
|
||||
});
|
||||
await writeFile(
|
||||
join(profile, "providers.json"),
|
||||
JSON.stringify({
|
||||
provider: {
|
||||
"rea-local": {
|
||||
baseURL: fixture.baseUrl,
|
||||
apiKey: false,
|
||||
models: {
|
||||
"rea-client-fixture": {
|
||||
contextWindow: 1_000_000,
|
||||
maxOutput: 4096,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
} catch (cause) {
|
||||
failure = cause;
|
||||
if (typeof cause?.stdout === "string")
|
||||
await writeFile(join(lab, `${mode}.stdout`), cause.stdout);
|
||||
if (typeof cause?.stderr === "string")
|
||||
await writeFile(join(lab, `${mode}.stderr`), cause.stderr);
|
||||
} finally {
|
||||
await fixture.close();
|
||||
const receipt = {
|
||||
client: "Command Code",
|
||||
clientVersion,
|
||||
mode,
|
||||
passed: failure === undefined,
|
||||
artifacts: lab,
|
||||
runtimeRoot,
|
||||
host: {
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
node: process.versions.node,
|
||||
const original =
|
||||
JSON.stringify(
|
||||
{
|
||||
mcpServers: {
|
||||
other: { command: "unrelated-server", enabled: false },
|
||||
},
|
||||
preserved: "keep unrelated settings",
|
||||
},
|
||||
null,
|
||||
2,
|
||||
) + "\n";
|
||||
await writeFile(configPath, original);
|
||||
const rea = (args) =>
|
||||
commands.exec(
|
||||
process.execPath,
|
||||
[join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],
|
||||
{ env: environment, cwd: workspace, timeout: 60_000, stdin: "pipe" },
|
||||
);
|
||||
clientVersion = (
|
||||
await commands.exec(command, ["--version"], {
|
||||
env: environment,
|
||||
cwd: workspace,
|
||||
timeout: 30_000,
|
||||
stdin: "pipe",
|
||||
})
|
||||
).stdout.trim();
|
||||
for (const args of [
|
||||
["setup", "--client", "commandcode", "--dry-run"],
|
||||
["setup", "--client", "commandcode", "--yes"],
|
||||
["doctor", "--client", "commandcode", "--skill"],
|
||||
]) {
|
||||
const response = await rea(args);
|
||||
await writeFile(
|
||||
join(
|
||||
lab,
|
||||
`${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`,
|
||||
),
|
||||
response.stdout,
|
||||
);
|
||||
const result = JSON.parse(response.stdout);
|
||||
if (args[0] === "doctor") assert.equal(result.healthy, true);
|
||||
else
|
||||
assert.equal(
|
||||
result.status,
|
||||
args.includes("--dry-run") ? "planned" : "ready",
|
||||
);
|
||||
if (args.includes("--dry-run")) {
|
||||
assert(
|
||||
result.plannedActions.some((action) => action.target === configPath),
|
||||
);
|
||||
assert(
|
||||
result.plannedActions.every(
|
||||
(action) =>
|
||||
action.target === configPath ||
|
||||
action.target === skillDirectory ||
|
||||
action.target.startsWith(skillDirectory + "/"),
|
||||
),
|
||||
"Refuse setup writes outside the disposable account's configuration and skill",
|
||||
);
|
||||
}
|
||||
}
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
const configured = await readFile(configPath, "utf8");
|
||||
assert.equal(JSON.parse(configured).preserved, "keep unrelated settings");
|
||||
assert.equal(
|
||||
JSON.parse(configured).mcpServers.other.command,
|
||||
"unrelated-server",
|
||||
);
|
||||
const repeated = JSON.parse(
|
||||
(await rea(["setup", "--client", "commandcode", "--yes"])).stdout,
|
||||
);
|
||||
assert.deepEqual(repeated.appliedActions, []);
|
||||
assert.equal(await readFile(configPath, "utf8"), configured);
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
expectedSkill = (await readFile(join(skillDirectory, "SKILL.md"), "utf8"))
|
||||
.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/, "")
|
||||
.trim();
|
||||
const response = await saveVerifierCommandOutput(
|
||||
() =>
|
||||
commands.exec(
|
||||
command,
|
||||
[
|
||||
"-p",
|
||||
mode === "chat"
|
||||
? "Say hello briefly."
|
||||
: `Use the REA skill to analyze ${target}.`,
|
||||
"-m",
|
||||
"rea-local/rea-client-fixture",
|
||||
"--local-only",
|
||||
"--skip-onboarding",
|
||||
"--trust",
|
||||
"--output-format",
|
||||
"json",
|
||||
"--max-turns",
|
||||
"8",
|
||||
"--yolo",
|
||||
],
|
||||
{
|
||||
env: environment,
|
||||
cwd: workspace,
|
||||
timeout: 120_000,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
stdin: "closed-pipe",
|
||||
},
|
||||
),
|
||||
{
|
||||
stdout: join(lab, `${mode}.stdout`),
|
||||
stderr: join(lab, `${mode}.stderr`),
|
||||
},
|
||||
);
|
||||
assert.equal(fixture.failure, undefined);
|
||||
const events = response.stdout
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
assert(
|
||||
events.some(
|
||||
(event) =>
|
||||
event.type === "result" &&
|
||||
event.subtype === "success" &&
|
||||
event.finalText === "REA_CLIENT_COMPATIBILITY_OK",
|
||||
),
|
||||
);
|
||||
assert(inventoryChecked && finalSeen);
|
||||
if (mode === "call")
|
||||
assert(
|
||||
skillLoaded &&
|
||||
schemasChecked === expectedNames.length &&
|
||||
evidenceSeen &&
|
||||
(!catalogArtifact || artifactProjectionSeen),
|
||||
);
|
||||
}, [
|
||||
() => commands.close(),
|
||||
async () => {
|
||||
await fixture?.close();
|
||||
},
|
||||
modelFixture:
|
||||
"loopback OpenAI completions/SSE, keyless BYOK, synthetic account-key gate and usage; no hosted authentication or live-model claim",
|
||||
discovery:
|
||||
"actual disposable OS account home; native user MCP configuration and default shared skill discovery; Windows unverified",
|
||||
interface:
|
||||
"default deferred schemas, complete exact-name search, native saved-catalog query when needed, actual JavaScript MCP call",
|
||||
inventoryChecked,
|
||||
catalogSize: inventoryChecked ? expectedNames.length : 0,
|
||||
schemasChecked,
|
||||
skillLoaded,
|
||||
catalogArtifact,
|
||||
catalogDigest,
|
||||
artifactProjectionSeen,
|
||||
evidenceSeen,
|
||||
evidenceBytes,
|
||||
finalSeen,
|
||||
requests: fixture.requests,
|
||||
probes: fixture.probes,
|
||||
fixtureFailure: fixture.failure,
|
||||
verifier: await completeVerifierRun(verifier),
|
||||
};
|
||||
await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
|
||||
console.log(JSON.stringify(receipt));
|
||||
}
|
||||
if (failure !== undefined) throw failure;
|
||||
]);
|
||||
passed = true;
|
||||
}, [
|
||||
async () => {
|
||||
const receipt = {
|
||||
client: "Command Code",
|
||||
clientVersion,
|
||||
mode,
|
||||
passed,
|
||||
artifacts: lab,
|
||||
runtimeRoot,
|
||||
host: {
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
node: process.versions.node,
|
||||
},
|
||||
modelFixture:
|
||||
"loopback OpenAI completions/SSE, keyless BYOK, synthetic account-key gate and usage; no hosted authentication or live-model claim",
|
||||
discovery:
|
||||
"actual disposable OS account home; native user MCP configuration and default shared skill discovery; Windows unverified",
|
||||
interface:
|
||||
"default deferred schemas, complete exact-name search, native saved-catalog query when needed, actual JavaScript MCP call",
|
||||
inventoryChecked,
|
||||
catalogSize,
|
||||
schemasChecked,
|
||||
skillLoaded,
|
||||
catalogArtifact,
|
||||
catalogDigest,
|
||||
artifactProjectionSeen,
|
||||
evidenceSeen,
|
||||
evidenceBytes,
|
||||
finalSeen,
|
||||
requests: fixture?.requests,
|
||||
probes: fixture?.probes,
|
||||
fixtureFailure: fixture?.failure,
|
||||
verifier: await completeVerifierRun(verifier),
|
||||
};
|
||||
await withVerifierCleanup(
|
||||
() =>
|
||||
writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)),
|
||||
[
|
||||
() => {
|
||||
console.log(JSON.stringify(receipt));
|
||||
},
|
||||
],
|
||||
);
|
||||
},
|
||||
]);
|
||||
|
||||
Reference in New Issue
Block a user