mirror/rea
1
0
Fork 0
mirror of https://github.com/morluto/rea synced 2026-10-25 22:08:11 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(verifiers): centralize failure and cleanup ownership

Browse Source 
Own operation results and exact rejections in withVerifierCleanup, including
undefined, null and independent cleanup failures. Remove the optional-primary
cleanup contract and migrate every caller, preserving returns, annotations,
signal restoration and dependent cleanup order.

Make Frida/JEB reporters fail on reasonless rejections and retain JEB fixture
files while provider cleanup is incomplete. Own the nested packaged WASM
verifier and expose its composite process/root cleanup capability for retry.
Update runnable docs and isolated fixture dependencies.

Validation: 547 focused cases in 20 files, eight isolated boundary unit checks,
static/build/docs/discovery gates and duplication check (3.58%). Node process,
SDK and descriptor cleanup are exercised; provider and packaging results in
orchestration tests are simulated. Pack/install subprocess ownership remains
with the existing shared package runner outside the nested-command change.
This commit is contained in:
morluto
2026-10-24 15:38:35 +08:00
parent 98ec00f406
commit 72ba1cd92b
48 changed files with 2380 additions and 1818 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/verification/javascript-analysis-effects-2026-10-15.md
+3 -12
View File
@@ -62,7 +62,7 @@ import assert from "node:assert/strict";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { toolAvailabilityInventory } from "./dist/contracts/toolOutputSchemaPrimitives.js";
import { closeVerifierResources } from "./scripts/lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./scripts/lib/verifier-cleanup.mjs";
const client = new Client({
name: "javascript-effect-verification",
@@ -73,8 +73,7 @@ const transport = new StdioClientTransport({
args: ["scripts/rea.mjs", "mcp"],
stderr: "pipe",
});
let primaryError;
try {
await withVerifierCleanup(async () => {
await client.connect(transport);
transport.stderr?.resume();
const { tools } = await client.listTools();
@@ -101,15 +100,7 @@ try {
catalog_tool_count: tools.length,
}),
);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => client.close(), () => transport.close()],
primaryError,
);
}
}, [() => client.close(), () => transport.close()]);
```
For the real workflow, follow the artifact acquisition/digest instructions in
docs/verification/javascript-retained-facts-2026-10-15.md
+1 -1
View File
@@ -305,7 +305,7 @@ returned `codex-cli 0.161.0`; historical trials retain their original metadata.
Cleanup retains the original `OwnedCommandFailure` or diagnostic-write aggregate,
its cleanupResult/resources, the unresolved cleanup owner and private workspace.
Shared `closeVerifierResources` combines a separate root-cleanup failure with
Shared `withVerifierCleanup` combines a separate root-cleanup failure with
the primary error. Snapshot diagnostics use actual retained/observed byte counts,
not nonexistent stream `truncated` fields. Signal handlers remain installed
through final cleanup; late cancellation preserves completed trial data and
experiments/analysis-efficiency/javascript-facts-agent.mjs
+229 -222
View File
@@ -5,7 +5,7 @@ import { join, isAbsolute } from "node:path";
import { pathToFileURL } from "node:url";
import { parseArgs } from "node:util";
import { performance } from "node:perf_hooks";
import { closeVerifierResources } from "../../scripts/lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs";
const { values } = parseArgs({
options: {
@@ -158,7 +158,6 @@ for (const [key, value] of Object.entries(config))
args.push(prompt);
const start = performance.now();
let cleanupOwner;
let primaryError;
let commandResult;
let report;
let cliVersion = null;
@@ -167,229 +166,237 @@ const controller = new AbortController();
const cancel = () => controller.abort();
process.on("SIGINT", cancel);
process.on("SIGTERM", cancel);
try {
const version = (commandResult = await runOwnedCommand(
{
command: "codex",
arguments: ["--version"],
cwd: workspace.path,
env: { ...process.env },
runId: `rea-facts-cli-version-${randomUUID()}`,
expectedCommand: null,
sweepTokenOwnedProcesses: true,
},
{ timeoutMs: 10_000, diagnosticBytes: 64 * 1024 },
{ signal: controller.signal },
));
const observedVersion = version.stdout.text.trim();
assert.ok(
observedVersion.length > 0,
"Codex CLI version probe returned no version",
);
cliVersion = observedVersion;
phase = "agent-trial";
await writeFile(schemaPath, JSON.stringify(schema));
const result = (commandResult = await runOwnedCommand(
{
command: "codex",
arguments: args,
cwd: workspace.path,
env: { ...process.env },
runId: `rea-facts-agent-${randomUUID()}`,
expectedCommand: null,
sweepTokenOwnedProcesses: true,
},
{
timeoutMs: 600_000,
diagnosticBytes: 16 * 1024 * 1024,
acceptNonZeroExit: true,
},
{ signal: controller.signal },
));
const events = result.stdout.text
.split("\n")
.filter(Boolean)
.map((line) => JSON.parse(line));
const completed = events.filter((event) => event.type === "item.completed");
const tools = completed
.filter(({ item }) =>
[
"mcp_tool_call",
"command_execution",
"file_change",
"web_search",
].includes(item?.type),
)
.map(({ item }) => item);
const notices = completed
.filter(
({ item }) =>
![
"reasoning",
"agent_message",
"mcp_tool_call",
"command_execution",
"file_change",
"web_search",
].includes(item?.type),
)
.map(({ item }) => item);
let answer = null;
try {
answer = JSON.parse(await readFile(answerPath, "utf8"));
} catch (cause) {
if (cause.code !== "ENOENT" && !(cause instanceof SyntaxError)) throw cause;
}
const metadata = {
task: values.task,
arm: values.arm,
requested_model: config.model,
reasoning: config.model_reasoning_effort,
harness_sha256: createHash("sha256")
.update(await readFile(new URL(import.meta.url)))
.digest("hex"),
cli_version: cliVersion,
enabled_tools: enabled,
code_mode_host: true,
prompt,
prompt_sha256: createHash("sha256").update(prompt).digest("hex"),
elapsed_ms: performance.now() - start,
exit_code: result.exitCode,
usage: events
.filter((event) => event.type === "turn.completed")
.map((event) => event.usage),
tool_calls: tools,
other_completed_items: notices,
answer,
failures: events.filter(
(event) => event.type === "error" || event.type === "turn.failed",
),
event_types: Object.fromEntries(
[...new Set(events.map((event) => event.type))].map((type) => [
type,
events.filter((event) => event.type === type).length,
]),
),
conditions:
"Actual Codex CLI, fresh process/MCP, same current backend and caller question. Normal existing auth; --ignore-user-config, no HOME/CODEX_HOME mutation or credential copy. Export/local-shell baseline and candidate share both existing tools; candidate additionally allows inspect_analysis_view. Client code-mode host remains enabled because this installed Codex requires it for all tool dispatch; the experimental code_mode and code_mode_only features remain disabled. Apps/plugins, skills discovery and other agents disabled in both. Workspace-write sandbox. Requested alias is not proof of underlying model identity. Source hashing/model billing unmeasured here; external observer supplies whole process-family costs. Regenerable exports removed with workspace.",
};
await writeFile(join(values.output, "events.jsonl"), result.stdout.text, {
mode: 0o600,
});
await writeFile(
join(values.output, "trial.json"),
JSON.stringify(metadata, null, 2) + "\n",
);
await writeFile(join(values.output, "stderr.log"), result.stderr.text, {
mode: 0o600,
});
report = {
task: values.task,
arm: values.arm,
elapsed_ms: metadata.elapsed_ms,
exit_code: result.exitCode,
usage: metadata.usage,
tool_calls: tools.length,
answered: answer !== null,
report: join(values.output, "trial.json"),
};
if (result.exitCode !== 0 || answer === null) process.exitCode = 1;
} catch (cause) {
primaryError = cause;
cleanupOwner =
cause instanceof OwnedCommandFailure ? cause.cleanupOwner : undefined;
const snapshot =
cause instanceof OwnedCommandFailure ? cause.snapshot : commandResult;
const writes = await Promise.allSettled([
writeFile(
join(values.output, "failure.json"),
JSON.stringify(
await withVerifierCleanup(async () => {
await withVerifierCleanup(async () => {
try {
const version = (commandResult = await runOwnedCommand(
{
task: values.task,
arm: values.arm,
classification: "incomplete-pilot",
phase,
cli_version: cliVersion,
requested_model: config.model,
reasoning: config.model_reasoning_effort,
message: cause.message,
reason:
cause instanceof OwnedCommandFailure
? cause.reason
: "verification",
elapsed_ms: performance.now() - start,
cleanup_owner_retained: cleanupOwner !== undefined,
retained_workspace:
cleanupOwner === undefined ? null : workspace.path,
cleanup_failure:
cause instanceof OwnedCommandFailure ? cause.cleanupFailure : null,
cleanup_result:
cause instanceof OwnedCommandFailure
? (cause.cleanupResult ?? null)
: null,
resources:
cause instanceof OwnedCommandFailure ? cause.resources : [],
collection_failure:
cause instanceof OwnedCommandFailure
? cause.collectionFailure
: null,
diagnostic_truncated: snapshot?.diagnosticTruncated ?? null,
output_incomplete: snapshot?.outputIncomplete ?? null,
stdout_truncated:
snapshot === null || snapshot === undefined
? null
: snapshot.stdout.observedBytes > snapshot.stdout.bytes,
stderr_truncated:
snapshot === null || snapshot === undefined
? null
: snapshot.stderr.observedBytes > snapshot.stderr.bytes,
command: "codex",
arguments: ["--version"],
cwd: workspace.path,
env: { ...process.env },
runId: `rea-facts-cli-version-${randomUUID()}`,
expectedCommand: null,
sweepTokenOwnedProcesses: true,
},
null,
2,
) + "\n",
),
...(snapshot
? [
writeFile(
join(values.output, "partial-events.jsonl"),
snapshot.stdout.text,
{ mode: 0o600 },
),
writeFile(join(values.output, "stderr.log"), snapshot.stderr.text, {
mode: 0o600,
}),
]
: []),
]);
const failures = writes
.filter((r) => r.status === "rejected")
.map((r) => r.reason);
if (failures.length) {
primaryError = new AggregateError(
[cause, ...failures],
"Pilot and failure recording failed",
);
}
throw primaryError;
} finally {
try {
await closeVerifierResources(
[
async () => {
if (cleanupOwner === undefined) await workspace.close();
},
],
primaryError,
);
if (primaryError === undefined && controller.signal.aborted)
throw new OwnedCommandFailure(
"cancelled",
"Agent pilot cancelled during finalization; completed trial data remains recorded.",
commandResult ?? null,
{ timeoutMs: 10_000, diagnosticBytes: 64 * 1024 },
{ signal: controller.signal },
));
const observedVersion = version.stdout.text.trim();
assert.ok(
observedVersion.length > 0,
"Codex CLI version probe returned no version",
);
} finally {
cliVersion = observedVersion;
phase = "agent-trial";
await writeFile(schemaPath, JSON.stringify(schema));
const result = (commandResult = await runOwnedCommand(
{
command: "codex",
arguments: args,
cwd: workspace.path,
env: { ...process.env },
runId: `rea-facts-agent-${randomUUID()}`,
expectedCommand: null,
sweepTokenOwnedProcesses: true,
},
{
timeoutMs: 600_000,
diagnosticBytes: 16 * 1024 * 1024,
acceptNonZeroExit: true,
},
{ signal: controller.signal },
));
const events = result.stdout.text
.split("\n")
.filter(Boolean)
.map((line) => JSON.parse(line));
const completed = events.filter(
(event) => event.type === "item.completed",
);
const tools = completed
.filter(({ item }) =>
[
"mcp_tool_call",
"command_execution",
"file_change",
"web_search",
].includes(item?.type),
)
.map(({ item }) => item);
const notices = completed
.filter(
({ item }) =>
![
"reasoning",
"agent_message",
"mcp_tool_call",
"command_execution",
"file_change",
"web_search",
].includes(item?.type),
)
.map(({ item }) => item);
let answer = null;
try {
answer = JSON.parse(await readFile(answerPath, "utf8"));
} catch (cause) {
if (cause.code !== "ENOENT" && !(cause instanceof SyntaxError))
throw cause;
}
const metadata = {
task: values.task,
arm: values.arm,
requested_model: config.model,
reasoning: config.model_reasoning_effort,
harness_sha256: createHash("sha256")
.update(await readFile(new URL(import.meta.url)))
.digest("hex"),
cli_version: cliVersion,
enabled_tools: enabled,
code_mode_host: true,
prompt,
prompt_sha256: createHash("sha256").update(prompt).digest("hex"),
elapsed_ms: performance.now() - start,
exit_code: result.exitCode,
usage: events
.filter((event) => event.type === "turn.completed")
.map((event) => event.usage),
tool_calls: tools,
other_completed_items: notices,
answer,
failures: events.filter(
(event) => event.type === "error" || event.type === "turn.failed",
),
event_types: Object.fromEntries(
[...new Set(events.map((event) => event.type))].map((type) => [
type,
events.filter((event) => event.type === type).length,
]),
),
conditions:
"Actual Codex CLI, fresh process/MCP, same current backend and caller question. Normal existing auth; --ignore-user-config, no HOME/CODEX_HOME mutation or credential copy. Export/local-shell baseline and candidate share both existing tools; candidate additionally allows inspect_analysis_view. Client code-mode host remains enabled because this installed Codex requires it for all tool dispatch; the experimental code_mode and code_mode_only features remain disabled. Apps/plugins, skills discovery and other agents disabled in both. Workspace-write sandbox. Requested alias is not proof of underlying model identity. Source hashing/model billing unmeasured here; external observer supplies whole process-family costs. Regenerable exports removed with workspace.",
};
await writeFile(join(values.output, "events.jsonl"), result.stdout.text, {
mode: 0o600,
});
await writeFile(
join(values.output, "trial.json"),
JSON.stringify(metadata, null, 2) + "\n",
);
await writeFile(join(values.output, "stderr.log"), result.stderr.text, {
mode: 0o600,
});
report = {
task: values.task,
arm: values.arm,
elapsed_ms: metadata.elapsed_ms,
exit_code: result.exitCode,
usage: metadata.usage,
tool_calls: tools.length,
answered: answer !== null,
report: join(values.output, "trial.json"),
};
if (result.exitCode !== 0 || answer === null) process.exitCode = 1;
} catch (cause) {
cleanupOwner =
cause instanceof OwnedCommandFailure ? cause.cleanupOwner : undefined;
const snapshot =
cause instanceof OwnedCommandFailure ? cause.snapshot : commandResult;
const writes = await Promise.allSettled([
writeFile(
join(values.output, "failure.json"),
JSON.stringify(
{
task: values.task,
arm: values.arm,
classification: "incomplete-pilot",
phase,
cli_version: cliVersion,
requested_model: config.model,
reasoning: config.model_reasoning_effort,
message: cause?.message,
reason:
cause instanceof OwnedCommandFailure
? cause.reason
: "verification",
elapsed_ms: performance.now() - start,
cleanup_owner_retained: cleanupOwner !== undefined,
retained_workspace:
cleanupOwner === undefined ? null : workspace.path,
cleanup_failure:
cause instanceof OwnedCommandFailure
? cause.cleanupFailure
: null,
cleanup_result:
cause instanceof OwnedCommandFailure
? (cause.cleanupResult ?? null)
: null,
resources:
cause instanceof OwnedCommandFailure ? cause.resources : [],
collection_failure:
cause instanceof OwnedCommandFailure
? cause.collectionFailure
: null,
diagnostic_truncated: snapshot?.diagnosticTruncated ?? null,
output_incomplete: snapshot?.outputIncomplete ?? null,
stdout_truncated:
snapshot === null || snapshot === undefined
? null
: snapshot.stdout.observedBytes > snapshot.stdout.bytes,
stderr_truncated:
snapshot === null || snapshot === undefined
? null
: snapshot.stderr.observedBytes > snapshot.stderr.bytes,
},
null,
2,
) + "\n",
),
...(snapshot
? [
writeFile(
join(values.output, "partial-events.jsonl"),
snapshot.stdout.text,
{ mode: 0o600 },
),
writeFile(
join(values.output, "stderr.log"),
snapshot.stderr.text,
{
mode: 0o600,
},
),
]
: []),
]);
const failures = writes
.filter((r) => r.status === "rejected")
.map((r) => r.reason);
if (failures.length)
throw new AggregateError(
[cause, ...failures],
"Pilot and failure recording failed",
);
throw cause;
}
}, [
async () => {
if (cleanupOwner === undefined) await workspace.close();
},
]);
if (controller.signal.aborted)
throw new OwnedCommandFailure(
"cancelled",
"Agent pilot cancelled during finalization; completed trial data remains recorded.",
commandResult ?? null,
);
}, [
() => {
process.off("SIGINT", cancel);
},
() => {
process.off("SIGTERM", cancel);
}
}
},
]);
console.log(JSON.stringify(report));
scripts/lib/artifact-e2e.mjs
+9 -13
View File
@@ -4,7 +4,7 @@ import { promisify } from "node:util";
import { fileURLToPath } from "node:url";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
// REA registers process-static tool contracts. Bind their parsed definitions to
// the SDK's negotiated connection identity, including when a Client reconnects.
@@ -127,8 +127,7 @@ const withOpenedArtifactMcp = async (target, openTool, verify) => {
stderr: "pipe",
});
const client = new Client({ name: "artifact-real-e2e", version: "1" });
let failure;
try {
return withVerifierCleanup(async () => {
await client.connect(transport);
const opened = await artifactMcpCall(client, {
name: openTool,
@@ -138,16 +137,13 @@ const withOpenedArtifactMcp = async (target, openTool, verify) => {
if (openTool === "open_bundle_resources")
assert.equal(opened.structuredContent?.result?.caller_path, target);
await verify(client);
} catch (cause) {
failure = cause;
throw cause;
} finally {
toolBindings.delete(client);
await closeVerifierResources(
[() => client.close(), () => transport.close()],
failure,
);
}
}, [
() => {
toolBindings.delete(client);
},
() => client.close(),
() => transport.close(),
]);
};
/** Run a real stdio MCP subprocess with an explicitly opened binary target. */
scripts/lib/browser-module-e2e.mjs
+13 -19
View File
@@ -14,7 +14,7 @@ import { webModuleTraceResultSchema } from "../../dist/domain/webModuleTrace.js"
import { NativeModuleResolver } from "../../dist/browser/modules/NativeModuleResolver.js";
import { startBrowserModuleSite } from "../fixtures/browser-module-site.mjs";
import { mcpTextValue } from "./mcp-verifier-results.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
/** Check native URL resolution against an independently loaded module graph through CLI/MCP. */
export async function verifyBrowserModules(
@@ -25,8 +25,7 @@ export async function verifyBrowserModules(
let root;
let transport;
let client;
let primaryError;
try {
return withVerifierCleanup(async () => {
site = await startBrowserModuleSite();
root = await mkdtemp(join(tmpdir(), "rea-browser-modules-"));
const env = {
@@ -304,18 +303,15 @@ export async function verifyBrowserModules(
map_identity_verified: true,
implicit_refetches: 0,
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
const cleanup = [];
if (client !== undefined) cleanup.push(() => client.close());
if (transport !== undefined) cleanup.push(() => transport.close());
if (site !== undefined) cleanup.push(() => site.close());
if (root !== undefined)
cleanup.push(() => rm(root, { recursive: true, force: true }));
await closeVerifierResources(cleanup, primaryError);
}
}, [
() => client?.close(),
() => transport?.close(),
() => site?.close(),
() =>
root === undefined
? undefined
: rm(root, { recursive: true, force: true }),
]);
}
const assertTrace = (evidence, oracle, site, scriptIndex) => {
@@ -366,7 +362,7 @@ const loadingOracle = async (executable, site) => {
: []),
],
});
try {
return withVerifierCleanup(async () => {
const page = await browser.newPage();
await page.goto(site.origin);
await page.waitForFunction(
@@ -394,7 +390,5 @@ const loadingOracle = async (executable, site) => {
),
);
return proof;
} finally {
await browser.close();
}
}, [() => browser.close()]);
};
scripts/lib/browser-nested-source-map-e2e.mjs
+126 -123
View File
@@ -11,7 +11,7 @@ import { parseEvidence } from "../../dist/domain/evidence.js";
import { traceSourceMap } from "../../dist/javascript/sourceMaps/TraceSourceMap.js";
import { startBrowserSourceMapSite } from "../fixtures/browser-source-map-site.mjs";
import { startRuntimeFixtureBrowser } from "./browser-runtime-fixture-lifecycle.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
const cases = [
{ name: "same-line nested column control", prefixes: ["\n\n ", " "] },
@@ -59,134 +59,137 @@ async function verifyCase(executable, entrypoint, scenario) {
});
let fixture;
let connection;
let primaryError;
let phase = "native-map-admission";
let browserVersion;
let expected;
let observed;
try {
const mapUrl = `${site.origin}/maps/app.js.map`;
// This independent existing decoder already composes valid nested offsets.
// It is a before-product admission control, not the expected browser oracle.
const point = traceSourceMap(site.map, mapUrl, site.position);
assert.equal(point.matches.length, 1);
assert.equal(
point.matches[0].original_position.line,
site.originalPosition.line,
);
assert.equal(
point.matches[0].original_position.column,
site.originalPosition.column,
);
expected = compilerMappingsAtPhysicalPositions(site, mapUrl);
assert.ok(
expected.length > 0,
"Native compiler leaf must carry point mappings",
);
return withVerifierCleanup(async () => {
try {
const mapUrl = `${site.origin}/maps/app.js.map`;
// This independent existing decoder already composes valid nested offsets.
// It is a before-product admission control, not the expected browser oracle.
const point = traceSourceMap(site.map, mapUrl, site.position);
assert.equal(point.matches.length, 1);
assert.equal(
point.matches[0].original_position.line,
site.originalPosition.line,
);
assert.equal(
point.matches[0].original_position.column,
site.originalPosition.column,
);
expected = compilerMappingsAtPhysicalPositions(site, mapUrl);
assert.ok(
expected.length > 0,
"Native compiler leaf must carry point mappings",
);
phase = "owned-browser-startup";
fixture = await startRuntimeFixtureBrowser(executable, site.origin);
const endpoint = `http://127.0.0.1:${fixture.port}`;
const provider = new CdpBrowserProvider();
const targets = await provider.listTargets(
listBrowserTargetsInputSchema.parse({
cdp_endpoint: endpoint,
allowed_origins: [site.origin],
}),
);
if (!targets.ok) throw targets.error;
const target = targets.value.targets[0];
assert.ok(target, "Owned fixture target missing");
const pages = await (await fetch(`${endpoint}/json/list`)).json();
const page = pages.find(({ id }) => id === target.target_id);
assert.ok(page, "Owned CDP fixture page missing");
connection = await CdpConnection.connect(
page.webSocketDebuggerUrl,
"inspect_web_page",
);
await ready(connection);
browserVersion = await connection.send("Browser.getVersion");
phase = "public-cli";
const { stdout } = await promisify(execFile)(
process.execPath,
[
entrypoint,
"analyze-web-bundle",
endpoint,
target.target_id,
"--allowed-origins",
site.origin,
"--fetch-source-maps",
"--observation-ms",
"0",
"--json",
],
{
env: { ...process.env, REA_LOG_LEVEL: "silent" },
timeout: 45_000,
maxBuffer: 16 * 1024 * 1024,
},
);
const evidence = parseEvidence(JSON.parse(stdout));
assert.equal(evidence.operation, "analyze_web_bundle");
const maps = evidence.normalized_result.observations.source_maps;
const item = maps.items.find(({ artifact }) => artifact?.text === site.map);
assert.ok(
item,
"Public browser analysis omitted the exact compiler-derived map",
);
phase = "public-map-assertions";
observed = item?.mappings;
assert.equal(item.status, "included");
assert.equal(item.mappings.length, expected.length);
for (const [index, coordinates] of expected.entries()) {
const mapping = item.mappings[index];
for (const [key, value] of Object.entries(coordinates))
assert.deepEqual(mapping[key], value);
// This compiler fixture has one source declaration in one wrapped leaf.
assert.equal(mapping.source_index, 0);
assert.equal(mapping.declaration_index, 0);
phase = "owned-browser-startup";
fixture = await startRuntimeFixtureBrowser(executable, site.origin);
const endpoint = `http://127.0.0.1:${fixture.port}`;
const provider = new CdpBrowserProvider();
const targets = await provider.listTargets(
listBrowserTargetsInputSchema.parse({
cdp_endpoint: endpoint,
allowed_origins: [site.origin],
}),
);
if (!targets.ok) throw targets.error;
const target = targets.value.targets[0];
assert.ok(target, "Owned fixture target missing");
const pages = await (await fetch(`${endpoint}/json/list`)).json();
const page = pages.find(({ id }) => id === target.target_id);
assert.ok(page, "Owned CDP fixture page missing");
connection = await CdpConnection.connect(
page.webSocketDebuggerUrl,
"inspect_web_page",
);
await ready(connection);
browserVersion = await connection.send("Browser.getVersion");
phase = "public-cli";
const { stdout } = await promisify(execFile)(
process.execPath,
[
entrypoint,
"analyze-web-bundle",
endpoint,
target.target_id,
"--allowed-origins",
site.origin,
"--fetch-source-maps",
"--observation-ms",
"0",
"--json",
],
{
env: { ...process.env, REA_LOG_LEVEL: "silent" },
timeout: 45_000,
maxBuffer: 16 * 1024 * 1024,
},
);
const evidence = parseEvidence(JSON.parse(stdout));
assert.equal(evidence.operation, "analyze_web_bundle");
const maps = evidence.normalized_result.observations.source_maps;
const item = maps.items.find(
({ artifact }) => artifact?.text === site.map,
);
assert.ok(
item,
"Public browser analysis omitted the exact compiler-derived map",
);
phase = "public-map-assertions";
observed = item?.mappings;
assert.equal(item.status, "included");
assert.equal(item.mappings.length, expected.length);
for (const [index, coordinates] of expected.entries()) {
const mapping = item.mappings[index];
for (const [key, value] of Object.entries(coordinates))
assert.deepEqual(mapping[key], value);
// This compiler fixture has one source declaration in one wrapped leaf.
assert.equal(mapping.source_index, 0);
assert.equal(mapping.declaration_index, 0);
}
assert.equal(item.original_sources.length, 1);
assert.equal(item.artifact.sha256, sha(site.map));
assert.equal(item.artifact.bytes, Buffer.byteLength(site.map));
assert.equal(
item.original_sources[0].artifact.sha256,
sha(site.original),
);
assert.ok(
site.mapRequests() > 0,
"Actual browser workflow did not fetch the declared map",
);
return {
compiler: site.compiler,
browser: browserVersion.product,
generated_sha256: sha(site.generated),
source_map_sha256: sha(site.map),
original_sha256: sha(site.original),
mappings: expected.length,
native_decoder_control: true,
};
} catch (cause) {
const failure = new Error(
`Nested source-map ${phase}: ${String(cause)}`,
{
cause,
},
);
failure.qualificationPhase = phase;
failure.qualificationReceipt = {
compiler: site.compiler,
browser: browserVersion?.product ?? null,
generated_sha256: sha(site.generated),
leaf_map_sha256: sha(site.leafMap),
source_map_sha256: sha(site.map),
original_sha256: sha(site.original),
expected_first_mapping: expected?.[0] ?? null,
observed_first_mapping: observed?.[0] ?? null,
};
throw failure;
}
assert.equal(item.original_sources.length, 1);
assert.equal(item.artifact.sha256, sha(site.map));
assert.equal(item.artifact.bytes, Buffer.byteLength(site.map));
assert.equal(item.original_sources[0].artifact.sha256, sha(site.original));
assert.ok(
site.mapRequests() > 0,
"Actual browser workflow did not fetch the declared map",
);
return {
compiler: site.compiler,
browser: browserVersion.product,
generated_sha256: sha(site.generated),
source_map_sha256: sha(site.map),
original_sha256: sha(site.original),
mappings: expected.length,
native_decoder_control: true,
};
} catch (cause) {
const failure = new Error(`Nested source-map ${phase}: ${String(cause)}`, {
cause,
});
failure.qualificationPhase = phase;
failure.qualificationReceipt = {
compiler: site.compiler,
browser: browserVersion?.product ?? null,
generated_sha256: sha(site.generated),
leaf_map_sha256: sha(site.leafMap),
source_map_sha256: sha(site.map),
original_sha256: sha(site.original),
expected_first_mapping: expected?.[0] ?? null,
observed_first_mapping: observed?.[0] ?? null,
};
primaryError = failure;
throw failure;
} finally {
await closeVerifierResources(
[() => connection?.close(), () => fixture?.close(), () => site.close()],
primaryError,
);
}
}, [() => connection?.close(), () => fixture?.close(), () => site.close()]);
}
function compilerMappingsAtPhysicalPositions(site, mapUrl) {
scripts/lib/browser-network-e2e.mjs
+3 -13
View File
@@ -14,7 +14,7 @@ import {
} from "../fixtures/browser-network-site.mjs";
import { assertBrowserNetworkEvidence } from "./browser-network-assertions.mjs";
import { scenarioProfiles } from "./browser-scenario-verifier.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
/** Exercise selected network content through real CLI and stdio MCP, one browser at a time. */
export async function verifyBrowserNetworkEvidence(
@@ -26,8 +26,7 @@ export async function verifyBrowserNetworkEvidence(
let profilesBefore;
let transport;
let client;
let primaryError;
try {
return withVerifierCleanup(async () => {
site = await startBrowserNetworkSite();
profilesBefore = await scenarioProfiles();
const scenario = browserScenarioSchema.parse({
@@ -178,14 +177,5 @@ export async function verifyBrowserNetworkEvidence(
profile_cleanup: true,
connected_network_verified: attachment !== undefined,
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
const cleanup = [];
if (client !== undefined) cleanup.push(() => client.close());
if (transport !== undefined) cleanup.push(() => transport.close());
if (site !== undefined) cleanup.push(() => site.close());
await closeVerifierResources(cleanup, primaryError);
}
}, [() => client?.close(), () => transport?.close(), () => site?.close()]);
}
scripts/lib/browser-runtime-e2e.mjs
+8 -17
View File
@@ -17,7 +17,7 @@ import { startBrowserRuntimeSite } from "../fixtures/browser-runtime-site.mjs";
import { requireOrdinaryWindowsVerifierCaller } from "./windows-ordinary-user-verifier.mjs";
import { mcpTextValue } from "./mcp-verifier-results.mjs";
import { startRuntimeFixtureBrowser } from "./browser-runtime-fixture-lifecycle.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
import { verifyRuntimeTargetTermination } from "./browser-runtime-termination-e2e.mjs";
import { verifyRuntimeInvalidSelectors } from "./browser-runtime-selector-e2e.mjs";
import {
@@ -40,10 +40,9 @@ export async function verifyBrowserRuntime(
await access(executable);
const site = await startBrowserRuntimeSite();
let fixture;
let primaryError;
let action;
let client;
try {
return withVerifierCleanup(async () => {
fixture = await startRuntimeFixtureBrowser(executable, site.origin);
const endpoint = `http://127.0.0.1:${fixture.port}`;
const targets = await new CdpBrowserProvider().listTargets(
@@ -176,20 +175,12 @@ export async function verifyBrowserRuntime(
fresh_mcp: freshMcpProof,
page_remained_open: true,
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
() => client?.close(),
() => action?.close(),
() => fixture?.close(),
() => site.close(),
],
primaryError,
);
}
}, [
() => client?.close(),
() => action?.close(),
() => fixture?.close(),
() => site.close(),
]);
}
async function executionMcp(
scripts/lib/browser-runtime-termination-e2e.mjs
+9 -18
View File
@@ -1,7 +1,7 @@
import assert from "node:assert/strict";
import { CdpConnection } from "../../dist/browser/CdpConnection.js";
import { mcpTextValue, requireMcpToolError } from "./mcp-verifier-results.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
/** Close a separately owned fixture page after actual MCP arming and inspect native cleanup diagnostics. */
export async function verifyRuntimeTargetTermination(client, input, origin) {
@@ -14,8 +14,7 @@ export async function verifyRuntimeTargetTermination(client, input, origin) {
);
let targetId;
let closed = false;
let primaryError;
try {
return withVerifierCleanup(async () => {
({ targetId } = await root.send("Target.createTarget", { url: origin }));
assert.equal(typeof targetId, "string");
await waitForTarget(input.cdp_endpoint, targetId, origin);
@@ -81,21 +80,13 @@ export async function verifyRuntimeTargetTermination(client, input, origin) {
cleanup: "unconfirmed_reported",
observed_window_end_reason: "session_detached",
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
() =>
targetId === undefined || closed
? undefined
: root.send("Target.closeTarget", { targetId }),
() => root.close(),
],
primaryError,
);
}
}, [
() =>
targetId === undefined || closed
? undefined
: root.send("Target.closeTarget", { targetId }),
() => root.close(),
]);
}
async function waitForTarget(endpoint, targetId, origin) {
scripts/lib/browser-scenario-environment-e2e.mjs
+5 -17
View File
@@ -11,7 +11,7 @@ import { browserScenarioSchema } from "../../dist/domain/browserScenario.js";
import { AnalysisCancelledError } from "../../dist/domain/analysisErrorCore.js";
import { runScenarioCli } from "./browser-scenario-verifier.mjs";
import { requireMcpEvidenceResult } from "./mcp-verifier-results.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
const expected = {
timezone: "Pacific/Honolulu",
@@ -140,8 +140,7 @@ export async function verifyScenarioEnvironment(endpoint, targetId, origin) {
return JSON.parse(value.result.value);
};
const provider = createBrowserScenarioProvider(process.env);
let verificationError;
try {
return withVerifierCleanup(async () => {
const baseline = await readEnvironment();
const callerLanguage = await readEnvironment(languageExpression);
const scenario = environmentScenario({
@@ -173,7 +172,7 @@ export async function verifyScenarioEnvironment(endpoint, targetId, origin) {
name: "browser-environment-e2e",
version: "1",
});
try {
await withVerifierCleanup(async () => {
await client.connect(transport);
const definition = (await client.listTools()).tools.find(
({ name }) => name === "capture_browser_scenario",
@@ -197,10 +196,7 @@ export async function verifyScenarioEnvironment(endpoint, targetId, origin) {
origin,
assertRestored,
});
} finally {
await client.close();
await transport.close();
}
}, [() => client.close(), () => transport.close()]);
await assertRestored();
const initializationFailure = await provider.captureScenario({
...scenario,
@@ -240,15 +236,7 @@ export async function verifyScenarioEnvironment(endpoint, targetId, origin) {
cancellation_cleanup: true,
external_target_preserved: true,
};
} catch (cause) {
verificationError = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => provider.close(), () => connection.close()],
verificationError,
);
}
}, [() => provider.close(), () => connection.close()]);
}
const verifyConcurrentMcpCapture = async ({
scripts/lib/browser-scenario-storage-e2e.mjs
+102 -114
View File
@@ -10,7 +10,7 @@ import {
runScenarioCli,
scenarioProfiles,
} from "./browser-scenario-verifier.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
const scenarioFor = (browser, origin) =>
browserScenarioSchema.parse({
@@ -203,7 +203,7 @@ export async function verifyScenarioStorage({
targetId,
}) {
const site = await startStorageSite();
try {
return withVerifierCleanup(async () => {
await verifyLaunchStorage(executable, site.origin);
await verifyConnectedStorage(endpoint, targetId, site.origin);
return {
@@ -217,9 +217,7 @@ export async function verifyScenarioStorage({
externalTargetSurvives: true,
cancellationCleanup: true,
};
} finally {
await site.close();
}
}, [() => site.close()]);
}
const verifyLaunchStorage = async (executable, origin) => {
@@ -300,8 +298,7 @@ const verifyConnectedStorage = async (endpoint, targetId, origin) => {
target.webSocketDebuggerUrl,
"capture_browser_scenario",
);
let failure;
try {
return withVerifierCleanup(async () => {
await assertStorageKeyPartition(observer, origin);
const result = await runScenarioCli(scenario);
const capture = result.normalized_result;
@@ -315,15 +312,7 @@ const verifyConnectedStorage = async (endpoint, targetId, origin) => {
assert.equal(capture.browser.cleanup, "disconnected-external");
await verifyCallerDebuggerPause(observer, origin, scenario);
await verifyCancelledStorage(observer, endpoint, targetId, scenario);
} catch (cause) {
failure = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => restoreTarget(observer, target.url), () => observer.close()],
failure,
);
}
}, [() => restoreTarget(observer, target.url), () => observer.close()]);
};
const assertStorageKeyPartition = async (connection, origin) => {
@@ -349,7 +338,7 @@ const assertStorageKeyPartition = async (connection, origin) => {
waitForDebuggerOnStart: false,
flatten: true,
});
try {
return withVerifierCleanup(async () => {
const start = await connection.send("Runtime.evaluate", {
expression: "performance.timeOrigin",
returnByValue: true,
@@ -412,14 +401,15 @@ const assertStorageKeyPartition = async (connection, origin) => {
topLevel.storageKey,
`Chrome did not expose distinct partition keys: child=${JSON.stringify(child)} top=${JSON.stringify(topLevel)} rootBefore=${JSON.stringify(beforeRoot)}`,
);
} finally {
unsubscribe();
await connection.send("Target.setAutoAttach", {
autoAttach: false,
waitForDebuggerOnStart: false,
flatten: true,
});
}
}, [
() => unsubscribe(),
() =>
connection.send("Target.setAutoAttach", {
autoAttach: false,
waitForDebuggerOnStart: false,
flatten: true,
}),
]);
};
const waitForPagePath = async (connection, path, priorTimeOrigin) => {
@@ -515,88 +505,90 @@ const verifyCallerDebuggerPause = async (caller, origin, scenario) => {
settled = true;
unsubscribe();
});
try {
const callFrameId = await raceWithTimeout(paused, 10_000, () => {
throw new Error(
`Caller debugger pause was not observed at ${debuggerUrl}; settled=${String(settled)} events=${JSON.stringify(protocolTrace)}`,
return withVerifierCleanup(async () => {
try {
const callFrameId = await raceWithTimeout(paused, 10_000, () => {
throw new Error(
`Caller debugger pause was not observed at ${debuggerUrl}; settled=${String(settled)} events=${JSON.stringify(protocolTrace)}`,
);
});
const pausedState = await caller.send("Debugger.evaluateOnCallFrame", {
callFrameId,
expression: "window.reaDebuggerAfter === true",
returnByValue: true,
});
assert.equal(
pausedState.result.value,
false,
"Caller-owned debugger frame resumed before the caller inspected it",
);
});
const pausedState = await caller.send("Debugger.evaluateOnCallFrame", {
callFrameId,
expression: "window.reaDebuggerAfter === true",
returnByValue: true,
});
assert.equal(
pausedState.result.value,
false,
"Caller-owned debugger frame resumed before the caller inspected it",
);
await delay(100);
assert.equal(
settled,
false,
"Storage observer resumed a caller-owned debugger pause",
);
await caller.send("Debugger.resume");
pauseObserved = false;
const result = await pending;
if (!result.ok) throw result.error;
const debuggerState = await caller.send("Runtime.evaluate", {
expression: "window.reaDebuggerAfter === true",
returnByValue: true,
});
assert.equal(debuggerState.result.value, true);
} catch (cause) {
if (!pauseObserved) controller.abort(cause);
else await caller.send("Debugger.resume").catch(() => undefined);
const completion = await raceWithTimeout(
pending.catch((pendingCause) => ({ ok: false, error: pendingCause })),
5_000,
);
if (completion === undefined)
throw new Error("Capture remained pending after cancellation", { cause });
const completionError =
typeof completion === "object" &&
completion !== null &&
"error" in completion
? completion.error
: typeof completion === "object" &&
completion !== null &&
"ok" in completion &&
completion.ok === false
await delay(100);
assert.equal(
settled,
false,
"Storage observer resumed a caller-owned debugger pause",
);
await caller.send("Debugger.resume");
pauseObserved = false;
const result = await pending;
if (!result.ok) throw result.error;
const debuggerState = await caller.send("Runtime.evaluate", {
expression: "window.reaDebuggerAfter === true",
returnByValue: true,
});
assert.equal(debuggerState.result.value, true);
} catch (cause) {
if (!pauseObserved) controller.abort(cause);
else await caller.send("Debugger.resume").catch(() => undefined);
const completion = await raceWithTimeout(
pending.catch((pendingCause) => ({ ok: false, error: pendingCause })),
5_000,
);
if (completion === undefined)
throw new Error("Capture remained pending after cancellation", {
cause,
});
const completionError =
typeof completion === "object" &&
completion !== null &&
"error" in completion
? completion.error
: undefined;
const completionDetail =
completionError === undefined
? "capture did not report an error"
: (() => {
const causes = [];
let current = completionError;
for (
let depth = 0;
depth < 4 && current !== undefined;
depth += 1
) {
causes.push({
name: current.name,
tag: current._tag,
reason: current.reason,
message: current.message,
});
current = current.cause;
}
return JSON.stringify(causes);
})();
throw new Error(`${String(cause)}; capture=${completionDetail}`, {
cause,
});
} finally {
unsubscribe();
await closeVerifierResources([
() => caller.send("Debugger.disable"),
() => provider.close(),
]);
}
: typeof completion === "object" &&
completion !== null &&
"ok" in completion &&
completion.ok === false
? completion.error
: undefined;
const completionDetail =
completionError === undefined
? "capture did not report an error"
: (() => {
const causes = [];
let current = completionError;
for (
let depth = 0;
depth < 4 && current !== undefined;
depth += 1
) {
causes.push({
name: current.name,
tag: current._tag,
reason: current.reason,
message: current.message,
});
current = current.cause;
}
return JSON.stringify(causes);
})();
throw new Error(`${String(cause)}; capture=${completionDetail}`, {
cause,
});
}
}, [
() => unsubscribe(),
() => caller.send("Debugger.disable"),
() => provider.close(),
]);
};
const verifyCancelledStorage = async (
@@ -626,8 +618,7 @@ const verifyCancelledStorage = async (
const pending = provider.captureScenario(cancelledScenario, {
signal: controller.signal,
});
let failure;
try {
return withVerifierCleanup(async () => {
await waitForNewDocument(
observer,
"/storage-main",
@@ -657,13 +648,10 @@ const verifyCancelledStorage = async (
targets.some(({ id }) => id === targetId),
"Cancelled connect-mode storage cleanup closed the external target",
);
} catch (cause) {
failure = cause;
throw cause;
} finally {
controller.abort(new Error("cancel failed storage verification"));
await closeVerifierResources([() => provider.close()], failure);
}
}, [
() => controller.abort(new Error("cancel failed storage verification")),
() => provider.close(),
]);
};
const waitForNewDocument = async (connection, path, priorTimeOrigin) => {
scripts/lib/browser-scenario-storage-fingerprints-e2e.mjs
+16 -23
View File
@@ -10,7 +10,7 @@ import {
runScenarioCli,
scenarioProfiles,
} from "./browser-scenario-verifier.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
const secretVariable = "REA_BROWSER_FINGERPRINT_VERIFIER_SECRET";
const secret = "rea-storage-fingerprint-private";
@@ -143,8 +143,7 @@ export async function verifyScenarioStorageFingerprints(executable, origin) {
name: "browser-storage-fingerprints-e2e",
version: "1",
});
let failure;
try {
return withVerifierCleanup(async () => {
const cli = await verifyValues(
(value) => runScenarioCli(scenarioFor(executable, origin, value)),
(before, after) =>
@@ -204,24 +203,18 @@ export async function verifyScenarioStorageFingerprints(executable, origin) {
retained_evidence: true,
versionless_captures_unknown: true,
};
} catch (cause) {
failure = cause;
throw cause;
} finally {
if (previousSecret === undefined) delete process.env[secretVariable];
else process.env[secretVariable] = previousSecret;
await closeVerifierResources(
[
() => client.close(),
() => transport.close(),
async () =>
assert.ok(
[...(await scenarioProfiles())].every((profile) =>
profilesBefore.has(profile),
),
),
],
failure,
);
}
}, [
() => {
if (previousSecret === undefined) delete process.env[secretVariable];
else process.env[secretVariable] = previousSecret;
},
() => client.close(),
() => transport.close(),
async () =>
assert.ok(
[...(await scenarioProfiles())].every((profile) =>
profilesBefore.has(profile),
),
),
]);
}
scripts/lib/browser-script-export-e2e.mjs
+11 -15
View File
@@ -15,7 +15,7 @@ import {
assertExportedScriptAnalysis,
} from "./browser-script-export-assertions.mjs";
import { mcpTextValue } from "./mcp-verifier-results.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
/** Verify capture → export → existing analysis through real CLI and stdio MCP. */
export async function verifyBrowserScriptExport(
@@ -27,8 +27,7 @@ export async function verifyBrowserScriptExport(
let root;
let transport;
let client;
let primaryError;
try {
return withVerifierCleanup(async () => {
site = await startBrowserScriptSite();
root = await mkdtemp(join(tmpdir(), "rea-browser-script-export-"));
const env = {
@@ -109,18 +108,15 @@ export async function verifyBrowserScriptExport(
query_variants_isolated: true,
implicit_refetches: 0,
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
const cleanup = [];
if (client !== undefined) cleanup.push(() => client.close());
if (transport !== undefined) cleanup.push(() => transport.close());
if (site !== undefined) cleanup.push(() => site.close());
if (root !== undefined)
cleanup.push(() => rm(root, { recursive: true, force: true }));
await closeVerifierResources(cleanup, primaryError);
}
}, [
() => client?.close(),
() => transport?.close(),
() => site?.close(),
() =>
root === undefined
? undefined
: rm(root, { recursive: true, force: true }),
]);
}
const verifyPassiveExport = async (attachment, { cli, call, save }) => {
scripts/lib/browser-source-map-e2e.mjs
+11 -15
View File
@@ -12,7 +12,7 @@ import { parseEvidence } from "../../dist/domain/evidence.js";
import { webSourceLocationResultSchema } from "../../dist/domain/webSourceLocation.js";
import { startBrowserSourceMapSite } from "../fixtures/browser-source-map-site.mjs";
import { mcpTextValue } from "./mcp-verifier-results.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
/** Verify real Chrome capture → export → original compiler point through packaged CLI/MCP. */
export async function verifyBrowserSourceMap(
@@ -23,8 +23,7 @@ export async function verifyBrowserSourceMap(
let root;
let transport;
let client;
let primaryError;
try {
return withVerifierCleanup(async () => {
site = await startBrowserSourceMapSite();
root = await mkdtemp(join(tmpdir(), "rea-browser-source-map-"));
const env = {
@@ -132,18 +131,15 @@ export async function verifyBrowserSourceMap(
selected_inert_userinfo_context_preserved: true,
public_point_cases: 4,
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
const cleanup = [];
if (client !== undefined) cleanup.push(() => client.close());
if (transport !== undefined) cleanup.push(() => transport.close());
if (site !== undefined) cleanup.push(() => site.close());
if (root !== undefined)
cleanup.push(() => rm(root, { recursive: true, force: true }));
await closeVerifierResources(cleanup, primaryError);
}
}, [
() => client?.close(),
() => transport?.close(),
() => site?.close(),
() =>
root === undefined
? undefined
: rm(root, { recursive: true, force: true }),
]);
}
function assertPoint(evidence, { site, input, capture, identity }) {
const result = webSourceLocationResultSchema.parse(
scripts/lib/browser-url-userinfo-e2e.mjs
+15 -24
View File
@@ -7,7 +7,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { CdpConnection } from "../../dist/browser/CdpConnection.js";
import { parseEvidence } from "../../dist/domain/evidence.js";
import { artifactCliEvidence } from "./artifact-e2e.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
/** Verify real V8 source labels, which retain spelling that navigation normalizes. */
export async function verifyBrowserUrlUserInfo(endpoint, origin) {
@@ -30,8 +30,7 @@ export async function verifyBrowserUrlUserInfo(endpoint, origin) {
let connection;
let transport;
let client;
let verificationError;
try {
return withVerifierCleanup(async () => {
connection = await CdpConnection.connect(
target.webSocketDebuggerUrl,
"inspect_web_page",
@@ -120,27 +119,19 @@ export async function verifyBrowserUrlUserInfo(endpoint, origin) {
original_source_digest: true,
browser_url_parser_normalizes_backslash_and_dot_segments: true,
};
} catch (cause) {
verificationError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
async () => client?.close(),
async () => transport?.close(),
async () => connection?.close(),
async () => {
const closed = await fetch(`${endpoint}/json/close/${target.id}`);
assert.equal(
closed.ok,
true,
"Browser retained the owned URL fixture target",
);
},
],
verificationError,
);
}
}, [
async () => client?.close(),
async () => transport?.close(),
async () => connection?.close(),
async () => {
const closed = await fetch(`${endpoint}/json/close/${target.id}`);
assert.equal(
closed.ok,
true,
"Browser retained the owned URL fixture target",
);
},
]);
}
async function loadFixturePage(connection, url) {
scripts/lib/verifier-cleanup.d.mts
+7 -2
View File
@@ -1,5 +1,10 @@
/** Attempt every owned cleanup and retain the primary verification failure. */
/** Attempt every independent cleanup and preserve each failure. */
export function closeVerifierResources(
tasks: readonly (() => void | Promise<void>)[],
primaryError?: unknown,
): Promise<void>;
/** Finish one operation and its cleanup without losing either failure. */
export function withVerifierCleanup<T>(
operation: () => T | Promise<T>,
cleanupTasks: readonly (() => void | Promise<void>)[],
): Promise<T>;
scripts/lib/verifier-cleanup.mjs
+28 -7
View File
@@ -1,5 +1,4 @@
/** Attempt every owned cleanup and retain the primary verification failure. */
export async function closeVerifierResources(tasks, primaryError) {
async function collectCleanupFailures(tasks) {
const failures = [];
for (const task of tasks) {
try {
@@ -9,9 +8,31 @@ export async function closeVerifierResources(tasks, primaryError) {
}
}
if (failures.length > 0)
throw new AggregateError(
primaryError === undefined ? failures : [primaryError, ...failures],
"Verifier cleanup failed",
);
return failures;
}
/** Attempt every independent cleanup and preserve each failure. */
export async function closeVerifierResources(tasks) {
const failures = await collectCleanupFailures(tasks);
if (failures.length > 0)
throw new AggregateError(failures, "Verifier cleanup failed");
}
/** Finish one operation and its cleanup without losing either failure. */
export async function withVerifierCleanup(operation, cleanupTasks) {
let value;
try {
value = await operation();
} catch (cause) {
const failures = await collectCleanupFailures(cleanupTasks);
if (failures.length > 0)
throw new AggregateError(
[cause, ...failures],
"Verifier cleanup failed",
{ cause },
);
throw cause;
}
await closeVerifierResources(cleanupTasks);
return value;
}
scripts/lib/verifier-file-resources.mjs
+5 -10
View File
@@ -1,7 +1,7 @@
import { hashStableFile } from "../../dist/filesystem/StableFileHash.js";
import { ArtifactResourceScope } from "../../dist/artifacts/ArtifactResourceScope.js";
import { AnalysisCleanupLease } from "../../dist/artifacts/AnalysisCleanupLease.js";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
/** Finish a prerequisite probe and release its exact retained cleanup owners. */
export async function withVerifierResourceOwner(operation) {
@@ -16,15 +16,10 @@ export async function withVerifierResourceOwner(operation) {
},
run: (work) => resources.run(work),
};
let primaryError;
try {
return await operation(resourceOwner);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources([() => resources.close()], primaryError);
}
return withVerifierCleanup(
() => operation(resourceOwner),
[() => resources.close()],
);
}
/** Hash one verifier input while retaining any descriptor whose close fails. */
scripts/lib/verify-wabt-cleanup.mjs
+5 -23
View File
@@ -14,14 +14,13 @@ import { execFile } from "node:child_process";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { requireMcpToolError } from "./mcp-verifier-results.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
import { WABT_OPERATION_TIMEOUT_MS } from "./wabt-verifier-timeouts.mjs";
const execute = promisify(execFile);
/** Real WABT inspection with authored cleanup denials and same-connection recovery. */
export async function verifyWabtCleanup(entrypoint, environment, parent) {
const root = await mkdtemp(join(parent, "wabt-cleanup-"));
let primaryFailure;
try {
return withVerifierCleanup(async () => {
const valid = join(root, "valid.wasm"),
invalid = join(root, "invalid.wasm");
await writeFile(
@@ -162,8 +161,7 @@ export async function verifyWabtCleanup(entrypoint, environment, parent) {
stderr: "pipe",
});
const calls = [];
let callFailure;
try {
await withVerifierCleanup(async () => {
await client.connect(transport);
const request = {
name: "inspect_wasm_artifact",
@@ -193,15 +191,7 @@ export async function verifyWabtCleanup(entrypoint, environment, parent) {
last.structuredContent.normalized_result.validation,
"valid",
);
} catch (cause) {
callFailure = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => client.close(), () => transport.close()],
callFailure,
);
}
}, [() => client.close(), () => transport.close()]);
const facts = await observations(c);
assert.deepEqual(
facts.remaining,
@@ -214,13 +204,5 @@ export async function verifyWabtCleanup(entrypoint, environment, parent) {
count + (record.kind === "cli" ? 1 : record.calls.length),
0,
);
} catch (cause) {
primaryFailure = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => rm(root, { recursive: true, force: true })],
primaryFailure,
);
}
}, [() => rm(root, { recursive: true, force: true })]);
}
scripts/verify-adb.mjs
+1 -4
View File
@@ -438,10 +438,7 @@ try {
verifierFailure = cause;
} finally {
try {
await closeVerifierResources(
[() => provider.close()],
verifierFailed ? verifierFailure : undefined,
);
await closeVerifierResources([() => provider.close()]);
} catch (cause) {
process.stderr.write(`[FAIL] verifier cleanup — ${formatFailure(cause)}\n`);
failures += 1;
scripts/verify-agent-experience.mjs
+326 -327
View File
@@ -15,7 +15,10 @@ import { tsImport } from "tsx/esm/api";
import { PRODUCT_IDENTITY } from "../dist/identity.js";
import { createAgentEvaluationCommands } from "./lib/agent-evaluation-commands.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import {
closeVerifierResources,
withVerifierCleanup,
} from "./lib/verifier-cleanup.mjs";
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
const { evaluateCodexEvents } = await tsImport(
@@ -99,341 +102,337 @@ const commands = createAgentEvaluationCommands({
});
const { runCodex, runProcess, readCommandVersion } = commands;
let primaryError;
try {
const codexVersion = await readCommandVersion(codex);
await Promise.all([
mkdir(fixtureRoot, { recursive: true }),
mkdir(account),
mkdir(codexHome, { mode: 0o700 }),
]);
// Reuse authentication only. User config, plugins, rules, history and skills
// must not make a fresh installed-skill trial pass accidentally.
try {
await copyFile(
join(process.env.CODEX_HOME ?? join(homedir(), ".codex"), "auth.json"),
authPath,
);
await chmod(authPath, 0o600);
} catch (cause) {
if (cause?.code !== "ENOENT") throw cause;
// Environment/API-key authentication can work without a local auth file.
}
await writeFile(
join(codexHome, "config.toml"),
'approval_policy = "never"\nsandbox_mode = "read-only"\nweb_search = "disabled"\n[features]\nmulti_agent = false\nshell_snapshot = false\n[history]\npersistence = "none"\n',
{ mode: 0o600 },
);
for (const approval of ["--dry-run", "--yes"]) {
await runProcess(
process.execPath,
[
join(repositoryRoot, "scripts/rea.mjs"),
"setup",
"--client",
"codex",
approval,
"--json",
],
evaluationRoot,
120_000,
evaluationEnvironment,
);
}
const installedSkill = await readFile(
join(skillDestination, "SKILL.md"),
"utf8",
);
if (!installedSkill.includes(` version: "${PRODUCT_IDENTITY.skillVersion}"`))
throw new Error("Setup did not install the current packaged REA skill");
const requestedScenarioIds = new Set(
(process.env.REA_AGENT_EVAL_SCENARIOS ?? "")
.split(",")
.map((value) => value.trim())
.filter((value) => value.length > 0),
);
const targets = await createTargets(
fixtureRoot,
requestedScenarioIds.size === 0 || requestedScenarioIds.has("managed"),
);
const allScenarios = [
nativeProviderScenario(targets.native),
{
id: "asar",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredAnswerTermGroups: [
["profileapi"],
["profile:read"],
["preload", "contextbridge"],
],
prompt: `Explain how the desktop application at ${targets.javascript} exposes APIs to its renderer. Base the answer on the shipped application artifact, and state what remains unknown.`,
},
{
id: "asar-zh",
fixtureScenarioId: "asar",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredAnswerTermGroups: [["profileapi"], ["profile:read"]],
prompt: `請靜態分析已交付的 Electron 程式 ${targets.javascript}，說明 renderer 如何透過 preload 讀取 profile，以及 main handler 的關係。不要執行目標或安裝分析引擎；引用完整的產出 Evidence，保留 runtime 未知事項。`,
},
{
id: "notes-export",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredAnswerTermGroups: [],
requiredToolSubsequence: [
"analyze_javascript_application",
"trace_application_feature",
"trace_javascript_semantics",
],
prompt: `Use REA on the shipped Electron application at ${targets.notesElectron} to investigate its notes export API, preload exposure and main handler. Do not read target files directly or launch Electron. Use the returned application Evidence to trace the exact notes:export IPC channel in both directions, then query semantic relations for the property invoke with forward influence, excluding ambiguous dynamic edges and without an expected sink. Report the selected static facts, source locations and coverage limits using the closed claim contract.`,
},
{
id: "javascript-module-view",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredToolSubsequence: [
"analyze_javascript_application",
"inspect_analysis_view",
"inspect_analysis_view",
],
requiredAnswerTermGroups: [
["main.js"],
["node", "identity"],
["observations", "recorded"],
],
prompt: `Inspect the recorded module identity, exports and observations for main.js in the shipped JavaScript tree ${targets.largeJavascript}. Use REA without directly reading target files or executing the target. Keep results focused on that module, cite Evidence, and state the limits of what its module view establishes.`,
},
MISSING_TARGET_SCENARIO,
{
id: "javascript-export-shape",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredToolSubsequence: [
"analyze_javascript_application",
"analyze_javascript_application",
"compare_javascript_export_shapes",
],
requiredAnswerTermGroups: [
["/depth", "depth"],
[
"literal value `1`",
"literal `1`",
"depth: 1",
'"depth": 1',
"depth = 1",
],
["static", "inferred"],
["runtime", "behavioral probes"],
],
prompt: `Without directly reading target files, compare the default export in parser.mjs between ${targets.javascriptShapeLeft} and ${targets.javascriptShapeRight}. Use REA to analyze each shipped artifact and then compare its exact static export return shapes. State the exact heading-shape change, cite produced Evidence, and distinguish static inference from runtime semantics.`,
},
{
id: "managed",
expectedFirstTool: "inspect_managed_artifact",
requiresEvidence: true,
requiredAnswerTermGroups: [["profile"], ["main", "entry point"]],
requiredToolSubsequence: [
"inspect_managed_artifact",
"inspect_managed_members",
],
prompt: `Explain the public managed types and entry point in ${targets.managed}. Use shipped-artifact evidence and distinguish unavailable behavior from observed metadata.`,
},
{
id: "browser",
expectedFirstTool: "list_browser_targets",
requiresEvidence: false,
requiredAnswerTermGroups: [
["unavailable", "could not", "connection", "endpoint"],
],
prompt:
"A user-owned page is already open at http://127.0.0.1:3000 through the approved local debugging endpoint http://127.0.0.1:9222. Inspect what is available without navigating or executing page code, and report any authority or availability limits.",
},
{
id: "navigation-context",
expectedFirstTool: "get_navigation_context",
requiresEvidence: false,
requiredAnswerTermGroups: [["unavailable", "could not", "session"]],
prompt:
"Report the selected document, current address, and containing procedure from the current native-analysis session in one coherent request. If no native session is available, report that limitation plainly.",
},
{
id: "address-context",
expectedFirstTool: "inspect_address_context",
requiresEvidence: false,
requiredAnswerTermGroups: [["unavailable", "could not", "session"]],
prompt:
"Inspect explicit address 0x401000 in the current native-analysis session. Report its analyzed name, containing procedure, regular comment, inline comment, and matching bookmarks. If no native session is available, report that limitation plainly.",
},
];
const scenarios =
requestedScenarioIds.size === 0
? allScenarios
: allScenarios.filter(({ id }) => requestedScenarioIds.has(id));
if (
requestedScenarioIds.size > 0 &&
scenarios.length !== requestedScenarioIds.size
)
throw new Error(
`Unknown or duplicate REA_AGENT_EVAL_SCENARIOS value: ${[...requestedScenarioIds].join(",")}`,
);
const results = [];
for (const scenario of scenarios) {
process.stderr.write(`Running Codex agent evaluation: ${scenario.id}\n`);
const fixtureId = scenario.fixtureScenarioId ?? scenario.id;
const fixtureClaims = agentFixtureClaims(fixtureId, targets);
const answerInstructions = agentFixtureAnswerInstructions(fixtureId);
const execution = await runCodex(
answerInstructions.length === 0
? scenario.prompt
: `${scenario.prompt}\n\nThis closed factual rubric needs complete producer Evidence in the transcript. Request complete detail on each initial inspection or analysis when supported; summary or view records alone are not accepted by this rubric.\n\n${answerInstructions}`,
scenario.approvedTools ?? [],
);
const transcriptDirectory = process.env.REA_AGENT_EVAL_TRANSCRIPT_DIR;
if (transcriptDirectory !== undefined) {
const transcriptPath = resolve(
transcriptDirectory,
`${scenario.id}.jsonl`,
await withVerifierCleanup(
async () => {
const codexVersion = await readCommandVersion(codex);
await Promise.all([
mkdir(fixtureRoot, { recursive: true }),
mkdir(account),
mkdir(codexHome, { mode: 0o700 }),
]);
// Reuse authentication only. User config, plugins, rules, history and skills
// must not make a fresh installed-skill trial pass accidentally.
try {
await copyFile(
join(process.env.CODEX_HOME ?? join(homedir(), ".codex"), "auth.json"),
authPath,
);
await mkdir(dirname(transcriptPath), { recursive: true });
await writeFile(
transcriptPath,
`${execution.events.map((event) => JSON.stringify(event)).join("\n")}\n`,
await chmod(authPath, 0o600);
} catch (cause) {
if (cause?.code !== "ENOENT") throw cause;
// Environment/API-key authentication can work without a local auth file.
}
await writeFile(
join(codexHome, "config.toml"),
'approval_policy = "never"\nsandbox_mode = "read-only"\nweb_search = "disabled"\n[features]\nmulti_agent = false\nshell_snapshot = false\n[history]\npersistence = "none"\n',
{ mode: 0o600 },
);
for (const approval of ["--dry-run", "--yes"]) {
await runProcess(
process.execPath,
[
join(repositoryRoot, "scripts/rea.mjs"),
"setup",
"--client",
"codex",
approval,
"--json",
],
evaluationRoot,
120_000,
evaluationEnvironment,
);
}
const metrics = evaluateCodexEvents(
execution.events,
scenario.expectedFirstTool,
const installedSkill = await readFile(
join(skillDestination, "SKILL.md"),
"utf8",
);
if (
!installedSkill.includes(` version: "${PRODUCT_IDENTITY.skillVersion}"`)
)
throw new Error("Setup did not install the current packaged REA skill");
const requestedScenarioIds = new Set(
(process.env.REA_AGENT_EVAL_SCENARIOS ?? "")
.split(",")
.map((value) => value.trim())
.filter((value) => value.length > 0),
);
const targets = await createTargets(
fixtureRoot,
requestedScenarioIds.size === 0 || requestedScenarioIds.has("managed"),
);
const allScenarios = [
nativeProviderScenario(targets.native),
{
requireEvidence: scenario.requiresEvidence,
id: "asar",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredAnswerTermGroups: [
["profileapi"],
["profile:read"],
["preload", "contextbridge"],
],
prompt: `Explain how the desktop application at ${targets.javascript} exposes APIs to its renderer. Base the answer on the shipped application artifact, and state what remains unknown.`,
},
{
id: "asar-zh",
fixtureScenarioId: "asar",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredAnswerTermGroups: [["profileapi"], ["profile:read"]],
prompt: `請靜態分析已交付的 Electron 程式 ${targets.javascript}，說明 renderer 如何透過 preload 讀取 profile，以及 main handler 的關係。不要執行目標或安裝分析引擎；引用完整的產出 Evidence，保留 runtime 未知事項。`,
},
{
id: "notes-export",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredAnswerTermGroups: [],
requiredToolSubsequence: [
"analyze_javascript_application",
"trace_application_feature",
"trace_javascript_semantics",
],
prompt: `Use REA on the shipped Electron application at ${targets.notesElectron} to investigate its notes export API, preload exposure and main handler. Do not read target files directly or launch Electron. Use the returned application Evidence to trace the exact notes:export IPC channel in both directions, then query semantic relations for the property invoke with forward influence, excluding ambiguous dynamic edges and without an expected sink. Report the selected static facts, source locations and coverage limits using the closed claim contract.`,
},
{
id: "javascript-module-view",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredToolSubsequence: [
"analyze_javascript_application",
"inspect_analysis_view",
"inspect_analysis_view",
],
requiredAnswerTermGroups: [
["main.js"],
["node", "identity"],
["observations", "recorded"],
],
prompt: `Inspect the recorded module identity, exports and observations for main.js in the shipped JavaScript tree ${targets.largeJavascript}. Use REA without directly reading target files or executing the target. Keep results focused on that module, cite Evidence, and state the limits of what its module view establishes.`,
},
MISSING_TARGET_SCENARIO,
{
id: "javascript-export-shape",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredToolSubsequence: [
"analyze_javascript_application",
"analyze_javascript_application",
"compare_javascript_export_shapes",
],
requiredAnswerTermGroups: [
["/depth", "depth"],
[
"literal value `1`",
"literal `1`",
"depth: 1",
'"depth": 1',
"depth = 1",
],
["static", "inferred"],
["runtime", "behavioral probes"],
],
prompt: `Without directly reading target files, compare the default export in parser.mjs between ${targets.javascriptShapeLeft} and ${targets.javascriptShapeRight}. Use REA to analyze each shipped artifact and then compare its exact static export return shapes. State the exact heading-shape change, cite produced Evidence, and distinguish static inference from runtime semantics.`,
},
{
id: "managed",
expectedFirstTool: "inspect_managed_artifact",
requiresEvidence: true,
requiredAnswerTermGroups: [["profile"], ["main", "entry point"]],
requiredToolSubsequence: [
"inspect_managed_artifact",
"inspect_managed_members",
],
prompt: `Explain the public managed types and entry point in ${targets.managed}. Use shipped-artifact evidence and distinguish unavailable behavior from observed metadata.`,
},
{
id: "browser",
expectedFirstTool: "list_browser_targets",
requiresEvidence: false,
requiredAnswerTermGroups: [
["unavailable", "could not", "connection", "endpoint"],
],
prompt:
"A user-owned page is already open at http://127.0.0.1:3000 through the approved local debugging endpoint http://127.0.0.1:9222. Inspect what is available without navigating or executing page code, and report any authority or availability limits.",
},
{
id: "navigation-context",
expectedFirstTool: "get_navigation_context",
requiresEvidence: false,
requiredAnswerTermGroups: [["unavailable", "could not", "session"]],
prompt:
"Report the selected document, current address, and containing procedure from the current native-analysis session in one coherent request. If no native session is available, report that limitation plainly.",
},
{
id: "address-context",
expectedFirstTool: "inspect_address_context",
requiresEvidence: false,
requiredAnswerTermGroups: [["unavailable", "could not", "session"]],
prompt:
"Inspect explicit address 0x401000 in the current native-analysis session. Report its analyzed name, containing procedure, regular comment, inline comment, and matching bookmarks. If no native session is available, report that limitation plainly.",
},
];
const scenarios =
requestedScenarioIds.size === 0
? allScenarios
: allScenarios.filter(({ id }) => requestedScenarioIds.has(id));
if (
requestedScenarioIds.size > 0 &&
scenarios.length !== requestedScenarioIds.size
)
throw new Error(
`Unknown or duplicate REA_AGENT_EVAL_SCENARIOS value: ${[...requestedScenarioIds].join(",")}`,
);
const results = [];
for (const scenario of scenarios) {
process.stderr.write(`Running Codex agent evaluation: ${scenario.id}\n`);
const fixtureId = scenario.fixtureScenarioId ?? scenario.id;
const fixtureClaims = agentFixtureClaims(fixtureId, targets);
const answerInstructions = agentFixtureAnswerInstructions(fixtureId);
const execution = await runCodex(
answerInstructions.length === 0
? scenario.prompt
: `${scenario.prompt}\n\nThis closed factual rubric needs complete producer Evidence in the transcript. Request complete detail on each initial inspection or analysis when supported; summary or view records alone are not accepted by this rubric.\n\n${answerInstructions}`,
scenario.approvedTools ?? [],
);
const transcriptDirectory = process.env.REA_AGENT_EVAL_TRANSCRIPT_DIR;
if (transcriptDirectory !== undefined) {
const transcriptPath = resolve(
transcriptDirectory,
`${scenario.id}.jsonl`,
);
await mkdir(dirname(transcriptPath), { recursive: true });
await writeFile(
transcriptPath,
`${execution.events.map((event) => JSON.stringify(event)).join("\n")}\n`,
);
}
const metrics = evaluateCodexEvents(
execution.events,
scenario.expectedFirstTool,
{
requireEvidence: scenario.requiresEvidence,
requiredAnswerTermGroups: scenario.requiredAnswerTermGroups,
requiredToolSubsequence: scenario.requiredToolSubsequence,
forbidInputValidationFailures: true,
...(fixtureClaims === undefined ? {} : { fixtureClaims }),
},
);
// Fixtures are deliberately visible beside prior investigation artifacts.
// Their existence does not identify the unnamed app in this request.
const targetClarificationPassed =
scenario.expectedFirstTool === null
? scenario.validTargetClarification(metrics.finalMessage)
: undefined;
const scenarioOutcome = scenario.assess?.(metrics);
const result = {
id: scenario.id,
expectedFirstTool: scenario.expectedFirstTool,
requiresEvidence: scenario.requiresEvidence,
requiredAnswerTermGroups: scenario.requiredAnswerTermGroups,
requiredToolSubsequence: scenario.requiredToolSubsequence,
forbidInputValidationFailures: true,
...(fixtureClaims === undefined ? {} : { fixtureClaims }),
requiredToolSubsequence: scenario.requiredToolSubsequence ?? [],
configuredClaimIds: fixtureClaims?.map(({ id }) => id) ?? [],
exitCode: execution.exitCode,
stderr: execution.stderr,
...metrics,
...(scenarioOutcome === undefined
? {}
: {
scenarioOutcome,
scenarioOutcomePassed: scenarioOutcome !== "failed",
}),
...(targetClarificationPassed === undefined
? {}
: { targetClarificationPassed }),
};
results.push(result);
process.stderr.write(
`${scenario.id}: first=${metrics.firstTool ?? "none"}, calls=${String(metrics.reaCalls.length)}, repeats=${String(metrics.repeatedCallCount)}, validation_failures=${String(metrics.inputValidationFailureCount)}, input_tokens=${String(metrics.inputTokens)}\n`,
);
}
const factualCorrectness = summarizeFactualCorrectness(results);
const summary = {
schemaVersion: 4,
evaluationScope: "routing_workflow_and_configured_fixture_claims",
factualCorrectness,
verifier_run: await completeVerifierRun(verifierRun),
codex,
codexVersion,
model: optionalModel ?? null,
installation: {
method: "rea_setup_codex",
isolatedHome: true,
isolatedCodexHome: true,
skillVersion: PRODUCT_IDENTITY.skillVersion,
authentication: "caller_auth_only_removed_after_run",
},
scenarios: results,
totals: {
scenarios: results.length,
naturalUse: results.filter(({ naturalUse }) => naturalUse).length,
correctFirstTool: results.filter(
({ correctFirstTool }) => correctFirstTool,
).length,
repeatedCallCount: results.reduce(
(total, { repeatedCallCount }) => total + repeatedCallCount,
0,
),
inputValidationFailureCount: results.reduce(
(total, { inputValidationFailureCount }) =>
total + inputValidationFailureCount,
0,
),
requiredToolSubsequence: results.filter(
({ requiredToolSubsequenceMet }) => requiredToolSubsequenceMet,
).length,
inputTokens: results.reduce(
(total, { inputTokens }) => total + inputTokens,
0,
),
cachedInputTokens: results.reduce(
(total, { cachedInputTokens }) => total + cachedInputTokens,
0,
),
answerHeuristicsMet: results.filter(({ answerHeuristicsMet }) =>
Boolean(answerHeuristicsMet),
).length,
epistemicCuePresent: results.filter(({ epistemicCuePresent }) =>
Boolean(epistemicCuePresent),
).length,
factualScenariosPassed: factualCorrectness.counts.passed,
factualScenariosFailed: factualCorrectness.counts.failed,
factualScenariosNotAssessed: factualCorrectness.counts.notAssessed,
},
);
// Fixtures are deliberately visible beside prior investigation artifacts.
// Their existence does not identify the unnamed app in this request.
const targetClarificationPassed =
scenario.expectedFirstTool === null
? scenario.validTargetClarification(metrics.finalMessage)
: undefined;
const scenarioOutcome = scenario.assess?.(metrics);
const result = {
id: scenario.id,
expectedFirstTool: scenario.expectedFirstTool,
requiresEvidence: scenario.requiresEvidence,
requiredAnswerTermGroups: scenario.requiredAnswerTermGroups,
requiredToolSubsequence: scenario.requiredToolSubsequence ?? [],
configuredClaimIds: fixtureClaims?.map(({ id }) => id) ?? [],
exitCode: execution.exitCode,
stderr: execution.stderr,
...metrics,
...(scenarioOutcome === undefined
? {}
: {
scenarioOutcome,
scenarioOutcomePassed: scenarioOutcome !== "failed",
}),
...(targetClarificationPassed === undefined
? {}
: { targetClarificationPassed }),
};
results.push(result);
process.stderr.write(
`${scenario.id}: first=${metrics.firstTool ?? "none"}, calls=${String(metrics.reaCalls.length)}, repeats=${String(metrics.repeatedCallCount)}, validation_failures=${String(metrics.inputValidationFailureCount)}, input_tokens=${String(metrics.inputTokens)}\n`,
);
}
const encoded = `${JSON.stringify(summary, null, 2)}\n`;
process.stdout.write(encoded);
const reportPath = process.env.REA_AGENT_EVAL_REPORT_PATH;
if (reportPath !== undefined) {
await mkdir(dirname(resolve(reportPath)), { recursive: true });
await writeFile(resolve(reportPath), encoded);
}
const factualCorrectness = summarizeFactualCorrectness(results);
const summary = {
schemaVersion: 4,
evaluationScope: "routing_workflow_and_configured_fixture_claims",
factualCorrectness,
verifier_run: await completeVerifierRun(verifierRun),
codex,
codexVersion,
model: optionalModel ?? null,
installation: {
method: "rea_setup_codex",
isolatedHome: true,
isolatedCodexHome: true,
skillVersion: PRODUCT_IDENTITY.skillVersion,
authentication: "caller_auth_only_removed_after_run",
},
scenarios: results,
totals: {
scenarios: results.length,
naturalUse: results.filter(({ naturalUse }) => naturalUse).length,
correctFirstTool: results.filter(
({ correctFirstTool }) => correctFirstTool,
).length,
repeatedCallCount: results.reduce(
(total, { repeatedCallCount }) => total + repeatedCallCount,
0,
),
inputValidationFailureCount: results.reduce(
(total, { inputValidationFailureCount }) =>
total + inputValidationFailureCount,
0,
),
requiredToolSubsequence: results.filter(
({ requiredToolSubsequenceMet }) => requiredToolSubsequenceMet,
).length,
inputTokens: results.reduce(
(total, { inputTokens }) => total + inputTokens,
0,
),
cachedInputTokens: results.reduce(
(total, { cachedInputTokens }) => total + cachedInputTokens,
0,
),
answerHeuristicsMet: results.filter(({ answerHeuristicsMet }) =>
Boolean(answerHeuristicsMet),
).length,
epistemicCuePresent: results.filter(({ epistemicCuePresent }) =>
Boolean(epistemicCuePresent),
).length,
factualScenariosPassed: factualCorrectness.counts.passed,
factualScenariosFailed: factualCorrectness.counts.failed,
factualScenariosNotAssessed: factualCorrectness.counts.notAssessed,
},
};
const encoded = `${JSON.stringify(summary, null, 2)}\n`;
process.stdout.write(encoded);
const reportPath = process.env.REA_AGENT_EVAL_REPORT_PATH;
if (reportPath !== undefined) {
await mkdir(dirname(resolve(reportPath)), { recursive: true });
await writeFile(resolve(reportPath), encoded);
}
const failed = results.filter((result) => !agentEvaluationPassed(result));
if (failed.length > 0)
throw new Error(
`Agent routing/workflow/answer checks failed: ${failed.map(({ id }) => id).join(", ")}`,
);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
const failed = results.filter((result) => !agentEvaluationPassed(result));
if (failed.length > 0)
throw new Error(
`Agent routing/workflow/answer checks failed: ${failed.map(({ id }) => id).join(", ")}`,
);
},
// Even a deliberately retained fixture directory must never retain auth.
await closeVerifierResources(
[
async () => {
// A failed process cleanup retains both its exact owner and its files.
await commands.close();
await closeVerifierResources([
() => rm(authPath, { force: true }),
...(process.env.REA_AGENT_EVAL_KEEP_FIXTURES === "true"
? []
: [() => rm(evaluationRoot, { recursive: true, force: true })]),
]);
},
],
primaryError,
);
}
[
async () => {
// A failed process cleanup retains both its exact owner and its files.
await commands.close();
await closeVerifierResources([
() => rm(authPath, { force: true }),
...(process.env.REA_AGENT_EVAL_KEEP_FIXTURES === "true"
? []
: [() => rm(evaluationRoot, { recursive: true, force: true })]),
]);
},
],
);
async function createTargets(root, includeManaged) {
const fixtures = await createAgentEvaluationFixtures(root, repositoryRoot);
scripts/verify-browser-dom-destinations.mjs
+10 -14
View File
@@ -6,7 +6,7 @@ import { join } from "node:path";
import { chromium } from "playwright-core";
import { startBrowserVerifierSite } from "./fixtures/browser-verifier-site.mjs";
import { verifyBrowserDomDestinations } from "./lib/browser-dom-destinations-e2e.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { requireOrdinaryWindowsVerifierCaller } from "./lib/windows-ordinary-user-verifier.mjs";
const executable = process.env.REA_BROWSER_EXECUTABLE;
@@ -17,8 +17,7 @@ let profile;
let site;
let context;
let report;
let primaryError;
try {
await withVerifierCleanup(async () => {
profile = await mkdtemp(join(tmpdir(), "rea-browser-dom-"));
site = await startBrowserVerifierSite();
context = await chromium.launchPersistentContext(profile, {
@@ -56,15 +55,12 @@ try {
native_oracle: true,
verified: true,
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
const cleanup = [];
if (context !== undefined) cleanup.push(() => context.close());
if (site !== undefined) cleanup.push(() => site.close());
if (profile !== undefined)
cleanup.push(() => rm(profile, { recursive: true, force: true }));
await closeVerifierResources(cleanup, primaryError);
}
}, [
() => context?.close(),
() => site?.close(),
async () => {
if (profile !== undefined)
await rm(profile, { recursive: true, force: true });
},
]);
process.stdout.write(`${JSON.stringify(report)}\n`);
scripts/verify-flutter.mjs
+153 -156
View File
@@ -9,7 +9,10 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { toolContract } from "../dist/contracts/toolContracts.js";
import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import {
closeVerifierResources,
withVerifierCleanup,
} from "./lib/verifier-cleanup.mjs";
import { hashVerifierFile } from "./lib/verifier-file-resources.mjs";
const options = parseArguments(process.argv.slice(2));
@@ -49,171 +52,165 @@ const transport = new StdioClientTransport({
stderr: "pipe",
});
const execute = promisify(execFile);
let primaryError;
try {
await client.connect(transport);
const apk = resolve(options.apk);
const identity = await hashVerifierFile(apk, resources);
const identified = await run("identify_flutter_build", { path: apk });
assert.deepEqual(identified.target, { path: apk, ...identity });
assert.equal(identified.flutter_detected, true);
assert.equal(identified.coverage, "complete");
const abis = identified.abis.filter((abi) => abi.libapp.present);
assert.ok(abis.length > 0);
const hashes = new Set();
for (const abi of abis) {
assert.equal(abi.libapp.elf_header_valid, true);
assert.equal(abi.libapp.snapshot_hash_sources, 2);
assert.ok(abi.libapp.snapshot_hash);
hashes.add(abi.libapp.snapshot_hash);
assert.equal(abi.libflutter.present, true);
assert.match(abi.libflutter.build_id ?? "", /^(?:[0-9a-f]{2}){4,64}$/u);
assert.deepEqual(abi.libflutter.build_id_candidates, [
abi.libflutter.build_id,
]);
}
assert.equal(hashes.size, 1, "Every ABI must agree on the snapshot hash");
if (expectations !== null) {
assert.equal(identity.sha256, expectations.target_sha256);
assert.deepEqual([...hashes], [expectations.snapshot_hash]);
assert.deepEqual(
abis.map((abi) => abi.abi).sort(),
Object.keys(expectations.engines).sort(),
);
await withVerifierCleanup(
async () => {
await client.connect(transport);
const apk = resolve(options.apk);
const identity = await hashVerifierFile(apk, resources);
const identified = await run("identify_flutter_build", { path: apk });
assert.deepEqual(identified.target, { path: apk, ...identity });
assert.equal(identified.flutter_detected, true);
assert.equal(identified.coverage, "complete");
const abis = identified.abis.filter((abi) => abi.libapp.present);
assert.ok(abis.length > 0);
const hashes = new Set();
for (const abi of abis) {
assert.equal(abi.libapp.elf_header_valid, true);
assert.equal(abi.libapp.snapshot_hash_sources, 2);
assert.ok(abi.libapp.snapshot_hash);
hashes.add(abi.libapp.snapshot_hash);
assert.equal(abi.libflutter.present, true);
assert.match(abi.libflutter.build_id ?? "", /^(?:[0-9a-f]{2}){4,64}$/u);
assert.deepEqual(abi.libflutter.build_id_candidates, [
abi.libflutter.build_id,
]);
}
assert.equal(hashes.size, 1, "Every ABI must agree on the snapshot hash");
if (expectations !== null) {
assert.equal(identity.sha256, expectations.target_sha256);
assert.deepEqual([...hashes], [expectations.snapshot_hash]);
assert.deepEqual(
{
dart_version: abi.libflutter.dart_version,
toolchain_lines: abi.libflutter.toolchain_lines,
},
expectations.engines[abi.abi],
abis.map((abi) => abi.abi).sort(),
Object.keys(expectations.engines).sort(),
);
assert.equal(abi.libflutter.dart_version_capped, false);
assert.equal(abi.libflutter.toolchain_capped, false);
}
process.stdout.write(
"[ok] Exact per-ABI engine labels match digest-bound expectations\n",
);
} else {
process.stdout.write(
"[unverified] Exact engine labels and their absence need --expectations JSON\n",
);
}
// The same checks apply to the default selection and each remaining ABI.
const preferred = await run("inspect_dart_aot", { path: apk });
for (const abi of abis) {
const aot =
abi.abi === preferred.abi
? preferred
: await run("inspect_dart_aot", { path: apk, abi: abi.abi });
assert.equal(aot.abi, abi.abi);
assert.equal(
aot.libapp.coverage,
"complete",
`Snapshot decoding must be complete on ${abi.abi}`,
);
assert.equal(aot.coverage, aot.string_pool.coverage);
if (aot.coverage !== "complete") {
for (const abi of abis) {
assert.deepEqual(
{
dart_version: abi.libflutter.dart_version,
toolchain_lines: abi.libflutter.toolchain_lines,
},
expectations.engines[abi.abi],
);
assert.equal(abi.libflutter.dart_version_capped, false);
assert.equal(abi.libflutter.toolchain_capped, false);
}
process.stdout.write(
`[unverified] ${abi.abi}: full string-pool projection (returned pool is bounded and partial)\n`,
);
}
assert.deepEqual(aot.target, identified.target);
assert.equal(aot.libapp.sha256, abi.libapp.sha256);
assert.equal(aot.libapp.bytes, abi.libapp.bytes);
assert.equal(aot.libapp.snapshot_hash_sources, 2);
assert.equal(aot.libapp.snapshot_hash, abi.libapp.snapshot_hash);
assert.deepEqual(
aot.libapp.sections.map((section) => section.name).sort(),
[
"_kDartVmSnapshotData",
"_kDartVmSnapshotInstructions",
"_kDartIsolateSnapshotData",
"_kDartIsolateSnapshotInstructions",
].sort(),
);
for (const section of aot.libapp.sections) {
assert.ok(section.offset + section.size <= aot.libapp.bytes);
if (!section.name.endsWith("Data")) continue;
assert.equal(section.magic_valid, true);
assert.equal(
section.kind,
3,
"This verifier requires an AOT app snapshot",
);
assert.ok(
section.header_length >= 48 &&
section.header_length + 4 <= section.size,
`Snapshot extent must fit ${section.name} on ${abi.abi}`,
);
}
assert.ok(aot.string_pool.package_uri_count_at_least > 0);
assert.ok(aot.string_pool.dart_uri_count_at_least > 0);
if (expectations?.string_pools !== undefined) {
const expected = expectations.string_pools[abi.abi];
assert.ok(
expected,
`Pinned string-pool counts are required on ${abi.abi}`,
);
assert.equal(
aot.string_pool.package_uri_count_at_least,
expected.package_uri_count_at_least,
);
assert.equal(
aot.string_pool.dart_uri_count_at_least,
expected.dart_uri_count_at_least,
);
process.stdout.write(
`[ok] ${abi.abi}: exact pinned package and Dart URI counts\n`,
"[ok] Exact per-ABI engine labels match digest-bound expectations\n",
);
} else {
process.stdout.write(
`[unverified] ${abi.abi}: exact string-pool counts need --expectations JSON\n`,
"[unverified] Exact engine labels and their absence need --expectations JSON\n",
);
}
process.stdout.write(
`[ok] ${abi.abi}: four symbols, bounded AOT headers, agreeing hashes and string pool\n`,
);
}
if (options.plainApk !== null) {
const plain = resolve(options.plainApk);
const negative = await run("identify_flutter_build", { path: plain });
assert.deepEqual(negative.target, {
path: plain,
...(await hashVerifierFile(plain, resources)),
});
assert.equal(negative.flutter_detected, false);
assert.equal(negative.coverage, "complete");
}
assert.deepEqual(
await hashVerifierFile(apk, resources),
identity,
"Verification must preserve its input",
);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
// Server shutdown retries retained provider cleanup and settles before success.
await closeVerifierResources(
[
async () => {
await closeVerifierResources([
() => client.close(),
() => transport.close(),
() => resources.close(),
]);
assert.deepEqual(
await readdir(root),
[],
`Provider snapshots remain under ${root}`,
// The same checks apply to the default selection and each remaining ABI.
const preferred = await run("inspect_dart_aot", { path: apk });
for (const abi of abis) {
const aot =
abi.abi === preferred.abi
? preferred
: await run("inspect_dart_aot", { path: apk, abi: abi.abi });
assert.equal(aot.abi, abi.abi);
assert.equal(
aot.libapp.coverage,
"complete",
`Snapshot decoding must be complete on ${abi.abi}`,
);
assert.equal(aot.coverage, aot.string_pool.coverage);
if (aot.coverage !== "complete") {
process.stdout.write(
`[unverified] ${abi.abi}: full string-pool projection (returned pool is bounded and partial)\n`,
);
await rm(root, { recursive: true, force: true });
},
],
primaryError,
);
}
}
assert.deepEqual(aot.target, identified.target);
assert.equal(aot.libapp.sha256, abi.libapp.sha256);
assert.equal(aot.libapp.bytes, abi.libapp.bytes);
assert.equal(aot.libapp.snapshot_hash_sources, 2);
assert.equal(aot.libapp.snapshot_hash, abi.libapp.snapshot_hash);
assert.deepEqual(
aot.libapp.sections.map((section) => section.name).sort(),
[
"_kDartVmSnapshotData",
"_kDartVmSnapshotInstructions",
"_kDartIsolateSnapshotData",
"_kDartIsolateSnapshotInstructions",
].sort(),
);
for (const section of aot.libapp.sections) {
assert.ok(section.offset + section.size <= aot.libapp.bytes);
if (!section.name.endsWith("Data")) continue;
assert.equal(section.magic_valid, true);
assert.equal(
section.kind,
3,
"This verifier requires an AOT app snapshot",
);
assert.ok(
section.header_length >= 48 &&
section.header_length + 4 <= section.size,
`Snapshot extent must fit ${section.name} on ${abi.abi}`,
);
}
assert.ok(aot.string_pool.package_uri_count_at_least > 0);
assert.ok(aot.string_pool.dart_uri_count_at_least > 0);
if (expectations?.string_pools !== undefined) {
const expected = expectations.string_pools[abi.abi];
assert.ok(
expected,
`Pinned string-pool counts are required on ${abi.abi}`,
);
assert.equal(
aot.string_pool.package_uri_count_at_least,
expected.package_uri_count_at_least,
);
assert.equal(
aot.string_pool.dart_uri_count_at_least,
expected.dart_uri_count_at_least,
);
process.stdout.write(
`[ok] ${abi.abi}: exact pinned package and Dart URI counts\n`,
);
} else {
process.stdout.write(
`[unverified] ${abi.abi}: exact string-pool counts need --expectations JSON\n`,
);
}
process.stdout.write(
`[ok] ${abi.abi}: four symbols, bounded AOT headers, agreeing hashes and string pool\n`,
);
}
if (options.plainApk !== null) {
const plain = resolve(options.plainApk);
const negative = await run("identify_flutter_build", { path: plain });
assert.deepEqual(negative.target, {
path: plain,
...(await hashVerifierFile(plain, resources)),
});
assert.equal(negative.flutter_detected, false);
assert.equal(negative.coverage, "complete");
}
assert.deepEqual(
await hashVerifierFile(apk, resources),
identity,
"Verification must preserve its input",
);
},
// Server shutdown retries retained provider cleanup and settles before success.
[
async () => {
await closeVerifierResources([
() => client.close(),
() => transport.close(),
() => resources.close(),
]);
assert.deepEqual(
await readdir(root),
[],
`Provider snapshots remain under ${root}`,
);
await rm(root, { recursive: true, force: true });
},
],
);
process.stdout.write(
"all asserted Flutter CLI/MCP checks passed; unverified facets are reported above\n",
);
scripts/verify-frida.mjs
+79 -93
View File
@@ -3,7 +3,7 @@ import { basename } from "node:path";
import { FridaInstrumentationService } from "../dist/application/frida/FridaInstrumentationService.js";
import { FridaInstrumentationManager } from "../dist/frida/FridaInstrumentationManager.js";
import { isValidFridaRemoteAddress } from "../dist/contracts/fridaRemoteAddress.js";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
const options = parseArguments(process.argv.slice(2));
if (options.help) {
@@ -31,103 +31,89 @@ const remote =
: { keepaliveInterval: options.keepaliveInterval }),
};
let primaryFailure;
let report;
try {
let targetProcess;
let input;
if (remote === undefined) {
input = {
mode: "spawn",
deviceId: "local",
program: process.execPath,
argv: ["-e", "setTimeout(() => {}, 1500)"],
source: {
sourceKind: "inline",
source: "send({ rea_frida_verification: 'ok', pid: Process.id });",
},
durationMs: 500,
};
targetProcess = { pid: undefined, name: basename(process.execPath) };
} else {
const inventory = await service.listProcesses({ remote });
if (!inventory.ok) throw new Error(inventory.error.message);
targetProcess = inventory.value.processes.find(
({ pid }) => pid === options.pid,
);
if (targetProcess === undefined)
throw new Error(
`Selected PID ${String(options.pid)} was not observed remotely`,
const report = await withVerifierCleanup(async () => {
let targetProcess;
let input;
if (remote === undefined) {
input = {
mode: "spawn",
deviceId: "local",
program: process.execPath,
argv: ["-e", "setTimeout(() => {}, 1500)"],
source: {
sourceKind: "inline",
source: "send({ rea_frida_verification: 'ok', pid: Process.id });",
},
durationMs: 500,
};
targetProcess = { pid: undefined, name: basename(process.execPath) };
} else {
const inventory = await service.listProcesses({ remote });
if (!inventory.ok) throw new Error(inventory.error.message);
targetProcess = inventory.value.processes.find(
({ pid }) => pid === options.pid,
);
input = {
mode: "attach",
remote,
pid: options.pid,
source: {
sourceKind: "inline",
source: "send({ rea_frida_verification: 'ok', pid: Process.id });",
},
durationMs: 500,
if (targetProcess === undefined)
throw new Error(
`Selected PID ${String(options.pid)} was not observed remotely`,
);
input = {
mode: "attach",
remote,
pid: options.pid,
source: {
sourceKind: "inline",
source: "send({ rea_frida_verification: 'ok', pid: Process.id });",
},
durationMs: 500,
};
}
const result = await service.instrument(input);
if (!result.ok) throw new Error(result.error.message);
const observed = result.value.evidence.normalized_result;
if (
typeof observed !== "object" ||
observed === null ||
Array.isArray(observed) ||
!Array.isArray(observed.messages)
)
throw new Error("Frida returned no message observation");
const messageObserved = observed.messages.some(
(message) =>
typeof message === "object" &&
message !== null &&
!Array.isArray(message) &&
message.type === "send" &&
typeof message.payload === "object" &&
message.payload !== null &&
!Array.isArray(message.payload) &&
message.payload.rea_frida_verification === "ok" &&
message.payload.pid === observed.pid,
);
if (!messageObserved)
throw new Error("Selected target did not return the Frida marker");
return {
ok: result.value.cleanupError === null,
provider: "frida",
host_platform: process.platform,
target_mode: remote === undefined ? "local_spawn" : "remote_attach",
target_pid: observed.pid,
target_name: targetProcess.name,
...(remote === undefined ? {} : { remote_target_observed: true }),
script_message_observed: true,
cleanup_error: result.value.cleanupError,
};
}
const result = await service.instrument(input);
if (!result.ok) throw new Error(result.error.message);
const observed = result.value.evidence.normalized_result;
if (
typeof observed !== "object" ||
observed === null ||
Array.isArray(observed) ||
!Array.isArray(observed.messages)
)
throw new Error("Frida returned no message observation");
const messageObserved = observed.messages.some(
(message) =>
typeof message === "object" &&
message !== null &&
!Array.isArray(message) &&
message.type === "send" &&
typeof message.payload === "object" &&
message.payload !== null &&
!Array.isArray(message.payload) &&
message.payload.rea_frida_verification === "ok" &&
message.payload.pid === observed.pid,
);
if (!messageObserved)
throw new Error("Selected target did not return the Frida marker");
report = {
ok: result.value.cleanupError === null,
provider: "frida",
host_platform: process.platform,
target_mode: remote === undefined ? "local_spawn" : "remote_attach",
target_pid: observed.pid,
target_name: targetProcess.name,
...(remote === undefined ? {} : { remote_target_observed: true }),
script_message_observed: true,
cleanup_error: result.value.cleanupError,
};
} catch (cause) {
primaryFailure = cause;
}
let cleanupFailure;
try {
await closeVerifierResources([() => manager.closeAll()], primaryFailure);
} catch (cause) {
cleanupFailure = cause;
}
if (primaryFailure !== undefined || cleanupFailure !== undefined) {
const causes =
cleanupFailure instanceof AggregateError
? cleanupFailure.errors
: [cleanupFailure ?? primaryFailure];
process.stderr.write(
`Frida real-provider verification failed: ${causes.map((cause) => (cause instanceof Error ? cause.message : String(cause))).join("; ")}\n`,
);
process.exitCode = 1;
} else {
}, [() => manager.closeAll()]);
process.stdout.write(`${JSON.stringify(report)}\n`);
if (!report.ok) process.exitCode = 1;
} catch (cause) {
const causes = cause instanceof AggregateError ? cause.errors : [cause];
process.stderr.write(
`Frida real-provider verification failed: ${causes.map((reason) => (reason instanceof Error ? reason.message : String(reason))).join("; ")}\n`,
);
process.exitCode = 1;
}
function parseArguments(args) {
scripts/verify-il2cpp-metadata.mjs
+3 -12
View File
@@ -9,7 +9,7 @@ import { toolContract } from "../dist/contracts/toolContracts.js";
import { analysisViewResultSchema } from "../dist/domain/analysisView/analysisView.js";
import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { hashVerifierFile } from "./lib/verifier-file-resources.mjs";
const args = process.argv.slice(2);
@@ -47,8 +47,7 @@ const call = async (name, input) => {
return JSON.parse(mcpTextValue(response));
};
const resources = new ArtifactResourceScope();
let primaryError;
try {
await withVerifierCleanup(async () => {
await client.connect(transport);
const { tools } = await client.listTools();
assert(tools.some((tool) => tool.name === operation));
@@ -182,13 +181,5 @@ try {
`[ok] summary and page parity; ${report.images.length} images, ${report.types.length} types, ${report.methods.length} methods; complete Evidence ${Buffer.byteLength(stdout)} bytes, summary ${Buffer.byteLength(JSON.stringify(summary))} bytes; CLI ${cliMs} ms`,
);
}
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => client.close(), () => transport.close(), () => resources.close()],
primaryError,
);
}
}, [() => client.close(), () => transport.close(), () => resources.close()]);
console.log("All asserted IL2CPP metadata CLI/MCP checks passed");
scripts/verify-jeb.mjs
+68 -78
View File
@@ -6,7 +6,7 @@ import { promisify } from "node:util";
import { JebAnalysisService } from "../dist/application/jeb/JebAnalysisService.js";
import { createJebAnalysisProvider } from "../dist/composition/jeb.js";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
const run = promisify(execFile);
@@ -27,92 +27,82 @@ if (options.help) {
const provider = createJebAnalysisProvider(process.env);
const service = new JebAnalysisService(provider);
let fixtureDirectory;
let primaryFailure;
let report;
try {
fixtureDirectory = await mkdtemp(join(tmpdir(), "rea-jeb-"));
const client = await service.execute("inspect_jeb_client", {});
if (!client.ok) throw new Error(client.error.message);
const engine = client.value.normalized_result.engine;
if (engine.version === null)
throw new Error("The JEB client did not report its version");
const report = await withVerifierCleanup(async () => {
fixtureDirectory = await mkdtemp(join(tmpdir(), "rea-jeb-"));
const client = await service.execute("inspect_jeb_client", {});
if (!client.ok) throw new Error(client.error.message);
const engine = client.value.normalized_result.engine;
if (engine.version === null)
throw new Error("The JEB client did not report its version");
// JEB 5.48 headless clients do not advertise open_project; a project opened
// at engine start is the supported target-selection route there. Attempt the
// tool and record the engine's exact refusal when it is unavailable.
let openRefusal = null;
const target = await resolveTarget(options, fixtureDirectory);
const opened = await service.execute("open_jeb_project", {
path: target.path,
});
if (opened.ok) {
const artifact = opened.value.normalized_result.input_files.find(
(file) => typeof file === "object" && file !== null,
);
if (
artifact === undefined ||
!/^[0-9a-f]{64}$/u.test(artifact.contents_sha256_hash)
)
throw new Error("The engine did not report an input artifact digest");
} else {
openRefusal = opened.error.message;
}
// JEB 5.48 headless clients do not advertise open_project; a project opened
// at engine start is the supported target-selection route there. Attempt the
// tool and record the engine's exact refusal when it is unavailable.
let openRefusal = null;
const target = await resolveTarget(options, fixtureDirectory);
const opened = await service.execute("open_jeb_project", {
path: target.path,
});
if (opened.ok) {
const artifact = opened.value.normalized_result.input_files.find(
(file) => typeof file === "object" && file !== null,
);
if (
artifact === undefined ||
!/^[0-9a-f]{64}$/u.test(artifact.contents_sha256_hash)
)
throw new Error("The engine did not report an input artifact digest");
} else {
openRefusal = opened.error.message;
}
const units = await service.execute("list_jeb_units", { count: 100 });
if (!units.ok) throw new Error(units.error.message);
if (units.value.normalized_result.units.length === 0)
throw new Error("The engine reported no project units");
const units = await service.execute("list_jeb_units", { count: 100 });
if (!units.ok) throw new Error(units.error.message);
if (units.value.normalized_result.units.length === 0)
throw new Error("The engine reported no project units");
const decompiled = await service.execute("decompile_jeb_item", {
item_address: target.itemAddress,
item_kind: "method",
});
if (!decompiled.ok) throw new Error(decompiled.error.message);
const text = decompiled.value.normalized_result.text;
if (!text.includes(target.marker))
throw new Error(
"Decompiled pseudo-code did not contain the fixture marker",
);
const decompiled = await service.execute("decompile_jeb_item", {
item_address: target.itemAddress,
item_kind: "method",
});
if (!decompiled.ok) throw new Error(decompiled.error.message);
const text = decompiled.value.normalized_result.text;
if (!text.includes(target.marker))
throw new Error(
"Decompiled pseudo-code did not contain the fixture marker",
);
report = {
ok: true,
provider: "jeb",
host_platform: process.platform,
engine_version: engine.version,
engine_gui: engine.gui_client,
endpoint: engine.endpoint,
open_project_observed: openRefusal === null,
...(openRefusal === null ? {} : { open_project_refusal: openRefusal }),
unit_count: units.value.normalized_result.units.length,
decompiled_marker_observed: true,
};
return {
ok: true,
provider: "jeb",
host_platform: process.platform,
engine_version: engine.version,
engine_gui: engine.gui_client,
endpoint: engine.endpoint,
open_project_observed: openRefusal === null,
...(openRefusal === null ? {} : { open_project_refusal: openRefusal }),
unit_count: units.value.normalized_result.units.length,
decompiled_marker_observed: true,
};
}, [
async () => {
await provider.close();
if (fixtureDirectory !== undefined) {
await rm(fixtureDirectory, { recursive: true, force: true });
fixtureDirectory = undefined;
}
},
]);
process.stdout.write(`${JSON.stringify(report)}\n`);
} catch (cause) {
primaryFailure = cause;
}
const cleanupTasks = [
() => provider.close(),
...(fixtureDirectory === undefined
? []
: [() => rm(fixtureDirectory, { recursive: true, force: true })]),
];
let cleanupFailure;
try {
await closeVerifierResources(cleanupTasks, primaryFailure);
} catch (cause) {
cleanupFailure = cause;
}
if (primaryFailure !== undefined || cleanupFailure !== undefined) {
const causes =
cleanupFailure instanceof AggregateError
? cleanupFailure.errors
: [cleanupFailure ?? primaryFailure];
if (fixtureDirectory !== undefined)
process.stderr.write(`JEB fixture retained at ${fixtureDirectory}\n`);
const causes = cause instanceof AggregateError ? cause.errors : [cause];
process.stderr.write(
`JEB real-provider verification failed: ${causes.map((cause) => (cause instanceof Error ? cause.message : String(cause))).join("; ")}\n`,
`JEB real-provider verification failed: ${causes.map((reason) => (reason instanceof Error ? reason.message : String(reason))).join("; ")}\n`,
);
process.exitCode = 1;
} else {
process.stdout.write(`${JSON.stringify(report)}\n`);
}
async function resolveTarget(selected, directory) {
scripts/verify-package-update.mjs
+118 -122
View File
@@ -5,7 +5,7 @@ import { dirname, join } from "node:path";
import { pathToFileURL } from "node:url";
import { exec, json } from "./lib/verify-package-core.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
/** Keep the live updater owner and its workspace when process cleanup is unproven. */
export class PackageUpdateCleanupFailure extends AggregateError {
@@ -175,63 +175,63 @@ export async function verifyPackageUpdate({
server.once("error", reject);
server.listen(0, "127.0.0.1", resolveListen);
});
let primary;
try {
const address = server.address();
if (typeof address !== "object" || address === null)
throw new Error("Registry fixture did not bind");
registry = `http://127.0.0.1:${address.port}`;
// Model an older installed release's identity while retaining the new updater
// under test. The registry supplies the unmodified packed target artifact.
const generatedPath = join(
packageRoot,
"dist",
"generatedPackageMetadata.js",
);
const generated = await readFile(generatedPath, "utf8");
const previousVersion = "0.0.0";
const previousGenerated = generated.replace(
`version: "${metadata.version}"`,
`version: "${previousVersion}"`,
);
if (previousGenerated === generated)
throw new Error("Could not create previous-release identity fixture");
await writeFile(generatedPath, previousGenerated);
await writeFile(
metadataPath,
JSON.stringify({ ...metadata, version: previousVersion }),
);
const runtimeEnvironment = {
...environment,
HOME: home,
USERPROFILE: home,
APPDATA: join(home, "AppData", "Roaming"),
LOCALAPPDATA: join(home, "AppData", "Local"),
CLAUDE_CONFIG_DIR: home,
CODEX_HOME: join(home, ".codex"),
COPILOT_HOME: join(home, ".copilot"),
GROK_HOME: join(home, ".grok"),
SAND_DATA_ROOT: join(home, ".grokbot"),
OMP_PROFILE: undefined,
PI_CODING_AGENT_DIR: undefined,
PI_CONFIG_DIR: undefined,
PI_PROFILE: undefined,
XDG_CONFIG_HOME: join(home, ".config"),
OPENCODE_CONFIG: join(home, ".config", "opencode", "opencode.json"),
npm_config_prefix: prefix,
npm_config_registry: registry,
};
const previous = (
await exec(process.execPath, [entry, "--version"], {
env: runtimeEnvironment,
})
).stdout.trim();
if (previous !== previousVersion)
throw new Error(`Previous-release fixture reported ${previous}`);
const harness = join(workspace, "verify-update.mjs");
await writeFile(
harness,
`
return withVerifierCleanup(async () => {
try {
const address = server.address();
if (typeof address !== "object" || address === null)
throw new Error("Registry fixture did not bind");
registry = `http://127.0.0.1:${address.port}`;
// Model an older installed release's identity while retaining the new updater
// under test. The registry supplies the unmodified packed target artifact.
const generatedPath = join(
packageRoot,
"dist",
"generatedPackageMetadata.js",
);
const generated = await readFile(generatedPath, "utf8");
const previousVersion = "0.0.0";
const previousGenerated = generated.replace(
`version: "${metadata.version}"`,
`version: "${previousVersion}"`,
);
if (previousGenerated === generated)
throw new Error("Could not create previous-release identity fixture");
await writeFile(generatedPath, previousGenerated);
await writeFile(
metadataPath,
JSON.stringify({ ...metadata, version: previousVersion }),
);
const runtimeEnvironment = {
...environment,
HOME: home,
USERPROFILE: home,
APPDATA: join(home, "AppData", "Roaming"),
LOCALAPPDATA: join(home, "AppData", "Local"),
CLAUDE_CONFIG_DIR: home,
CODEX_HOME: join(home, ".codex"),
COPILOT_HOME: join(home, ".copilot"),
GROK_HOME: join(home, ".grok"),
SAND_DATA_ROOT: join(home, ".grokbot"),
OMP_PROFILE: undefined,
PI_CODING_AGENT_DIR: undefined,
PI_CONFIG_DIR: undefined,
PI_PROFILE: undefined,
XDG_CONFIG_HOME: join(home, ".config"),
OPENCODE_CONFIG: join(home, ".config", "opencode", "opencode.json"),
npm_config_prefix: prefix,
npm_config_registry: registry,
};
const previous = (
await exec(process.execPath, [entry, "--version"], {
env: runtimeEnvironment,
})
).stdout.trim();
if (previous !== previousVersion)
throw new Error(`Previous-release fixture reported ${previous}`);
const harness = join(workspace, "verify-update.mjs");
await writeFile(
harness,
`
import { runUpdate } from ${JSON.stringify(pathToFileURL(join(packageRoot, "dist/application/Update.js")).href)};
import { systemUpdateHost } from ${JSON.stringify(pathToFileURL(join(packageRoot, "dist/application/UpdateRuntime.js")).href)};
const host = systemUpdateHost(${JSON.stringify(packageRoot)}, ${JSON.stringify(home)});
@@ -239,72 +239,68 @@ const result = await runUpdate(${JSON.stringify(previousVersion)}, host, "struct
process.stdout.write(JSON.stringify(result));
if (result.status === "failed") process.exitCode = 1;
`,
);
const result = json(
await runUpdater({
harness,
environment: runtimeEnvironment,
signal: fixture.signal,
workspace,
retainWorkspace,
}),
);
if (
result.status !== "updated" ||
result.installedVersion !== metadata.version ||
result.command.at(-1) !== `${metadata.name}@${metadata.version}` ||
result.maintenance.status !== "planned" ||
result.maintenance.scope.clients.join(",") !== "codex" ||
result.maintenance.scope.skill !== false
)
throw new Error(
`Packaged updater failed its exact-release contract: ${JSON.stringify(result)}`,
);
if ((await readFile(codexPath, "utf8")) !== originalConfig)
throw new Error(
"Update applied integration maintenance without approval",
const result = json(
await runUpdater({
harness,
environment: runtimeEnvironment,
signal: fixture.signal,
workspace,
retainWorkspace,
}),
);
const current = (
await exec(process.execPath, [entry, "--version"], {
env: runtimeEnvironment,
})
).stdout.trim();
if (current !== metadata.version)
throw new Error(`Updated executable reported ${current}`);
const help = (
await exec(process.execPath, [entry, "--help"], {
env: runtimeEnvironment,
})
).stdout;
if (!/^\s{2}update\s/mu.test(help) || /^\s{2}upgrade\s/mu.test(help))
throw new Error(
"Updated CLI did not expose update as its sole maintenance command",
);
if (fixtureFailure !== undefined) throw fixtureFailure;
return {
installed_version: result.installedVersion,
exact_release: true,
maintenance_read_only: true,
};
} catch (error) {
primary =
error instanceof PackageUpdateCleanupFailure
if (
result.status !== "updated" ||
result.installedVersion !== metadata.version ||
result.command.at(-1) !== `${metadata.name}@${metadata.version}` ||
result.maintenance.status !== "planned" ||
result.maintenance.scope.clients.join(",") !== "codex" ||
result.maintenance.scope.skill !== false
)
throw new Error(
`Packaged updater failed its exact-release contract: ${JSON.stringify(result)}`,
);
if ((await readFile(codexPath, "utf8")) !== originalConfig)
throw new Error(
"Update applied integration maintenance without approval",
);
const current = (
await exec(process.execPath, [entry, "--version"], {
env: runtimeEnvironment,
})
).stdout.trim();
if (current !== metadata.version)
throw new Error(`Updated executable reported ${current}`);
const help = (
await exec(process.execPath, [entry, "--help"], {
env: runtimeEnvironment,
})
).stdout;
if (!/^\s{2}update\s/mu.test(help) || /^\s{2}upgrade\s/mu.test(help))
throw new Error(
"Updated CLI did not expose update as its sole maintenance command",
);
if (fixtureFailure !== undefined) throw fixtureFailure;
return {
installed_version: result.installedVersion,
exact_release: true,
maintenance_read_only: true,
};
} catch (error) {
// A retained process owner takes precedence over its triggering fixture
// failure; otherwise report the first registry/proxy failure exactly.
throw error instanceof PackageUpdateCleanupFailure
? error
: (fixtureFailure ?? error);
throw primary;
} finally {
await closeVerifierResources(
[
async () => {
server.closeAllConnections();
await new Promise((resolveClose, reject) =>
server.close((error) =>
error === undefined ? resolveClose() : reject(error),
),
);
},
],
primary,
);
}
}
}, [
async () => {
server.closeAllConnections();
await new Promise((resolveClose, reject) =>
server.close((error) =>
error === undefined ? resolveClose() : reject(error),
),
);
},
]);
}
scripts/verify-package.mjs
+14 -11
View File
@@ -22,6 +22,7 @@ import { verifyPackageSetup } from "./verify-package-setup.mjs";
import { verifyRepositorySkill } from "./verify-repository-skill.mjs";
import { verifyManaged } from "./verify-package-managed.mjs";
import { verifyUnknownProvider } from "./verify-package-unknown-provider.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
import {
createVerifiedPackageMetadata,
@@ -60,7 +61,7 @@ const runStage = async (name, action) => {
return result;
};
try {
await withVerifierCleanup(async () => {
if (retentionDirectory !== undefined && retentionDirectory !== "") {
const sourceSha = (
await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: root })
@@ -258,16 +259,18 @@ try {
metadata: retentionMetadata,
});
}
} finally {
if (updateWorkspaceRetained)
process.stderr.write(
`Package updater cleanup incomplete; retained ${workspace}\n`,
);
else
await runStage("workspace cleanup", () =>
rm(workspace, { recursive: true, force: true }),
);
}
}, [
async () => {
if (updateWorkspaceRetained)
process.stderr.write(
`Package updater cleanup incomplete; retained ${workspace}\n`,
);
else
await runStage("workspace cleanup", () =>
rm(workspace, { recursive: true, force: true }),
);
},
]);
const verifierLineage = await runStage("verifier lineage", () =>
completeVerifierRun(verifierRun),
);
scripts/verify-packaged-wasm-artifact.mjs
+73 -20
View File
@@ -1,17 +1,57 @@
#!/usr/bin/env node
import { join } from "node:path";
import { mkdir } from "node:fs/promises";
import { randomUUID } from "node:crypto";
import { fileURLToPath } from "node:url";
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { exec, execNpm } from "./lib/verify-package-core.mjs";
import {
OwnedCommandFailure,
runOwnedCommand,
} from "../dist/process/OwnedCommand.js";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { execNpm } from "./lib/verify-package-core.mjs";
import { WABT_WORKFLOW_TIMEOUT_MS } from "./lib/wabt-verifier-timeouts.mjs";
import { verifyPackagePack } from "./verify-package-pack.mjs";
/** Preserve the actual dependent process/root owner across failed cleanup. */
class PackagedWasmCleanupFailure extends Error {
constructor(cause, cleanupOwner, runtimeRoot) {
super(
`Packaged WASM verifier cleanup incomplete; retained ${runtimeRoot.path}: ${cause instanceof Error ? cause.message : String(cause)}`,
{ cause },
);
this.name = "PackagedWasmCleanupFailure";
this.cleanupOwner = cleanupOwner;
this.runtimeRoot = runtimeRoot;
}
}
const root = await PrivateRuntimeRoot.create({
prefix: "rea-wabt-package-verifier-",
});
let primaryError;
try {
let commandCleanupOwner;
let closing;
const cleanupOwner = {
close() {
if (closing !== undefined) return closing;
let attempt;
attempt = (async () => {
if (commandCleanupOwner !== undefined) {
const stopped = await commandCleanupOwner.stop();
if (stopped.status === "incomplete")
throw new Error(stopped.reason, { cause: stopped });
commandCleanupOwner = undefined;
}
await root.close();
})().catch((cause) => {
if (closing === attempt) closing = undefined;
throw new PackagedWasmCleanupFailure(cause, cleanupOwner, root);
});
closing = attempt;
return attempt;
},
};
const verifiedOutput = await withVerifierCleanup(async () => {
const { tarball } = await verifyPackagePack({
root: process.cwd(),
workspace: root.path,
@@ -48,19 +88,32 @@ try {
"scripts",
"rea.mjs",
);
const verified = await exec(
process.execPath,
["scripts/verify-wasm-artifact.mjs", entrypoint, "--require-tools"],
{
env: environment,
timeout: WABT_WORKFLOW_TIMEOUT_MS,
maxBuffer: 4 * 1024 * 1024,
},
);
process.stdout.write(verified.stdout);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources([() => root.close()], primaryError);
}
let verified;
try {
verified = await runOwnedCommand(
{
command: process.execPath,
arguments: [
fileURLToPath(new URL("./verify-wasm-artifact.mjs", import.meta.url)),
entrypoint,
"--require-tools",
],
cwd: process.cwd(),
hostEnvironment: environment,
expectedCommand: process.execPath,
runId: `rea-packaged-wabt-${randomUUID()}`,
sweepTokenOwnedProcesses: true,
},
{
timeoutMs: WABT_WORKFLOW_TIMEOUT_MS,
diagnosticBytes: 4 * 1024 * 1024,
},
);
} catch (cause) {
if (cause instanceof OwnedCommandFailure)
commandCleanupOwner = cause.cleanupOwner;
throw cause;
}
return verified.stdout.text;
}, [() => cleanupOwner.close()]);
process.stdout.write(verifiedOutput);
scripts/verify-packet-capture.mjs
+3 -12
View File
@@ -12,7 +12,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { Ajv2020 } from "ajv/dist/2020.js";
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
import { parseEvidence } from "../dist/domain/evidence.js";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
if (process.platform !== "linux")
@@ -107,8 +107,7 @@ const expectedAddressSpace = Math.min(
.map((value) => (value === "unlimited" ? Infinity : Number(value))),
);
let primaryError;
try {
await withVerifierCleanup(async () => {
await writeFile(path, source);
await writeFile(bluetooth, bluetoothSource);
await client.connect(transport);
@@ -510,15 +509,7 @@ try {
"network with spaces.pcap",
"paged.pcap",
]);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => client.close(), () => transport.close(), () => root.close()],
primaryError,
);
}
}, [() => client.close(), () => transport.close(), () => root.close()]);
console.log(
JSON.stringify(
{
scripts/verify-pcap.mjs
+20 -26
View File
@@ -9,7 +9,10 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { toolContract } from "../dist/contracts/toolContracts.js";
import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import {
closeVerifierResources,
withVerifierCleanup,
} from "./lib/verifier-cleanup.mjs";
import { hashVerifierFile } from "./lib/verifier-file-resources.mjs";
const args = process.argv.slice(2);
@@ -44,8 +47,7 @@ const transport = new StdioClientTransport({
stderr: "pipe",
});
transport.stderr?.resume();
let primaryError;
try {
await withVerifierCleanup(async () => {
await client.connect(transport);
const readiness = await run("inspect_pcap_client", {});
assert(readiness.client.tshark_version && readiness.client.capinfos_version);
@@ -87,29 +89,21 @@ try {
`[ok] ${result.packet_count} packets; ${result.endpoint_count} addresses; coverage=${result.coverage}`,
);
}
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
async () => {
await closeVerifierResources([
() => client.close(),
() => transport.close(),
() => resources.close(),
]);
assert.deepEqual(
await readdir(root),
[],
`Owned capture resources remain under ${root}`,
);
await rmdir(root);
},
],
primaryError,
);
}
}, [
async () => {
await closeVerifierResources([
() => client.close(),
() => transport.close(),
() => resources.close(),
]);
assert.deepEqual(
await readdir(root),
[],
`Owned capture resources remain under ${root}`,
);
await rmdir(root);
},
]);
console.log("All asserted pcap CLI/MCP checks passed");
async function run(name, input) {
scripts/verify-real-browser.mjs
+21 -27
View File
@@ -39,7 +39,7 @@ import {
verifyScenarioFailureEvidence,
} from "./lib/browser-scenario-verifier.mjs";
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { requireOrdinaryWindowsVerifierCaller } from "./lib/windows-ordinary-user-verifier.mjs";
import { verifyLargeScreenshotE2e } from "./lib/browser-screenshot-e2e.mjs";
import { verifyViewportScreenshotE2e } from "./lib/browser-viewport-screenshot-e2e.mjs";
@@ -70,8 +70,7 @@ let site;
let browser;
let pageProxy;
let report;
let verificationError;
try {
await withVerifierCleanup(async () => {
profile = await mkdtemp(join(tmpdir(), "rea-real-browser-"));
site = await startBrowserVerifierSite();
browser = spawn(
@@ -444,40 +443,35 @@ try {
viewportScreenshot,
verified: true,
};
} catch (cause) {
verificationError = cause;
throw cause;
} finally {
const cleanup = [];
if (pageProxy !== undefined) {
cleanup.push(async () => {
}, [
async () => {
if (pageProxy !== undefined) {
process.stderr.write("Browser verifier cleanup: closing page proxy\n");
await pageProxy.close();
});
}
if (browser !== undefined) {
cleanup.push(async () => {
}
},
async () => {
if (browser !== undefined) {
process.stderr.write("Browser verifier cleanup: stopping owned Chrome\n");
await stopProcess(browser);
});
}
if (site !== undefined) {
cleanup.push(async () => {
}
},
async () => {
if (site !== undefined) {
process.stderr.write("Browser verifier cleanup: closing fixture site\n");
await site.close();
});
}
if (profile !== undefined)
cleanup.push(() =>
rm(profile, {
}
},
async () => {
if (profile !== undefined)
await rm(profile, {
recursive: true,
force: true,
maxRetries: 10,
retryDelay: 100,
}),
);
await closeVerifierResources(cleanup, verificationError);
}
});
},
]);
process.stdout.write(
`${JSON.stringify({
verifier_run: await completeVerifierRun(verifierRun),
scripts/verify-reconstruction-readiness.mjs
+3 -12
View File
@@ -11,7 +11,7 @@ import { tsImport } from "tsx/esm/api";
import { toolContract } from "../dist/contracts/toolContracts.js";
import { parseEvidence } from "../dist/domain/evidence.js";
import { createEvidenceBundle } from "../dist/domain/evidenceBundle.js";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
const [
{ RECONSTRUCTION_READINESS_EXAMPLE },
@@ -69,8 +69,7 @@ const client = new Client({
});
let verificationReport;
let primaryError;
try {
await withVerifierCleanup(async () => {
await client.connect(transport);
const catalog = await client.listTools();
const sessionTool = catalog.tools.find(
@@ -149,15 +148,7 @@ try {
},
evaluator_conformance: verifySyntheticEvaluator(),
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => client.close(), () => transport.close()],
primaryError,
);
}
}, [() => client.close(), () => transport.close()]);
if (process.env.REA_READINESS_REPORT_PATH !== undefined)
await writeFile(
scripts/verify-signal-recording.mjs
+3 -12
View File
@@ -12,7 +12,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { Ajv2020 } from "ajv/dist/2020.js";
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
import { parseEvidence } from "../dist/domain/evidence.js";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
if (process.platform !== "linux")
@@ -103,8 +103,7 @@ const expectedAddressSpace = Math.min(
.map((value) => (value === "unlimited" ? Infinity : Number(value))),
);
let primaryError;
try {
await withVerifierCleanup(async () => {
await writeFile(path, source);
await client.connect(transport);
const advertised = (await client.listTools()).tools.find(
@@ -426,15 +425,7 @@ try {
"paged.vcd",
"recording with spaces.vcd",
]);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => client.close(), () => transport.close(), () => root.close()],
primaryError,
);
}
}, [() => client.close(), () => transport.close(), () => root.close()]);
console.log(
JSON.stringify(
{
scripts/verify-test-temp-hygiene.mjs
+11 -9
View File
@@ -4,6 +4,7 @@ import { dirname, join, relative, resolve } from "node:path";
import { spawnSync } from "node:child_process";
import { fileURLToPath } from "node:url";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { isOwnedTestTemporaryName } from "./lib/test-temporary-ownership.mjs";
const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
@@ -28,7 +29,7 @@ const isolatedTemporaryRoot = await mkdtemp(
join(canonicalTemporaryRoot, "rea-test-hygiene-"),
);
try {
await withVerifierCleanup(async () => {
const result = spawnSync(
process.execPath,
[vitestPath, "run", "--coverage.enabled=false", ...process.argv.slice(2)],
@@ -60,14 +61,15 @@ try {
}
if (failures.length > 0) throw new Error(failures.join("\n\n"));
} finally {
await rm(isolatedTemporaryRoot, {
recursive: true,
force: true,
maxRetries: 3,
retryDelay: 25,
});
}
}, [
() =>
rm(isolatedTemporaryRoot, {
recursive: true,
force: true,
maxRetries: 3,
retryDelay: 25,
}),
]);
process.stdout.write(
"Test temporary-directory hygiene verified: no owned residues remain.\n",
);
scripts/verify-windows-native.mjs
+16 -33
View File
@@ -23,7 +23,7 @@ import { pathToFileURL } from "node:url";
import { verifyWindowsProtocolInput } from "./lib/windows-protocol-input.mjs";
import { verifyWindowsJobMemoryLimit } from "./lib/windows-job-memory-limit.mjs";
import { verifyVolumeRootAppendLease } from "./lib/windows-volume-root-append-lease.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
if (process.platform !== "win32")
throw new Error(
@@ -608,8 +608,7 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe
);
const heldPath = join(retryRuntime.observation.path, "snapshot.bin");
let holder;
let primaryError;
try {
await withVerifierCleanup(async () => {
await retryRuntime.snapshot(source, "snapshot.bin");
holder = await startDeleteSharingDeniedReader(heldPath);
// The reader denies delete sharing. Cleanup must retain ownership when
@@ -636,20 +635,12 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe
/No native private runtime/u,
);
report.controls.cleanupFailureRetainsOwnershipForRetry = true;
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
...(holder === undefined
? []
: [() => stopDeleteSharingDeniedReader(holder)]),
() => retryRuntime.close(),
],
primaryError,
);
}
}, [
async () => {
if (holder !== undefined) await stopDeleteSharingDeniedReader(holder);
},
() => retryRuntime.close(),
]);
const transientRuntime = WindowsPrivateRuntime.create(
workspace,
"cleanup-transient-",
@@ -659,8 +650,7 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe
"snapshot.bin",
);
let transientHolder;
let transientPrimaryError;
try {
await withVerifierCleanup(async () => {
await transientRuntime.snapshot(source, "snapshot.bin");
transientHolder = await startDeleteSharingDeniedReader(transientHeldPath);
// Keep the reader held through the first close attempt. Without retries,
@@ -690,20 +680,13 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe
/No native private runtime/u,
);
report.controls.cleanupTransientConflictRetries = true;
} catch (cause) {
transientPrimaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
...(transientHolder === undefined
? []
: [() => stopDeleteSharingDeniedReader(transientHolder)]),
() => transientRuntime.close(),
],
transientPrimaryError,
);
}
}, [
async () => {
if (transientHolder !== undefined)
await stopDeleteSharingDeniedReader(transientHolder);
},
() => transientRuntime.close(),
]);
report.controls = {
...report.controls,
protectedDaclReadback: true,
scripts/verify/firmware/analysis.mjs
+14 -20
View File
@@ -11,7 +11,10 @@ import { firmwareResultSchemas } from "../../../dist/domain/firmware/firmwareAna
import { parseEvidence } from "../../../dist/domain/evidence.js";
import { ArtifactResourceScope } from "../../../dist/artifacts/ArtifactResourceScope.js";
import { hashVerifierFile } from "../../lib/verifier-file-resources.mjs";
import { closeVerifierResources } from "../../lib/verifier-cleanup.mjs";
import {
closeVerifierResources,
withVerifierCleanup,
} from "../../lib/verifier-cleanup.mjs";
const execFileAsync = promisify(execFile);
const { join, resolve, dirname } = nodePath;
@@ -377,8 +380,7 @@ async function stageGhidra(mcpFiles) {
// Entry point
// ---------------------------------------------------------------------------
let primaryError;
try {
await withVerifierCleanup(async () => {
assert.equal(
(await hashVerifierFile(firmwarePath, fileResources)).sha256,
oracle.sha256,
@@ -396,20 +398,12 @@ try {
oracle.sha256,
"Original firmware bytes must remain unchanged",
);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
async () => {
await closeVerifierResources([
() => mcpClient.close(),
() => fileResources.close(),
]);
await fs.rm(workDir, { recursive: true, force: true });
},
],
primaryError,
);
}
}, [
async () => {
await closeVerifierResources([
() => mcpClient.close(),
() => fileResources.close(),
]);
await fs.rm(workDir, { recursive: true, force: true });
},
]);
scripts/verify/javascript/module-references.mjs
+14 -23
View File
@@ -10,7 +10,7 @@ import { parseArgs } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { artifactCliEvidence } from "../../lib/artifact-e2e.mjs";
import { closeVerifierResources } from "../../lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "../../lib/verifier-cleanup.mjs";
import { PrivateRuntimeRoot } from "../../../dist/process/PrivateRuntimeRoot.js";
import {
OwnedCommandFailure,
@@ -82,9 +82,8 @@ const normalize = (rows) =>
),
}));
let temporary;
let primaryError;
let retainTemporary = false;
try {
await withVerifierCleanup(async () => {
await client.connect(transport);
transport.stderr?.resume();
const tools = (await client.listTools()).tools;
@@ -393,23 +392,15 @@ print(json.dumps(dict(tokenizer='cl100k_base',tiktoken_version=tiktoken.__versio
"First query is after shared artifact analysis; subsequent queries share one MCP session. Legacy item reads are serial; a concurrent legacy baseline is unmeasured. Filesystem/build warmness uncontrolled. Token costs count one post-change catalog, preparation request/summary and each request/structured reply once; client rendering, replay, reasoning, final answer and model latency are unmeasured. Preparation and parity/tokenization overhead are outside per-query elapsed times; external family-resource receipts include the complete verifier and children.",
})}\n`,
);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
() => client.close(),
() => transport.close(),
async () => {
if (temporary === undefined) return;
if (retainTemporary)
throw new Error(
`Command cleanup is incomplete; retaining its private input/Evidence files at ${temporary.path}.`,
);
await temporary.close();
},
],
primaryError,
);
}
}, [
() => client.close(),
() => transport.close(),
async () => {
if (temporary === undefined) return;
if (retainTemporary)
throw new Error(
`Command cleanup is incomplete; retaining its private input/Evidence files at ${temporary.path}.`,
);
await temporary.close();
},
]);
scripts/verify/javascript/native-handoffs.mjs
+13 -22
View File
@@ -16,7 +16,7 @@ import {
OwnedCommandFailure,
runOwnedCommand,
} from "../../../dist/process/OwnedCommand.js";
import { closeVerifierResources } from "../../lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "../../lib/verifier-cleanup.mjs";
import {
matchesStructuredScopeExpectation,
structuredScopeExpectationsSchema,
@@ -122,7 +122,6 @@ const simulatedNativeEvidence = (path, sha256) =>
},
);
let temporary;
let primaryError;
let retainTemporary = false;
let report;
const ownedCommand = async (command, arguments_, timeoutMs = 60_000) => {
@@ -145,7 +144,7 @@ const ownedCommand = async (command, arguments_, timeoutMs = 60_000) => {
throw cause;
}
};
try {
await withVerifierCleanup(async () => {
const archiveSha256 = await fileDigest(values.artifact);
const companion = join(`${values.artifact}.unpacked`, values["addon-path"]);
let companionSha256 = null;
@@ -460,23 +459,15 @@ print(json.dumps(dict(tokenizer='cl100k_base',tiktoken_version=tiktoken.__versio
assert.ok(safeIdentity, "Native identity/linkage oracle failed");
assert.ok(safeApplicability, "File analysis applicability oracle failed");
}
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
() => client.close(),
() => transport.close(),
async () => {
if (retainTemporary)
throw new Error(
`Command cleanup incomplete; retaining owned inputs at ${temporary?.path}.`,
);
await temporary?.close();
},
],
primaryError,
);
}
}, [
() => client.close(),
() => transport.close(),
async () => {
if (retainTemporary)
throw new Error(
`Command cleanup incomplete; retaining owned inputs at ${temporary?.path}.`,
);
await temporary?.close();
},
]);
console.log(serialize(report));
scripts/verify/javascript/retained-facts.mjs
+13 -22
View File
@@ -15,7 +15,7 @@ import {
OwnedCommandFailure,
runOwnedCommand,
} from "../../../dist/process/OwnedCommand.js";
import { closeVerifierResources } from "../../lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "../../lib/verifier-cleanup.mjs";
import { selectExportedJavaScriptFacts } from "../../lib/javascript-facts-reference.mjs";
const { values } = parseArgs({
@@ -56,7 +56,6 @@ const client = new Client({
version: "1",
});
let temporary;
let primaryError;
let retainTemporary = false;
let report;
const serialize = (value) => JSON.stringify(value);
@@ -106,7 +105,7 @@ const resourceSnapshot = async () => {
};
};
try {
await withVerifierCleanup(async () => {
const start = performance.now();
const artifactHash = createHash("sha256");
for await (const chunk of createReadStream(values.artifact))
@@ -344,23 +343,15 @@ try {
conditions:
"Fresh MCP and input hashing (file cache warmed); build/OS caches uncontrolled. First export sample includes export plus caller projection code, later samples reuse that immutable export. View samples precede diagnostic export. Task token projections include preparation, first answer, selected/full name/description/inputSchema catalog and the exact baseline program; exec tool schema, host rendering, output schemas, reasoning and billing unmeasured. At-answer cgroup counters include observer/startup overhead, before repeats/export diagnostics/parity/tokenization; final external observer measures all validation work. No application or native provider executes.",
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
() => client.close(),
() => transport.close(),
async () => {
if (retainTemporary)
throw new Error(
`Command cleanup incomplete; retaining owned inputs at ${temporary?.path}.`,
);
await temporary?.close();
},
],
primaryError,
);
}
}, [
() => client.close(),
() => transport.close(),
async () => {
if (retainTemporary)
throw new Error(
`Command cleanup incomplete; retaining owned inputs at ${temporary?.path}.`,
);
await temporary?.close();
},
]);
console.log(serialize(report));
tests/boundary/filesystem/testTempHygiene.test.ts
+1
View File
@@ -34,6 +34,7 @@ it.each(["clean", "file", "directory", "cache", "symlink", "failed-test"])(
for (const path of [
"scripts/verify-test-temp-hygiene.mjs",
"scripts/lib/test-temporary-ownership.mjs",
"scripts/lib/verifier-cleanup.mjs",
])
await copyFile(join(process.cwd(), path), join(root, path));
await mkdir(join(root, "external"));
tests/boundary/process/agentEvaluationCleanup.test.ts
+23 -32
View File
@@ -5,7 +5,7 @@ import { fileURLToPath } from "node:url";
import { expect, it } from "vitest";
import { z } from "zod";
import { closeVerifierResources } from "../../../scripts/lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs";
import { resolveSwiftcCompileTimeoutMs } from "../../../src/process/DarwinProcessRunTokenReader.js";
import {
execFileOutput,
@@ -60,8 +60,7 @@ process.exitCode = 1;
{ mode: 0o700 },
);
let run: { root: string; codex: string } | undefined;
let primaryError: unknown;
try {
await withVerifierCleanup(async () => {
let stderr: string;
try {
await execFileOutput(
@@ -125,35 +124,27 @@ process.exitCode = 1;
await expect(stat(run.root)).rejects.toMatchObject({
code: "ENOENT",
});
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
// Restore this test's deliberately denied subtree before fixture teardown.
if (denied)
await closeVerifierResources(
[
async () => {
try {
run ??= receiptSchema.parse(
JSON.parse(await readFile(receipt, "utf8")),
);
} catch (cause) {
// Startup can fail before the executable creates its receipt.
if (
cause instanceof Error &&
"code" in cause &&
cause.code === "ENOENT"
)
return;
throw cause;
}
await chmod(run.codex, 0o700);
},
],
primaryError,
);
}
}, [
async () => {
// Restore this test's deliberately denied subtree before fixture teardown.
if (!denied) return;
try {
run ??= receiptSchema.parse(
JSON.parse(await readFile(receipt, "utf8")),
);
} catch (cause) {
// Startup can fail before the executable creates its receipt.
if (
cause instanceof Error &&
"code" in cause &&
cause.code === "ENOENT"
)
return;
throw cause;
}
await chmod(run.codex, 0o700);
},
]);
},
);
}
tests/boundary/process/agentEvaluationCommands.test.ts
+3 -9
View File
@@ -68,7 +68,7 @@ const { createAgentEvaluationCommands } = await import(${JSON.stringify(
const { OwnedCommandFailure, prepareOwnedCommand } = await import(${JSON.stringify(
new URL("../../../dist/process/OwnedCommand.js", import.meta.url).href,
)});
const { closeVerifierResources } = await import(${JSON.stringify(
const { withVerifierCleanup } = await import(${JSON.stringify(
new URL("../../../scripts/lib/verifier-cleanup.mjs", import.meta.url)
.href,
)});
@@ -76,8 +76,7 @@ await prepareOwnedCommand();
const root = ${JSON.stringify(root)}, fixture = ${JSON.stringify(fixture)};
const commands = createAgentEvaluationCommands({ codex: fixture, evaluationRoot: root,
evaluationEnvironment: process.env, timeoutMs: 1000, repositoryRoot: root });
let primaryError;
try {
await withVerifierCleanup(async () => {
for (const name of ["runCodex", "runProcess", "readCommandVersion"]) {
await assert.rejects(() => name === "runCodex" ? commands.runCodex("fixture", []) :
name === "runProcess" ? commands.runProcess(fixture, [], root, 1000) : commands.readCommandVersion(fixture), error => {
@@ -109,12 +108,7 @@ await assert.rejects(() => commands.runProcess(fixture, [], root, 2000), error =
assert.ok(error.snapshot.stderr.bytes <= 32768);
return true;
});
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources([() => commands.close()], primaryError);
}
}, [() => commands.close()]);
console.log(JSON.stringify({ passed: true, timeoutHelpers: 3, controls: 7, childrenClosed: children.every(x => x.closed) }));
`,
);
tests/boundary/process/packagedWasmVerifierOwnership.test.ts
+358
View File
@@ -0,0 +1,358 @@
import { randomUUID } from "node:crypto";
import { copyFile, readFile, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
import { pathToFileURL } from "node:url";
import { expect, it } from "vitest";
import { cleanupOwnedProcessGroup } from "../../../src/process/ProcessOwnership.js";
import {
ProviderProcessSupervisor,
spawnOwnedProviderProcess,
} from "../../../src/process/ProviderProcess.js";
import { waitForProviderProcessReady } from "../../fixtures/providerProcess.js";
import {
createTestWorkspace,
removeTestWorkspace,
} from "../../support/workspace/workspaceFixture.js";
const builtModule = (path: string): string =>
pathToFileURL(resolve("dist", path)).href;
const PACKAGE_PACK_FIXTURE = `
import { appendFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
export async function verifyPackagePack({ root, workspace }) {
appendFileSync(process.env.OWNERSHIP_EVENTS, JSON.stringify({ kind: "pack", root, workspace }) + "\\n");
const tarball = join(workspace, "authored.tgz");
writeFileSync(tarball, "authored archive boundary");
return { tarball };
}
`;
const PACKAGE_INSTALL_FIXTURE = `
import assert from "node:assert/strict";
import { appendFileSync } from "node:fs";
export async function execNpm(args, { env }) {
assert.equal(args[0], "install");
assert.ok(args.includes("--ignore-scripts"));
appendFileSync(process.env.OWNERSHIP_EVENTS, JSON.stringify({ kind: "install", prefix: args[args.indexOf("--prefix") + 1], home: env.HOME }) + "\\n");
return { stdout: "", stderr: "" };
}
`;
const PRIVATE_ROOT_OBSERVER = `
import { appendFileSync, existsSync, readFileSync } from "node:fs";
import { PrivateRuntimeRoot as ActualPrivateRoot } from ${JSON.stringify(builtModule("process/PrivateRuntimeRoot.js"))};
import { systemProcessOwnershipHost } from ${JSON.stringify(builtModule("process/ProcessOwnershipObservation.js"))};
import { liveProcesses } from ${JSON.stringify(builtModule("process/ProcessOwnershipProcessTree.js"))};
export const PrivateRuntimeRoot = {
async create(options) {
const root = await ActualPrivateRoot.create(options);
const actualClose = root.close.bind(root);
root.close = async () => {
const events = readFileSync(process.env.OWNERSHIP_EVENTS, "utf8").trim().split("\\n").map(JSON.parse);
const ready = events.find((event) => event.kind === "ready");
const readyPids = ready === undefined ? null : [ready.leader, ready.descendant];
const processes = liveProcesses(await systemProcessOwnershipHost.listProcesses());
const livePids = readyPids === null ? null : processes.filter(({ pid }) => readyPids.includes(pid)).map(({ pid }) => pid);
appendFileSync(process.env.OWNERSHIP_EVENTS, JSON.stringify({ kind: "root-close", runtime: root.path, rootExists: existsSync(root.path), readyPids, livePids }) + "\\n");
await actualClose();
appendFileSync(process.env.OWNERSHIP_EVENTS, JSON.stringify({ kind: "root-closed", runtime: root.path, rootExists: existsSync(root.path) }) + "\\n");
};
return root;
},
};
`;
const OWNED_COMMAND_FIXTURE = `
import { appendFileSync, existsSync } from "node:fs";
import { runOwnedCommand as actualRun, OwnedCommandFailure } from ${JSON.stringify(builtModule("process/OwnedCommand.js"))};
import { spawnOwnedProviderProcess } from ${JSON.stringify(builtModule("process/ProviderProcess.js"))};
import { cleanupOwnedProcessGroup } from ${JSON.stringify(builtModule("process/ProcessOwnership.js"))};
export { OwnedCommandFailure };
export const runOwnedCommand = (spawn, limits) => actualRun(spawn, limits, {
launcher: async (input) => {
const acquired = await spawnOwnedProviderProcess(input);
return {
...acquired,
cleanup: async () => {
const denied = process.env.OWNERSHIP_DENY === "1" && !existsSync(process.env.OWNERSHIP_ALLOW);
appendFileSync(process.env.OWNERSHIP_EVENTS, JSON.stringify({ kind: "cleanup", denied }) + "\\n");
if (denied) return { cleaned: false, reason: "authored process cleanup denial" };
return acquired.cleanup === undefined ? cleanupOwnedProcessGroup(acquired.ownership) : acquired.cleanup();
},
};
},
});
`;
const NESTED_VERIFIER_FIXTURE = `
import { spawn } from "node:child_process";
import { appendFileSync, existsSync } from "node:fs";
import { dirname } from "node:path";
const runtime = dirname(process.env.HOME);
const record = (entry) => appendFileSync(process.env.OWNERSHIP_EVENTS, JSON.stringify(entry) + "\\n");
const childSource = ${JSON.stringify(`
const { appendFileSync, existsSync } = require("node:fs");
const { dirname } = require("node:path");
process.on("SIGTERM", () => {
appendFileSync(process.env.OWNERSHIP_EVENTS, JSON.stringify({ kind: "descendant-stopped", rootExists: existsSync(dirname(process.env.HOME)) }) + "\\n");
process.exit(0);
});
process.send({ pid: process.pid, runId: process.env.REA_PROCESS_RUN_ID });
setInterval(() => {}, 1000);
`)};
const child = spawn(process.execPath, ["--eval", childSource], { stdio: ["ignore", "ignore", "ignore", "ipc"] });
const ready = await new Promise((resolve, reject) => {
child.once("message", resolve);
child.once("error", reject);
});
process.on("SIGTERM", () => {
record({ kind: "leader-stopped", rootExists: existsSync(runtime) });
process.exit(0);
});
record({ kind: "ready", runtime, leader: process.pid, descendant: ready.pid, runId: process.env.REA_PROCESS_RUN_ID, descendantRunId: ready.runId });
process.stdout.write("nested ready\\n");
setInterval(() => {}, 1000);
`;
const HARNESS_FIXTURE = `
import assert from "node:assert/strict";
import { existsSync } from "node:fs";
import { readFile, writeFile } from "node:fs/promises";
import { OwnedCommandFailure, prepareOwnedCommand } from ${JSON.stringify(builtModule("process/OwnedCommand.js"))};
import { systemProcessOwnershipHost } from ${JSON.stringify(builtModule("process/ProcessOwnershipObservation.js"))};
import { liveProcesses } from ${JSON.stringify(builtModule("process/ProcessOwnershipProcessTree.js"))};
import { withVerifierCleanup } from "./scripts/lib/verifier-cleanup.mjs";
await prepareOwnedCommand();
let failure;
try { await import("./scripts/verify-packaged-wasm-artifact.mjs"); }
catch (cause) { failure = cause; }
// Capture actual capabilities before any fixture parsing or assertion can fail.
const failures = failure instanceof AggregateError ? failure.errors : [failure];
const primary = failures.find((error) => error instanceof OwnedCommandFailure);
const cleanup = failures.find((error) => error !== null && typeof error === "object" &&
(error.runtimeRoot !== undefined || typeof error.cleanupOwner?.close === "function"));
const compositeOwner = typeof cleanup?.cleanupOwner?.close === "function" ? cleanup.cleanupOwner : undefined;
const retainedProcess = primary?.cleanupOwner;
const retainedRoot = cleanup?.runtimeRoot;
await withVerifierCleanup(async () => {
if (process.env.OWNERSHIP_HARNESS_ASSERT === "1") throw new Error("authored harness assertion failure");
const events = (await readFile(process.env.OWNERSHIP_EVENTS, "utf8")).trim().split("\\n").map(JSON.parse);
const ready = events.find((entry) => entry.kind === "ready");
assert.ok(ready, "The actual descendant must acknowledge startup before the command deadline");
assert.ok(primary instanceof OwnedCommandFailure);
assert.equal(primary.reason, "timeout");
const pids = [ready.leader, ready.descendant];
const remaining = async () => liveProcesses(await systemProcessOwnershipHost.listProcesses()).filter(({ pid }) => pids.includes(pid)).map(({ pid }) => pid);
const result = { reason: primary.reason, runtime: ready.runtime, rootExists: existsSync(ready.runtime), livePids: await remaining(), pids, runId: ready.runId, descendantRunId: ready.descendantRunId };
if (process.env.OWNERSHIP_DENY === "1") {
assert.ok(failure instanceof AggregateError);
assert.equal(failure.cause, primary);
assert.equal(failure.errors[1], cleanup);
assert.equal(cleanup.name, "PackagedWasmCleanupFailure");
assert.equal(cleanup.runtimeRoot.path, ready.runtime);
assert.equal(typeof cleanup.cleanupOwner.close, "function");
assert.ok(primary.cleanupOwner);
result.compositeOwner = true;
}
await writeFile(process.env.OWNERSHIP_REPORT, JSON.stringify(result));
process.stdout.write("ready\\n");
if (cleanup !== undefined) {
await new Promise((resolve) => process.stdin.once("data", resolve));
process.stdin.destroy();
const owner = cleanup.cleanupOwner;
await owner.close();
await owner.close();
result.recovered = { rootExists: existsSync(ready.runtime), livePids: await remaining() };
await writeFile(process.env.OWNERSHIP_REPORT, JSON.stringify(result));
}
}, [async () => {
await writeFile(process.env.OWNERSHIP_ALLOW, "harness cleanup restores process authority");
if (compositeOwner !== undefined) {
await compositeOwner.close();
} else if (retainedProcess !== undefined) {
const stopped = await retainedProcess.stop();
if (stopped.status === "incomplete") throw new Error(stopped.reason);
if (typeof retainedRoot?.close === "function") await retainedRoot.close();
}
}]);
`;
const expectRootCleanupAfterProcessExit = (
events: readonly Record<string, unknown>[],
runtime: string,
readyPids: readonly number[],
): void => {
expect(events.filter(({ kind }) => kind === "root-close")).toEqual([
{ kind: "root-close", runtime, rootExists: true, readyPids, livePids: [] },
]);
expect(events.filter(({ kind }) => kind === "root-closed")).toEqual([
{ kind: "root-closed", runtime, rootExists: false },
]);
};
const expectHarnessFailureCleanup = async (
supervisor: ProviderProcessSupervisor,
eventsPath: string,
): Promise<void> => {
expect(await supervisor.waitForOutputClose(25_000)).toBe(true);
const snapshot = supervisor.snapshot();
expect(snapshot).toMatchObject({ exitCode: 1, signal: null });
expect(snapshot.stderr.text).toContain("authored harness assertion failure");
const events = (await readFile(eventsPath, "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line));
const ready = events.find(({ kind }) => kind === "ready");
expect(ready).toBeDefined();
expect(ready.runId).toMatch(/^rea-packaged-wabt-/);
expect(ready.descendantRunId).toBe(ready.runId);
expectRootCleanupAfterProcessExit(events, ready.runtime, [
ready.leader,
ready.descendant,
]);
};
// Packaging, installation, and WABT are authored boundaries. The copied verifier,
// private runtime root, Node processes, ownership inspection, and cleanup are real.
it.skipIf(process.platform === "win32").each([
{ denied: false, harnessFailure: false },
{ denied: true, harnessFailure: false },
{ denied: true, harnessFailure: true },
])(
"joins a real packaged-verifier descendant before root cleanup (denied: $denied, assertion failure: $harnessFailure)",
async ({ denied, harnessFailure }) => {
const workspace = await createTestWorkspace("rea-packaged-wasm-owner-");
const events = workspace.path("events.jsonl");
const report = workspace.path("report.json");
const allow = workspace.path("allow-cleanup");
const runtimeParent = await workspace.mkdir("runtime");
await workspace.mkdir("scripts/lib");
await workspace.mkdir("dist/process");
await Promise.all([
workspace.write("package.json", '{"type":"module"}\n'),
copyFile(
resolve("scripts/verify-packaged-wasm-artifact.mjs"),
workspace.path("scripts/verify-packaged-wasm-artifact.mjs"),
),
copyFile(
resolve("scripts/lib/verifier-cleanup.mjs"),
workspace.path("scripts/lib/verifier-cleanup.mjs"),
),
workspace.write(
"scripts/lib/wabt-verifier-timeouts.mjs",
"export const WABT_WORKFLOW_TIMEOUT_MS = 1000;\n",
),
workspace.write(
"dist/process/PrivateRuntimeRoot.js",
PRIVATE_ROOT_OBSERVER,
),
workspace.write("scripts/verify-package-pack.mjs", PACKAGE_PACK_FIXTURE),
workspace.write(
"scripts/lib/verify-package-core.mjs",
PACKAGE_INSTALL_FIXTURE,
),
workspace.write("dist/process/OwnedCommand.js", OWNED_COMMAND_FIXTURE),
workspace.write(
"scripts/verify-wasm-artifact.mjs",
NESTED_VERIFIER_FIXTURE,
),
workspace.write("harness.mjs", HARNESS_FIXTURE),
]);
let supervisor: ProviderProcessSupervisor | undefined;
let child:
| Awaited<ReturnType<typeof spawnOwnedProviderProcess>>
| undefined;
try {
child = await spawnOwnedProviderProcess({
command: process.execPath,
arguments: [workspace.path("harness.mjs")],
cwd: workspace.root,
runId: `rea-packaged-wasm-test-${randomUUID()}`,
sweepTokenOwnedProcesses: true,
stdin: "pipe",
hostEnvironment: {
...process.env,
...workspace.environment,
TMPDIR: runtimeParent,
TMP: runtimeParent,
TEMP: runtimeParent,
OWNERSHIP_EVENTS: events,
OWNERSHIP_REPORT: report,
OWNERSHIP_ALLOW: allow,
OWNERSHIP_DENY: denied ? "1" : "0",
OWNERSHIP_HARNESS_ASSERT: harnessFailure ? "1" : "0",
},
});
supervisor = new ProviderProcessSupervisor(
{
...child,
ownsProcessLifetime: true,
cleanup:
child.cleanup ?? (() => cleanupOwnedProcessGroup(child!.ownership)),
},
{ maxDiagnosticBytes: 64 * 1024 },
);
if (harnessFailure) {
await expectHarnessFailureCleanup(supervisor, events);
return;
}
await waitForProviderProcessReady(child.process, 25_000);
const observed = JSON.parse(await readFile(report, "utf8"));
expect(observed.reason).toBe("timeout");
expect(observed.pids).toHaveLength(2);
expect(observed.runId).toMatch(/^rea-packaged-wabt-/);
expect(observed.descendantRunId).toBe(observed.runId);
expect(observed.rootExists).toBe(denied);
expect(observed.livePids.sort()).toEqual(
denied ? [...observed.pids].sort() : [],
);
if (denied) {
expect(observed.compositeOwner).toBe(true);
await writeFile(allow, "restore actual process cleanup");
child.process.stdin?.write("retry\n");
}
expect(await supervisor.waitForOutputClose(10_000)).toBe(true);
expect(supervisor.snapshot()).toMatchObject({
exitCode: 0,
signal: null,
});
const final = JSON.parse(await readFile(report, "utf8"));
if (denied)
expect(final.recovered).toEqual({ rootExists: false, livePids: [] });
const recorded = (await readFile(events, "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line));
expect(recorded.filter(({ kind }) => kind === "pack")).toEqual([
expect.objectContaining({
root: workspace.root,
workspace: observed.runtime,
}),
]);
expect(recorded.filter(({ kind }) => kind === "install")).toHaveLength(1);
expectRootCleanupAfterProcessExit(
recorded,
observed.runtime,
observed.pids,
);
expect(
recorded.filter(
({ kind, denied: refusal }) =>
kind === "cleanup" && refusal === false,
),
).toHaveLength(1);
} finally {
await writeFile(allow, "teardown restores cleanup");
if (child?.process.exitCode === null && child.process.signalCode === null)
child.process.stdin?.write("retry\n");
await supervisor?.waitForOutputClose(5_000);
if (supervisor !== undefined)
expect((await supervisor.stop()).status).not.toBe("incomplete");
await removeTestWorkspace(workspace.root);
}
},
45_000,
);
tests/boundary/process/providerVerifierCleanup.test.ts
+178
View File
@@ -0,0 +1,178 @@
import {
copyFile,
mkdir,
readFile,
readdir,
writeFile,
} from "node:fs/promises";
import { join } from "node:path";
import { expect } from "vitest";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { processTest } from "../../support/process/processFixture.js";
type Provider = "frida" | "jeb";
type Primary = "undefined" | "null" | "success";
const cases = (["frida", "jeb"] as const).flatMap((provider) =>
[
{ primary: "undefined", closeFails: false },
{ primary: "null", closeFails: false },
{ primary: "undefined", closeFails: true },
{ primary: "success", closeFails: true },
].map(({ primary, closeFails }) => ({
provider,
primary: primary as Primary,
closeFails,
})),
);
/** Copy the actual script; substitute only the engine-facing module boundary. */
const verifierFixture = async (provider: Provider) => {
const root = await createTestTempDirectory("rea-provider-verifier-cleanup-");
const temporary = join(root, "temporary");
await mkdir(temporary);
await mkdir(join(root, "scripts", "lib"), { recursive: true });
await mkdir(join(root, "dist", "application", provider), { recursive: true });
await writeFile(join(root, "package.json"), '{"type":"module"}\n');
const script = join(root, "scripts", `verify-${provider}.mjs`);
await copyFile(
new URL(`../../../scripts/verify-${provider}.mjs`, import.meta.url),
script,
);
await copyFile(
new URL("../../../scripts/lib/verifier-cleanup.mjs", import.meta.url),
join(root, "scripts", "lib", "verifier-cleanup.mjs"),
);
const closeBody = `
await appendFile(process.env.REA_TEST_VERIFIER_CLOSE_LOG, "${provider}\\n");
if (process.env.REA_TEST_VERIFIER_CLOSE_FAIL === "1")
throw new Error("controlled ${provider} cleanup denial");
`;
const primaryBody = `
if (process.env.REA_TEST_VERIFIER_PRIMARY === "undefined") throw undefined;
if (process.env.REA_TEST_VERIFIER_PRIMARY === "null") throw null;
`;
if (provider === "frida") {
await mkdir(join(root, "dist", "frida"));
await mkdir(join(root, "dist", "contracts"));
await writeFile(
join(root, "dist", "frida", "FridaInstrumentationManager.js"),
`import { appendFile } from "node:fs/promises";
export class FridaInstrumentationManager {
async closeAll() { ${closeBody} }
}
`,
);
await writeFile(
join(
root,
"dist",
"application",
"frida",
"FridaInstrumentationService.js",
),
`export class FridaInstrumentationService {
async instrument() {
${primaryBody}
return { ok: true, value: {
cleanupError: null,
evidence: { normalized_result: {
pid: 77,
messages: [{ type: "send", payload: { rea_frida_verification: "ok", pid: 77 } }],
} },
} };
}
}
`,
);
await writeFile(
join(root, "dist", "contracts", "fridaRemoteAddress.js"),
"export const isValidFridaRemoteAddress = () => false;\n",
);
} else {
await mkdir(join(root, "dist", "composition"));
await writeFile(
join(root, "dist", "composition", "jeb.js"),
`import { appendFile } from "node:fs/promises";
export const createJebAnalysisProvider = () => ({
async close() { ${closeBody} },
});
`,
);
await writeFile(
join(root, "dist", "application", "jeb", "JebAnalysisService.js"),
`export class JebAnalysisService {
async execute(operation) {
${primaryBody}
const result = {
inspect_jeb_client: { engine: { version: "simulated", gui_client: false, endpoint: "simulated" } },
open_jeb_project: { input_files: [{ contents_sha256_hash: "${"a".repeat(64)}" }] },
list_jeb_units: { units: [{ id: "simulated-unit" }] },
decompile_jeb_item: { text: "simulated marker" },
}[operation];
if (result === undefined) throw new Error("Unexpected simulated operation: " + operation);
return { ok: true, value: { normalized_result: result } };
}
}
`,
);
}
return { root, script, temporary, closeLog: join(root, "close.log") };
};
// These are simulated provider outcomes exercising real verifier orchestration;
// they establish no Frida instrumentation or JEB engine behavior.
processTest.for(cases)(
"$provider verifier preserves primary=$primary with closeFails=$closeFails",
async ({ provider, primary, closeFails }, { processes }) => {
const fixture = await verifierFixture(provider);
const result = await processes.run(
process.execPath,
[
fixture.script,
...(provider === "jeb"
? ["--item-address", "simulated-item", "--marker", "simulated marker"]
: []),
],
{
cwd: fixture.root,
env: {
...process.env,
TMPDIR: fixture.temporary,
TEMP: fixture.temporary,
TMP: fixture.temporary,
REA_TEST_VERIFIER_PRIMARY: primary,
REA_TEST_VERIFIER_CLOSE_FAIL: closeFails ? "1" : "0",
REA_TEST_VERIFIER_CLOSE_LOG: fixture.closeLog,
},
timeoutMs: 10_000,
},
);
expect(result.signal, result.stderr).toBeNull();
expect(result.exitCode, result.stderr).toBe(1);
const reasons = [
...(primary === "success" ? [] : [primary]),
...(closeFails ? [`controlled ${provider} cleanup denial`] : []),
];
const label = provider === "frida" ? "Frida" : "JEB";
const retained = await readdir(fixture.temporary);
let retainedMessage = "";
if (provider === "jeb" && closeFails) {
expect(retained).toHaveLength(1);
const name = retained[0];
if (name === undefined) throw new Error("Missing retained JEB fixture");
expect(name).toMatch(/^rea-jeb-/u);
retainedMessage = `JEB fixture retained at ${join(fixture.temporary, name)}\n`;
} else expect(retained).toEqual([]);
expect(result.stderr).toBe(
`${retainedMessage}${label} real-provider verification failed: ${reasons.join("; ")}\n`,
);
expect(result.stdout).toBe("");
expect(await readFile(fixture.closeLog, "utf8")).toBe(`${provider}\n`);
},
);
tests/boundary/process/verifierCleanup.test.ts
+146 -22
View File
@@ -1,5 +1,8 @@
import { expect, it } from "vitest";
import { closeVerifierResources } from "../../../scripts/lib/verifier-cleanup.mjs";
import {
closeVerifierResources,
withVerifierCleanup,
} from "../../../scripts/lib/verifier-cleanup.mjs";
it("attempts every cleanup and reports failures when there is no primary error", async () => {
const attempted: string[] = [];
@@ -42,11 +45,14 @@ it("retains primary verification and independent cleanup failures", async () =>
let failure: unknown;
try {
await closeVerifierResources(tasks, primary);
await withVerifierCleanup(() => {
throw primary;
}, tasks);
} catch (cause) {
failure = cause;
}
expect(failure).toBeInstanceOf(AggregateError);
expect((failure as AggregateError).cause).toBe(primary);
expect((failure as AggregateError).errors).toEqual([
primary,
expect.objectContaining({ message: "provider close failed" }),
@@ -61,33 +67,151 @@ it("keeps a parent fixture when any owned child cleanup remains incomplete", asy
let parentRemoved = false;
let failure: unknown;
try {
await closeVerifierResources(
[
async () => {
await closeVerifierResources([
async () => {
attempted.push("client");
},
async () => {
attempted.push("transport");
throw new Error("transport close failed");
},
async () => {
attempted.push("file-scope");
},
]);
parentRemoved = true;
},
],
primary,
);
await withVerifierCleanup(() => {
throw primary;
}, [
async () => {
await closeVerifierResources([
async () => {
attempted.push("client");
},
async () => {
attempted.push("transport");
throw new Error("transport close failed");
},
async () => {
attempted.push("file-scope");
},
]);
parentRemoved = true;
},
]);
} catch (cause) {
failure = cause;
}
expect(failure).toBeInstanceOf(AggregateError);
expect((failure as AggregateError).errors[0]).toBe(primary);
expect((failure as AggregateError).cause).toBe(primary);
expect((failure as AggregateError).errors[1]).toBeInstanceOf(AggregateError);
expect(attempted).toEqual(["client", "transport", "file-scope"]);
expect(parentRemoved).toBe(false);
});
it.each([undefined, null, new Error("exact operation failure")])(
"preserves an exact operation rejection after successful cleanup: %s",
async (primary) => {
const attempted: string[] = [];
await expect(
withVerifierCleanup(() => {
attempted.push("operation");
throw primary;
}, [
() => {
attempted.push("cleanup");
},
]),
).rejects.toBe(primary);
expect(attempted).toEqual(["operation", "cleanup"]);
},
);
it.each([undefined, null])(
"retains an opaque operation rejection with its independent cleanup failure: %s",
async (primary) => {
const cleanup = new Error("independent cleanup failure");
const outcome = await withVerifierCleanup(async () => {
throw primary;
}, [
() => {
throw cleanup;
},
]).then(
(value) => ({ kind: "fulfilled" as const, value }),
(reason: unknown) => ({ kind: "rejected" as const, reason }),
);
if (outcome.kind !== "rejected")
throw new Error("Expected both operation and cleanup failures");
expect(outcome.reason).toBeInstanceOf(AggregateError);
const error = outcome.reason as AggregateError;
expect(error.errors).toEqual([primary, cleanup]);
expect(error.errors[1]).toBe(cleanup);
expect(Object.hasOwn(error, "cause")).toBe(true);
expect(error.cause).toBe(primary);
},
);
it.each([undefined, { observation: "retained value" }])(
"returns the exact operation value after cleanup: %s",
async (value) => {
const attempted: string[] = [];
const result = await withVerifierCleanup(() => {
attempted.push("operation");
return value;
}, [
async () => {
attempted.push("cleanup");
},
]);
expect(result).toBe(value);
expect(attempted).toEqual(["operation", "cleanup"]);
},
);
it("does not repeat cleanup or invent a primary when cleanup alone fails", async () => {
const attempted: string[] = [];
const cleanup = new Error("cleanup after successful observation failed");
const outcome = await withVerifierCleanup(() => {
attempted.push("operation");
return "completed";
}, [
async () => {
attempted.push("provider");
throw cleanup;
},
() => {
attempted.push("fixture");
throw undefined;
},
() => {
attempted.push("independent resource");
},
]).catch((reason: unknown) => reason);
expect(outcome).toBeInstanceOf(AggregateError);
expect((outcome as AggregateError).errors).toEqual([cleanup, undefined]);
expect((outcome as AggregateError).errors[0]).toBe(cleanup);
expect(Object.hasOwn(outcome as AggregateError, "cause")).toBe(false);
expect(attempted).toEqual([
"operation",
"provider",
"fixture",
"independent resource",
]);
});
it("preserves nested aggregates and independent failures with the same identity", async () => {
const shared = new Error("same reason from independent owners");
const primary = new AggregateError([shared], "nested operation failure");
const nested = new AggregateError([undefined, shared], "nested cleanup");
const outcome = await withVerifierCleanup(() => {
throw primary;
}, [
() => {
throw nested;
},
() => {
throw shared;
},
() => {
throw shared;
},
]).catch((reason: unknown) => reason);
expect(outcome).toBeInstanceOf(AggregateError);
const error = outcome as AggregateError;
expect(error.cause).toBe(primary);
expect(error.errors).toEqual([primary, nested, shared, shared]);
expect(error.errors[0]).toBe(primary);
expect(error.errors[1]).toBe(nested);
expect(error.errors[2]).toBe(shared);
expect(error.errors[3]).toBe(shared);
});
tests/boundary/process/verifierFileResources.test.ts
+86
View File
@@ -11,6 +11,92 @@ import type { validateVerifierJadxConfiguration } from "../../../scripts/lib/jad
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { writeJadxJarInventory } from "../../fixtures/android/jadxJar.js";
it("retains an undefined verifier failure alongside real descriptor cleanup and retries its exact owner", async () => {
const directory = await createTestTempDirectory("rea-verifier-undefined-");
const path = join(directory, "retained.bin");
const peerPath = join(directory, "released.bin");
await writeFile(path, "retained verifier input");
await writeFile(peerPath, "independent verifier input");
const script = String.raw`
import assert from 'node:assert/strict';
import { open } from 'node:fs/promises';
import { withVerifierResourceOwner } from ${JSON.stringify(new URL("../../../scripts/lib/verifier-file-resources.mjs", import.meta.url).href)};
import { ArtifactResourceRetryFailure } from ${JSON.stringify(new URL("../../../dist/artifacts/ArtifactResourceScope.js", import.meta.url).href)};
const [path, peerPath] = process.argv.slice(1);
const retained = await open(path, 'r');
const closeRetained = retained.close.bind(retained);
let peer;
let closePeer;
let denied = true;
let retainedAttempts = 0;
let peerAttempts = 0;
const cleanupFailure = new Error('controlled retained descriptor close denial');
try {
peer = await open(peerPath, 'r');
closePeer = peer.close.bind(peer);
retained.close = async () => {
retainedAttempts++;
if (denied) throw cleanupFailure;
await closeRetained();
};
peer.close = async () => {
peerAttempts++;
await closePeer();
};
const descriptor = retained.fd;
const outcome = await withVerifierResourceOwner(async (owner) => {
owner.retain(retained, path);
owner.retain(peer, peerPath);
assert.equal(await retained.readFile('utf8'), 'retained verifier input');
assert.equal(await peer.readFile('utf8'), 'independent verifier input');
throw undefined;
}).then(
value => ({ status: 'fulfilled', value }),
reason => ({ status: 'rejected', reason }),
);
assert.equal(outcome.status, 'rejected');
const error = outcome.reason;
assert.ok(error instanceof AggregateError);
assert.equal(error.errors.length, 2, 'Undefined primary and descriptor cleanup must both survive');
assert.equal(error.errors[0], undefined);
assert.ok(Object.hasOwn(error, 'cause'));
assert.equal(error.cause, undefined);
const cleanup = error.errors[1];
assert.ok(cleanup instanceof ArtifactResourceRetryFailure);
assert.deepEqual(cleanup.cleanup.resources, [path]);
assert.ok(cleanup.cause instanceof AggregateError);
assert.deepEqual(cleanup.cause.errors, [cleanupFailure]);
assert.equal(retained.fd, descriptor);
assert.ok(descriptor >= 0);
assert.equal(peer.fd, -1);
assert.equal(retainedAttempts, 1);
assert.equal(peerAttempts, 1);
assert.equal(cleanup.cleanupLeases.length, 1);
denied = false;
for (const lease of cleanup.cleanupLeases) await lease.release();
assert.equal(retained.fd, -1);
assert.equal(retainedAttempts, 2);
assert.equal(peerAttempts, 1);
console.log(JSON.stringify({ primaryRetained: true, descriptorRetried: true, peerClosedOnce: true }));
} finally {
denied = false;
await closeRetained();
await closePeer?.();
}
`;
const result = spawnSync(
process.execPath,
["--input-type=module", "-e", script, path, peerPath],
{ encoding: "utf8", timeout: 10_000 },
);
expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0);
expect(JSON.parse(result.stdout.trim())).toEqual({
primaryRetained: true,
descriptorRetried: true,
peerClosedOnce: true,
});
});
it("retains and retries the exact descriptor after a verifier hash close failure", async () => {
const directory = await createTestTempDirectory("rea-verifier-file-");
const path = join(directory, "input.bin");
tests/fixtures/artifactMcpVerifierProbe.mjs
Vendored
+3 -12
View File
@@ -4,7 +4,7 @@ import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import * as artifact from "../../scripts/lib/artifact-e2e.mjs";
import { closeVerifierResources } from "../../scripts/lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs";
const [mode, target] = process.argv.slice(2);
assert.ok(
@@ -138,8 +138,7 @@ async function standaloneConnections() {
},
stderr: "pipe",
});
let failure;
try {
await withVerifierCleanup(async () => {
await client.connect(transport);
const version = client.getServerVersion();
assert.ok(version);
@@ -152,15 +151,7 @@ async function standaloneConnections() {
if (mode === "concurrent")
await Promise.all([status(client), status(client)]);
else await status(client);
} catch (cause) {
failure = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => client.close(), () => transport.close()],
failure,
);
}
}, [() => client.close(), () => transport.close()]);
}
}