mirror/rea
1
0
Fork 0
mirror of https://github.com/morluto/rea synced 2026-10-25 22:08:11 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(verifiers): centralize failure and cleanup ownership

Browse Source 
Own operation results and exact rejections in withVerifierCleanup, including
undefined, null and independent cleanup failures. Remove the optional-primary
cleanup contract and migrate every caller, preserving returns, annotations,
signal restoration and dependent cleanup order.

Make Frida/JEB reporters fail on reasonless rejections and retain JEB fixture
files while provider cleanup is incomplete. Own the nested packaged WASM
verifier and expose its composite process/root cleanup capability for retry.
Update runnable docs and isolated fixture dependencies.

Validation: 547 focused cases in 20 files, eight isolated boundary unit checks,
static/build/docs/discovery gates and duplication check (3.58%). Node process,
SDK and descriptor cleanup are exercised; provider and packaging results in
orchestration tests are simulated. Pack/install subprocess ownership remains
with the existing shared package runner outside the nested-command change.
This commit is contained in:
morluto
2026-10-24 15:38:35 +08:00
parent 98ec00f406
commit 72ba1cd92b
48 changed files with 2380 additions and 1818 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/lib/artifact-e2e.mjs
+9 -13
View File
@@ -4,7 +4,7 @@ import { promisify } from "node:util";
import { fileURLToPath } from "node:url";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
// REA registers process-static tool contracts. Bind their parsed definitions to
// the SDK's negotiated connection identity, including when a Client reconnects.
@@ -127,8 +127,7 @@ const withOpenedArtifactMcp = async (target, openTool, verify) => {
stderr: "pipe",
});
const client = new Client({ name: "artifact-real-e2e", version: "1" });
let failure;
try {
return withVerifierCleanup(async () => {
await client.connect(transport);
const opened = await artifactMcpCall(client, {
name: openTool,
@@ -138,16 +137,13 @@ const withOpenedArtifactMcp = async (target, openTool, verify) => {
if (openTool === "open_bundle_resources")
assert.equal(opened.structuredContent?.result?.caller_path, target);
await verify(client);
} catch (cause) {
failure = cause;
throw cause;
} finally {
toolBindings.delete(client);
await closeVerifierResources(
[() => client.close(), () => transport.close()],
failure,
);
}
}, [
() => {
toolBindings.delete(client);
},
() => client.close(),
() => transport.close(),
]);
};
/** Run a real stdio MCP subprocess with an explicitly opened binary target. */
scripts/lib/browser-module-e2e.mjs
+13 -19
View File
@@ -14,7 +14,7 @@ import { webModuleTraceResultSchema } from "../../dist/domain/webModuleTrace.js"
import { NativeModuleResolver } from "../../dist/browser/modules/NativeModuleResolver.js";
import { startBrowserModuleSite } from "../fixtures/browser-module-site.mjs";
import { mcpTextValue } from "./mcp-verifier-results.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
/** Check native URL resolution against an independently loaded module graph through CLI/MCP. */
export async function verifyBrowserModules(
@@ -25,8 +25,7 @@ export async function verifyBrowserModules(
let root;
let transport;
let client;
let primaryError;
try {
return withVerifierCleanup(async () => {
site = await startBrowserModuleSite();
root = await mkdtemp(join(tmpdir(), "rea-browser-modules-"));
const env = {
@@ -304,18 +303,15 @@ export async function verifyBrowserModules(
map_identity_verified: true,
implicit_refetches: 0,
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
const cleanup = [];
if (client !== undefined) cleanup.push(() => client.close());
if (transport !== undefined) cleanup.push(() => transport.close());
if (site !== undefined) cleanup.push(() => site.close());
if (root !== undefined)
cleanup.push(() => rm(root, { recursive: true, force: true }));
await closeVerifierResources(cleanup, primaryError);
}
}, [
() => client?.close(),
() => transport?.close(),
() => site?.close(),
() =>
root === undefined
? undefined
: rm(root, { recursive: true, force: true }),
]);
}
const assertTrace = (evidence, oracle, site, scriptIndex) => {
@@ -366,7 +362,7 @@ const loadingOracle = async (executable, site) => {
: []),
],
});
try {
return withVerifierCleanup(async () => {
const page = await browser.newPage();
await page.goto(site.origin);
await page.waitForFunction(
@@ -394,7 +390,5 @@ const loadingOracle = async (executable, site) => {
),
);
return proof;
} finally {
await browser.close();
}
}, [() => browser.close()]);
};
scripts/lib/browser-nested-source-map-e2e.mjs
+126 -123
View File
@@ -11,7 +11,7 @@ import { parseEvidence } from "../../dist/domain/evidence.js";
import { traceSourceMap } from "../../dist/javascript/sourceMaps/TraceSourceMap.js";
import { startBrowserSourceMapSite } from "../fixtures/browser-source-map-site.mjs";
import { startRuntimeFixtureBrowser } from "./browser-runtime-fixture-lifecycle.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
const cases = [
{ name: "same-line nested column control", prefixes: ["\n\n ", " "] },
@@ -59,134 +59,137 @@ async function verifyCase(executable, entrypoint, scenario) {
});
let fixture;
let connection;
let primaryError;
let phase = "native-map-admission";
let browserVersion;
let expected;
let observed;
try {
const mapUrl = `${site.origin}/maps/app.js.map`;
// This independent existing decoder already composes valid nested offsets.
// It is a before-product admission control, not the expected browser oracle.
const point = traceSourceMap(site.map, mapUrl, site.position);
assert.equal(point.matches.length, 1);
assert.equal(
point.matches[0].original_position.line,
site.originalPosition.line,
);
assert.equal(
point.matches[0].original_position.column,
site.originalPosition.column,
);
expected = compilerMappingsAtPhysicalPositions(site, mapUrl);
assert.ok(
expected.length > 0,
"Native compiler leaf must carry point mappings",
);
return withVerifierCleanup(async () => {
try {
const mapUrl = `${site.origin}/maps/app.js.map`;
// This independent existing decoder already composes valid nested offsets.
// It is a before-product admission control, not the expected browser oracle.
const point = traceSourceMap(site.map, mapUrl, site.position);
assert.equal(point.matches.length, 1);
assert.equal(
point.matches[0].original_position.line,
site.originalPosition.line,
);
assert.equal(
point.matches[0].original_position.column,
site.originalPosition.column,
);
expected = compilerMappingsAtPhysicalPositions(site, mapUrl);
assert.ok(
expected.length > 0,
"Native compiler leaf must carry point mappings",
);
phase = "owned-browser-startup";
fixture = await startRuntimeFixtureBrowser(executable, site.origin);
const endpoint = `http://127.0.0.1:${fixture.port}`;
const provider = new CdpBrowserProvider();
const targets = await provider.listTargets(
listBrowserTargetsInputSchema.parse({
cdp_endpoint: endpoint,
allowed_origins: [site.origin],
}),
);
if (!targets.ok) throw targets.error;
const target = targets.value.targets[0];
assert.ok(target, "Owned fixture target missing");
const pages = await (await fetch(`${endpoint}/json/list`)).json();
const page = pages.find(({ id }) => id === target.target_id);
assert.ok(page, "Owned CDP fixture page missing");
connection = await CdpConnection.connect(
page.webSocketDebuggerUrl,
"inspect_web_page",
);
await ready(connection);
browserVersion = await connection.send("Browser.getVersion");
phase = "public-cli";
const { stdout } = await promisify(execFile)(
process.execPath,
[
entrypoint,
"analyze-web-bundle",
endpoint,
target.target_id,
"--allowed-origins",
site.origin,
"--fetch-source-maps",
"--observation-ms",
"0",
"--json",
],
{
env: { ...process.env, REA_LOG_LEVEL: "silent" },
timeout: 45_000,
maxBuffer: 16 * 1024 * 1024,
},
);
const evidence = parseEvidence(JSON.parse(stdout));
assert.equal(evidence.operation, "analyze_web_bundle");
const maps = evidence.normalized_result.observations.source_maps;
const item = maps.items.find(({ artifact }) => artifact?.text === site.map);
assert.ok(
item,
"Public browser analysis omitted the exact compiler-derived map",
);
phase = "public-map-assertions";
observed = item?.mappings;
assert.equal(item.status, "included");
assert.equal(item.mappings.length, expected.length);
for (const [index, coordinates] of expected.entries()) {
const mapping = item.mappings[index];
for (const [key, value] of Object.entries(coordinates))
assert.deepEqual(mapping[key], value);
// This compiler fixture has one source declaration in one wrapped leaf.
assert.equal(mapping.source_index, 0);
assert.equal(mapping.declaration_index, 0);
phase = "owned-browser-startup";
fixture = await startRuntimeFixtureBrowser(executable, site.origin);
const endpoint = `http://127.0.0.1:${fixture.port}`;
const provider = new CdpBrowserProvider();
const targets = await provider.listTargets(
listBrowserTargetsInputSchema.parse({
cdp_endpoint: endpoint,
allowed_origins: [site.origin],
}),
);
if (!targets.ok) throw targets.error;
const target = targets.value.targets[0];
assert.ok(target, "Owned fixture target missing");
const pages = await (await fetch(`${endpoint}/json/list`)).json();
const page = pages.find(({ id }) => id === target.target_id);
assert.ok(page, "Owned CDP fixture page missing");
connection = await CdpConnection.connect(
page.webSocketDebuggerUrl,
"inspect_web_page",
);
await ready(connection);
browserVersion = await connection.send("Browser.getVersion");
phase = "public-cli";
const { stdout } = await promisify(execFile)(
process.execPath,
[
entrypoint,
"analyze-web-bundle",
endpoint,
target.target_id,
"--allowed-origins",
site.origin,
"--fetch-source-maps",
"--observation-ms",
"0",
"--json",
],
{
env: { ...process.env, REA_LOG_LEVEL: "silent" },
timeout: 45_000,
maxBuffer: 16 * 1024 * 1024,
},
);
const evidence = parseEvidence(JSON.parse(stdout));
assert.equal(evidence.operation, "analyze_web_bundle");
const maps = evidence.normalized_result.observations.source_maps;
const item = maps.items.find(
({ artifact }) => artifact?.text === site.map,
);
assert.ok(
item,
"Public browser analysis omitted the exact compiler-derived map",
);
phase = "public-map-assertions";
observed = item?.mappings;
assert.equal(item.status, "included");
assert.equal(item.mappings.length, expected.length);
for (const [index, coordinates] of expected.entries()) {
const mapping = item.mappings[index];
for (const [key, value] of Object.entries(coordinates))
assert.deepEqual(mapping[key], value);
// This compiler fixture has one source declaration in one wrapped leaf.
assert.equal(mapping.source_index, 0);
assert.equal(mapping.declaration_index, 0);
}
assert.equal(item.original_sources.length, 1);
assert.equal(item.artifact.sha256, sha(site.map));
assert.equal(item.artifact.bytes, Buffer.byteLength(site.map));
assert.equal(
item.original_sources[0].artifact.sha256,
sha(site.original),
);
assert.ok(
site.mapRequests() > 0,
"Actual browser workflow did not fetch the declared map",
);
return {
compiler: site.compiler,
browser: browserVersion.product,
generated_sha256: sha(site.generated),
source_map_sha256: sha(site.map),
original_sha256: sha(site.original),
mappings: expected.length,
native_decoder_control: true,
};
} catch (cause) {
const failure = new Error(
`Nested source-map ${phase}: ${String(cause)}`,
{
cause,
},
);
failure.qualificationPhase = phase;
failure.qualificationReceipt = {
compiler: site.compiler,
browser: browserVersion?.product ?? null,
generated_sha256: sha(site.generated),
leaf_map_sha256: sha(site.leafMap),
source_map_sha256: sha(site.map),
original_sha256: sha(site.original),
expected_first_mapping: expected?.[0] ?? null,
observed_first_mapping: observed?.[0] ?? null,
};
throw failure;
}
assert.equal(item.original_sources.length, 1);
assert.equal(item.artifact.sha256, sha(site.map));
assert.equal(item.artifact.bytes, Buffer.byteLength(site.map));
assert.equal(item.original_sources[0].artifact.sha256, sha(site.original));
assert.ok(
site.mapRequests() > 0,
"Actual browser workflow did not fetch the declared map",
);
return {
compiler: site.compiler,
browser: browserVersion.product,
generated_sha256: sha(site.generated),
source_map_sha256: sha(site.map),
original_sha256: sha(site.original),
mappings: expected.length,
native_decoder_control: true,
};
} catch (cause) {
const failure = new Error(`Nested source-map ${phase}: ${String(cause)}`, {
cause,
});
failure.qualificationPhase = phase;
failure.qualificationReceipt = {
compiler: site.compiler,
browser: browserVersion?.product ?? null,
generated_sha256: sha(site.generated),
leaf_map_sha256: sha(site.leafMap),
source_map_sha256: sha(site.map),
original_sha256: sha(site.original),
expected_first_mapping: expected?.[0] ?? null,
observed_first_mapping: observed?.[0] ?? null,
};
primaryError = failure;
throw failure;
} finally {
await closeVerifierResources(
[() => connection?.close(), () => fixture?.close(), () => site.close()],
primaryError,
);
}
}, [() => connection?.close(), () => fixture?.close(), () => site.close()]);
}
function compilerMappingsAtPhysicalPositions(site, mapUrl) {
scripts/lib/browser-network-e2e.mjs
+3 -13
View File
@@ -14,7 +14,7 @@ import {
} from "../fixtures/browser-network-site.mjs";
import { assertBrowserNetworkEvidence } from "./browser-network-assertions.mjs";
import { scenarioProfiles } from "./browser-scenario-verifier.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
/** Exercise selected network content through real CLI and stdio MCP, one browser at a time. */
export async function verifyBrowserNetworkEvidence(
@@ -26,8 +26,7 @@ export async function verifyBrowserNetworkEvidence(
let profilesBefore;
let transport;
let client;
let primaryError;
try {
return withVerifierCleanup(async () => {
site = await startBrowserNetworkSite();
profilesBefore = await scenarioProfiles();
const scenario = browserScenarioSchema.parse({
@@ -178,14 +177,5 @@ export async function verifyBrowserNetworkEvidence(
profile_cleanup: true,
connected_network_verified: attachment !== undefined,
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
const cleanup = [];
if (client !== undefined) cleanup.push(() => client.close());
if (transport !== undefined) cleanup.push(() => transport.close());
if (site !== undefined) cleanup.push(() => site.close());
await closeVerifierResources(cleanup, primaryError);
}
}, [() => client?.close(), () => transport?.close(), () => site?.close()]);
}
scripts/lib/browser-runtime-e2e.mjs
+8 -17
View File
@@ -17,7 +17,7 @@ import { startBrowserRuntimeSite } from "../fixtures/browser-runtime-site.mjs";
import { requireOrdinaryWindowsVerifierCaller } from "./windows-ordinary-user-verifier.mjs";
import { mcpTextValue } from "./mcp-verifier-results.mjs";
import { startRuntimeFixtureBrowser } from "./browser-runtime-fixture-lifecycle.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
import { verifyRuntimeTargetTermination } from "./browser-runtime-termination-e2e.mjs";
import { verifyRuntimeInvalidSelectors } from "./browser-runtime-selector-e2e.mjs";
import {
@@ -40,10 +40,9 @@ export async function verifyBrowserRuntime(
await access(executable);
const site = await startBrowserRuntimeSite();
let fixture;
let primaryError;
let action;
let client;
try {
return withVerifierCleanup(async () => {
fixture = await startRuntimeFixtureBrowser(executable, site.origin);
const endpoint = `http://127.0.0.1:${fixture.port}`;
const targets = await new CdpBrowserProvider().listTargets(
@@ -176,20 +175,12 @@ export async function verifyBrowserRuntime(
fresh_mcp: freshMcpProof,
page_remained_open: true,
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
() => client?.close(),
() => action?.close(),
() => fixture?.close(),
() => site.close(),
],
primaryError,
);
}
}, [
() => client?.close(),
() => action?.close(),
() => fixture?.close(),
() => site.close(),
]);
}
async function executionMcp(
scripts/lib/browser-runtime-termination-e2e.mjs
+9 -18
View File
@@ -1,7 +1,7 @@
import assert from "node:assert/strict";
import { CdpConnection } from "../../dist/browser/CdpConnection.js";
import { mcpTextValue, requireMcpToolError } from "./mcp-verifier-results.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
/** Close a separately owned fixture page after actual MCP arming and inspect native cleanup diagnostics. */
export async function verifyRuntimeTargetTermination(client, input, origin) {
@@ -14,8 +14,7 @@ export async function verifyRuntimeTargetTermination(client, input, origin) {
);
let targetId;
let closed = false;
let primaryError;
try {
return withVerifierCleanup(async () => {
({ targetId } = await root.send("Target.createTarget", { url: origin }));
assert.equal(typeof targetId, "string");
await waitForTarget(input.cdp_endpoint, targetId, origin);
@@ -81,21 +80,13 @@ export async function verifyRuntimeTargetTermination(client, input, origin) {
cleanup: "unconfirmed_reported",
observed_window_end_reason: "session_detached",
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
() =>
targetId === undefined || closed
? undefined
: root.send("Target.closeTarget", { targetId }),
() => root.close(),
],
primaryError,
);
}
}, [
() =>
targetId === undefined || closed
? undefined
: root.send("Target.closeTarget", { targetId }),
() => root.close(),
]);
}
async function waitForTarget(endpoint, targetId, origin) {
scripts/lib/browser-scenario-environment-e2e.mjs
+5 -17
View File
@@ -11,7 +11,7 @@ import { browserScenarioSchema } from "../../dist/domain/browserScenario.js";
import { AnalysisCancelledError } from "../../dist/domain/analysisErrorCore.js";
import { runScenarioCli } from "./browser-scenario-verifier.mjs";
import { requireMcpEvidenceResult } from "./mcp-verifier-results.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
const expected = {
timezone: "Pacific/Honolulu",
@@ -140,8 +140,7 @@ export async function verifyScenarioEnvironment(endpoint, targetId, origin) {
return JSON.parse(value.result.value);
};
const provider = createBrowserScenarioProvider(process.env);
let verificationError;
try {
return withVerifierCleanup(async () => {
const baseline = await readEnvironment();
const callerLanguage = await readEnvironment(languageExpression);
const scenario = environmentScenario({
@@ -173,7 +172,7 @@ export async function verifyScenarioEnvironment(endpoint, targetId, origin) {
name: "browser-environment-e2e",
version: "1",
});
try {
await withVerifierCleanup(async () => {
await client.connect(transport);
const definition = (await client.listTools()).tools.find(
({ name }) => name === "capture_browser_scenario",
@@ -197,10 +196,7 @@ export async function verifyScenarioEnvironment(endpoint, targetId, origin) {
origin,
assertRestored,
});
} finally {
await client.close();
await transport.close();
}
}, [() => client.close(), () => transport.close()]);
await assertRestored();
const initializationFailure = await provider.captureScenario({
...scenario,
@@ -240,15 +236,7 @@ export async function verifyScenarioEnvironment(endpoint, targetId, origin) {
cancellation_cleanup: true,
external_target_preserved: true,
};
} catch (cause) {
verificationError = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => provider.close(), () => connection.close()],
verificationError,
);
}
}, [() => provider.close(), () => connection.close()]);
}
const verifyConcurrentMcpCapture = async ({
scripts/lib/browser-scenario-storage-e2e.mjs
+102 -114
View File
@@ -10,7 +10,7 @@ import {
runScenarioCli,
scenarioProfiles,
} from "./browser-scenario-verifier.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
const scenarioFor = (browser, origin) =>
browserScenarioSchema.parse({
@@ -203,7 +203,7 @@ export async function verifyScenarioStorage({
targetId,
}) {
const site = await startStorageSite();
try {
return withVerifierCleanup(async () => {
await verifyLaunchStorage(executable, site.origin);
await verifyConnectedStorage(endpoint, targetId, site.origin);
return {
@@ -217,9 +217,7 @@ export async function verifyScenarioStorage({
externalTargetSurvives: true,
cancellationCleanup: true,
};
} finally {
await site.close();
}
}, [() => site.close()]);
}
const verifyLaunchStorage = async (executable, origin) => {
@@ -300,8 +298,7 @@ const verifyConnectedStorage = async (endpoint, targetId, origin) => {
target.webSocketDebuggerUrl,
"capture_browser_scenario",
);
let failure;
try {
return withVerifierCleanup(async () => {
await assertStorageKeyPartition(observer, origin);
const result = await runScenarioCli(scenario);
const capture = result.normalized_result;
@@ -315,15 +312,7 @@ const verifyConnectedStorage = async (endpoint, targetId, origin) => {
assert.equal(capture.browser.cleanup, "disconnected-external");
await verifyCallerDebuggerPause(observer, origin, scenario);
await verifyCancelledStorage(observer, endpoint, targetId, scenario);
} catch (cause) {
failure = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => restoreTarget(observer, target.url), () => observer.close()],
failure,
);
}
}, [() => restoreTarget(observer, target.url), () => observer.close()]);
};
const assertStorageKeyPartition = async (connection, origin) => {
@@ -349,7 +338,7 @@ const assertStorageKeyPartition = async (connection, origin) => {
waitForDebuggerOnStart: false,
flatten: true,
});
try {
return withVerifierCleanup(async () => {
const start = await connection.send("Runtime.evaluate", {
expression: "performance.timeOrigin",
returnByValue: true,
@@ -412,14 +401,15 @@ const assertStorageKeyPartition = async (connection, origin) => {
topLevel.storageKey,
`Chrome did not expose distinct partition keys: child=${JSON.stringify(child)} top=${JSON.stringify(topLevel)} rootBefore=${JSON.stringify(beforeRoot)}`,
);
} finally {
unsubscribe();
await connection.send("Target.setAutoAttach", {
autoAttach: false,
waitForDebuggerOnStart: false,
flatten: true,
});
}
}, [
() => unsubscribe(),
() =>
connection.send("Target.setAutoAttach", {
autoAttach: false,
waitForDebuggerOnStart: false,
flatten: true,
}),
]);
};
const waitForPagePath = async (connection, path, priorTimeOrigin) => {
@@ -515,88 +505,90 @@ const verifyCallerDebuggerPause = async (caller, origin, scenario) => {
settled = true;
unsubscribe();
});
try {
const callFrameId = await raceWithTimeout(paused, 10_000, () => {
throw new Error(
`Caller debugger pause was not observed at ${debuggerUrl}; settled=${String(settled)} events=${JSON.stringify(protocolTrace)}`,
return withVerifierCleanup(async () => {
try {
const callFrameId = await raceWithTimeout(paused, 10_000, () => {
throw new Error(
`Caller debugger pause was not observed at ${debuggerUrl}; settled=${String(settled)} events=${JSON.stringify(protocolTrace)}`,
);
});
const pausedState = await caller.send("Debugger.evaluateOnCallFrame", {
callFrameId,
expression: "window.reaDebuggerAfter === true",
returnByValue: true,
});
assert.equal(
pausedState.result.value,
false,
"Caller-owned debugger frame resumed before the caller inspected it",
);
});
const pausedState = await caller.send("Debugger.evaluateOnCallFrame", {
callFrameId,
expression: "window.reaDebuggerAfter === true",
returnByValue: true,
});
assert.equal(
pausedState.result.value,
false,
"Caller-owned debugger frame resumed before the caller inspected it",
);
await delay(100);
assert.equal(
settled,
false,
"Storage observer resumed a caller-owned debugger pause",
);
await caller.send("Debugger.resume");
pauseObserved = false;
const result = await pending;
if (!result.ok) throw result.error;
const debuggerState = await caller.send("Runtime.evaluate", {
expression: "window.reaDebuggerAfter === true",
returnByValue: true,
});
assert.equal(debuggerState.result.value, true);
} catch (cause) {
if (!pauseObserved) controller.abort(cause);
else await caller.send("Debugger.resume").catch(() => undefined);
const completion = await raceWithTimeout(
pending.catch((pendingCause) => ({ ok: false, error: pendingCause })),
5_000,
);
if (completion === undefined)
throw new Error("Capture remained pending after cancellation", { cause });
const completionError =
typeof completion === "object" &&
completion !== null &&
"error" in completion
? completion.error
: typeof completion === "object" &&
completion !== null &&
"ok" in completion &&
completion.ok === false
await delay(100);
assert.equal(
settled,
false,
"Storage observer resumed a caller-owned debugger pause",
);
await caller.send("Debugger.resume");
pauseObserved = false;
const result = await pending;
if (!result.ok) throw result.error;
const debuggerState = await caller.send("Runtime.evaluate", {
expression: "window.reaDebuggerAfter === true",
returnByValue: true,
});
assert.equal(debuggerState.result.value, true);
} catch (cause) {
if (!pauseObserved) controller.abort(cause);
else await caller.send("Debugger.resume").catch(() => undefined);
const completion = await raceWithTimeout(
pending.catch((pendingCause) => ({ ok: false, error: pendingCause })),
5_000,
);
if (completion === undefined)
throw new Error("Capture remained pending after cancellation", {
cause,
});
const completionError =
typeof completion === "object" &&
completion !== null &&
"error" in completion
? completion.error
: undefined;
const completionDetail =
completionError === undefined
? "capture did not report an error"
: (() => {
const causes = [];
let current = completionError;
for (
let depth = 0;
depth < 4 && current !== undefined;
depth += 1
) {
causes.push({
name: current.name,
tag: current._tag,
reason: current.reason,
message: current.message,
});
current = current.cause;
}
return JSON.stringify(causes);
})();
throw new Error(`${String(cause)}; capture=${completionDetail}`, {
cause,
});
} finally {
unsubscribe();
await closeVerifierResources([
() => caller.send("Debugger.disable"),
() => provider.close(),
]);
}
: typeof completion === "object" &&
completion !== null &&
"ok" in completion &&
completion.ok === false
? completion.error
: undefined;
const completionDetail =
completionError === undefined
? "capture did not report an error"
: (() => {
const causes = [];
let current = completionError;
for (
let depth = 0;
depth < 4 && current !== undefined;
depth += 1
) {
causes.push({
name: current.name,
tag: current._tag,
reason: current.reason,
message: current.message,
});
current = current.cause;
}
return JSON.stringify(causes);
})();
throw new Error(`${String(cause)}; capture=${completionDetail}`, {
cause,
});
}
}, [
() => unsubscribe(),
() => caller.send("Debugger.disable"),
() => provider.close(),
]);
};
const verifyCancelledStorage = async (
@@ -626,8 +618,7 @@ const verifyCancelledStorage = async (
const pending = provider.captureScenario(cancelledScenario, {
signal: controller.signal,
});
let failure;
try {
return withVerifierCleanup(async () => {
await waitForNewDocument(
observer,
"/storage-main",
@@ -657,13 +648,10 @@ const verifyCancelledStorage = async (
targets.some(({ id }) => id === targetId),
"Cancelled connect-mode storage cleanup closed the external target",
);
} catch (cause) {
failure = cause;
throw cause;
} finally {
controller.abort(new Error("cancel failed storage verification"));
await closeVerifierResources([() => provider.close()], failure);
}
}, [
() => controller.abort(new Error("cancel failed storage verification")),
() => provider.close(),
]);
};
const waitForNewDocument = async (connection, path, priorTimeOrigin) => {
scripts/lib/browser-scenario-storage-fingerprints-e2e.mjs
+16 -23
View File
@@ -10,7 +10,7 @@ import {
runScenarioCli,
scenarioProfiles,
} from "./browser-scenario-verifier.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
const secretVariable = "REA_BROWSER_FINGERPRINT_VERIFIER_SECRET";
const secret = "rea-storage-fingerprint-private";
@@ -143,8 +143,7 @@ export async function verifyScenarioStorageFingerprints(executable, origin) {
name: "browser-storage-fingerprints-e2e",
version: "1",
});
let failure;
try {
return withVerifierCleanup(async () => {
const cli = await verifyValues(
(value) => runScenarioCli(scenarioFor(executable, origin, value)),
(before, after) =>
@@ -204,24 +203,18 @@ export async function verifyScenarioStorageFingerprints(executable, origin) {
retained_evidence: true,
versionless_captures_unknown: true,
};
} catch (cause) {
failure = cause;
throw cause;
} finally {
if (previousSecret === undefined) delete process.env[secretVariable];
else process.env[secretVariable] = previousSecret;
await closeVerifierResources(
[
() => client.close(),
() => transport.close(),
async () =>
assert.ok(
[...(await scenarioProfiles())].every((profile) =>
profilesBefore.has(profile),
),
),
],
failure,
);
}
}, [
() => {
if (previousSecret === undefined) delete process.env[secretVariable];
else process.env[secretVariable] = previousSecret;
},
() => client.close(),
() => transport.close(),
async () =>
assert.ok(
[...(await scenarioProfiles())].every((profile) =>
profilesBefore.has(profile),
),
),
]);
}
scripts/lib/browser-script-export-e2e.mjs
+11 -15
View File
@@ -15,7 +15,7 @@ import {
assertExportedScriptAnalysis,
} from "./browser-script-export-assertions.mjs";
import { mcpTextValue } from "./mcp-verifier-results.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
/** Verify capture → export → existing analysis through real CLI and stdio MCP. */
export async function verifyBrowserScriptExport(
@@ -27,8 +27,7 @@ export async function verifyBrowserScriptExport(
let root;
let transport;
let client;
let primaryError;
try {
return withVerifierCleanup(async () => {
site = await startBrowserScriptSite();
root = await mkdtemp(join(tmpdir(), "rea-browser-script-export-"));
const env = {
@@ -109,18 +108,15 @@ export async function verifyBrowserScriptExport(
query_variants_isolated: true,
implicit_refetches: 0,
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
const cleanup = [];
if (client !== undefined) cleanup.push(() => client.close());
if (transport !== undefined) cleanup.push(() => transport.close());
if (site !== undefined) cleanup.push(() => site.close());
if (root !== undefined)
cleanup.push(() => rm(root, { recursive: true, force: true }));
await closeVerifierResources(cleanup, primaryError);
}
}, [
() => client?.close(),
() => transport?.close(),
() => site?.close(),
() =>
root === undefined
? undefined
: rm(root, { recursive: true, force: true }),
]);
}
const verifyPassiveExport = async (attachment, { cli, call, save }) => {
scripts/lib/browser-source-map-e2e.mjs
+11 -15
View File
@@ -12,7 +12,7 @@ import { parseEvidence } from "../../dist/domain/evidence.js";
import { webSourceLocationResultSchema } from "../../dist/domain/webSourceLocation.js";
import { startBrowserSourceMapSite } from "../fixtures/browser-source-map-site.mjs";
import { mcpTextValue } from "./mcp-verifier-results.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
/** Verify real Chrome capture → export → original compiler point through packaged CLI/MCP. */
export async function verifyBrowserSourceMap(
@@ -23,8 +23,7 @@ export async function verifyBrowserSourceMap(
let root;
let transport;
let client;
let primaryError;
try {
return withVerifierCleanup(async () => {
site = await startBrowserSourceMapSite();
root = await mkdtemp(join(tmpdir(), "rea-browser-source-map-"));
const env = {
@@ -132,18 +131,15 @@ export async function verifyBrowserSourceMap(
selected_inert_userinfo_context_preserved: true,
public_point_cases: 4,
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
const cleanup = [];
if (client !== undefined) cleanup.push(() => client.close());
if (transport !== undefined) cleanup.push(() => transport.close());
if (site !== undefined) cleanup.push(() => site.close());
if (root !== undefined)
cleanup.push(() => rm(root, { recursive: true, force: true }));
await closeVerifierResources(cleanup, primaryError);
}
}, [
() => client?.close(),
() => transport?.close(),
() => site?.close(),
() =>
root === undefined
? undefined
: rm(root, { recursive: true, force: true }),
]);
}
function assertPoint(evidence, { site, input, capture, identity }) {
const result = webSourceLocationResultSchema.parse(
scripts/lib/browser-url-userinfo-e2e.mjs
+15 -24
View File
@@ -7,7 +7,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { CdpConnection } from "../../dist/browser/CdpConnection.js";
import { parseEvidence } from "../../dist/domain/evidence.js";
import { artifactCliEvidence } from "./artifact-e2e.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
/** Verify real V8 source labels, which retain spelling that navigation normalizes. */
export async function verifyBrowserUrlUserInfo(endpoint, origin) {
@@ -30,8 +30,7 @@ export async function verifyBrowserUrlUserInfo(endpoint, origin) {
let connection;
let transport;
let client;
let verificationError;
try {
return withVerifierCleanup(async () => {
connection = await CdpConnection.connect(
target.webSocketDebuggerUrl,
"inspect_web_page",
@@ -120,27 +119,19 @@ export async function verifyBrowserUrlUserInfo(endpoint, origin) {
original_source_digest: true,
browser_url_parser_normalizes_backslash_and_dot_segments: true,
};
} catch (cause) {
verificationError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
async () => client?.close(),
async () => transport?.close(),
async () => connection?.close(),
async () => {
const closed = await fetch(`${endpoint}/json/close/${target.id}`);
assert.equal(
closed.ok,
true,
"Browser retained the owned URL fixture target",
);
},
],
verificationError,
);
}
}, [
async () => client?.close(),
async () => transport?.close(),
async () => connection?.close(),
async () => {
const closed = await fetch(`${endpoint}/json/close/${target.id}`);
assert.equal(
closed.ok,
true,
"Browser retained the owned URL fixture target",
);
},
]);
}
async function loadFixturePage(connection, url) {
scripts/lib/verifier-cleanup.d.mts
+7 -2
View File
@@ -1,5 +1,10 @@
/** Attempt every owned cleanup and retain the primary verification failure. */
/** Attempt every independent cleanup and preserve each failure. */
export function closeVerifierResources(
tasks: readonly (() => void | Promise<void>)[],
primaryError?: unknown,
): Promise<void>;
/** Finish one operation and its cleanup without losing either failure. */
export function withVerifierCleanup<T>(
operation: () => T | Promise<T>,
cleanupTasks: readonly (() => void | Promise<void>)[],
): Promise<T>;
scripts/lib/verifier-cleanup.mjs
+28 -7
View File
@@ -1,5 +1,4 @@
/** Attempt every owned cleanup and retain the primary verification failure. */
export async function closeVerifierResources(tasks, primaryError) {
async function collectCleanupFailures(tasks) {
const failures = [];
for (const task of tasks) {
try {
@@ -9,9 +8,31 @@ export async function closeVerifierResources(tasks, primaryError) {
}
}
if (failures.length > 0)
throw new AggregateError(
primaryError === undefined ? failures : [primaryError, ...failures],
"Verifier cleanup failed",
);
return failures;
}
/** Attempt every independent cleanup and preserve each failure. */
export async function closeVerifierResources(tasks) {
const failures = await collectCleanupFailures(tasks);
if (failures.length > 0)
throw new AggregateError(failures, "Verifier cleanup failed");
}
/** Finish one operation and its cleanup without losing either failure. */
export async function withVerifierCleanup(operation, cleanupTasks) {
let value;
try {
value = await operation();
} catch (cause) {
const failures = await collectCleanupFailures(cleanupTasks);
if (failures.length > 0)
throw new AggregateError(
[cause, ...failures],
"Verifier cleanup failed",
{ cause },
);
throw cause;
}
await closeVerifierResources(cleanupTasks);
return value;
}
scripts/lib/verifier-file-resources.mjs
+5 -10
View File
@@ -1,7 +1,7 @@
import { hashStableFile } from "../../dist/filesystem/StableFileHash.js";
import { ArtifactResourceScope } from "../../dist/artifacts/ArtifactResourceScope.js";
import { AnalysisCleanupLease } from "../../dist/artifacts/AnalysisCleanupLease.js";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
/** Finish a prerequisite probe and release its exact retained cleanup owners. */
export async function withVerifierResourceOwner(operation) {
@@ -16,15 +16,10 @@ export async function withVerifierResourceOwner(operation) {
},
run: (work) => resources.run(work),
};
let primaryError;
try {
return await operation(resourceOwner);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources([() => resources.close()], primaryError);
}
return withVerifierCleanup(
() => operation(resourceOwner),
[() => resources.close()],
);
}
/** Hash one verifier input while retaining any descriptor whose close fails. */
scripts/lib/verify-wabt-cleanup.mjs
+5 -23
View File
@@ -14,14 +14,13 @@ import { execFile } from "node:child_process";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { requireMcpToolError } from "./mcp-verifier-results.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
import { WABT_OPERATION_TIMEOUT_MS } from "./wabt-verifier-timeouts.mjs";
const execute = promisify(execFile);
/** Real WABT inspection with authored cleanup denials and same-connection recovery. */
export async function verifyWabtCleanup(entrypoint, environment, parent) {
const root = await mkdtemp(join(parent, "wabt-cleanup-"));
let primaryFailure;
try {
return withVerifierCleanup(async () => {
const valid = join(root, "valid.wasm"),
invalid = join(root, "invalid.wasm");
await writeFile(
@@ -162,8 +161,7 @@ export async function verifyWabtCleanup(entrypoint, environment, parent) {
stderr: "pipe",
});
const calls = [];
let callFailure;
try {
await withVerifierCleanup(async () => {
await client.connect(transport);
const request = {
name: "inspect_wasm_artifact",
@@ -193,15 +191,7 @@ export async function verifyWabtCleanup(entrypoint, environment, parent) {
last.structuredContent.normalized_result.validation,
"valid",
);
} catch (cause) {
callFailure = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => client.close(), () => transport.close()],
callFailure,
);
}
}, [() => client.close(), () => transport.close()]);
const facts = await observations(c);
assert.deepEqual(
facts.remaining,
@@ -214,13 +204,5 @@ export async function verifyWabtCleanup(entrypoint, environment, parent) {
count + (record.kind === "cli" ? 1 : record.calls.length),
0,
);
} catch (cause) {
primaryFailure = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => rm(root, { recursive: true, force: true })],
primaryFailure,
);
}
}, [() => rm(root, { recursive: true, force: true })]);
}
scripts/verify-adb.mjs
+1 -4
View File
@@ -438,10 +438,7 @@ try {
verifierFailure = cause;
} finally {
try {
await closeVerifierResources(
[() => provider.close()],
verifierFailed ? verifierFailure : undefined,
);
await closeVerifierResources([() => provider.close()]);
} catch (cause) {
process.stderr.write(`[FAIL] verifier cleanup — ${formatFailure(cause)}\n`);
failures += 1;
scripts/verify-agent-experience.mjs
+326 -327
View File
@@ -15,7 +15,10 @@ import { tsImport } from "tsx/esm/api";
import { PRODUCT_IDENTITY } from "../dist/identity.js";
import { createAgentEvaluationCommands } from "./lib/agent-evaluation-commands.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import {
closeVerifierResources,
withVerifierCleanup,
} from "./lib/verifier-cleanup.mjs";
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
const { evaluateCodexEvents } = await tsImport(
@@ -99,341 +102,337 @@ const commands = createAgentEvaluationCommands({
});
const { runCodex, runProcess, readCommandVersion } = commands;
let primaryError;
try {
const codexVersion = await readCommandVersion(codex);
await Promise.all([
mkdir(fixtureRoot, { recursive: true }),
mkdir(account),
mkdir(codexHome, { mode: 0o700 }),
]);
// Reuse authentication only. User config, plugins, rules, history and skills
// must not make a fresh installed-skill trial pass accidentally.
try {
await copyFile(
join(process.env.CODEX_HOME ?? join(homedir(), ".codex"), "auth.json"),
authPath,
);
await chmod(authPath, 0o600);
} catch (cause) {
if (cause?.code !== "ENOENT") throw cause;
// Environment/API-key authentication can work without a local auth file.
}
await writeFile(
join(codexHome, "config.toml"),
'approval_policy = "never"\nsandbox_mode = "read-only"\nweb_search = "disabled"\n[features]\nmulti_agent = false\nshell_snapshot = false\n[history]\npersistence = "none"\n',
{ mode: 0o600 },
);
for (const approval of ["--dry-run", "--yes"]) {
await runProcess(
process.execPath,
[
join(repositoryRoot, "scripts/rea.mjs"),
"setup",
"--client",
"codex",
approval,
"--json",
],
evaluationRoot,
120_000,
evaluationEnvironment,
);
}
const installedSkill = await readFile(
join(skillDestination, "SKILL.md"),
"utf8",
);
if (!installedSkill.includes(` version: "${PRODUCT_IDENTITY.skillVersion}"`))
throw new Error("Setup did not install the current packaged REA skill");
const requestedScenarioIds = new Set(
(process.env.REA_AGENT_EVAL_SCENARIOS ?? "")
.split(",")
.map((value) => value.trim())
.filter((value) => value.length > 0),
);
const targets = await createTargets(
fixtureRoot,
requestedScenarioIds.size === 0 || requestedScenarioIds.has("managed"),
);
const allScenarios = [
nativeProviderScenario(targets.native),
{
id: "asar",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredAnswerTermGroups: [
["profileapi"],
["profile:read"],
["preload", "contextbridge"],
],
prompt: `Explain how the desktop application at ${targets.javascript} exposes APIs to its renderer. Base the answer on the shipped application artifact, and state what remains unknown.`,
},
{
id: "asar-zh",
fixtureScenarioId: "asar",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredAnswerTermGroups: [["profileapi"], ["profile:read"]],
prompt: `請靜態分析已交付的 Electron 程式 ${targets.javascript}，說明 renderer 如何透過 preload 讀取 profile，以及 main handler 的關係。不要執行目標或安裝分析引擎；引用完整的產出 Evidence，保留 runtime 未知事項。`,
},
{
id: "notes-export",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredAnswerTermGroups: [],
requiredToolSubsequence: [
"analyze_javascript_application",
"trace_application_feature",
"trace_javascript_semantics",
],
prompt: `Use REA on the shipped Electron application at ${targets.notesElectron} to investigate its notes export API, preload exposure and main handler. Do not read target files directly or launch Electron. Use the returned application Evidence to trace the exact notes:export IPC channel in both directions, then query semantic relations for the property invoke with forward influence, excluding ambiguous dynamic edges and without an expected sink. Report the selected static facts, source locations and coverage limits using the closed claim contract.`,
},
{
id: "javascript-module-view",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredToolSubsequence: [
"analyze_javascript_application",
"inspect_analysis_view",
"inspect_analysis_view",
],
requiredAnswerTermGroups: [
["main.js"],
["node", "identity"],
["observations", "recorded"],
],
prompt: `Inspect the recorded module identity, exports and observations for main.js in the shipped JavaScript tree ${targets.largeJavascript}. Use REA without directly reading target files or executing the target. Keep results focused on that module, cite Evidence, and state the limits of what its module view establishes.`,
},
MISSING_TARGET_SCENARIO,
{
id: "javascript-export-shape",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredToolSubsequence: [
"analyze_javascript_application",
"analyze_javascript_application",
"compare_javascript_export_shapes",
],
requiredAnswerTermGroups: [
["/depth", "depth"],
[
"literal value `1`",
"literal `1`",
"depth: 1",
'"depth": 1',
"depth = 1",
],
["static", "inferred"],
["runtime", "behavioral probes"],
],
prompt: `Without directly reading target files, compare the default export in parser.mjs between ${targets.javascriptShapeLeft} and ${targets.javascriptShapeRight}. Use REA to analyze each shipped artifact and then compare its exact static export return shapes. State the exact heading-shape change, cite produced Evidence, and distinguish static inference from runtime semantics.`,
},
{
id: "managed",
expectedFirstTool: "inspect_managed_artifact",
requiresEvidence: true,
requiredAnswerTermGroups: [["profile"], ["main", "entry point"]],
requiredToolSubsequence: [
"inspect_managed_artifact",
"inspect_managed_members",
],
prompt: `Explain the public managed types and entry point in ${targets.managed}. Use shipped-artifact evidence and distinguish unavailable behavior from observed metadata.`,
},
{
id: "browser",
expectedFirstTool: "list_browser_targets",
requiresEvidence: false,
requiredAnswerTermGroups: [
["unavailable", "could not", "connection", "endpoint"],
],
prompt:
"A user-owned page is already open at http://127.0.0.1:3000 through the approved local debugging endpoint http://127.0.0.1:9222. Inspect what is available without navigating or executing page code, and report any authority or availability limits.",
},
{
id: "navigation-context",
expectedFirstTool: "get_navigation_context",
requiresEvidence: false,
requiredAnswerTermGroups: [["unavailable", "could not", "session"]],
prompt:
"Report the selected document, current address, and containing procedure from the current native-analysis session in one coherent request. If no native session is available, report that limitation plainly.",
},
{
id: "address-context",
expectedFirstTool: "inspect_address_context",
requiresEvidence: false,
requiredAnswerTermGroups: [["unavailable", "could not", "session"]],
prompt:
"Inspect explicit address 0x401000 in the current native-analysis session. Report its analyzed name, containing procedure, regular comment, inline comment, and matching bookmarks. If no native session is available, report that limitation plainly.",
},
];
const scenarios =
requestedScenarioIds.size === 0
? allScenarios
: allScenarios.filter(({ id }) => requestedScenarioIds.has(id));
if (
requestedScenarioIds.size > 0 &&
scenarios.length !== requestedScenarioIds.size
)
throw new Error(
`Unknown or duplicate REA_AGENT_EVAL_SCENARIOS value: ${[...requestedScenarioIds].join(",")}`,
);
const results = [];
for (const scenario of scenarios) {
process.stderr.write(`Running Codex agent evaluation: ${scenario.id}\n`);
const fixtureId = scenario.fixtureScenarioId ?? scenario.id;
const fixtureClaims = agentFixtureClaims(fixtureId, targets);
const answerInstructions = agentFixtureAnswerInstructions(fixtureId);
const execution = await runCodex(
answerInstructions.length === 0
? scenario.prompt
: `${scenario.prompt}\n\nThis closed factual rubric needs complete producer Evidence in the transcript. Request complete detail on each initial inspection or analysis when supported; summary or view records alone are not accepted by this rubric.\n\n${answerInstructions}`,
scenario.approvedTools ?? [],
);
const transcriptDirectory = process.env.REA_AGENT_EVAL_TRANSCRIPT_DIR;
if (transcriptDirectory !== undefined) {
const transcriptPath = resolve(
transcriptDirectory,
`${scenario.id}.jsonl`,
await withVerifierCleanup(
async () => {
const codexVersion = await readCommandVersion(codex);
await Promise.all([
mkdir(fixtureRoot, { recursive: true }),
mkdir(account),
mkdir(codexHome, { mode: 0o700 }),
]);
// Reuse authentication only. User config, plugins, rules, history and skills
// must not make a fresh installed-skill trial pass accidentally.
try {
await copyFile(
join(process.env.CODEX_HOME ?? join(homedir(), ".codex"), "auth.json"),
authPath,
);
await mkdir(dirname(transcriptPath), { recursive: true });
await writeFile(
transcriptPath,
`${execution.events.map((event) => JSON.stringify(event)).join("\n")}\n`,
await chmod(authPath, 0o600);
} catch (cause) {
if (cause?.code !== "ENOENT") throw cause;
// Environment/API-key authentication can work without a local auth file.
}
await writeFile(
join(codexHome, "config.toml"),
'approval_policy = "never"\nsandbox_mode = "read-only"\nweb_search = "disabled"\n[features]\nmulti_agent = false\nshell_snapshot = false\n[history]\npersistence = "none"\n',
{ mode: 0o600 },
);
for (const approval of ["--dry-run", "--yes"]) {
await runProcess(
process.execPath,
[
join(repositoryRoot, "scripts/rea.mjs"),
"setup",
"--client",
"codex",
approval,
"--json",
],
evaluationRoot,
120_000,
evaluationEnvironment,
);
}
const metrics = evaluateCodexEvents(
execution.events,
scenario.expectedFirstTool,
const installedSkill = await readFile(
join(skillDestination, "SKILL.md"),
"utf8",
);
if (
!installedSkill.includes(` version: "${PRODUCT_IDENTITY.skillVersion}"`)
)
throw new Error("Setup did not install the current packaged REA skill");
const requestedScenarioIds = new Set(
(process.env.REA_AGENT_EVAL_SCENARIOS ?? "")
.split(",")
.map((value) => value.trim())
.filter((value) => value.length > 0),
);
const targets = await createTargets(
fixtureRoot,
requestedScenarioIds.size === 0 || requestedScenarioIds.has("managed"),
);
const allScenarios = [
nativeProviderScenario(targets.native),
{
requireEvidence: scenario.requiresEvidence,
id: "asar",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredAnswerTermGroups: [
["profileapi"],
["profile:read"],
["preload", "contextbridge"],
],
prompt: `Explain how the desktop application at ${targets.javascript} exposes APIs to its renderer. Base the answer on the shipped application artifact, and state what remains unknown.`,
},
{
id: "asar-zh",
fixtureScenarioId: "asar",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredAnswerTermGroups: [["profileapi"], ["profile:read"]],
prompt: `請靜態分析已交付的 Electron 程式 ${targets.javascript}，說明 renderer 如何透過 preload 讀取 profile，以及 main handler 的關係。不要執行目標或安裝分析引擎；引用完整的產出 Evidence，保留 runtime 未知事項。`,
},
{
id: "notes-export",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredAnswerTermGroups: [],
requiredToolSubsequence: [
"analyze_javascript_application",
"trace_application_feature",
"trace_javascript_semantics",
],
prompt: `Use REA on the shipped Electron application at ${targets.notesElectron} to investigate its notes export API, preload exposure and main handler. Do not read target files directly or launch Electron. Use the returned application Evidence to trace the exact notes:export IPC channel in both directions, then query semantic relations for the property invoke with forward influence, excluding ambiguous dynamic edges and without an expected sink. Report the selected static facts, source locations and coverage limits using the closed claim contract.`,
},
{
id: "javascript-module-view",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredToolSubsequence: [
"analyze_javascript_application",
"inspect_analysis_view",
"inspect_analysis_view",
],
requiredAnswerTermGroups: [
["main.js"],
["node", "identity"],
["observations", "recorded"],
],
prompt: `Inspect the recorded module identity, exports and observations for main.js in the shipped JavaScript tree ${targets.largeJavascript}. Use REA without directly reading target files or executing the target. Keep results focused on that module, cite Evidence, and state the limits of what its module view establishes.`,
},
MISSING_TARGET_SCENARIO,
{
id: "javascript-export-shape",
expectedFirstTool: "analyze_javascript_application",
requiresEvidence: true,
requiredToolSubsequence: [
"analyze_javascript_application",
"analyze_javascript_application",
"compare_javascript_export_shapes",
],
requiredAnswerTermGroups: [
["/depth", "depth"],
[
"literal value `1`",
"literal `1`",
"depth: 1",
'"depth": 1',
"depth = 1",
],
["static", "inferred"],
["runtime", "behavioral probes"],
],
prompt: `Without directly reading target files, compare the default export in parser.mjs between ${targets.javascriptShapeLeft} and ${targets.javascriptShapeRight}. Use REA to analyze each shipped artifact and then compare its exact static export return shapes. State the exact heading-shape change, cite produced Evidence, and distinguish static inference from runtime semantics.`,
},
{
id: "managed",
expectedFirstTool: "inspect_managed_artifact",
requiresEvidence: true,
requiredAnswerTermGroups: [["profile"], ["main", "entry point"]],
requiredToolSubsequence: [
"inspect_managed_artifact",
"inspect_managed_members",
],
prompt: `Explain the public managed types and entry point in ${targets.managed}. Use shipped-artifact evidence and distinguish unavailable behavior from observed metadata.`,
},
{
id: "browser",
expectedFirstTool: "list_browser_targets",
requiresEvidence: false,
requiredAnswerTermGroups: [
["unavailable", "could not", "connection", "endpoint"],
],
prompt:
"A user-owned page is already open at http://127.0.0.1:3000 through the approved local debugging endpoint http://127.0.0.1:9222. Inspect what is available without navigating or executing page code, and report any authority or availability limits.",
},
{
id: "navigation-context",
expectedFirstTool: "get_navigation_context",
requiresEvidence: false,
requiredAnswerTermGroups: [["unavailable", "could not", "session"]],
prompt:
"Report the selected document, current address, and containing procedure from the current native-analysis session in one coherent request. If no native session is available, report that limitation plainly.",
},
{
id: "address-context",
expectedFirstTool: "inspect_address_context",
requiresEvidence: false,
requiredAnswerTermGroups: [["unavailable", "could not", "session"]],
prompt:
"Inspect explicit address 0x401000 in the current native-analysis session. Report its analyzed name, containing procedure, regular comment, inline comment, and matching bookmarks. If no native session is available, report that limitation plainly.",
},
];
const scenarios =
requestedScenarioIds.size === 0
? allScenarios
: allScenarios.filter(({ id }) => requestedScenarioIds.has(id));
if (
requestedScenarioIds.size > 0 &&
scenarios.length !== requestedScenarioIds.size
)
throw new Error(
`Unknown or duplicate REA_AGENT_EVAL_SCENARIOS value: ${[...requestedScenarioIds].join(",")}`,
);
const results = [];
for (const scenario of scenarios) {
process.stderr.write(`Running Codex agent evaluation: ${scenario.id}\n`);
const fixtureId = scenario.fixtureScenarioId ?? scenario.id;
const fixtureClaims = agentFixtureClaims(fixtureId, targets);
const answerInstructions = agentFixtureAnswerInstructions(fixtureId);
const execution = await runCodex(
answerInstructions.length === 0
? scenario.prompt
: `${scenario.prompt}\n\nThis closed factual rubric needs complete producer Evidence in the transcript. Request complete detail on each initial inspection or analysis when supported; summary or view records alone are not accepted by this rubric.\n\n${answerInstructions}`,
scenario.approvedTools ?? [],
);
const transcriptDirectory = process.env.REA_AGENT_EVAL_TRANSCRIPT_DIR;
if (transcriptDirectory !== undefined) {
const transcriptPath = resolve(
transcriptDirectory,
`${scenario.id}.jsonl`,
);
await mkdir(dirname(transcriptPath), { recursive: true });
await writeFile(
transcriptPath,
`${execution.events.map((event) => JSON.stringify(event)).join("\n")}\n`,
);
}
const metrics = evaluateCodexEvents(
execution.events,
scenario.expectedFirstTool,
{
requireEvidence: scenario.requiresEvidence,
requiredAnswerTermGroups: scenario.requiredAnswerTermGroups,
requiredToolSubsequence: scenario.requiredToolSubsequence,
forbidInputValidationFailures: true,
...(fixtureClaims === undefined ? {} : { fixtureClaims }),
},
);
// Fixtures are deliberately visible beside prior investigation artifacts.
// Their existence does not identify the unnamed app in this request.
const targetClarificationPassed =
scenario.expectedFirstTool === null
? scenario.validTargetClarification(metrics.finalMessage)
: undefined;
const scenarioOutcome = scenario.assess?.(metrics);
const result = {
id: scenario.id,
expectedFirstTool: scenario.expectedFirstTool,
requiresEvidence: scenario.requiresEvidence,
requiredAnswerTermGroups: scenario.requiredAnswerTermGroups,
requiredToolSubsequence: scenario.requiredToolSubsequence,
forbidInputValidationFailures: true,
...(fixtureClaims === undefined ? {} : { fixtureClaims }),
requiredToolSubsequence: scenario.requiredToolSubsequence ?? [],
configuredClaimIds: fixtureClaims?.map(({ id }) => id) ?? [],
exitCode: execution.exitCode,
stderr: execution.stderr,
...metrics,
...(scenarioOutcome === undefined
? {}
: {
scenarioOutcome,
scenarioOutcomePassed: scenarioOutcome !== "failed",
}),
...(targetClarificationPassed === undefined
? {}
: { targetClarificationPassed }),
};
results.push(result);
process.stderr.write(
`${scenario.id}: first=${metrics.firstTool ?? "none"}, calls=${String(metrics.reaCalls.length)}, repeats=${String(metrics.repeatedCallCount)}, validation_failures=${String(metrics.inputValidationFailureCount)}, input_tokens=${String(metrics.inputTokens)}\n`,
);
}
const factualCorrectness = summarizeFactualCorrectness(results);
const summary = {
schemaVersion: 4,
evaluationScope: "routing_workflow_and_configured_fixture_claims",
factualCorrectness,
verifier_run: await completeVerifierRun(verifierRun),
codex,
codexVersion,
model: optionalModel ?? null,
installation: {
method: "rea_setup_codex",
isolatedHome: true,
isolatedCodexHome: true,
skillVersion: PRODUCT_IDENTITY.skillVersion,
authentication: "caller_auth_only_removed_after_run",
},
scenarios: results,
totals: {
scenarios: results.length,
naturalUse: results.filter(({ naturalUse }) => naturalUse).length,
correctFirstTool: results.filter(
({ correctFirstTool }) => correctFirstTool,
).length,
repeatedCallCount: results.reduce(
(total, { repeatedCallCount }) => total + repeatedCallCount,
0,
),
inputValidationFailureCount: results.reduce(
(total, { inputValidationFailureCount }) =>
total + inputValidationFailureCount,
0,
),
requiredToolSubsequence: results.filter(
({ requiredToolSubsequenceMet }) => requiredToolSubsequenceMet,
).length,
inputTokens: results.reduce(
(total, { inputTokens }) => total + inputTokens,
0,
),
cachedInputTokens: results.reduce(
(total, { cachedInputTokens }) => total + cachedInputTokens,
0,
),
answerHeuristicsMet: results.filter(({ answerHeuristicsMet }) =>
Boolean(answerHeuristicsMet),
).length,
epistemicCuePresent: results.filter(({ epistemicCuePresent }) =>
Boolean(epistemicCuePresent),
).length,
factualScenariosPassed: factualCorrectness.counts.passed,
factualScenariosFailed: factualCorrectness.counts.failed,
factualScenariosNotAssessed: factualCorrectness.counts.notAssessed,
},
);
// Fixtures are deliberately visible beside prior investigation artifacts.
// Their existence does not identify the unnamed app in this request.
const targetClarificationPassed =
scenario.expectedFirstTool === null
? scenario.validTargetClarification(metrics.finalMessage)
: undefined;
const scenarioOutcome = scenario.assess?.(metrics);
const result = {
id: scenario.id,
expectedFirstTool: scenario.expectedFirstTool,
requiresEvidence: scenario.requiresEvidence,
requiredAnswerTermGroups: scenario.requiredAnswerTermGroups,
requiredToolSubsequence: scenario.requiredToolSubsequence ?? [],
configuredClaimIds: fixtureClaims?.map(({ id }) => id) ?? [],
exitCode: execution.exitCode,
stderr: execution.stderr,
...metrics,
...(scenarioOutcome === undefined
? {}
: {
scenarioOutcome,
scenarioOutcomePassed: scenarioOutcome !== "failed",
}),
...(targetClarificationPassed === undefined
? {}
: { targetClarificationPassed }),
};
results.push(result);
process.stderr.write(
`${scenario.id}: first=${metrics.firstTool ?? "none"}, calls=${String(metrics.reaCalls.length)}, repeats=${String(metrics.repeatedCallCount)}, validation_failures=${String(metrics.inputValidationFailureCount)}, input_tokens=${String(metrics.inputTokens)}\n`,
);
}
const encoded = `${JSON.stringify(summary, null, 2)}\n`;
process.stdout.write(encoded);
const reportPath = process.env.REA_AGENT_EVAL_REPORT_PATH;
if (reportPath !== undefined) {
await mkdir(dirname(resolve(reportPath)), { recursive: true });
await writeFile(resolve(reportPath), encoded);
}
const factualCorrectness = summarizeFactualCorrectness(results);
const summary = {
schemaVersion: 4,
evaluationScope: "routing_workflow_and_configured_fixture_claims",
factualCorrectness,
verifier_run: await completeVerifierRun(verifierRun),
codex,
codexVersion,
model: optionalModel ?? null,
installation: {
method: "rea_setup_codex",
isolatedHome: true,
isolatedCodexHome: true,
skillVersion: PRODUCT_IDENTITY.skillVersion,
authentication: "caller_auth_only_removed_after_run",
},
scenarios: results,
totals: {
scenarios: results.length,
naturalUse: results.filter(({ naturalUse }) => naturalUse).length,
correctFirstTool: results.filter(
({ correctFirstTool }) => correctFirstTool,
).length,
repeatedCallCount: results.reduce(
(total, { repeatedCallCount }) => total + repeatedCallCount,
0,
),
inputValidationFailureCount: results.reduce(
(total, { inputValidationFailureCount }) =>
total + inputValidationFailureCount,
0,
),
requiredToolSubsequence: results.filter(
({ requiredToolSubsequenceMet }) => requiredToolSubsequenceMet,
).length,
inputTokens: results.reduce(
(total, { inputTokens }) => total + inputTokens,
0,
),
cachedInputTokens: results.reduce(
(total, { cachedInputTokens }) => total + cachedInputTokens,
0,
),
answerHeuristicsMet: results.filter(({ answerHeuristicsMet }) =>
Boolean(answerHeuristicsMet),
).length,
epistemicCuePresent: results.filter(({ epistemicCuePresent }) =>
Boolean(epistemicCuePresent),
).length,
factualScenariosPassed: factualCorrectness.counts.passed,
factualScenariosFailed: factualCorrectness.counts.failed,
factualScenariosNotAssessed: factualCorrectness.counts.notAssessed,
},
};
const encoded = `${JSON.stringify(summary, null, 2)}\n`;
process.stdout.write(encoded);
const reportPath = process.env.REA_AGENT_EVAL_REPORT_PATH;
if (reportPath !== undefined) {
await mkdir(dirname(resolve(reportPath)), { recursive: true });
await writeFile(resolve(reportPath), encoded);
}
const failed = results.filter((result) => !agentEvaluationPassed(result));
if (failed.length > 0)
throw new Error(
`Agent routing/workflow/answer checks failed: ${failed.map(({ id }) => id).join(", ")}`,
);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
const failed = results.filter((result) => !agentEvaluationPassed(result));
if (failed.length > 0)
throw new Error(
`Agent routing/workflow/answer checks failed: ${failed.map(({ id }) => id).join(", ")}`,
);
},
// Even a deliberately retained fixture directory must never retain auth.
await closeVerifierResources(
[
async () => {
// A failed process cleanup retains both its exact owner and its files.
await commands.close();
await closeVerifierResources([
() => rm(authPath, { force: true }),
...(process.env.REA_AGENT_EVAL_KEEP_FIXTURES === "true"
? []
: [() => rm(evaluationRoot, { recursive: true, force: true })]),
]);
},
],
primaryError,
);
}
[
async () => {
// A failed process cleanup retains both its exact owner and its files.
await commands.close();
await closeVerifierResources([
() => rm(authPath, { force: true }),
...(process.env.REA_AGENT_EVAL_KEEP_FIXTURES === "true"
? []
: [() => rm(evaluationRoot, { recursive: true, force: true })]),
]);
},
],
);
async function createTargets(root, includeManaged) {
const fixtures = await createAgentEvaluationFixtures(root, repositoryRoot);
scripts/verify-browser-dom-destinations.mjs
+10 -14
View File
@@ -6,7 +6,7 @@ import { join } from "node:path";
import { chromium } from "playwright-core";
import { startBrowserVerifierSite } from "./fixtures/browser-verifier-site.mjs";
import { verifyBrowserDomDestinations } from "./lib/browser-dom-destinations-e2e.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { requireOrdinaryWindowsVerifierCaller } from "./lib/windows-ordinary-user-verifier.mjs";
const executable = process.env.REA_BROWSER_EXECUTABLE;
@@ -17,8 +17,7 @@ let profile;
let site;
let context;
let report;
let primaryError;
try {
await withVerifierCleanup(async () => {
profile = await mkdtemp(join(tmpdir(), "rea-browser-dom-"));
site = await startBrowserVerifierSite();
context = await chromium.launchPersistentContext(profile, {
@@ -56,15 +55,12 @@ try {
native_oracle: true,
verified: true,
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
const cleanup = [];
if (context !== undefined) cleanup.push(() => context.close());
if (site !== undefined) cleanup.push(() => site.close());
if (profile !== undefined)
cleanup.push(() => rm(profile, { recursive: true, force: true }));
await closeVerifierResources(cleanup, primaryError);
}
}, [
() => context?.close(),
() => site?.close(),
async () => {
if (profile !== undefined)
await rm(profile, { recursive: true, force: true });
},
]);
process.stdout.write(`${JSON.stringify(report)}\n`);
scripts/verify-flutter.mjs
+153 -156
View File
@@ -9,7 +9,10 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { toolContract } from "../dist/contracts/toolContracts.js";
import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import {
closeVerifierResources,
withVerifierCleanup,
} from "./lib/verifier-cleanup.mjs";
import { hashVerifierFile } from "./lib/verifier-file-resources.mjs";
const options = parseArguments(process.argv.slice(2));
@@ -49,171 +52,165 @@ const transport = new StdioClientTransport({
stderr: "pipe",
});
const execute = promisify(execFile);
let primaryError;
try {
await client.connect(transport);
const apk = resolve(options.apk);
const identity = await hashVerifierFile(apk, resources);
const identified = await run("identify_flutter_build", { path: apk });
assert.deepEqual(identified.target, { path: apk, ...identity });
assert.equal(identified.flutter_detected, true);
assert.equal(identified.coverage, "complete");
const abis = identified.abis.filter((abi) => abi.libapp.present);
assert.ok(abis.length > 0);
const hashes = new Set();
for (const abi of abis) {
assert.equal(abi.libapp.elf_header_valid, true);
assert.equal(abi.libapp.snapshot_hash_sources, 2);
assert.ok(abi.libapp.snapshot_hash);
hashes.add(abi.libapp.snapshot_hash);
assert.equal(abi.libflutter.present, true);
assert.match(abi.libflutter.build_id ?? "", /^(?:[0-9a-f]{2}){4,64}$/u);
assert.deepEqual(abi.libflutter.build_id_candidates, [
abi.libflutter.build_id,
]);
}
assert.equal(hashes.size, 1, "Every ABI must agree on the snapshot hash");
if (expectations !== null) {
assert.equal(identity.sha256, expectations.target_sha256);
assert.deepEqual([...hashes], [expectations.snapshot_hash]);
assert.deepEqual(
abis.map((abi) => abi.abi).sort(),
Object.keys(expectations.engines).sort(),
);
await withVerifierCleanup(
async () => {
await client.connect(transport);
const apk = resolve(options.apk);
const identity = await hashVerifierFile(apk, resources);
const identified = await run("identify_flutter_build", { path: apk });
assert.deepEqual(identified.target, { path: apk, ...identity });
assert.equal(identified.flutter_detected, true);
assert.equal(identified.coverage, "complete");
const abis = identified.abis.filter((abi) => abi.libapp.present);
assert.ok(abis.length > 0);
const hashes = new Set();
for (const abi of abis) {
assert.equal(abi.libapp.elf_header_valid, true);
assert.equal(abi.libapp.snapshot_hash_sources, 2);
assert.ok(abi.libapp.snapshot_hash);
hashes.add(abi.libapp.snapshot_hash);
assert.equal(abi.libflutter.present, true);
assert.match(abi.libflutter.build_id ?? "", /^(?:[0-9a-f]{2}){4,64}$/u);
assert.deepEqual(abi.libflutter.build_id_candidates, [
abi.libflutter.build_id,
]);
}
assert.equal(hashes.size, 1, "Every ABI must agree on the snapshot hash");
if (expectations !== null) {
assert.equal(identity.sha256, expectations.target_sha256);
assert.deepEqual([...hashes], [expectations.snapshot_hash]);
assert.deepEqual(
{
dart_version: abi.libflutter.dart_version,
toolchain_lines: abi.libflutter.toolchain_lines,
},
expectations.engines[abi.abi],
abis.map((abi) => abi.abi).sort(),
Object.keys(expectations.engines).sort(),
);
assert.equal(abi.libflutter.dart_version_capped, false);
assert.equal(abi.libflutter.toolchain_capped, false);
}
process.stdout.write(
"[ok] Exact per-ABI engine labels match digest-bound expectations\n",
);
} else {
process.stdout.write(
"[unverified] Exact engine labels and their absence need --expectations JSON\n",
);
}
// The same checks apply to the default selection and each remaining ABI.
const preferred = await run("inspect_dart_aot", { path: apk });
for (const abi of abis) {
const aot =
abi.abi === preferred.abi
? preferred
: await run("inspect_dart_aot", { path: apk, abi: abi.abi });
assert.equal(aot.abi, abi.abi);
assert.equal(
aot.libapp.coverage,
"complete",
`Snapshot decoding must be complete on ${abi.abi}`,
);
assert.equal(aot.coverage, aot.string_pool.coverage);
if (aot.coverage !== "complete") {
for (const abi of abis) {
assert.deepEqual(
{
dart_version: abi.libflutter.dart_version,
toolchain_lines: abi.libflutter.toolchain_lines,
},
expectations.engines[abi.abi],
);
assert.equal(abi.libflutter.dart_version_capped, false);
assert.equal(abi.libflutter.toolchain_capped, false);
}
process.stdout.write(
`[unverified] ${abi.abi}: full string-pool projection (returned pool is bounded and partial)\n`,
);
}
assert.deepEqual(aot.target, identified.target);
assert.equal(aot.libapp.sha256, abi.libapp.sha256);
assert.equal(aot.libapp.bytes, abi.libapp.bytes);
assert.equal(aot.libapp.snapshot_hash_sources, 2);
assert.equal(aot.libapp.snapshot_hash, abi.libapp.snapshot_hash);
assert.deepEqual(
aot.libapp.sections.map((section) => section.name).sort(),
[
"_kDartVmSnapshotData",
"_kDartVmSnapshotInstructions",
"_kDartIsolateSnapshotData",
"_kDartIsolateSnapshotInstructions",
].sort(),
);
for (const section of aot.libapp.sections) {
assert.ok(section.offset + section.size <= aot.libapp.bytes);
if (!section.name.endsWith("Data")) continue;
assert.equal(section.magic_valid, true);
assert.equal(
section.kind,
3,
"This verifier requires an AOT app snapshot",
);
assert.ok(
section.header_length >= 48 &&
section.header_length + 4 <= section.size,
`Snapshot extent must fit ${section.name} on ${abi.abi}`,
);
}
assert.ok(aot.string_pool.package_uri_count_at_least > 0);
assert.ok(aot.string_pool.dart_uri_count_at_least > 0);
if (expectations?.string_pools !== undefined) {
const expected = expectations.string_pools[abi.abi];
assert.ok(
expected,
`Pinned string-pool counts are required on ${abi.abi}`,
);
assert.equal(
aot.string_pool.package_uri_count_at_least,
expected.package_uri_count_at_least,
);
assert.equal(
aot.string_pool.dart_uri_count_at_least,
expected.dart_uri_count_at_least,
);
process.stdout.write(
`[ok] ${abi.abi}: exact pinned package and Dart URI counts\n`,
"[ok] Exact per-ABI engine labels match digest-bound expectations\n",
);
} else {
process.stdout.write(
`[unverified] ${abi.abi}: exact string-pool counts need --expectations JSON\n`,
"[unverified] Exact engine labels and their absence need --expectations JSON\n",
);
}
process.stdout.write(
`[ok] ${abi.abi}: four symbols, bounded AOT headers, agreeing hashes and string pool\n`,
);
}
if (options.plainApk !== null) {
const plain = resolve(options.plainApk);
const negative = await run("identify_flutter_build", { path: plain });
assert.deepEqual(negative.target, {
path: plain,
...(await hashVerifierFile(plain, resources)),
});
assert.equal(negative.flutter_detected, false);
assert.equal(negative.coverage, "complete");
}
assert.deepEqual(
await hashVerifierFile(apk, resources),
identity,
"Verification must preserve its input",
);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
// Server shutdown retries retained provider cleanup and settles before success.
await closeVerifierResources(
[
async () => {
await closeVerifierResources([
() => client.close(),
() => transport.close(),
() => resources.close(),
]);
assert.deepEqual(
await readdir(root),
[],
`Provider snapshots remain under ${root}`,
// The same checks apply to the default selection and each remaining ABI.
const preferred = await run("inspect_dart_aot", { path: apk });
for (const abi of abis) {
const aot =
abi.abi === preferred.abi
? preferred
: await run("inspect_dart_aot", { path: apk, abi: abi.abi });
assert.equal(aot.abi, abi.abi);
assert.equal(
aot.libapp.coverage,
"complete",
`Snapshot decoding must be complete on ${abi.abi}`,
);
assert.equal(aot.coverage, aot.string_pool.coverage);
if (aot.coverage !== "complete") {
process.stdout.write(
`[unverified] ${abi.abi}: full string-pool projection (returned pool is bounded and partial)\n`,
);
await rm(root, { recursive: true, force: true });
},
],
primaryError,
);
}
}
assert.deepEqual(aot.target, identified.target);
assert.equal(aot.libapp.sha256, abi.libapp.sha256);
assert.equal(aot.libapp.bytes, abi.libapp.bytes);
assert.equal(aot.libapp.snapshot_hash_sources, 2);
assert.equal(aot.libapp.snapshot_hash, abi.libapp.snapshot_hash);
assert.deepEqual(
aot.libapp.sections.map((section) => section.name).sort(),
[
"_kDartVmSnapshotData",
"_kDartVmSnapshotInstructions",
"_kDartIsolateSnapshotData",
"_kDartIsolateSnapshotInstructions",
].sort(),
);
for (const section of aot.libapp.sections) {
assert.ok(section.offset + section.size <= aot.libapp.bytes);
if (!section.name.endsWith("Data")) continue;
assert.equal(section.magic_valid, true);
assert.equal(
section.kind,
3,
"This verifier requires an AOT app snapshot",
);
assert.ok(
section.header_length >= 48 &&
section.header_length + 4 <= section.size,
`Snapshot extent must fit ${section.name} on ${abi.abi}`,
);
}
assert.ok(aot.string_pool.package_uri_count_at_least > 0);
assert.ok(aot.string_pool.dart_uri_count_at_least > 0);
if (expectations?.string_pools !== undefined) {
const expected = expectations.string_pools[abi.abi];
assert.ok(
expected,
`Pinned string-pool counts are required on ${abi.abi}`,
);
assert.equal(
aot.string_pool.package_uri_count_at_least,
expected.package_uri_count_at_least,
);
assert.equal(
aot.string_pool.dart_uri_count_at_least,
expected.dart_uri_count_at_least,
);
process.stdout.write(
`[ok] ${abi.abi}: exact pinned package and Dart URI counts\n`,
);
} else {
process.stdout.write(
`[unverified] ${abi.abi}: exact string-pool counts need --expectations JSON\n`,
);
}
process.stdout.write(
`[ok] ${abi.abi}: four symbols, bounded AOT headers, agreeing hashes and string pool\n`,
);
}
if (options.plainApk !== null) {
const plain = resolve(options.plainApk);
const negative = await run("identify_flutter_build", { path: plain });
assert.deepEqual(negative.target, {
path: plain,
...(await hashVerifierFile(plain, resources)),
});
assert.equal(negative.flutter_detected, false);
assert.equal(negative.coverage, "complete");
}
assert.deepEqual(
await hashVerifierFile(apk, resources),
identity,
"Verification must preserve its input",
);
},
// Server shutdown retries retained provider cleanup and settles before success.
[
async () => {
await closeVerifierResources([
() => client.close(),
() => transport.close(),
() => resources.close(),
]);
assert.deepEqual(
await readdir(root),
[],
`Provider snapshots remain under ${root}`,
);
await rm(root, { recursive: true, force: true });
},
],
);
process.stdout.write(
"all asserted Flutter CLI/MCP checks passed; unverified facets are reported above\n",
);
scripts/verify-frida.mjs
+79 -93
View File
@@ -3,7 +3,7 @@ import { basename } from "node:path";
import { FridaInstrumentationService } from "../dist/application/frida/FridaInstrumentationService.js";
import { FridaInstrumentationManager } from "../dist/frida/FridaInstrumentationManager.js";
import { isValidFridaRemoteAddress } from "../dist/contracts/fridaRemoteAddress.js";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
const options = parseArguments(process.argv.slice(2));
if (options.help) {
@@ -31,103 +31,89 @@ const remote =
: { keepaliveInterval: options.keepaliveInterval }),
};
let primaryFailure;
let report;
try {
let targetProcess;
let input;
if (remote === undefined) {
input = {
mode: "spawn",
deviceId: "local",
program: process.execPath,
argv: ["-e", "setTimeout(() => {}, 1500)"],
source: {
sourceKind: "inline",
source: "send({ rea_frida_verification: 'ok', pid: Process.id });",
},
durationMs: 500,
};
targetProcess = { pid: undefined, name: basename(process.execPath) };
} else {
const inventory = await service.listProcesses({ remote });
if (!inventory.ok) throw new Error(inventory.error.message);
targetProcess = inventory.value.processes.find(
({ pid }) => pid === options.pid,
);
if (targetProcess === undefined)
throw new Error(
`Selected PID ${String(options.pid)} was not observed remotely`,
const report = await withVerifierCleanup(async () => {
let targetProcess;
let input;
if (remote === undefined) {
input = {
mode: "spawn",
deviceId: "local",
program: process.execPath,
argv: ["-e", "setTimeout(() => {}, 1500)"],
source: {
sourceKind: "inline",
source: "send({ rea_frida_verification: 'ok', pid: Process.id });",
},
durationMs: 500,
};
targetProcess = { pid: undefined, name: basename(process.execPath) };
} else {
const inventory = await service.listProcesses({ remote });
if (!inventory.ok) throw new Error(inventory.error.message);
targetProcess = inventory.value.processes.find(
({ pid }) => pid === options.pid,
);
input = {
mode: "attach",
remote,
pid: options.pid,
source: {
sourceKind: "inline",
source: "send({ rea_frida_verification: 'ok', pid: Process.id });",
},
durationMs: 500,
if (targetProcess === undefined)
throw new Error(
`Selected PID ${String(options.pid)} was not observed remotely`,
);
input = {
mode: "attach",
remote,
pid: options.pid,
source: {
sourceKind: "inline",
source: "send({ rea_frida_verification: 'ok', pid: Process.id });",
},
durationMs: 500,
};
}
const result = await service.instrument(input);
if (!result.ok) throw new Error(result.error.message);
const observed = result.value.evidence.normalized_result;
if (
typeof observed !== "object" ||
observed === null ||
Array.isArray(observed) ||
!Array.isArray(observed.messages)
)
throw new Error("Frida returned no message observation");
const messageObserved = observed.messages.some(
(message) =>
typeof message === "object" &&
message !== null &&
!Array.isArray(message) &&
message.type === "send" &&
typeof message.payload === "object" &&
message.payload !== null &&
!Array.isArray(message.payload) &&
message.payload.rea_frida_verification === "ok" &&
message.payload.pid === observed.pid,
);
if (!messageObserved)
throw new Error("Selected target did not return the Frida marker");
return {
ok: result.value.cleanupError === null,
provider: "frida",
host_platform: process.platform,
target_mode: remote === undefined ? "local_spawn" : "remote_attach",
target_pid: observed.pid,
target_name: targetProcess.name,
...(remote === undefined ? {} : { remote_target_observed: true }),
script_message_observed: true,
cleanup_error: result.value.cleanupError,
};
}
const result = await service.instrument(input);
if (!result.ok) throw new Error(result.error.message);
const observed = result.value.evidence.normalized_result;
if (
typeof observed !== "object" ||
observed === null ||
Array.isArray(observed) ||
!Array.isArray(observed.messages)
)
throw new Error("Frida returned no message observation");
const messageObserved = observed.messages.some(
(message) =>
typeof message === "object" &&
message !== null &&
!Array.isArray(message) &&
message.type === "send" &&
typeof message.payload === "object" &&
message.payload !== null &&
!Array.isArray(message.payload) &&
message.payload.rea_frida_verification === "ok" &&
message.payload.pid === observed.pid,
);
if (!messageObserved)
throw new Error("Selected target did not return the Frida marker");
report = {
ok: result.value.cleanupError === null,
provider: "frida",
host_platform: process.platform,
target_mode: remote === undefined ? "local_spawn" : "remote_attach",
target_pid: observed.pid,
target_name: targetProcess.name,
...(remote === undefined ? {} : { remote_target_observed: true }),
script_message_observed: true,
cleanup_error: result.value.cleanupError,
};
} catch (cause) {
primaryFailure = cause;
}
let cleanupFailure;
try {
await closeVerifierResources([() => manager.closeAll()], primaryFailure);
} catch (cause) {
cleanupFailure = cause;
}
if (primaryFailure !== undefined || cleanupFailure !== undefined) {
const causes =
cleanupFailure instanceof AggregateError
? cleanupFailure.errors
: [cleanupFailure ?? primaryFailure];
process.stderr.write(
`Frida real-provider verification failed: ${causes.map((cause) => (cause instanceof Error ? cause.message : String(cause))).join("; ")}\n`,
);
process.exitCode = 1;
} else {
}, [() => manager.closeAll()]);
process.stdout.write(`${JSON.stringify(report)}\n`);
if (!report.ok) process.exitCode = 1;
} catch (cause) {
const causes = cause instanceof AggregateError ? cause.errors : [cause];
process.stderr.write(
`Frida real-provider verification failed: ${causes.map((reason) => (reason instanceof Error ? reason.message : String(reason))).join("; ")}\n`,
);
process.exitCode = 1;
}
function parseArguments(args) {
scripts/verify-il2cpp-metadata.mjs
+3 -12
View File
@@ -9,7 +9,7 @@ import { toolContract } from "../dist/contracts/toolContracts.js";
import { analysisViewResultSchema } from "../dist/domain/analysisView/analysisView.js";
import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { hashVerifierFile } from "./lib/verifier-file-resources.mjs";
const args = process.argv.slice(2);
@@ -47,8 +47,7 @@ const call = async (name, input) => {
return JSON.parse(mcpTextValue(response));
};
const resources = new ArtifactResourceScope();
let primaryError;
try {
await withVerifierCleanup(async () => {
await client.connect(transport);
const { tools } = await client.listTools();
assert(tools.some((tool) => tool.name === operation));
@@ -182,13 +181,5 @@ try {
`[ok] summary and page parity; ${report.images.length} images, ${report.types.length} types, ${report.methods.length} methods; complete Evidence ${Buffer.byteLength(stdout)} bytes, summary ${Buffer.byteLength(JSON.stringify(summary))} bytes; CLI ${cliMs} ms`,
);
}
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => client.close(), () => transport.close(), () => resources.close()],
primaryError,
);
}
}, [() => client.close(), () => transport.close(), () => resources.close()]);
console.log("All asserted IL2CPP metadata CLI/MCP checks passed");
scripts/verify-jeb.mjs
+68 -78
View File
@@ -6,7 +6,7 @@ import { promisify } from "node:util";
import { JebAnalysisService } from "../dist/application/jeb/JebAnalysisService.js";
import { createJebAnalysisProvider } from "../dist/composition/jeb.js";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
const run = promisify(execFile);
@@ -27,92 +27,82 @@ if (options.help) {
const provider = createJebAnalysisProvider(process.env);
const service = new JebAnalysisService(provider);
let fixtureDirectory;
let primaryFailure;
let report;
try {
fixtureDirectory = await mkdtemp(join(tmpdir(), "rea-jeb-"));
const client = await service.execute("inspect_jeb_client", {});
if (!client.ok) throw new Error(client.error.message);
const engine = client.value.normalized_result.engine;
if (engine.version === null)
throw new Error("The JEB client did not report its version");
const report = await withVerifierCleanup(async () => {
fixtureDirectory = await mkdtemp(join(tmpdir(), "rea-jeb-"));
const client = await service.execute("inspect_jeb_client", {});
if (!client.ok) throw new Error(client.error.message);
const engine = client.value.normalized_result.engine;
if (engine.version === null)
throw new Error("The JEB client did not report its version");
// JEB 5.48 headless clients do not advertise open_project; a project opened
// at engine start is the supported target-selection route there. Attempt the
// tool and record the engine's exact refusal when it is unavailable.
let openRefusal = null;
const target = await resolveTarget(options, fixtureDirectory);
const opened = await service.execute("open_jeb_project", {
path: target.path,
});
if (opened.ok) {
const artifact = opened.value.normalized_result.input_files.find(
(file) => typeof file === "object" && file !== null,
);
if (
artifact === undefined ||
!/^[0-9a-f]{64}$/u.test(artifact.contents_sha256_hash)
)
throw new Error("The engine did not report an input artifact digest");
} else {
openRefusal = opened.error.message;
}
// JEB 5.48 headless clients do not advertise open_project; a project opened
// at engine start is the supported target-selection route there. Attempt the
// tool and record the engine's exact refusal when it is unavailable.
let openRefusal = null;
const target = await resolveTarget(options, fixtureDirectory);
const opened = await service.execute("open_jeb_project", {
path: target.path,
});
if (opened.ok) {
const artifact = opened.value.normalized_result.input_files.find(
(file) => typeof file === "object" && file !== null,
);
if (
artifact === undefined ||
!/^[0-9a-f]{64}$/u.test(artifact.contents_sha256_hash)
)
throw new Error("The engine did not report an input artifact digest");
} else {
openRefusal = opened.error.message;
}
const units = await service.execute("list_jeb_units", { count: 100 });
if (!units.ok) throw new Error(units.error.message);
if (units.value.normalized_result.units.length === 0)
throw new Error("The engine reported no project units");
const units = await service.execute("list_jeb_units", { count: 100 });
if (!units.ok) throw new Error(units.error.message);
if (units.value.normalized_result.units.length === 0)
throw new Error("The engine reported no project units");
const decompiled = await service.execute("decompile_jeb_item", {
item_address: target.itemAddress,
item_kind: "method",
});
if (!decompiled.ok) throw new Error(decompiled.error.message);
const text = decompiled.value.normalized_result.text;
if (!text.includes(target.marker))
throw new Error(
"Decompiled pseudo-code did not contain the fixture marker",
);
const decompiled = await service.execute("decompile_jeb_item", {
item_address: target.itemAddress,
item_kind: "method",
});
if (!decompiled.ok) throw new Error(decompiled.error.message);
const text = decompiled.value.normalized_result.text;
if (!text.includes(target.marker))
throw new Error(
"Decompiled pseudo-code did not contain the fixture marker",
);
report = {
ok: true,
provider: "jeb",
host_platform: process.platform,
engine_version: engine.version,
engine_gui: engine.gui_client,
endpoint: engine.endpoint,
open_project_observed: openRefusal === null,
...(openRefusal === null ? {} : { open_project_refusal: openRefusal }),
unit_count: units.value.normalized_result.units.length,
decompiled_marker_observed: true,
};
return {
ok: true,
provider: "jeb",
host_platform: process.platform,
engine_version: engine.version,
engine_gui: engine.gui_client,
endpoint: engine.endpoint,
open_project_observed: openRefusal === null,
...(openRefusal === null ? {} : { open_project_refusal: openRefusal }),
unit_count: units.value.normalized_result.units.length,
decompiled_marker_observed: true,
};
}, [
async () => {
await provider.close();
if (fixtureDirectory !== undefined) {
await rm(fixtureDirectory, { recursive: true, force: true });
fixtureDirectory = undefined;
}
},
]);
process.stdout.write(`${JSON.stringify(report)}\n`);
} catch (cause) {
primaryFailure = cause;
}
const cleanupTasks = [
() => provider.close(),
...(fixtureDirectory === undefined
? []
: [() => rm(fixtureDirectory, { recursive: true, force: true })]),
];
let cleanupFailure;
try {
await closeVerifierResources(cleanupTasks, primaryFailure);
} catch (cause) {
cleanupFailure = cause;
}
if (primaryFailure !== undefined || cleanupFailure !== undefined) {
const causes =
cleanupFailure instanceof AggregateError
? cleanupFailure.errors
: [cleanupFailure ?? primaryFailure];
if (fixtureDirectory !== undefined)
process.stderr.write(`JEB fixture retained at ${fixtureDirectory}\n`);
const causes = cause instanceof AggregateError ? cause.errors : [cause];
process.stderr.write(
`JEB real-provider verification failed: ${causes.map((cause) => (cause instanceof Error ? cause.message : String(cause))).join("; ")}\n`,
`JEB real-provider verification failed: ${causes.map((reason) => (reason instanceof Error ? reason.message : String(reason))).join("; ")}\n`,
);
process.exitCode = 1;
} else {
process.stdout.write(`${JSON.stringify(report)}\n`);
}
async function resolveTarget(selected, directory) {
scripts/verify-package-update.mjs
+118 -122
View File
@@ -5,7 +5,7 @@ import { dirname, join } from "node:path";
import { pathToFileURL } from "node:url";
import { exec, json } from "./lib/verify-package-core.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
/** Keep the live updater owner and its workspace when process cleanup is unproven. */
export class PackageUpdateCleanupFailure extends AggregateError {
@@ -175,63 +175,63 @@ export async function verifyPackageUpdate({
server.once("error", reject);
server.listen(0, "127.0.0.1", resolveListen);
});
let primary;
try {
const address = server.address();
if (typeof address !== "object" || address === null)
throw new Error("Registry fixture did not bind");
registry = `http://127.0.0.1:${address.port}`;
// Model an older installed release's identity while retaining the new updater
// under test. The registry supplies the unmodified packed target artifact.
const generatedPath = join(
packageRoot,
"dist",
"generatedPackageMetadata.js",
);
const generated = await readFile(generatedPath, "utf8");
const previousVersion = "0.0.0";
const previousGenerated = generated.replace(
`version: "${metadata.version}"`,
`version: "${previousVersion}"`,
);
if (previousGenerated === generated)
throw new Error("Could not create previous-release identity fixture");
await writeFile(generatedPath, previousGenerated);
await writeFile(
metadataPath,
JSON.stringify({ ...metadata, version: previousVersion }),
);
const runtimeEnvironment = {
...environment,
HOME: home,
USERPROFILE: home,
APPDATA: join(home, "AppData", "Roaming"),
LOCALAPPDATA: join(home, "AppData", "Local"),
CLAUDE_CONFIG_DIR: home,
CODEX_HOME: join(home, ".codex"),
COPILOT_HOME: join(home, ".copilot"),
GROK_HOME: join(home, ".grok"),
SAND_DATA_ROOT: join(home, ".grokbot"),
OMP_PROFILE: undefined,
PI_CODING_AGENT_DIR: undefined,
PI_CONFIG_DIR: undefined,
PI_PROFILE: undefined,
XDG_CONFIG_HOME: join(home, ".config"),
OPENCODE_CONFIG: join(home, ".config", "opencode", "opencode.json"),
npm_config_prefix: prefix,
npm_config_registry: registry,
};
const previous = (
await exec(process.execPath, [entry, "--version"], {
env: runtimeEnvironment,
})
).stdout.trim();
if (previous !== previousVersion)
throw new Error(`Previous-release fixture reported ${previous}`);
const harness = join(workspace, "verify-update.mjs");
await writeFile(
harness,
`
return withVerifierCleanup(async () => {
try {
const address = server.address();
if (typeof address !== "object" || address === null)
throw new Error("Registry fixture did not bind");
registry = `http://127.0.0.1:${address.port}`;
// Model an older installed release's identity while retaining the new updater
// under test. The registry supplies the unmodified packed target artifact.
const generatedPath = join(
packageRoot,
"dist",
"generatedPackageMetadata.js",
);
const generated = await readFile(generatedPath, "utf8");
const previousVersion = "0.0.0";
const previousGenerated = generated.replace(
`version: "${metadata.version}"`,
`version: "${previousVersion}"`,
);
if (previousGenerated === generated)
throw new Error("Could not create previous-release identity fixture");
await writeFile(generatedPath, previousGenerated);
await writeFile(
metadataPath,
JSON.stringify({ ...metadata, version: previousVersion }),
);
const runtimeEnvironment = {
...environment,
HOME: home,
USERPROFILE: home,
APPDATA: join(home, "AppData", "Roaming"),
LOCALAPPDATA: join(home, "AppData", "Local"),
CLAUDE_CONFIG_DIR: home,
CODEX_HOME: join(home, ".codex"),
COPILOT_HOME: join(home, ".copilot"),
GROK_HOME: join(home, ".grok"),
SAND_DATA_ROOT: join(home, ".grokbot"),
OMP_PROFILE: undefined,
PI_CODING_AGENT_DIR: undefined,
PI_CONFIG_DIR: undefined,
PI_PROFILE: undefined,
XDG_CONFIG_HOME: join(home, ".config"),
OPENCODE_CONFIG: join(home, ".config", "opencode", "opencode.json"),
npm_config_prefix: prefix,
npm_config_registry: registry,
};
const previous = (
await exec(process.execPath, [entry, "--version"], {
env: runtimeEnvironment,
})
).stdout.trim();
if (previous !== previousVersion)
throw new Error(`Previous-release fixture reported ${previous}`);
const harness = join(workspace, "verify-update.mjs");
await writeFile(
harness,
`
import { runUpdate } from ${JSON.stringify(pathToFileURL(join(packageRoot, "dist/application/Update.js")).href)};
import { systemUpdateHost } from ${JSON.stringify(pathToFileURL(join(packageRoot, "dist/application/UpdateRuntime.js")).href)};
const host = systemUpdateHost(${JSON.stringify(packageRoot)}, ${JSON.stringify(home)});
@@ -239,72 +239,68 @@ const result = await runUpdate(${JSON.stringify(previousVersion)}, host, "struct
process.stdout.write(JSON.stringify(result));
if (result.status === "failed") process.exitCode = 1;
`,
);
const result = json(
await runUpdater({
harness,
environment: runtimeEnvironment,
signal: fixture.signal,
workspace,
retainWorkspace,
}),
);
if (
result.status !== "updated" ||
result.installedVersion !== metadata.version ||
result.command.at(-1) !== `${metadata.name}@${metadata.version}` ||
result.maintenance.status !== "planned" ||
result.maintenance.scope.clients.join(",") !== "codex" ||
result.maintenance.scope.skill !== false
)
throw new Error(
`Packaged updater failed its exact-release contract: ${JSON.stringify(result)}`,
);
if ((await readFile(codexPath, "utf8")) !== originalConfig)
throw new Error(
"Update applied integration maintenance without approval",
const result = json(
await runUpdater({
harness,
environment: runtimeEnvironment,
signal: fixture.signal,
workspace,
retainWorkspace,
}),
);
const current = (
await exec(process.execPath, [entry, "--version"], {
env: runtimeEnvironment,
})
).stdout.trim();
if (current !== metadata.version)
throw new Error(`Updated executable reported ${current}`);
const help = (
await exec(process.execPath, [entry, "--help"], {
env: runtimeEnvironment,
})
).stdout;
if (!/^\s{2}update\s/mu.test(help) || /^\s{2}upgrade\s/mu.test(help))
throw new Error(
"Updated CLI did not expose update as its sole maintenance command",
);
if (fixtureFailure !== undefined) throw fixtureFailure;
return {
installed_version: result.installedVersion,
exact_release: true,
maintenance_read_only: true,
};
} catch (error) {
primary =
error instanceof PackageUpdateCleanupFailure
if (
result.status !== "updated" ||
result.installedVersion !== metadata.version ||
result.command.at(-1) !== `${metadata.name}@${metadata.version}` ||
result.maintenance.status !== "planned" ||
result.maintenance.scope.clients.join(",") !== "codex" ||
result.maintenance.scope.skill !== false
)
throw new Error(
`Packaged updater failed its exact-release contract: ${JSON.stringify(result)}`,
);
if ((await readFile(codexPath, "utf8")) !== originalConfig)
throw new Error(
"Update applied integration maintenance without approval",
);
const current = (
await exec(process.execPath, [entry, "--version"], {
env: runtimeEnvironment,
})
).stdout.trim();
if (current !== metadata.version)
throw new Error(`Updated executable reported ${current}`);
const help = (
await exec(process.execPath, [entry, "--help"], {
env: runtimeEnvironment,
})
).stdout;
if (!/^\s{2}update\s/mu.test(help) || /^\s{2}upgrade\s/mu.test(help))
throw new Error(
"Updated CLI did not expose update as its sole maintenance command",
);
if (fixtureFailure !== undefined) throw fixtureFailure;
return {
installed_version: result.installedVersion,
exact_release: true,
maintenance_read_only: true,
};
} catch (error) {
// A retained process owner takes precedence over its triggering fixture
// failure; otherwise report the first registry/proxy failure exactly.
throw error instanceof PackageUpdateCleanupFailure
? error
: (fixtureFailure ?? error);
throw primary;
} finally {
await closeVerifierResources(
[
async () => {
server.closeAllConnections();
await new Promise((resolveClose, reject) =>
server.close((error) =>
error === undefined ? resolveClose() : reject(error),
),
);
},
],
primary,
);
}
}
}, [
async () => {
server.closeAllConnections();
await new Promise((resolveClose, reject) =>
server.close((error) =>
error === undefined ? resolveClose() : reject(error),
),
);
},
]);
}
scripts/verify-package.mjs
+14 -11
View File
@@ -22,6 +22,7 @@ import { verifyPackageSetup } from "./verify-package-setup.mjs";
import { verifyRepositorySkill } from "./verify-repository-skill.mjs";
import { verifyManaged } from "./verify-package-managed.mjs";
import { verifyUnknownProvider } from "./verify-package-unknown-provider.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
import {
createVerifiedPackageMetadata,
@@ -60,7 +61,7 @@ const runStage = async (name, action) => {
return result;
};
try {
await withVerifierCleanup(async () => {
if (retentionDirectory !== undefined && retentionDirectory !== "") {
const sourceSha = (
await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: root })
@@ -258,16 +259,18 @@ try {
metadata: retentionMetadata,
});
}
} finally {
if (updateWorkspaceRetained)
process.stderr.write(
`Package updater cleanup incomplete; retained ${workspace}\n`,
);
else
await runStage("workspace cleanup", () =>
rm(workspace, { recursive: true, force: true }),
);
}
}, [
async () => {
if (updateWorkspaceRetained)
process.stderr.write(
`Package updater cleanup incomplete; retained ${workspace}\n`,
);
else
await runStage("workspace cleanup", () =>
rm(workspace, { recursive: true, force: true }),
);
},
]);
const verifierLineage = await runStage("verifier lineage", () =>
completeVerifierRun(verifierRun),
);
scripts/verify-packaged-wasm-artifact.mjs
+73 -20
View File
@@ -1,17 +1,57 @@
#!/usr/bin/env node
import { join } from "node:path";
import { mkdir } from "node:fs/promises";
import { randomUUID } from "node:crypto";
import { fileURLToPath } from "node:url";
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { exec, execNpm } from "./lib/verify-package-core.mjs";
import {
OwnedCommandFailure,
runOwnedCommand,
} from "../dist/process/OwnedCommand.js";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { execNpm } from "./lib/verify-package-core.mjs";
import { WABT_WORKFLOW_TIMEOUT_MS } from "./lib/wabt-verifier-timeouts.mjs";
import { verifyPackagePack } from "./verify-package-pack.mjs";
/** Preserve the actual dependent process/root owner across failed cleanup. */
class PackagedWasmCleanupFailure extends Error {
constructor(cause, cleanupOwner, runtimeRoot) {
super(
`Packaged WASM verifier cleanup incomplete; retained ${runtimeRoot.path}: ${cause instanceof Error ? cause.message : String(cause)}`,
{ cause },
);
this.name = "PackagedWasmCleanupFailure";
this.cleanupOwner = cleanupOwner;
this.runtimeRoot = runtimeRoot;
}
}
const root = await PrivateRuntimeRoot.create({
prefix: "rea-wabt-package-verifier-",
});
let primaryError;
try {
let commandCleanupOwner;
let closing;
const cleanupOwner = {
close() {
if (closing !== undefined) return closing;
let attempt;
attempt = (async () => {
if (commandCleanupOwner !== undefined) {
const stopped = await commandCleanupOwner.stop();
if (stopped.status === "incomplete")
throw new Error(stopped.reason, { cause: stopped });
commandCleanupOwner = undefined;
}
await root.close();
})().catch((cause) => {
if (closing === attempt) closing = undefined;
throw new PackagedWasmCleanupFailure(cause, cleanupOwner, root);
});
closing = attempt;
return attempt;
},
};
const verifiedOutput = await withVerifierCleanup(async () => {
const { tarball } = await verifyPackagePack({
root: process.cwd(),
workspace: root.path,
@@ -48,19 +88,32 @@ try {
"scripts",
"rea.mjs",
);
const verified = await exec(
process.execPath,
["scripts/verify-wasm-artifact.mjs", entrypoint, "--require-tools"],
{
env: environment,
timeout: WABT_WORKFLOW_TIMEOUT_MS,
maxBuffer: 4 * 1024 * 1024,
},
);
process.stdout.write(verified.stdout);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources([() => root.close()], primaryError);
}
let verified;
try {
verified = await runOwnedCommand(
{
command: process.execPath,
arguments: [
fileURLToPath(new URL("./verify-wasm-artifact.mjs", import.meta.url)),
entrypoint,
"--require-tools",
],
cwd: process.cwd(),
hostEnvironment: environment,
expectedCommand: process.execPath,
runId: `rea-packaged-wabt-${randomUUID()}`,
sweepTokenOwnedProcesses: true,
},
{
timeoutMs: WABT_WORKFLOW_TIMEOUT_MS,
diagnosticBytes: 4 * 1024 * 1024,
},
);
} catch (cause) {
if (cause instanceof OwnedCommandFailure)
commandCleanupOwner = cause.cleanupOwner;
throw cause;
}
return verified.stdout.text;
}, [() => cleanupOwner.close()]);
process.stdout.write(verifiedOutput);
scripts/verify-packet-capture.mjs
+3 -12
View File
@@ -12,7 +12,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { Ajv2020 } from "ajv/dist/2020.js";
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
import { parseEvidence } from "../dist/domain/evidence.js";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
if (process.platform !== "linux")
@@ -107,8 +107,7 @@ const expectedAddressSpace = Math.min(
.map((value) => (value === "unlimited" ? Infinity : Number(value))),
);
let primaryError;
try {
await withVerifierCleanup(async () => {
await writeFile(path, source);
await writeFile(bluetooth, bluetoothSource);
await client.connect(transport);
@@ -510,15 +509,7 @@ try {
"network with spaces.pcap",
"paged.pcap",
]);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => client.close(), () => transport.close(), () => root.close()],
primaryError,
);
}
}, [() => client.close(), () => transport.close(), () => root.close()]);
console.log(
JSON.stringify(
{
scripts/verify-pcap.mjs
+20 -26
View File
@@ -9,7 +9,10 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { toolContract } from "../dist/contracts/toolContracts.js";
import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import {
closeVerifierResources,
withVerifierCleanup,
} from "./lib/verifier-cleanup.mjs";
import { hashVerifierFile } from "./lib/verifier-file-resources.mjs";
const args = process.argv.slice(2);
@@ -44,8 +47,7 @@ const transport = new StdioClientTransport({
stderr: "pipe",
});
transport.stderr?.resume();
let primaryError;
try {
await withVerifierCleanup(async () => {
await client.connect(transport);
const readiness = await run("inspect_pcap_client", {});
assert(readiness.client.tshark_version && readiness.client.capinfos_version);
@@ -87,29 +89,21 @@ try {
`[ok] ${result.packet_count} packets; ${result.endpoint_count} addresses; coverage=${result.coverage}`,
);
}
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
async () => {
await closeVerifierResources([
() => client.close(),
() => transport.close(),
() => resources.close(),
]);
assert.deepEqual(
await readdir(root),
[],
`Owned capture resources remain under ${root}`,
);
await rmdir(root);
},
],
primaryError,
);
}
}, [
async () => {
await closeVerifierResources([
() => client.close(),
() => transport.close(),
() => resources.close(),
]);
assert.deepEqual(
await readdir(root),
[],
`Owned capture resources remain under ${root}`,
);
await rmdir(root);
},
]);
console.log("All asserted pcap CLI/MCP checks passed");
async function run(name, input) {
scripts/verify-real-browser.mjs
+21 -27
View File
@@ -39,7 +39,7 @@ import {
verifyScenarioFailureEvidence,
} from "./lib/browser-scenario-verifier.mjs";
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { requireOrdinaryWindowsVerifierCaller } from "./lib/windows-ordinary-user-verifier.mjs";
import { verifyLargeScreenshotE2e } from "./lib/browser-screenshot-e2e.mjs";
import { verifyViewportScreenshotE2e } from "./lib/browser-viewport-screenshot-e2e.mjs";
@@ -70,8 +70,7 @@ let site;
let browser;
let pageProxy;
let report;
let verificationError;
try {
await withVerifierCleanup(async () => {
profile = await mkdtemp(join(tmpdir(), "rea-real-browser-"));
site = await startBrowserVerifierSite();
browser = spawn(
@@ -444,40 +443,35 @@ try {
viewportScreenshot,
verified: true,
};
} catch (cause) {
verificationError = cause;
throw cause;
} finally {
const cleanup = [];
if (pageProxy !== undefined) {
cleanup.push(async () => {
}, [
async () => {
if (pageProxy !== undefined) {
process.stderr.write("Browser verifier cleanup: closing page proxy\n");
await pageProxy.close();
});
}
if (browser !== undefined) {
cleanup.push(async () => {
}
},
async () => {
if (browser !== undefined) {
process.stderr.write("Browser verifier cleanup: stopping owned Chrome\n");
await stopProcess(browser);
});
}
if (site !== undefined) {
cleanup.push(async () => {
}
},
async () => {
if (site !== undefined) {
process.stderr.write("Browser verifier cleanup: closing fixture site\n");
await site.close();
});
}
if (profile !== undefined)
cleanup.push(() =>
rm(profile, {
}
},
async () => {
if (profile !== undefined)
await rm(profile, {
recursive: true,
force: true,
maxRetries: 10,
retryDelay: 100,
}),
);
await closeVerifierResources(cleanup, verificationError);
}
});
},
]);
process.stdout.write(
`${JSON.stringify({
verifier_run: await completeVerifierRun(verifierRun),
scripts/verify-reconstruction-readiness.mjs
+3 -12
View File
@@ -11,7 +11,7 @@ import { tsImport } from "tsx/esm/api";
import { toolContract } from "../dist/contracts/toolContracts.js";
import { parseEvidence } from "../dist/domain/evidence.js";
import { createEvidenceBundle } from "../dist/domain/evidenceBundle.js";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
const [
{ RECONSTRUCTION_READINESS_EXAMPLE },
@@ -69,8 +69,7 @@ const client = new Client({
});
let verificationReport;
let primaryError;
try {
await withVerifierCleanup(async () => {
await client.connect(transport);
const catalog = await client.listTools();
const sessionTool = catalog.tools.find(
@@ -149,15 +148,7 @@ try {
},
evaluator_conformance: verifySyntheticEvaluator(),
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => client.close(), () => transport.close()],
primaryError,
);
}
}, [() => client.close(), () => transport.close()]);
if (process.env.REA_READINESS_REPORT_PATH !== undefined)
await writeFile(
scripts/verify-signal-recording.mjs
+3 -12
View File
@@ -12,7 +12,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { Ajv2020 } from "ajv/dist/2020.js";
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
import { parseEvidence } from "../dist/domain/evidence.js";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
if (process.platform !== "linux")
@@ -103,8 +103,7 @@ const expectedAddressSpace = Math.min(
.map((value) => (value === "unlimited" ? Infinity : Number(value))),
);
let primaryError;
try {
await withVerifierCleanup(async () => {
await writeFile(path, source);
await client.connect(transport);
const advertised = (await client.listTools()).tools.find(
@@ -426,15 +425,7 @@ try {
"paged.vcd",
"recording with spaces.vcd",
]);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[() => client.close(), () => transport.close(), () => root.close()],
primaryError,
);
}
}, [() => client.close(), () => transport.close(), () => root.close()]);
console.log(
JSON.stringify(
{
scripts/verify-test-temp-hygiene.mjs
+11 -9
View File
@@ -4,6 +4,7 @@ import { dirname, join, relative, resolve } from "node:path";
import { spawnSync } from "node:child_process";
import { fileURLToPath } from "node:url";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { isOwnedTestTemporaryName } from "./lib/test-temporary-ownership.mjs";
const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
@@ -28,7 +29,7 @@ const isolatedTemporaryRoot = await mkdtemp(
join(canonicalTemporaryRoot, "rea-test-hygiene-"),
);
try {
await withVerifierCleanup(async () => {
const result = spawnSync(
process.execPath,
[vitestPath, "run", "--coverage.enabled=false", ...process.argv.slice(2)],
@@ -60,14 +61,15 @@ try {
}
if (failures.length > 0) throw new Error(failures.join("\n\n"));
} finally {
await rm(isolatedTemporaryRoot, {
recursive: true,
force: true,
maxRetries: 3,
retryDelay: 25,
});
}
}, [
() =>
rm(isolatedTemporaryRoot, {
recursive: true,
force: true,
maxRetries: 3,
retryDelay: 25,
}),
]);
process.stdout.write(
"Test temporary-directory hygiene verified: no owned residues remain.\n",
);
scripts/verify-windows-native.mjs
+16 -33
View File
@@ -23,7 +23,7 @@ import { pathToFileURL } from "node:url";
import { verifyWindowsProtocolInput } from "./lib/windows-protocol-input.mjs";
import { verifyWindowsJobMemoryLimit } from "./lib/windows-job-memory-limit.mjs";
import { verifyVolumeRootAppendLease } from "./lib/windows-volume-root-append-lease.mjs";
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
if (process.platform !== "win32")
throw new Error(
@@ -608,8 +608,7 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe
);
const heldPath = join(retryRuntime.observation.path, "snapshot.bin");
let holder;
let primaryError;
try {
await withVerifierCleanup(async () => {
await retryRuntime.snapshot(source, "snapshot.bin");
holder = await startDeleteSharingDeniedReader(heldPath);
// The reader denies delete sharing. Cleanup must retain ownership when
@@ -636,20 +635,12 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe
/No native private runtime/u,
);
report.controls.cleanupFailureRetainsOwnershipForRetry = true;
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
...(holder === undefined
? []
: [() => stopDeleteSharingDeniedReader(holder)]),
() => retryRuntime.close(),
],
primaryError,
);
}
}, [
async () => {
if (holder !== undefined) await stopDeleteSharingDeniedReader(holder);
},
() => retryRuntime.close(),
]);
const transientRuntime = WindowsPrivateRuntime.create(
workspace,
"cleanup-transient-",
@@ -659,8 +650,7 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe
"snapshot.bin",
);
let transientHolder;
let transientPrimaryError;
try {
await withVerifierCleanup(async () => {
await transientRuntime.snapshot(source, "snapshot.bin");
transientHolder = await startDeleteSharingDeniedReader(transientHeldPath);
// Keep the reader held through the first close attempt. Without retries,
@@ -690,20 +680,13 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe
/No native private runtime/u,
);
report.controls.cleanupTransientConflictRetries = true;
} catch (cause) {
transientPrimaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
...(transientHolder === undefined
? []
: [() => stopDeleteSharingDeniedReader(transientHolder)]),
() => transientRuntime.close(),
],
transientPrimaryError,
);
}
}, [
async () => {
if (transientHolder !== undefined)
await stopDeleteSharingDeniedReader(transientHolder);
},
() => transientRuntime.close(),
]);
report.controls = {
...report.controls,
protectedDaclReadback: true,
scripts/verify/firmware/analysis.mjs
+14 -20
View File
@@ -11,7 +11,10 @@ import { firmwareResultSchemas } from "../../../dist/domain/firmware/firmwareAna
import { parseEvidence } from "../../../dist/domain/evidence.js";
import { ArtifactResourceScope } from "../../../dist/artifacts/ArtifactResourceScope.js";
import { hashVerifierFile } from "../../lib/verifier-file-resources.mjs";
import { closeVerifierResources } from "../../lib/verifier-cleanup.mjs";
import {
closeVerifierResources,
withVerifierCleanup,
} from "../../lib/verifier-cleanup.mjs";
const execFileAsync = promisify(execFile);
const { join, resolve, dirname } = nodePath;
@@ -377,8 +380,7 @@ async function stageGhidra(mcpFiles) {
// Entry point
// ---------------------------------------------------------------------------
let primaryError;
try {
await withVerifierCleanup(async () => {
assert.equal(
(await hashVerifierFile(firmwarePath, fileResources)).sha256,
oracle.sha256,
@@ -396,20 +398,12 @@ try {
oracle.sha256,
"Original firmware bytes must remain unchanged",
);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
async () => {
await closeVerifierResources([
() => mcpClient.close(),
() => fileResources.close(),
]);
await fs.rm(workDir, { recursive: true, force: true });
},
],
primaryError,
);
}
}, [
async () => {
await closeVerifierResources([
() => mcpClient.close(),
() => fileResources.close(),
]);
await fs.rm(workDir, { recursive: true, force: true });
},
]);
scripts/verify/javascript/module-references.mjs
+14 -23
View File
@@ -10,7 +10,7 @@ import { parseArgs } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { artifactCliEvidence } from "../../lib/artifact-e2e.mjs";
import { closeVerifierResources } from "../../lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "../../lib/verifier-cleanup.mjs";
import { PrivateRuntimeRoot } from "../../../dist/process/PrivateRuntimeRoot.js";
import {
OwnedCommandFailure,
@@ -82,9 +82,8 @@ const normalize = (rows) =>
),
}));
let temporary;
let primaryError;
let retainTemporary = false;
try {
await withVerifierCleanup(async () => {
await client.connect(transport);
transport.stderr?.resume();
const tools = (await client.listTools()).tools;
@@ -393,23 +392,15 @@ print(json.dumps(dict(tokenizer='cl100k_base',tiktoken_version=tiktoken.__versio
"First query is after shared artifact analysis; subsequent queries share one MCP session. Legacy item reads are serial; a concurrent legacy baseline is unmeasured. Filesystem/build warmness uncontrolled. Token costs count one post-change catalog, preparation request/summary and each request/structured reply once; client rendering, replay, reasoning, final answer and model latency are unmeasured. Preparation and parity/tokenization overhead are outside per-query elapsed times; external family-resource receipts include the complete verifier and children.",
})}\n`,
);
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
() => client.close(),
() => transport.close(),
async () => {
if (temporary === undefined) return;
if (retainTemporary)
throw new Error(
`Command cleanup is incomplete; retaining its private input/Evidence files at ${temporary.path}.`,
);
await temporary.close();
},
],
primaryError,
);
}
}, [
() => client.close(),
() => transport.close(),
async () => {
if (temporary === undefined) return;
if (retainTemporary)
throw new Error(
`Command cleanup is incomplete; retaining its private input/Evidence files at ${temporary.path}.`,
);
await temporary.close();
},
]);
scripts/verify/javascript/native-handoffs.mjs
+13 -22
View File
@@ -16,7 +16,7 @@ import {
OwnedCommandFailure,
runOwnedCommand,
} from "../../../dist/process/OwnedCommand.js";
import { closeVerifierResources } from "../../lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "../../lib/verifier-cleanup.mjs";
import {
matchesStructuredScopeExpectation,
structuredScopeExpectationsSchema,
@@ -122,7 +122,6 @@ const simulatedNativeEvidence = (path, sha256) =>
},
);
let temporary;
let primaryError;
let retainTemporary = false;
let report;
const ownedCommand = async (command, arguments_, timeoutMs = 60_000) => {
@@ -145,7 +144,7 @@ const ownedCommand = async (command, arguments_, timeoutMs = 60_000) => {
throw cause;
}
};
try {
await withVerifierCleanup(async () => {
const archiveSha256 = await fileDigest(values.artifact);
const companion = join(`${values.artifact}.unpacked`, values["addon-path"]);
let companionSha256 = null;
@@ -460,23 +459,15 @@ print(json.dumps(dict(tokenizer='cl100k_base',tiktoken_version=tiktoken.__versio
assert.ok(safeIdentity, "Native identity/linkage oracle failed");
assert.ok(safeApplicability, "File analysis applicability oracle failed");
}
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
() => client.close(),
() => transport.close(),
async () => {
if (retainTemporary)
throw new Error(
`Command cleanup incomplete; retaining owned inputs at ${temporary?.path}.`,
);
await temporary?.close();
},
],
primaryError,
);
}
}, [
() => client.close(),
() => transport.close(),
async () => {
if (retainTemporary)
throw new Error(
`Command cleanup incomplete; retaining owned inputs at ${temporary?.path}.`,
);
await temporary?.close();
},
]);
console.log(serialize(report));
scripts/verify/javascript/retained-facts.mjs
+13 -22
View File
@@ -15,7 +15,7 @@ import {
OwnedCommandFailure,
runOwnedCommand,
} from "../../../dist/process/OwnedCommand.js";
import { closeVerifierResources } from "../../lib/verifier-cleanup.mjs";
import { withVerifierCleanup } from "../../lib/verifier-cleanup.mjs";
import { selectExportedJavaScriptFacts } from "../../lib/javascript-facts-reference.mjs";
const { values } = parseArgs({
@@ -56,7 +56,6 @@ const client = new Client({
version: "1",
});
let temporary;
let primaryError;
let retainTemporary = false;
let report;
const serialize = (value) => JSON.stringify(value);
@@ -106,7 +105,7 @@ const resourceSnapshot = async () => {
};
};
try {
await withVerifierCleanup(async () => {
const start = performance.now();
const artifactHash = createHash("sha256");
for await (const chunk of createReadStream(values.artifact))
@@ -344,23 +343,15 @@ try {
conditions:
"Fresh MCP and input hashing (file cache warmed); build/OS caches uncontrolled. First export sample includes export plus caller projection code, later samples reuse that immutable export. View samples precede diagnostic export. Task token projections include preparation, first answer, selected/full name/description/inputSchema catalog and the exact baseline program; exec tool schema, host rendering, output schemas, reasoning and billing unmeasured. At-answer cgroup counters include observer/startup overhead, before repeats/export diagnostics/parity/tokenization; final external observer measures all validation work. No application or native provider executes.",
};
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources(
[
() => client.close(),
() => transport.close(),
async () => {
if (retainTemporary)
throw new Error(
`Command cleanup incomplete; retaining owned inputs at ${temporary?.path}.`,
);
await temporary?.close();
},
],
primaryError,
);
}
}, [
() => client.close(),
() => transport.close(),
async () => {
if (retainTemporary)
throw new Error(
`Command cleanup incomplete; retaining owned inputs at ${temporary?.path}.`,
);
await temporary?.close();
},
]);
console.log(serialize(report));