fix(verify): report final success only after resource cleanup

This commit is contained in:
morluto
2026-10-09 03:30:07 +08:00
parent 3eb7dbdc16
commit ee043e2cbf
17 changed files with 277 additions and 195 deletions
+8 -3
View File
@@ -13,6 +13,7 @@ if (!executable)
const profile = await mkdtemp(join(tmpdir(), "rea-browser-dom-"));
const site = await startBrowserVerifierSite();
let context;
let report;
try {
context = await chromium.launchPersistentContext(profile, {
executablePath: executable,
@@ -43,11 +44,15 @@ try {
process.argv[2],
page.url(),
);
process.stdout.write(
`${JSON.stringify({ ...result, browser: context.browser().version(), native_oracle: true, verified: true })}\n`,
);
report = {
...result,
browser: context.browser().version(),
native_oracle: true,
verified: true,
};
} finally {
await context?.close();
await site.close();
await rm(profile, { recursive: true, force: true });
}
process.stdout.write(`${JSON.stringify(report)}\n`);
+14 -3
View File
@@ -15,6 +15,7 @@ if (process.platform !== "darwin")
const root = await mkdtemp(join(tmpdir(), "rea-ui-fixture-"));
let child;
let fixturePid;
let report;
try {
const childRetrievalTest = join(root, "child-retrieval-test");
await promisify(execFile)("/usr/bin/xcrun", [
@@ -187,9 +188,18 @@ try {
);
if (mismatch.ok || !mismatch.error.message.includes("target-mismatch"))
throw new Error("Real helper did not reject changed target bytes");
process.stdout.write(
`${JSON.stringify({ ok: true, verification_status: observation.ok ? "passed" : "permission-boundary-only", positive_e2e: observation.ok, observation: observation.ok ? "captured-selected-fixture-window" : "os-permission-denial-verified", scenario: scenarioStatus, permission_failure: observation.ok ? null : observation.error.message, mismatch_rejected: true, target_owned: true })}\n`,
);
report = {
ok: true,
verification_status: observation.ok ? "passed" : "permission-boundary-only",
positive_e2e: observation.ok,
observation: observation.ok
? "captured-selected-fixture-window"
: "os-permission-denial-verified",
scenario: scenarioStatus,
permission_failure: observation.ok ? null : observation.error.message,
mismatch_rejected: true,
target_owned: true,
};
} finally {
if (fixturePid !== undefined) {
try {
@@ -205,3 +215,4 @@ try {
}
await rm(root, { recursive: true, force: true });
}
process.stdout.write(`${JSON.stringify(report)}\n`);
+39 -3
View File
@@ -34,6 +34,7 @@ let tarball;
let mcpStartup;
let mcpModuleLoading;
let update;
let report;
try {
({ tarball } = await verifyPackagePack({ root, workspace }));
@@ -132,9 +133,44 @@ try {
evidenceRoot,
artifactArchive,
});
process.stdout.write(
`${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), cli: true, analysisCli: true, artifactCli: true, managedCli: true, managedReconstructionCli: true, managedNativeVerificationCli: true, managedApplicationGraphCli: true, evidenceCli: true, incurMcpCommand: PRODUCT_IDENTITY.mcpCommand, lifecycleScriptsRequired: false, doctor: "platform-appropriate", setup: supportedSetupHost ? "planned-then-idempotent" : "unsupported-host-rejected", setupPlanReadOnly: supportedSetupHost, existingHopperPreserved: supportedSetupHost, clients: supportedSetupHost ? 4 : 0, backupReadback: supportedSetupHost, failureRecovery: supportedSetupHost, configSymlinkLifecycle: supportedSetupHost, skill: supportedSetupHost, skillReferences: supportedSetupHost, mcpTools: TOOL_CONTRACTS.length, mcpPrompts: prompts.names.length, promptCompletion: true, promptCompletionLifecycle: true, evidenceMcp: true, targetFree: true, targetLifecycle: true, boundedRegexBridge: true, update, mcpStartup, mcpModuleLoading })}\n`,
);
report = {
cli: true,
analysisCli: true,
artifactCli: true,
managedCli: true,
managedReconstructionCli: true,
managedNativeVerificationCli: true,
managedApplicationGraphCli: true,
evidenceCli: true,
incurMcpCommand: PRODUCT_IDENTITY.mcpCommand,
lifecycleScriptsRequired: false,
doctor: "platform-appropriate",
setup: supportedSetupHost
? "planned-then-idempotent"
: "unsupported-host-rejected",
setupPlanReadOnly: supportedSetupHost,
existingHopperPreserved: supportedSetupHost,
clients: supportedSetupHost ? 4 : 0,
backupReadback: supportedSetupHost,
failureRecovery: supportedSetupHost,
configSymlinkLifecycle: supportedSetupHost,
skill: supportedSetupHost,
skillReferences: supportedSetupHost,
mcpTools: TOOL_CONTRACTS.length,
mcpPrompts: prompts.names.length,
promptCompletion: true,
promptCompletionLifecycle: true,
evidenceMcp: true,
targetFree: true,
targetLifecycle: true,
boundedRegexBridge: true,
update,
mcpStartup,
mcpModuleLoading,
};
} finally {
await rm(workspace, { recursive: true, force: true });
}
process.stdout.write(
`${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), ...report })}\n`,
);
+33 -29
View File
@@ -52,6 +52,7 @@ const profile = await mkdtemp(join(tmpdir(), "rea-real-browser-"));
const site = await startBrowserVerifierSite();
let browser;
let pageProxy;
let report;
try {
browser = spawn(
executable,
@@ -338,35 +339,32 @@ try {
target_id: target,
});
const moduleTrace = await verifyBrowserModules(executable);
process.stdout.write(
`${JSON.stringify({
verifier_run: await completeVerifierRun(verifierRun),
browser: observed.value.browser.product,
endpoint,
target,
domNodes: observed.value.dom.nodes.length,
accessibilityNodes: observed.value.accessibility.nodes.length,
scripts: observed.value.scripts.items.length,
networkRequests: observed.value.network.requests.length,
consoleEvents: observed.value.console.events.length,
websocketEvents: observed.value.network.websocket_events.length,
bundleScripts: bundle.value.capture.scripts_analyzed,
sourceMaps: bundle.value.observations.source_maps.processed,
sessionEvents: session.value.timeline.length,
pageScopedTransport: true,
screenshotBytes: screenshot.value.artifact.bytes,
largeScreenshot,
popupEvents,
networkEvidence,
scriptExport,
moduleTrace,
domDestinations,
browserScenarioCli: true,
browserScenarioAttachCleanup: "disconnected-external",
browserScenarioLaunchCleanup: "terminated-owned-process",
verified: true,
})}\n`,
);
report = {
browser: observed.value.browser.product,
endpoint,
target,
domNodes: observed.value.dom.nodes.length,
accessibilityNodes: observed.value.accessibility.nodes.length,
scripts: observed.value.scripts.items.length,
networkRequests: observed.value.network.requests.length,
consoleEvents: observed.value.console.events.length,
websocketEvents: observed.value.network.websocket_events.length,
bundleScripts: bundle.value.capture.scripts_analyzed,
sourceMaps: bundle.value.observations.source_maps.processed,
sessionEvents: session.value.timeline.length,
pageScopedTransport: true,
screenshotBytes: screenshot.value.artifact.bytes,
largeScreenshot,
popupEvents,
networkEvidence,
scriptExport,
moduleTrace,
domDestinations,
browserScenarioCli: true,
browserScenarioAttachCleanup: "disconnected-external",
browserScenarioLaunchCleanup: "terminated-owned-process",
verified: true,
};
} finally {
if (pageProxy !== undefined) await pageProxy.close();
if (browser !== undefined) await stopProcess(browser);
@@ -378,6 +376,12 @@ try {
retryDelay: 100,
});
}
process.stdout.write(
`${JSON.stringify({
verifier_run: await completeVerifierRun(verifierRun),
...report,
})}\n`,
);
async function verifyPageScopedTransport(provider, proxy, origin) {
const target = await pageTarget(provider, proxy.endpoint);
+25 -24
View File
@@ -43,6 +43,7 @@ import {
} from "./verify-real-ghidra-function.mjs";
const exec = promisify(execFile);
const verifierRun = createVerifierRun();
let report;
const installDir = process.env.GHIDRA_INSTALL_DIR;
if (installDir === undefined || !isAbsolute(installDir))
throw new Error(
@@ -265,33 +266,33 @@ try {
const customPath = process.env.GHIDRA_TARGET_PATH;
const custom =
customPath === undefined ? null : await verifyTarget(customPath, "custom");
process.stdout.write(
`${JSON.stringify({
verifier_run: await completeVerifierRun(verifierRun),
ok: true,
provider: { id: "ghidra", version: SUPPORTED_GHIDRA_VERSION },
verification_lane: aarch64JumpTableOnly
? "aarch64-jump-table"
: crossFormat
? "cross-format"
: "host-native",
fixture_sources:
crossFormat || aarch64JumpTableOnly
? [sourcePath, crossFormatSourcePath]
: [sourcePath],
fixtures: [debug, stripped, layout, ...crossResults].map(summary),
malformed_target: "rejected-before-provider-start",
native_api_cli: aarch64JumpTableOnly
? (crossResults[0]?.native_api_cli ?? null)
: debug.native_api_cli,
native_value_e2e: debug.native_values,
custom_target: custom === null ? null : summary(custom),
cleanup: "complete",
})}\n`,
);
report = {
ok: true,
provider: { id: "ghidra", version: SUPPORTED_GHIDRA_VERSION },
verification_lane: aarch64JumpTableOnly
? "aarch64-jump-table"
: crossFormat
? "cross-format"
: "host-native",
fixture_sources:
crossFormat || aarch64JumpTableOnly
? [sourcePath, crossFormatSourcePath]
: [sourcePath],
fixtures: [debug, stripped, layout, ...crossResults].map(summary),
malformed_target: "rejected-before-provider-start",
native_api_cli: aarch64JumpTableOnly
? (crossResults[0]?.native_api_cli ?? null)
: debug.native_api_cli,
native_value_e2e: debug.native_values,
custom_target: custom === null ? null : summary(custom),
cleanup: "complete",
};
} finally {
await rm(fixtureRoot, { recursive: true, force: true });
}
process.stdout.write(
`${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), ...report })}\n`,
);
function nativeFixtureTarget(platform, architecture) {
const supportedTargets = {
+4 -2
View File
@@ -10,6 +10,7 @@ const execute = promisify(execFile);
const root = resolve(import.meta.dirname, "..");
const temporary = await mkdtemp(join(tmpdir(), "rea-test-discovery-"));
let discovered;
try {
const output = join(temporary, "discovery.json");
await execute(
@@ -58,8 +59,9 @@ try {
) {
console.error(JSON.stringify(report, null, 2));
process.exitCode = 1;
} else
console.log(`Discovered all ${report.discovered} test files exactly once.`);
} else discovered = report.discovered;
} finally {
await rm(temporary, { recursive: true, force: true });
}
if (discovered !== undefined)
console.log(`Discovered all ${discovered} test files exactly once.`);
+3 -3
View File
@@ -59,9 +59,6 @@ try {
}
if (failures.length > 0) throw new Error(failures.join("\n\n"));
process.stdout.write(
"Test temporary-directory hygiene verified: no owned residues remain.\n",
);
} finally {
await rm(isolatedTemporaryRoot, {
recursive: true,
@@ -70,3 +67,6 @@ try {
retryDelay: 25,
});
}
process.stdout.write(
"Test temporary-directory hygiene verified: no owned residues remain.\n",
);
+15 -14
View File
@@ -27,6 +27,7 @@ const verifierRun = createVerifierRun();
const root = process.cwd();
const workspace = await mkdtemp(join(tmpdir(), "rea-windows-package-"));
const prefix = join(workspace, "prefix");
let report;
try {
const packed = JSON.parse(
@@ -212,23 +213,23 @@ try {
environment,
});
process.stdout.write(
`${JSON.stringify({
verifier_run: await completeVerifierRun(verifierRun),
ok: true,
platform: process.platform,
package: packageResult.filename,
tools: toolCount,
ghidra_bridge: "present",
agent_setup: ["opencode", "vscode", "copilot_cli"],
setup_idempotent: true,
uninstall: true,
update,
})}\n`,
);
report = {
ok: true,
platform: process.platform,
package: packageResult.filename,
tools: toolCount,
ghidra_bridge: "present",
agent_setup: ["opencode", "vscode", "copilot_cli"],
setup_idempotent: true,
uninstall: true,
update,
};
} finally {
await rm(workspace, { recursive: true, force: true });
}
process.stdout.write(
`${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), ...report })}\n`,
);
function npm(arguments_, cwd) {
const npmExecPath = process.env.npm_execpath;
+11 -3
View File
@@ -21,6 +21,7 @@ await exec("/usr/bin/xcrun", ["--find", "actool"]);
await exec("/usr/bin/xcrun", ["--find", "clang"]);
await exec("/usr/bin/assetutil", ["--version"]);
const root = await mkdtemp(join(tmpdir(), "rea-asset-e2e-"));
let report;
try {
const app = join(root, "Fixture.app");
const resources = join(app, "Contents", "Resources");
@@ -98,9 +99,16 @@ try {
});
assert.equal(invalid.isError, true);
});
process.stdout.write(
`${JSON.stringify({ ok: true, mocked: false, cli: true, stdio_mcp: true, records: raw.length, pagination: true, malformed_input_rejected: true })}\n`,
);
report = {
ok: true,
mocked: false,
cli: true,
stdio_mcp: true,
records: raw.length,
pagination: true,
malformed_input_rejected: true,
};
} finally {
await rm(root, { recursive: true, force: true });
}
process.stdout.write(`${JSON.stringify(report)}\n`);
+3 -3
View File
@@ -142,6 +142,7 @@ const checkSwift = (metadata, mode) => {
};
const root = await mkdtemp(join(tmpdir(), "rea-dispatch-"));
let report;
try {
const results = [];
for (const mode of LINK_MODES) {
@@ -197,9 +198,8 @@ try {
});
}
}
process.stdout.write(
`${JSON.stringify({ ok: true, target_executed: false, fixtures: results })}\n`,
);
report = { ok: true, target_executed: false, fixtures: results };
} finally {
await rm(root, { recursive: true, force: true });
}
process.stdout.write(`${JSON.stringify(report)}\n`);
+15 -15
View File
@@ -21,6 +21,7 @@ const sourceRoot = fileURLToPath(
new URL("../../../tests/conformance/interface-builder/", import.meta.url),
);
let report;
try {
await mkdir(join(appPath, "Contents", "MacOS"), { recursive: true });
await mkdir(resourcesPath, { recursive: true });
@@ -83,21 +84,20 @@ try {
);
assert.equal(decoded.graph.truncated, false);
process.stdout.write(
`${JSON.stringify({
ok: true,
fixture: "compiled-appkit-nib",
provider: execution.value.provider,
target_sha256: decoded.target_sha256,
archive_sha256: decoded.documents[0].archive_sha256,
objects: decoded.documents[0].object_count,
connections: decoded.documents[0].connection_count,
recovered_control: button.name,
recovered_action: action.name,
hierarchy_complete: decoded.documents[0].hierarchy_complete,
truncated: decoded.graph.truncated,
})}\n`,
);
report = {
ok: true,
fixture: "compiled-appkit-nib",
provider: execution.value.provider,
target_sha256: decoded.target_sha256,
archive_sha256: decoded.documents[0].archive_sha256,
objects: decoded.documents[0].object_count,
connections: decoded.documents[0].connection_count,
recovered_control: button.name,
recovered_action: action.name,
hierarchy_complete: decoded.documents[0].hierarchy_complete,
truncated: decoded.graph.truncated,
};
} finally {
await rm(fixtureRoot, { recursive: true, force: true });
}
process.stdout.write(`${JSON.stringify(report)}\n`);
+16 -3
View File
@@ -17,6 +17,7 @@ if (process.platform !== "darwin")
"Real Foundation keyed archive verification requires macOS and Xcode Swift tools",
);
const root = await mkdtemp(join(tmpdir(), "rea-keyed-fixture-"));
let report;
try {
const archive = join(root, "model.plist");
await promisify(execFile)("/usr/bin/xcrun", [
@@ -150,9 +151,21 @@ try {
[{ source: null, path: ["UID"], target: 1, status: "resolved" }],
);
}
process.stdout.write(
`${JSON.stringify({ ok: true, mocked: false, cli: true, stdio_mcp: true, xml_golden: true, uid_named_root: true, format: graph.archive_format, objects: graph.total_objects, references: graph.total_references, shared_identity: true, cyclic_identity: true, target_classes_instantiated_by_reader: false })}\n`,
);
report = {
ok: true,
mocked: false,
cli: true,
stdio_mcp: true,
xml_golden: true,
uid_named_root: true,
format: graph.archive_format,
objects: graph.total_objects,
references: graph.total_references,
shared_identity: true,
cyclic_identity: true,
target_classes_instantiated_by_reader: false,
};
} finally {
await rm(root, { recursive: true, force: true });
}
process.stdout.write(`${JSON.stringify(report)}\n`);
+13 -13
View File
@@ -65,6 +65,7 @@ const EXPECTED_BUNDLES = [
["Contents/XPCServices/Svc.xpc", "xpc-service", "macos-deep", "."],
];
let report;
try {
const { app } = await buildMacosBundleFixture(root);
const zip = join(root, "MacFixture.zip");
@@ -155,19 +156,18 @@ try {
const dylibs = await verifyDylibResolution(app);
const dyldEnvironment = await verifyDyldEnvironment(root);
process.stdout.write(
`${JSON.stringify({
ok: true,
mocked: false,
cli: true,
stdio_mcp: true,
containers: containers.map(({ format }) => format),
bundles: EXPECTED_BUNDLES.length,
dmg_detached: true,
dylib_resolution: dylibs,
dyld_environment: dyldEnvironment,
})}\n`,
);
report = {
ok: true,
mocked: false,
cli: true,
stdio_mcp: true,
containers: containers.map(({ format }) => format),
bundles: EXPECTED_BUNDLES.length,
dmg_detached: true,
dylib_resolution: dylibs,
dyld_environment: dyldEnvironment,
};
} finally {
await rm(root, { recursive: true, force: true });
}
process.stdout.write(`${JSON.stringify(report)}\n`);
+11 -11
View File
@@ -34,6 +34,7 @@ const sampleRss = () => {
};
const sampler = setInterval(sampleRss, 25);
sampler.unref();
let finalReport;
try {
const evidence = createEvidence(
@@ -112,17 +113,15 @@ try {
assert.equal(digest, expectedHash.digest("hex"));
assert.equal((await stat(path)).size, expectedBytes);
sampleRss();
process.stdout.write(
`${JSON.stringify({
json_output_verified: true,
format,
output_bytes: expectedBytes,
engine_string_limit: constants.MAX_STRING_LENGTH,
output_sha256: digest,
evidence_id: evidence.evidence_id,
rss: { method: "node-memoryUsage-rss-sampled", peak_bytes: peakRss },
})}\n`,
);
finalReport = {
json_output_verified: true,
format,
output_bytes: expectedBytes,
engine_string_limit: constants.MAX_STRING_LENGTH,
output_sha256: digest,
evidence_id: evidence.evidence_id,
rss: { method: "node-memoryUsage-rss-sampled", peak_bytes: peakRss },
};
} finally {
clearInterval(sampler);
if (destination !== undefined && !destination.closed) {
@@ -132,3 +131,4 @@ try {
}
await rm(workspace, { recursive: true, force: true });
}
process.stdout.write(`${JSON.stringify(finalReport)}\n`);
+8 -8
View File
@@ -49,6 +49,7 @@ const call = async (name, args) => {
};
const fingerprint = (bytes) => createHash("sha256").update(bytes).digest("hex");
const counts = [];
let finalReport;
try {
const fixtures = await buildRecoveryFixtures(
root,
@@ -128,19 +129,18 @@ try {
});
}
}
console.log(
JSON.stringify({
engine: "wakaru 1.13.0",
toolchains: fixtures.toolchains,
results: counts,
verified: true,
}),
);
finalReport = {
engine: "wakaru 1.13.0",
toolchains: fixtures.toolchains,
results: counts,
verified: true,
};
} finally {
await client.close();
await transport.close();
await rm(root, { recursive: true, force: true });
}
console.log(JSON.stringify(finalReport));
async function assertRecoveredArtifacts(evidence, result, fixture) {
assert.deepEqual(result.options, {
+56 -58
View File
@@ -34,6 +34,8 @@ import {
const verifierRun = createVerifierRun();
const workspace = await mkdtemp(join(tmpdir(), "rea-managed-conformance-"));
let completionReportInput;
let finalReport;
const {
fixture,
fixtureBytes,
@@ -378,65 +380,61 @@ try {
const manifestSelfTest = await runManagedAppManifestSelfTest();
const operatorManifest = await runOptionalManagedAppManifest();
const ilspyOracle = await runOptionalIlspyOracle();
const completionReport = createManagedCompletionReport(
{
modern,
framework,
nativeFunctionTarget,
readyToRun,
obfuscated,
left,
right,
nativeOnly,
malformed,
manifestSelfTest,
operatorManifest,
ilspyOracle,
completionReportInput = {
modern,
framework,
nativeFunctionTarget,
readyToRun,
obfuscated,
left,
right,
nativeOnly,
malformed,
manifestSelfTest,
operatorManifest,
ilspyOracle,
};
finalReport = {
verified:
11 + (operatorManifest === null ? 0 : 1) + (ilspyOracle === null ? 0 : 1),
managedSurfaces: [
"inspect_managed_artifact",
"inspect_managed_members",
"inspect_managed_native_boundaries",
"compare_managed_members",
"verify_managed_native_boundaries",
"import_managed_reconstruction",
"project_managed_application_graph",
],
coverage: [
"modern-dotnet-anycpu",
"dotnet-framework-x86-pinvoke",
"x64-ready-to-run-native-body",
"managed-native-verification",
"managed-application-graph-projection",
"unicode-obfuscated-identifiers",
"decompiler-reconstruction-import",
"mvid-and-token-drift",
"not-managed",
"malformed-metadata",
"operator-local-managed-manifest",
...(ilspyOracle === null ? [] : ["ilspy-reconstruction-oracle"]),
],
managedAppManifest: {
env: "REA_MANAGED_APP_MANIFEST_PATH",
selfTest: manifestSelfTest,
operator: operatorManifest ?? { configured: false },
},
await completeVerifierRun(verifierRun),
);
process.stdout.write(
`${JSON.stringify({
verified:
11 +
(operatorManifest === null ? 0 : 1) +
(ilspyOracle === null ? 0 : 1),
managedSurfaces: [
"inspect_managed_artifact",
"inspect_managed_members",
"inspect_managed_native_boundaries",
"compare_managed_members",
"verify_managed_native_boundaries",
"import_managed_reconstruction",
"project_managed_application_graph",
],
coverage: [
"modern-dotnet-anycpu",
"dotnet-framework-x86-pinvoke",
"x64-ready-to-run-native-body",
"managed-native-verification",
"managed-application-graph-projection",
"unicode-obfuscated-identifiers",
"decompiler-reconstruction-import",
"mvid-and-token-drift",
"not-managed",
"malformed-metadata",
"operator-local-managed-manifest",
...(ilspyOracle === null ? [] : ["ilspy-reconstruction-oracle"]),
],
managedAppManifest: {
env: "REA_MANAGED_APP_MANIFEST_PATH",
selfTest: manifestSelfTest,
operator: operatorManifest ?? { configured: false },
},
ilspyOracle: ilspyOracle ?? {
env: "REA_ILSPY_CMD_PATH",
configured: false,
},
completionReport,
})}\n`,
);
ilspyOracle: ilspyOracle ?? {
env: "REA_ILSPY_CMD_PATH",
configured: false,
},
};
} finally {
await rm(workspace, { recursive: true, force: true });
}
finalReport.completionReport = createManagedCompletionReport(
completionReportInput,
await completeVerifierRun(verifierRun),
);
process.stdout.write(`${JSON.stringify(finalReport)}\n`);