diff --git a/docs/testing.md b/docs/testing.md index aa73ede80c..b6dbf2ac7d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1539,7 +1539,12 @@ NUL-containing names. It compares exact WAT, section output and selected/tool digests through the compiled CLI and real MCP SDK, validates advertised schemas, checks identical CLI/MCP Evidence IDs and session bundle readback, retains invalid/truncated module diagnostics, preserves -all inputs and checks workspace cleanup. Pass an installed package's +all inputs and checks workspace cleanup. +The lane also injects one cleanup denial for exact private runtime roots and +input descriptors, checks recovered CLI errors, and verifies retry on the same +MCP connection after repeated runtime-removal denial. These are controlled +faults, not naturally occurring permission failures; no WASM target is executed. +Pass an installed package's `scripts/rea.mjs` as the first argument to run the same lane against the package. `npm run verify:wasm:package` packs and installs REA into an isolated temporary prefix with lifecycle scripts disabled and runs the same required real-tool lane. diff --git a/scripts/lib/verify-wabt-cleanup.mjs b/scripts/lib/verify-wabt-cleanup.mjs new file mode 100644 index 0000000000..73bbd0eccc --- /dev/null +++ b/scripts/lib/verify-wabt-cleanup.mjs @@ -0,0 +1,226 @@ +import assert from "node:assert/strict"; +import { + mkdtemp, + mkdir, + readFile, + writeFile, + readdir, + rm, +} from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { execFile } from "node:child_process"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { requireMcpToolError } from "./mcp-verifier-results.mjs"; +import { closeVerifierResources } from "./verifier-cleanup.mjs"; +import { WABT_OPERATION_TIMEOUT_MS } from "./wabt-verifier-timeouts.mjs"; +const execute = promisify(execFile); +/** Real WABT inspection with authored cleanup denials and same-connection recovery. */ +export async function verifyWabtCleanup(entrypoint, environment, parent) { + const root = await mkdtemp(join(parent, "wabt-cleanup-")); + let primaryFailure; + try { + const valid = join(root, "valid.wasm"), + invalid = join(root, "invalid.wasm"); + await writeFile( + valid, + Buffer.from("0061736d01000000000a066f7061717565616263", "hex"), + ); + await writeFile(invalid, Buffer.from("0061736d01000000ff", "hex")); + const preload = fileURLToPath( + new URL("../../tests/fixtures/wasm/cleanupPreload.mjs", import.meta.url), + ); + const records = []; + async function setup(name, mode, input) { + const temporary = join(root, name); + await mkdir(temporary); + const trace = join(root, name + ".jsonl"); + return { + temporary, + trace, + env: { + ...environment, + HOME: root, + USERPROFILE: root, + REA_LOG_LEVEL: "silent", + TMPDIR: temporary, + TMP: temporary, + TEMP: temporary, + REA_CLEANUP_PROBE_ROOT: temporary, + REA_CLEANUP_PROBE_TRACE: trace, + REA_CLEANUP_PROBE_MODE: mode, + REA_CLEANUP_PROBE_INPUT: input, + }, + }; + } + async function observations(configuration) { + const trace = await readFile(configuration.trace, "utf8").catch((e) => { + if (e.code === "ENOENT") return ""; + throw e; + }); + return { + trace: + trace.trim() === "" ? [] : trace.trim().split("\n").map(JSON.parse), + remaining: await readdir(configuration.temporary), + }; + } + for (const [name, mode, input] of [ + ["normal", "normal", valid], + ["valid-recovered", "runtime-once", valid], + ["invalid-recovered", "runtime-once", invalid], + ["descriptor-recovered", "input-once", valid], + ]) { + const c = await setup(name, mode, input); + let response, + status = 0; + try { + response = await execute( + process.execPath, + [ + "--import", + preload, + entrypoint, + "inspect-wasm-artifact", + input, + "--json", + ], + { + env: c.env, + timeout: WABT_OPERATION_TIMEOUT_MS, + maxBuffer: 64 * 1024 * 1024, + }, + ); + } catch (e) { + if (typeof e.code !== "number" || typeof e.stdout !== "string") throw e; + response = e; + status = e.code; + } + const output = JSON.parse(response.stdout); + const facts = await observations(c); + assert.deepEqual(facts.remaining, []); + if (mode === "normal") { + assert.equal(status, 0); + assert.equal(output.normalized_result.validation, "valid"); + } else { + assert.equal(status, 1); + assert.equal( + output.code, + name === "valid-recovered" ? "execution_failure" : "invalid_request", + ); + if (name === "valid-recovered") { + assert.equal( + output.details?.partial_observation?.normalized_result?.validation, + "valid", + ); + assert.equal( + output.details?.partial_observation?.evidence_id, + records[0].output.evidence_id, + ); + } + if (mode === "input-once") { + assert.equal( + output.details?.issues?.[0]?.message, + "authored descriptor cleanup denial", + ); + const closes = facts.trace.filter( + (fact) => fact.kind === "input-close", + ); + assert.ok(closes.length >= 2); + assert.equal(closes[0].path, input); + assert.equal(closes[0].denied, true); + assert.ok(Number.isInteger(closes[0].descriptor)); + for (const retry of closes.slice(1)) { + assert.equal(retry.path, input); + assert.equal(retry.descriptor, closes[0].descriptor); + assert.equal(retry.denied, false); + } + } + assert.notEqual(output.details?.cleanup, "incomplete"); + assert.equal(output.details?.resources, undefined); + if (input === invalid) { + assert.equal(output.code, "invalid_request"); + assert.ok(output.details?.captured_output?.stderr); + } + } + if (mode === "runtime-once") { + assert.equal(facts.trace.length, 2); + assert.equal(facts.trace[0].path, facts.trace[1].path); + assert.equal(facts.trace[0].denied, true); + assert.equal(facts.trace[1].exists, false); + } + records.push({ kind: "cli", name, status, output, ...facts }); + } + for (const mode of ["runtime-once", "runtime-until-allowed"]) { + const c = await setup("mcp-" + mode, mode, valid); + const client = new Client({ name: "wabt-cleanup-probe", version: "1" }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: ["--import", preload, entrypoint, "mcp"], + env: c.env, + stderr: "pipe", + }); + const calls = []; + let callFailure; + try { + await client.connect(transport); + const request = { + name: "inspect_wasm_artifact", + arguments: { path: valid }, + }; + const call = () => + client.callTool(request, { timeout: WABT_OPERATION_TIMEOUT_MS }); + const first = await call(); + calls.push(first); + assert.equal(requireMcpToolError(first).code, "cleanup_incomplete"); + if (mode === "runtime-until-allowed") { + const second = await call(); + calls.push(second); + assert.equal(requireMcpToolError(second).code, "cleanup_incomplete"); + assert.equal( + (await observations(c)).remaining.filter((name) => + name.startsWith("rea-wabt-"), + ).length, + 1, + ); + await writeFile(join(c.temporary, "allow"), "restore"); + } + const last = await call(); + calls.push(last); + assert.notEqual(last.isError, true, JSON.stringify(last)); + assert.equal( + last.structuredContent.normalized_result.validation, + "valid", + ); + } catch (cause) { + callFailure = cause; + throw cause; + } finally { + await closeVerifierResources( + [() => client.close(), () => transport.close()], + callFailure, + ); + } + const facts = await observations(c); + assert.deepEqual( + facts.remaining, + mode === "runtime-until-allowed" ? ["allow"] : [], + ); + records.push({ kind: "mcp", mode, calls, ...facts }); + } + return records.reduce( + (count, record) => + count + (record.kind === "cli" ? 1 : record.calls.length), + 0, + ); + } catch (cause) { + primaryFailure = cause; + throw cause; + } finally { + await closeVerifierResources( + [() => rm(root, { recursive: true, force: true })], + primaryFailure, + ); + } +} diff --git a/scripts/verify-wasm-artifact.mjs b/scripts/verify-wasm-artifact.mjs index 9bc5436aff..7b9c08071b 100644 --- a/scripts/verify-wasm-artifact.mjs +++ b/scripts/verify-wasm-artifact.mjs @@ -26,6 +26,8 @@ import { import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { WABT_OPERATION_TIMEOUT_MS } from "./lib/wabt-verifier-timeouts.mjs"; +import { verifyWabtCleanup } from "./lib/verify-wabt-cleanup.mjs"; + const directory = process.env.REA_WABT_BIN_DIRECTORY; if (!isAbsolute(directory ?? "")) { if (process.argv.includes("--require-tools")) @@ -405,6 +407,7 @@ try { cases += 2; } } + cases += await verifyWabtCleanup(entrypoint, environment, childTempRoot); } catch (cause) { failures.push(cause); } diff --git a/src/wasm/WabtArtifactProvider.test.ts b/src/wasm/WabtArtifactProvider.test.ts index dc448dcb2d..b09d6ea2be 100644 --- a/src/wasm/WabtArtifactProvider.test.ts +++ b/src/wasm/WabtArtifactProvider.test.ts @@ -17,6 +17,7 @@ const fileCloseFailure = vi.hoisted(() => ({ selectedPath: "", attempts: 0, failFirst: false, + failRead: undefined as (() => Error) | undefined, })); vi.mock("node:fs/promises", async (importOriginal) => { @@ -27,6 +28,13 @@ vi.mock("node:fs/promises", async (importOriginal) => { const handle = await fs.open(...args); if (String(args[0]) !== fileCloseFailure.selectedPath) return handle; const close = handle.close.bind(handle); + const failure = fileCloseFailure.failRead; + if (failure !== undefined) { + fileCloseFailure.failRead = undefined; + vi.spyOn(handle, "stat").mockImplementationOnce(() => { + throw failure(); + }); + } handle.close = async () => { fileCloseFailure.attempts += 1; if (fileCloseFailure.failFirst) { @@ -44,6 +52,9 @@ const outcomeModes = [ "success", "invalid", "invalid-close", + "invalid-close-failure", + "snapshot-close-failure", + "snapshot-read-failure-late-abort", "malformed", "timeout", "cancelled", @@ -59,6 +70,9 @@ type OutcomeMode = (typeof outcomeModes)[number]; const failureTags: Readonly, string>> = { invalid: "AnalysisInputError", "invalid-close": "AnalysisInputError", + "invalid-close-failure": "ProviderAdapterError", + "snapshot-close-failure": "AnalysisOutputError", + "snapshot-read-failure-late-abort": "AnalysisOutputError", timeout: "AnalysisTimeoutError", cancelled: "AnalysisCancelledError", "cancelled-close": "AnalysisCancelledError", @@ -105,6 +119,12 @@ const assertCleanupRecovery = async (options: { await expect(access(options.runtimePath)).rejects.toMatchObject({ code: "ENOENT", }); + expect(options.result).toMatchObject({ + error: { cleanupIncomplete: false, cleanupResources: [] }, + }); + expect(blocked).toMatchObject({ + error: { cleanupIncomplete: false, cleanupResources: [] }, + }); }; const assertCancellationDuringRootRelease = async (options: { @@ -113,6 +133,7 @@ const assertCancellationDuringRootRelease = async (options: { readonly provider: WabtArtifactProvider; readonly runtimePath: string; readonly cleanupAttempts: () => number; + readonly cancellationReason: unknown; }): Promise => { if (options.mode === "cancelled-close") { expect(options.result).toMatchObject({ @@ -123,6 +144,8 @@ const assertCancellationDuringRootRelease = async (options: { cleanupIncomplete: false, }, }); + if (!options.result.ok) + expect(options.result.error.cause).toBe(options.cancellationReason); return; } expect(options.result).toMatchObject({ @@ -132,13 +155,20 @@ const assertCancellationDuringRootRelease = async (options: { cleanupIncomplete: true, cleanupResources: [options.runtimePath], partialObservation: { normalized_result: { validation: "valid" } }, - diagnostics: { previous_error: { code: "cancelled" } }, + diagnostics: { primary_error: { code: "cancelled" } }, }, }); expect(options.cleanupAttempts()).toBe(1); + if (!options.result.ok) + expect(options.result.error.cleanupPrimary?.cause).toBe( + options.cancellationReason, + ); await expect(access(options.runtimePath)).resolves.toBeUndefined(); await options.provider.close(); expect(options.cleanupAttempts()).toBe(2); + expect(options.result).toMatchObject({ + error: { cleanupIncomplete: false, cleanupResources: [] }, + }); }; const createWabtArtifactFixture = async () => { @@ -278,6 +308,9 @@ it("retries a retained input descriptor before admitting the next WABT operation }); expect(retried.ok).toBe(true); expect(fileCloseFailure.attempts).toBeGreaterThan(1); + expect(first).toMatchObject({ + error: { cleanupIncomplete: false, cleanupResources: [] }, + }); fileCloseFailure.failFirst = true; const closeRetry = await provider.inspect({ @@ -292,6 +325,9 @@ it("retries a retained input descriptor before admitting the next WABT operation const beforeClose = fileCloseFailure.attempts; await provider.close(); expect(fileCloseFailure.attempts).toBeGreaterThan(beforeClose); + expect(closeRetry).toMatchObject({ + error: { cleanupIncomplete: false, cleanupResources: [] }, + }); } finally { fileCloseFailure.selectedPath = ""; fileCloseFailure.failFirst = false; @@ -378,7 +414,86 @@ const assertGlueObservations = ( } }; -const runOwnedArtifactOutcome = async (mode: OutcomeMode): Promise => { +const launchWabtOutcome = async ( + input: Parameters[0], + mode: OutcomeMode, + fixture: { bytes: Buffer; dump: string; controller: AbortController }, +) => { + const { bytes, dump, controller } = fixture; + const version = input.arguments[0] === "--version"; + if (!version) { + expect(await readFile(join(input.cwd ?? "", "module.wasm"))).toEqual(bytes); + if (mode === "changed-snapshot") { + const snapshot = join(input.cwd ?? "", "module.wasm"); + await chmod(snapshot, 0o600); + await writeFile(snapshot, Buffer.alloc(76, 2)); + } + if (mode === "timeout") + throw new OwnedCommandFailure("timeout", "Source-owned deadline seam"); + } + const stdout = version + ? "1.0.42\n" + : input.command.includes("objdump") + ? mode === "malformed" + ? "unexpected producer output" + : dump + : input.command.includes("wasm2wat") + ? mode === "malformed-wat" + ? "unexpected decoder output\n" + : '(module (@custom "test" "payload"))\n' + : ""; + const invalid = + !version && + (mode === "invalid" || + mode === "invalid-close" || + mode === "invalid-close-failure"); + const stderr = invalid + ? "module.wasm: invalid bytes\n" + : !version && input.command.includes("wasm2wat") && mode === "wat-warning" + ? "decoder warning\n" + : ""; + const script = + !version && mode === "cancelled" + ? "setInterval(() => undefined, 1000)" + : `process.stdout.write(${JSON.stringify(stdout)}); process.stderr.write(${JSON.stringify(stderr)}); process.exit(${invalid ? 1 : 0});`; + const launched = await spawnOwnedProviderProcess({ + ...input, + command: process.execPath, + arguments: ["-e", script], + expectedCommand: null, + }); + if (!version && mode === "cancelled") { + setImmediate(() => controller.abort()); + } + return launched; +}; + +const glueSourcesForOutcome = (mode: OutcomeMode) => { + const literal = 'fetch("./selected.wasm");'; + return mode === "success" + ? [ + { source: literal, line: 1, column: 6 }, + { source: `\uFEFF${literal}`, line: 1, column: 7 }, + { source: `\uFEFF\n${literal}`, line: 2, column: 6 }, + { source: `#!/usr/bin/env node\n${literal}`, line: 2, column: 6 }, + { + source: `\uFEFF#!/usr/bin/env node\n${literal}`, + line: 2, + column: 6, + }, + { + source: String.raw`"./selected\u002ewasm";`, + line: 1, + column: 0, + }, + ] + : []; +}; + +const runOwnedArtifactOutcome = async ( + mode: OutcomeMode, + cancellationReason: unknown = "authored finalization cancellation", +): Promise => { const { selectedRoot, path, bytes, dump } = await createWabtArtifactFixture(); let runtimePath = ""; let runtimeCreations = 0; @@ -388,56 +503,13 @@ const runOwnedArtifactOutcome = async (mode: OutcomeMode): Promise => { const provider = new WabtArtifactProvider( { REA_WABT_BIN_DIRECTORY: selectedRoot }, async (input) => { - const version = input.arguments[0] === "--version"; - if (!version) { - expect(await readFile(join(input.cwd ?? "", "module.wasm"))).toEqual( - bytes, - ); - if (mode === "changed-snapshot") { - const snapshot = join(input.cwd ?? "", "module.wasm"); - await chmod(snapshot, 0o600); - await writeFile(snapshot, Buffer.alloc(76, 2)); - } - if (mode === "timeout") - throw new OwnedCommandFailure( - "timeout", - "Source-owned deadline seam", - ); - } - const stdout = version - ? "1.0.42\n" - : input.command.includes("objdump") - ? mode === "malformed" - ? "unexpected producer output" - : dump - : input.command.includes("wasm2wat") - ? mode === "malformed-wat" - ? "unexpected decoder output\n" - : '(module (@custom "test" "payload"))\n' - : ""; - const invalid = - !version && (mode === "invalid" || mode === "invalid-close"); - const stderr = invalid - ? "module.wasm: invalid bytes\n" - : !version && - input.command.includes("wasm2wat") && - mode === "wat-warning" - ? "decoder warning\n" - : ""; - const script = - !version && mode === "cancelled" - ? "setInterval(() => undefined, 1000)" - : `process.stdout.write(${JSON.stringify(stdout)}); process.stderr.write(${JSON.stringify(stderr)}); process.exit(${invalid ? 1 : 0});`; - const launched = await spawnOwnedProviderProcess({ - ...input, - command: process.execPath, - arguments: ["-e", script], - expectedCommand: null, + const launched = await launchWabtOutcome(input, mode, { + bytes, + dump, + controller, }); - if (!version && mode === "cancelled") { + if (mode === "cancelled" && input.arguments[0] !== "--version") acquiredPid = launched.process.pid; - setImmediate(() => controller.abort()); - } return launched; }, async () => { @@ -446,43 +518,41 @@ const runOwnedArtifactOutcome = async (mode: OutcomeMode): Promise => { prefix: "rea-wabt-owned-test-", }); runtimePath = root.path; + if ( + mode === "snapshot-close-failure" || + mode === "snapshot-read-failure-late-abort" + ) { + fileCloseFailure.selectedPath = join(root.path, "module.wasm"); + fileCloseFailure.failFirst = true; + fileCloseFailure.attempts = 0; + if (mode === "snapshot-read-failure-late-abort") + fileCloseFailure.failRead = () => { + controller.abort("late abort after physical read failure"); + return Object.assign(new Error("Authored snapshot read failure"), { + code: "EACCES", + }); + }; + } return { path: root.path, close: async () => { if (mode === "cancelled-close-failure") { - controller.abort(); + controller.abort(cancellationReason); if (++cleanupAttempts === 1) throw new Error("Source-owned cleanup uncertainty"); } + if (mode === "invalid-close-failure" && ++cleanupAttempts === 1) + throw new Error("Source-owned cleanup uncertainty"); if (mode === "cleanup" && ++cleanupAttempts <= 2) throw new Error("Source-owned cleanup uncertainty"); await root.close(); if (mode === "cancelled-close" || mode === "invalid-close") - controller.abort(); + controller.abort(cancellationReason); }, }; }, ); - const literal = 'fetch("./selected.wasm");'; - const glueSources = - mode === "success" - ? [ - { source: literal, line: 1, column: 6 }, - { source: `\uFEFF${literal}`, line: 1, column: 7 }, - { source: `\uFEFF\n${literal}`, line: 2, column: 6 }, - { source: `#!/usr/bin/env node\n${literal}`, line: 2, column: 6 }, - { - source: `\uFEFF#!/usr/bin/env node\n${literal}`, - line: 2, - column: 6, - }, - { - source: String.raw`"./selected\u002ewasm";`, - line: 1, - column: 0, - }, - ] - : []; + const glueSources = glueSourcesForOutcome(mode); const glueInputs = glueSources.map(({ source, line, column }, index) => ({ path: join(selectedRoot, `glue-${index}.js`), source, @@ -520,7 +590,49 @@ const runOwnedArtifactOutcome = async (mode: OutcomeMode): Promise => { provider, runtimePath, cleanupAttempts: () => cleanupAttempts, + cancellationReason: controller.signal.reason, }); + if (mode === "invalid-close-failure") { + expect(result).toMatchObject({ error: { cleanupIncomplete: true } }); + await provider.close(); + expect(cleanupAttempts).toBe(2); + if (!result.ok) { + expect(result.error.cleanupIncomplete).toBe(false); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "invalid_request", + details: { + captured_output: { stderr: "module.wasm: invalid bytes\n" }, + cleanup_history: expect.any(Object), + }, + }); + } + } + if ( + mode === "snapshot-close-failure" || + mode === "snapshot-read-failure-late-abort" + ) { + try { + expect(result).toMatchObject({ + error: { + cleanupIncomplete: true, + cleanupResources: [join(runtimePath, "module.wasm")], + }, + }); + } finally { + fileCloseFailure.failFirst = false; + fileCloseFailure.failRead = undefined; + await provider.close(); + fileCloseFailure.selectedPath = ""; + } + expect(result).toMatchObject({ + error: { cleanupIncomplete: false, cleanupResources: [] }, + }); + if (!result.ok && mode === "snapshot-read-failure-late-abort") + expect(result.error).toMatchObject({ + _tag: "AnalysisOutputError", + reason: expect.stringContaining("Authored snapshot read failure"), + }); + } await expect(access(runtimePath)).rejects.toMatchObject({ code: "ENOENT" }); expect(await readFile(path)).toEqual(bytes); if (acquiredPid !== undefined) { @@ -531,7 +643,11 @@ const runOwnedArtifactOutcome = async (mode: OutcomeMode): Promise => { it.each(outcomeModes)( "owns private artifact bytes and retains %s outcome", - runOwnedArtifactOutcome, + (mode) => runOwnedArtifactOutcome(mode), +); +it.each([null, { reason: "caller-owned object" }, undefined])( + "preserves finalization abort reason %j through cleanup recovery", + (reason) => runOwnedArtifactOutcome("cancelled-close-failure", reason), ); const outputLimitFailure = ({ stdout = "(module", diff --git a/src/wasm/WabtArtifactProvider.ts b/src/wasm/WabtArtifactProvider.ts index cceb14ef0b..2436a387f0 100644 --- a/src/wasm/WabtArtifactProvider.ts +++ b/src/wasm/WabtArtifactProvider.ts @@ -9,7 +9,12 @@ import type { WasmArtifactPort, } from "../application/wasm/WasmArtifactPort.js"; import { readStableArtifact } from "../artifacts/readStableArtifact.js"; -import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { + ArtifactResourceScope, + ArtifactResourceRetryFailure, +} from "../artifacts/ArtifactResourceScope.js"; +import { AnalysisCleanupLeaseGroup } from "../artifacts/AnalysisCleanupLease.js"; +import { bindArtifactFileCleanup } from "../artifacts/ArtifactFileOperation.js"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import type { AnalysisError, @@ -23,6 +28,7 @@ import { AnalysisOutputError, } from "../domain/analysisErrorCore.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; import { createEvidence } from "../domain/evidence.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { @@ -32,6 +38,7 @@ import { import { err, ok, type Result } from "../domain/result.js"; import { runOwnedCommand } from "../process/OwnedCommand.js"; import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; +import { WabtRuntime } from "./WabtRuntime.js"; import { WABT_PROVIDER_IDENTITY, WABT_LIMITS } from "./WabtRelease.js"; import { wabtFailure } from "./WabtFailures.js"; import { parseWabtOutput } from "./WabtOutput.js"; @@ -57,18 +64,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { try { await this.#resources.close(); } catch (cause: unknown) { - const cleanup = - cause instanceof ArtifactReaderFailure ? cause.cleanup : undefined; - throw new ProviderCleanupError( - WABT_PROVIDER_IDENTITY.id, - cleanup?.resources ?? ["artifact-resources"], - { - reason: - cleanup?.reason ?? - (cause instanceof Error ? cause.message : String(cause)), - }, - { operation: OPERATION, cause }, - ); + throw resourceCleanupFailure(cause); } } async inspect( @@ -78,6 +74,8 @@ export class WabtArtifactProvider implements WasmArtifactPort { try { return await this.#resources.run(() => this.#inspect(input, options)); } catch (cause: unknown) { + if (cause instanceof ArtifactResourceRetryFailure) + return err(resourceCleanupFailure(cause)); if ( cause instanceof ArtifactReaderFailure && cause.reason === "unavailable" && @@ -100,7 +98,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { input: InspectWasmArtifactInput, options?: ExecutionOptions, ): Promise> { - let root: Pick | undefined; + let runtime: WabtRuntime | undefined; let phase: "configuration" | "artifact" | "producer" = "configuration"; let result: Result; let capturedOutput: AnalysisCapturedOutput | undefined; @@ -144,36 +142,25 @@ export class WabtArtifactProvider implements WasmArtifactPort { : ["--enable-all", "module.wasm"], }); } - root = await this.createRuntime(); + const root = await this.createRuntime(); + runtime = new WabtRuntime(root); const snapshotPath = join(root.path, "module.wasm"); let snapshotHash: string | undefined; const verifySnapshot = async () => { if (snapshotHash === undefined) return; - try { - const bytes = await readStableArtifact( - snapshotPath, - WABT_LIMITS.inputBytes, - options?.signal, - undefined, - this.#resources, - ); - if (bytes.sha256 !== snapshotHash) - throw new Error("Private WASM snapshot bytes changed."); - } catch (cause: unknown) { - const cleanup = - cause instanceof ArtifactReaderFailure ? cause.cleanup : undefined; - if (options?.signal?.aborted) - throw new AnalysisCancelledError(OPERATION, { - cause: options.signal.reason, - ...(cleanup === undefined ? {} : { cleanup }), - }); + const bytes = await readStableArtifact( + snapshotPath, + WABT_LIMITS.inputBytes, + options?.signal, + undefined, + this.#resources, + ); + if (bytes.sha256 !== snapshotHash) throw new AnalysisArtifactChangedError( OPERATION, input.path, - "Private WASM snapshot no longer matches selected byte identity.", - { cause, ...(cleanup === undefined ? {} : { cleanup }) }, + "Private WASM snapshot bytes changed.", ); - } }; const execute = async ( command: (typeof commands)[number], @@ -186,7 +173,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { { command: command.path, arguments: args, - ...(root === undefined ? {} : { cwd: root.path }), + cwd: root.path, expectedCommand: command.path, runId: `rea-wabt-${randomUUID()}`, hostEnvironment: this.environment, @@ -431,22 +418,25 @@ export class WabtArtifactProvider implements WasmArtifactPort { ); } catch (cause: unknown) { const failure = wabtFailure(cause, phase, input.path); + runtime?.retainCommandFailure(cause, failure, this.#resources); result = err( failure instanceof AnalysisOutputError && failure.capturedOutput === undefined && capturedOutput !== undefined ? new AnalysisOutputError(OPERATION, failure.reason, { - cause, + cause: failure, capturedOutput, - }) + cleanup: failure.cleanup, + diagnostics: failure.diagnostics, + }).bindCleanupDependencies([failure]) : failure, ); } - if (root !== undefined) { + if (runtime !== undefined) { const cleanup = await this.#resources.release({ - kind: "private-runtime-root", - resource: root.path, - root, + kind: "lease", + resource: runtime.path, + lease: runtime.lease, }); let partialObservation = result.ok ? undefined @@ -470,12 +460,10 @@ export class WabtArtifactProvider implements WasmArtifactPort { } if (result.ok && options?.signal?.aborted) { result = err( - new AnalysisCancelledError( - OPERATION, - partialObservation === undefined - ? undefined - : { partialObservation }, - ), + new AnalysisCancelledError(OPERATION, { + cause: options?.signal?.reason, + ...(partialObservation === undefined ? {} : { partialObservation }), + }), ); } if (cleanup.kind === "failed") { @@ -483,27 +471,53 @@ export class WabtArtifactProvider implements WasmArtifactPort { const retainedObservation = result.ok ? partialObservation : result.error.partialObservation; - result = err( - new ProviderCleanupError( - WABT_PROVIDER_IDENTITY.id, - [root.path], - { - reason: cause instanceof Error ? cause.message : String(cause), - previous_error: result.ok - ? null - : projectAnalysisError(result.error), - }, - { - operation: OPERATION, - cause, - ...(retainedObservation === undefined - ? {} - : { partialObservation: retainedObservation }), - }, - ), + const failure = new ProviderCleanupError( + WABT_PROVIDER_IDENTITY.id, + [runtime.path], + { + reason: cause instanceof Error ? cause.message : String(cause), + previous_error: result.ok + ? null + : projectAnalysisError(result.error), + }, + { + operation: OPERATION, + cause, + ...(retainedObservation === undefined + ? {} + : { partialObservation: retainedObservation }), + }, ); + new AnalysisCleanupLeaseGroup(failure, [runtime.lease]); + result = result.ok + ? err(failure) + : err( + analysisErrorWithCleanupFailure( + result.error, + failure, + OPERATION, + ).bindCleanupDependencies([result.error, failure]), + ); } } return result; } } + +/** Retry errors follow their exact retained leases, including partially released groups. */ +const resourceCleanupFailure = (cause: unknown): ProviderCleanupError => { + const cleanup = + cause instanceof ArtifactReaderFailure ? cause.cleanup : undefined; + const failure = new ProviderCleanupError( + WABT_PROVIDER_IDENTITY.id, + cleanup?.resources ?? ["artifact-resources"], + { + reason: + cleanup?.reason ?? + (cause instanceof Error ? cause.message : String(cause)), + }, + { operation: OPERATION, cause }, + ); + bindArtifactFileCleanup(cause, failure); + return failure; +}; diff --git a/src/wasm/WabtFailures.ts b/src/wasm/WabtFailures.ts index e737163c7d..8dfb9b774e 100644 --- a/src/wasm/WabtFailures.ts +++ b/src/wasm/WabtFailures.ts @@ -1,5 +1,8 @@ import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; -import { artifactFileCancellationCause } from "../artifacts/ArtifactFileOperation.js"; +import { + artifactFileCancellationCause, + bindArtifactFileCleanup, +} from "../artifacts/ArtifactFileOperation.js"; import { boundedUtf8Text } from "../process/boundedUtf8Text.js"; import type { ProviderProcessSnapshot } from "../process/ProviderProcess.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; @@ -37,6 +40,16 @@ export const wabtFailure = ( cause: unknown, phase: "configuration" | "artifact" | "producer", path: string, +): AnalysisError => { + const failure = classifyWabtFailure(cause, phase, path); + bindArtifactFileCleanup(cause, failure); + return failure; +}; + +const classifyWabtFailure = ( + cause: unknown, + phase: "configuration" | "artifact" | "producer", + path: string, ): AnalysisError => { if (cause instanceof AnalysisError) return cause; const artifactCleanup = diff --git a/src/wasm/WabtRuntime.test.ts b/src/wasm/WabtRuntime.test.ts new file mode 100644 index 0000000000..4663554070 --- /dev/null +++ b/src/wasm/WabtRuntime.test.ts @@ -0,0 +1,119 @@ +import { EventEmitter } from "node:events"; +import { PassThrough } from "node:stream"; +import { expect, it, vi } from "vitest"; +import { AnalysisCleanupLeaseGroup } from "../artifacts/AnalysisCleanupLease.js"; +import { + ArtifactResourceScope, + ArtifactResourceRetryFailure, +} from "../artifacts/ArtifactResourceScope.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { OwnedCommandFailure } from "../process/OwnedCommand.js"; +import { ProviderProcessSupervisor } from "../process/ProviderProcess.js"; +import { WabtRuntime } from "./WabtRuntime.js"; +import { wabtFailure } from "./WabtFailures.js"; + +it("retries exact command owners before runtime removal and preserves mixed outcomes", async () => { + const resources = new ArtifactResourceScope(); + const close = vi.fn(async () => undefined); + const runtime = new WabtRuntime({ path: "owned-runtime", close }); + const controls = [true, false].map((allowed, index) => { + const process = Object.assign(new EventEmitter(), { + stdout: new PassThrough(), + stderr: new PassThrough(), + exitCode: null, + signalCode: null, + kill: () => false, + }); + const owner = new ProviderProcessSupervisor({ + process, + ownsProcessLifetime: false, + }); + const state = { allowed }; + const stop = vi + .spyOn(owner, "stop") + .mockImplementation(async () => + state.allowed + ? { status: "already-exited" } + : { status: "incomplete", reason: "authored cleanup denial" }, + ); + const cause = new OwnedCommandFailure( + "process", + "authored command failure", + null, + "authored cleanup denial", + undefined, + [`command-${String(index)}`], + owner, + ); + const error = wabtFailure(cause, "producer", "input.wasm"); + runtime.retainCommandFailure(cause, error, resources); + return { owner, state, stop, error }; + }); + const rootError = new ProviderCleanupError( + "wabt-artifact", + [runtime.path], + {}, + ); + new AnalysisCleanupLeaseGroup(rootError, [runtime.lease]); + try { + expect( + await resources.release({ + kind: "lease", + resource: runtime.path, + lease: runtime.lease, + }), + ).toMatchObject({ kind: "failed" }); + expect(close).not.toHaveBeenCalled(); + let aggregate: ProviderCleanupError | undefined; + try { + await resources.close(); + } catch (cause) { + expect(cause).toBeInstanceOf(ArtifactResourceRetryFailure); + if ( + !(cause instanceof ArtifactResourceRetryFailure) || + cause.cleanupLeases === undefined + ) + throw cause; + aggregate = new ProviderCleanupError("wabt-artifact", [], {}); + new AnalysisCleanupLeaseGroup(aggregate, cause.cleanupLeases); + } + expect(controls[0]?.error.cleanupIncomplete).toBe(false); + expect(controls[1]?.error.cleanupIncomplete).toBe(true); + expect(aggregate?.cleanupResources).toEqual(["command-1", "owned-runtime"]); + expect(rootError.cleanupIncomplete).toBe(true); + expect(close).not.toHaveBeenCalled(); + controls[1]!.state.allowed = true; + await resources.close(); + expect(controls.map((control) => control.error.cleanupIncomplete)).toEqual([ + false, + false, + ]); + expect(aggregate?.cleanupIncomplete).toBe(false); + expect(rootError.cleanupIncomplete).toBe(false); + expect(close).toHaveBeenCalledOnce(); + expect(controls.map((control) => control.stop.mock.calls.length)).toEqual([ + 1, 2, + ]); + } finally { + for (const control of controls) control.owner.dispose(); + } +}); + +it("keeps runtime removal unconfirmed when a failed command supplies no ownership capability", async () => { + const close = vi.fn(async () => undefined); + const runtime = new WabtRuntime({ path: "owned-runtime", close }); + const cause = new OwnedCommandFailure( + "process", + "unconfirmed command", + null, + "no cleanup owner", + ); + runtime.retainCommandFailure( + cause, + wabtFailure(cause, "producer", "input.wasm"), + new ArtifactResourceScope(), + ); + await expect(runtime.lease.release()).rejects.toThrow("commands must stop"); + expect(runtime.lease.released).toBe(false); + expect(close).not.toHaveBeenCalled(); +}); diff --git a/src/wasm/WabtRuntime.ts b/src/wasm/WabtRuntime.ts new file mode 100644 index 0000000000..ffd9589c23 --- /dev/null +++ b/src/wasm/WabtRuntime.ts @@ -0,0 +1,84 @@ +import { + AnalysisCleanupLease, + AnalysisCleanupLeaseGroup, +} from "../artifacts/AnalysisCleanupLease.js"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import type { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; +import type { ProviderProcessSupervisor } from "../process/ProviderProcess.js"; +import { OwnedCommandFailure } from "../process/OwnedCommand.js"; + +/** A private runtime cannot be removed until every retained command has stopped. */ +export class WabtRuntime { + readonly lease: AnalysisCleanupLease; + readonly path: string; + readonly #commands = new Map< + ProviderProcessSupervisor, + AnalysisCleanupLease + >(); + #unknownCommand: string | undefined; + + constructor(root: Pick) { + this.path = root.path; + this.lease = new AnalysisCleanupLease(async () => { + const pending = [...this.#commands.values()].filter( + (lease) => !lease.released, + ); + if (pending.length > 0 || this.#unknownCommand !== undefined) + throw new ArtifactReaderFailure( + "unavailable", + "WABT commands must stop before runtime removal", + { + cleanup: { + reason: + this.#unknownCommand ?? + "WABT command cleanup remains incomplete", + resources: [ + ...new Set([ + root.path, + ...pending.flatMap((lease) => lease.cleanupResources ?? []), + ]), + ], + }, + }, + ); + await root.close(); + }, [root.path]); + } + + /** Adopt only the supervisor supplied by the command boundary, never a PID or label. */ + retainCommandFailure( + cause: unknown, + failure: AnalysisError, + resources: ArtifactResourceScope, + ): void { + if ( + !(cause instanceof OwnedCommandFailure) || + cause.cleanupFailure === null + ) + return; + const owner = cause.cleanupOwner; + if (owner === undefined) { + this.#unknownCommand = cause.cleanupFailure; + return; + } + let lease = this.#commands.get(owner); + if (lease === undefined) { + lease = new AnalysisCleanupLease(async () => { + const stopped = await owner.stop(); + if (stopped.status === "incomplete") + throw new ArtifactReaderFailure("unavailable", stopped.reason, { + cleanup: { reason: stopped.reason, resources: cause.resources }, + }); + }, cause.resources); + this.#commands.set(owner, lease); + resources.retain({ + kind: "lease", + resource: cause.resources[0] ?? this.path, + lease, + }); + } + new AnalysisCleanupLeaseGroup(failure, [lease]); + } +} diff --git a/tests/fixtures/wasm/cleanupPreload.mjs b/tests/fixtures/wasm/cleanupPreload.mjs new file mode 100644 index 0000000000..60372a7762 --- /dev/null +++ b/tests/fixtures/wasm/cleanupPreload.mjs @@ -0,0 +1,69 @@ +import fs from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +import { dirname, basename, resolve } from "node:path"; +const root = process.env.REA_CLEANUP_PROBE_ROOT; +const trace = process.env.REA_CLEANUP_PROBE_TRACE; +const mode = process.env.REA_CLEANUP_PROBE_MODE; +const selected = process.env.REA_CLEANUP_PROBE_INPUT; +let removals = 0; +let deniedInput = false; +const record = (value) => + fs.appendFileSync(trace, JSON.stringify(value) + "\n"); +const rm = fs.promises.rm.bind(fs.promises); +fs.promises.rm = async (path, ...args) => { + const value = resolve(String(path)); + if (dirname(value) !== root || !basename(value).startsWith("rea-wabt-")) + return rm(path, ...args); + const deny = + (mode === "runtime-once" && removals++ === 0) || + (mode === "runtime-until-allowed" && !fs.existsSync(root + "/allow")); + if (deny) { + record({ + kind: "runtime", + path: value, + denied: true, + exists: fs.existsSync(value), + }); + throw Object.assign(new Error("authored runtime cleanup denial"), { + code: "EACCES", + }); + } + await rm(path, ...args); + record({ + kind: "runtime", + path: value, + denied: false, + exists: fs.existsSync(value), + }); +}; +const open = fs.promises.open.bind(fs.promises); +fs.promises.open = async (path, ...args) => { + const handle = await open(path, ...args); + if (mode === "input-once" && resolve(String(path)) === selected) { + const close = handle.close.bind(handle); + const descriptor = handle.fd; + handle.close = async () => { + if (!deniedInput) { + deniedInput = true; + record({ + kind: "input-close", + path: selected, + descriptor, + denied: true, + }); + throw Object.assign(new Error("authored descriptor cleanup denial"), { + code: "EACCES", + }); + } + await close(); + record({ + kind: "input-close", + path: selected, + descriptor, + denied: false, + }); + }; + } + return handle; +}; +syncBuiltinESMExports();