diff --git a/docs/process-capture.md b/docs/process-capture.md index 69eef8fb04..df9741bdbd 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -450,6 +450,13 @@ and process residual unknowns. Successful cleanup verifies the owned group; it does not attribute those unrelated processes. Related or otherwise unexplained unreadable processes still prevent successful cleanup. +Failed cleanup retains its exact resources for the owning session to retry. +The CLI retries before projecting its error; MCP retries pending cleanup before +admitting another capture and during session cleanup. Once every attributed +owner confirms release, the live error status clears `cleanup_incomplete`. +The original failure, observations and `cleanup_report` retain the facts from +the earlier attempt, including any cleanup failure recorded then. + On macOS, changing a Node process's `process.title` can make its run token unreadable. Node documents that [setting the title overwrites argv memory](https://nodejs.org/download/release/v24.18.0/docs/api/process.html#processtitle). With the pinned toolchain, a live child retained a readable start identity diff --git a/src/process/capture/ProcessCaptureError.test.ts b/src/process/capture/ProcessCaptureError.test.ts index 2eaa56b6a6..6670470a60 100644 --- a/src/process/capture/ProcessCaptureError.test.ts +++ b/src/process/capture/ProcessCaptureError.test.ts @@ -6,13 +6,21 @@ import { normalizeCaptureFailure, ProcessCaptureError, } from "./ProcessCaptureError.js"; -import { resolveProcessResult } from "./ProcessCaptureLifecycle.js"; +import { + resolveProcessResult, + type ProcessCaptureCleanupOutcome, +} from "./ProcessCaptureLifecycle.js"; +import type { ProcessCaptureCleanupReport } from "../../domain/process/processCapture.js"; const cleanupReport = { owned_process_group: { state: "not_required" as const, reason: null }, terminal_renderer: { state: "not_required" as const, reason: null }, temporary_root: { state: "cleaned" as const, reason: null }, }; +const historicalCleanupOutcome = ( + report: ProcessCaptureCleanupReport, +): ProcessCaptureCleanupOutcome => ({ report, cleanupLease: undefined }); + const fixture = () => { const cleanup = new ProviderCleanupError( "process-ownership", @@ -70,7 +78,11 @@ it("retains compiler facts and cleanup dependency through capture finalization", const { cleanup, error } = fixture(); let finalized: unknown; try { - resolveProcessResult(undefined, error, cleanupReport); + resolveProcessResult( + undefined, + error, + historicalCleanupOutcome(cleanupReport), + ); } catch (cause: unknown) { finalized = cause; } @@ -98,13 +110,17 @@ it("keeps later capture cleanup unresolved after compiler cleanup succeeds", () const { cleanup, error } = fixture(); let finalized: unknown; try { - resolveProcessResult(undefined, error, { - ...cleanupReport, - temporary_root: { - state: "failed", - reason: "capture root removal denied", - }, - }); + resolveProcessResult( + undefined, + error, + historicalCleanupOutcome({ + ...cleanupReport, + temporary_root: { + state: "failed", + reason: "capture root removal denied", + }, + }), + ); } catch (cause: unknown) { finalized = cause; } diff --git a/src/process/capture/ProcessCaptureLifecycle.ts b/src/process/capture/ProcessCaptureLifecycle.ts index b92a156dd7..f78248eff3 100644 --- a/src/process/capture/ProcessCaptureLifecycle.ts +++ b/src/process/capture/ProcessCaptureLifecycle.ts @@ -5,6 +5,10 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { setImmediate, setTimeout as delay } from "node:timers/promises"; import type { IPty } from "@lydell/node-pty"; +import { + AnalysisCleanupLease, + AnalysisCleanupLeaseGroup, +} from "../../artifacts/AnalysisCleanupLease.js"; import type { ProcessCapture } from "../../domain/process/processCaptureParsing.js"; import type { ProcessScenario } from "../../domain/process/processScenario.js"; @@ -1095,8 +1099,16 @@ export interface ProcessCaptureResourceCleanupOptions { readonly host?: ProcessCaptureCleanupHost; } +/** Historical cleanup facts and the exact authority that can confirm recovery. */ +export interface ProcessCaptureCleanupOutcome { + readonly report: ProcessCaptureCleanupReport; + /** Undefined means that only historical facts, not an owner, are available. */ + readonly cleanupLease: AnalysisCleanupLease | undefined; +} + /** One capture's exact resource owner, with completion tracked per resource. */ class ProcessCaptureResourceOwner { + readonly cleanupLease: AnalysisCleanupLease; readonly #host: ProcessCaptureCleanupHost; #terminalRenderer: | ProcessCaptureCleanupReport["terminal_renderer"] @@ -1112,14 +1124,27 @@ class ProcessCaptureResourceOwner { this.#terminalRenderer = { state: "not_required", reason: null }; if (options.terminal === undefined) this.#ownedProcessGroup = { state: "not_required", reason: null }; + this.cleanupLease = new AnalysisCleanupLease(async () => { + const report = await this.close(); + const failure = cleanupReportFailure(report); + if (failure === undefined) return; + const error = new ProcessCaptureError(failure, { + reason: "cleanup_incomplete", + cleanup: { reason: failure, resources: this.cleanupResources }, + cleanupReport: report, + }); + new AnalysisCleanupLeaseGroup(error, [this.cleanupLease]); + throw error; + }, this); } - async close(): Promise { - for (const timer of this.options.timers) timer.cancel(); - await this.#releaseRenderer(); - await this.#releaseOwnedProcessGroup(); - await this.#releaseTemporaryRoot(); - return { + get cleanupResources(): readonly string[] { + return unresolvedCleanupResources(this.report); + } + + get report(): ProcessCaptureCleanupReport { + // Reports are observations; consumers cannot edit the owner's completion state. + return structuredClone({ owned_process_group: this.#ownedProcessGroup ?? { state: "unverified", reason: "owned process cleanup was not attempted", @@ -1132,7 +1157,29 @@ class ProcessCaptureResourceOwner { state: "unverified", reason: "temporary root cleanup was not attempted", }, - }; + }); + } + + async close(): Promise { + for (const timer of this.options.timers) timer.cancel(); + await this.#releaseRenderer(); + await this.#releaseOwnedProcessGroup(); + await this.#releaseTemporaryRoot(); + return this.report; + } + + async release(): Promise { + try { + await this.cleanupLease.release(); + return { report: this.report, cleanupLease: this.cleanupLease }; + } catch (cause: unknown) { + if ( + !(cause instanceof ProcessCaptureError) || + cause.cleanupReport === undefined + ) + throw cause; + return { report: cause.cleanupReport, cleanupLease: this.cleanupLease }; + } } async #releaseRenderer(): Promise { @@ -1205,12 +1252,12 @@ const cleanupFailureMessage = (cause: unknown, fallback: string): string => /** Release one capture's resources, retaining the exact owner for scoped retry. */ export const releaseProcessResources = async ( options: ProcessCaptureResourceCleanupOptions, -): Promise => - new ProcessCaptureResourceOwner(options).close(); +): Promise => + new ProcessCaptureResourceOwner(options).release(); /** Run-local owner collection for resources whose first cleanup did not finish. */ export class ProcessCaptureResourceScope { - readonly #pending = new Set(); + readonly #pending = new Set(); readonly #active = new Set>(); #draining: Promise | undefined; #closed = false; @@ -1237,13 +1284,12 @@ export class ProcessCaptureResourceScope { async release( options: ProcessCaptureResourceCleanupOptions, - ): Promise { + ): Promise { const owner = new ProcessCaptureResourceOwner(options); - const report = await owner.close(); - if (cleanupReportFailure(report) !== undefined) { - this.#pending.add(owner); - } - return report; + this.#pending.add(owner.cleanupLease); + const outcome = await owner.release(); + if (owner.cleanupLease.released) this.#pending.delete(owner.cleanupLease); + return outcome; } /** Drain active captures, then retry their retained resources without closing admission. */ @@ -1263,35 +1309,64 @@ export class ProcessCaptureResourceScope { async #drainAndRetry(): Promise { while (this.#active.size > 0) await Promise.allSettled(this.#active); - const failures: string[] = []; - for (const owner of this.#pending) { + const failures: { + readonly lease: AnalysisCleanupLease; + readonly cause: unknown; + }[] = []; + for (const lease of this.#pending) { try { - const report = await owner.close(); - if (cleanupReportFailure(report) === undefined) - this.#pending.delete(owner); - else - failures.push(cleanupReportFailure(report) ?? "cleanup incomplete"); + await lease.release(); + this.#pending.delete(lease); } catch (cause: unknown) { - failures.push( - cause instanceof Error ? cause.message || cause.name : String(cause), - ); + failures.push({ lease, cause }); } } - if (failures.length > 0) - throw new AggregateError( - failures, - `Process capture cleanup remains incomplete: ${failures.join("; ")}`, - ); + if (failures.length === 0) return; + const reason = failures + .map(({ cause }) => cleanupFailureMessage(cause, "cleanup incomplete")) + .join("; "); + const error = new ProcessCaptureError( + `Process capture cleanup remains incomplete: ${reason}`, + { + cause: new AggregateError( + failures.map(({ cause }) => cause), + "Process capture resources remain owned", + ), + reason: "cleanup_incomplete", + cleanup: { + reason, + resources: [ + ...new Set( + failures.flatMap(({ lease }) => lease.cleanupResources ?? []), + ), + ], + }, + }, + ); + new AnalysisCleanupLeaseGroup( + error, + failures.map(({ lease }) => lease), + ); + throw error; } } +const unresolvedCleanupOutcomes = ( + report: ProcessCaptureCleanupReport, +): [string, ProcessCaptureCleanupReport[keyof ProcessCaptureCleanupReport]][] => + Object.entries(report).filter( + ([, outcome]) => + outcome.state === "failed" || outcome.state === "unverified", + ); + +const unresolvedCleanupResources = ( + report: ProcessCaptureCleanupReport, +): string[] => unresolvedCleanupOutcomes(report).map(([resource]) => resource); + export const cleanupReportFailure = ( report: ProcessCaptureCleanupReport, ): string | undefined => { - const failures = Object.entries(report).filter( - ([, outcome]) => - outcome.state === "failed" || outcome.state === "unverified", - ); + const failures = unresolvedCleanupOutcomes(report); return failures.length === 0 ? undefined : failures @@ -1374,13 +1449,14 @@ export interface TerminalCaptureRetention { export const resolveProcessResult = ( capture: PendingProcessCapture | ProcessCapture | undefined, executionFailure: unknown, - cleanup: ProcessCaptureCleanupReport, + cleanupOutcome: ProcessCaptureCleanupOutcome, observations?: IncompleteProcessCaptureObservations, partialContext?: { readonly scenario: ProcessScenario; readonly rootPid?: number; }, ): ProcessCapture => { + const cleanup = cleanupOutcome.report; const cleanupFailure = cleanupReportFailure(cleanup); const executionFailureReason = describeProcessCaptureExecutionFailure(executionFailure); @@ -1429,12 +1505,7 @@ export const resolveProcessResult = ( } } if (cleanupFailure !== undefined) { - const cleanupResources = Object.entries(cleanup) - .filter( - ([, outcome]) => - outcome.state === "failed" || outcome.state === "unverified", - ) - .map(([resource]) => resource); + const cleanupResources = unresolvedCleanupResources(cleanup); const cleanupError = new ProcessCaptureError(cleanupFailure, { cause: executionFailure, reason: "cleanup_incomplete", @@ -1445,6 +1516,10 @@ export const resolveProcessResult = ( ...(partialObservation === undefined ? {} : { partialObservation }), cleanupReport: cleanup, }); + if (cleanupOutcome.cleanupLease !== undefined) + new AnalysisCleanupLeaseGroup(cleanupError, [ + cleanupOutcome.cleanupLease, + ]); if (executionFailure instanceof ProcessCaptureError) throw executionFailure.withFinalization({ ...(executionFailureReason === undefined diff --git a/src/process/capture/ProcessHarness.test.ts b/src/process/capture/ProcessHarness.test.ts index 12cbab964a..6c7ed7cb67 100644 --- a/src/process/capture/ProcessHarness.test.ts +++ b/src/process/capture/ProcessHarness.test.ts @@ -25,6 +25,7 @@ import { createProcessCaptureObservationBuffer, prepareProcessCapture, resolveProcessResult, + type ProcessCaptureCleanupOutcome, type ProcessCaptureCleanupHost, } from "./ProcessCaptureLifecycle.js"; import { DarwinProcessOwnershipInspectionError } from "../DarwinProcessRunTokenReader.js"; @@ -32,6 +33,7 @@ import { DarwinProcessOwnershipInspectionError } from "../DarwinProcessRunTokenR import { partialProcessCaptureObservationSchema, processCaptureSchema, + type ProcessCaptureCleanupReport, } from "../../domain/process/processCapture.js"; const emptyFilesystemCoverage = emptyProcessCapture().truncation_details.filesystem_before; @@ -40,6 +42,10 @@ const incompleteFilesystemCoverage = { enumeration_truncated: true, enumeration_reasons: ["identity_changed"], } satisfies typeof emptyFilesystemCoverage; +const historicalCleanupOutcome = ( + report: ProcessCaptureCleanupReport, +): ProcessCaptureCleanupOutcome => ({ report, cleanupLease: undefined }); + it("retains cancellation and cleanup facts when ownership baseline inspection aborts", async () => { const controller = new AbortController(); const reason = new Error("ownership baseline caller cancellation"); @@ -194,7 +200,7 @@ it("projects complete after-snapshot metadata when a later capture stage fails", resolveProcessResult( undefined, new Error("capture manifest failed after final snapshot"), - cleanup, + historicalCleanupOutcome(cleanup), observations, { scenario }, ); @@ -318,7 +324,7 @@ it("keeps empty cleanup exception messages actionable in the report", async () = throw new Error(""); }, }; - const report = await releaseProcessResources({ + const outcome = await releaseProcessResources({ timers: new Set(), terminal: undefined, renderer: undefined, @@ -327,7 +333,10 @@ it("keeps empty cleanup exception messages actionable in the report", async () = host, }); - expect(report.temporary_root).toEqual({ state: "failed", reason: "Error" }); + expect(outcome.report.temporary_root).toEqual({ + state: "failed", + reason: "Error", + }); }); it.each(["termination", "verification"] as const)( @@ -365,16 +374,16 @@ it.each(["termination", "verification"] as const)( }); expect(removeTemporaryRoot).not.toHaveBeenCalled(); - expect(cleanup.temporary_root).toEqual({ + expect(cleanup.report.temporary_root).toEqual({ state: "unverified", reason: `temporary root retained because owned process cleanup is failed: ${cleanupFailure.message}`, }); - expect(cleanup.owned_process_group).toMatchObject({ + expect(cleanup.report.owned_process_group).toMatchObject({ state: "failed", reason: cleanupFailure.message, }); if (stage === "verification") - expect(cleanup.owned_process_group.unverified_processes).toEqual([ + expect(cleanup.report.owned_process_group.unverified_processes).toEqual([ { pid: 900, reason: "environment unavailable" }, ]); @@ -397,7 +406,7 @@ it.each(["termination", "verification"] as const)( ).toMatchObject({ code: "cleanup_incomplete", details: { - cleanup_report: cleanup, + cleanup_report: cleanup.report, execution_failure: "capture snapshot failed", partial_observation: { capture: { frames: [{ data: "collected before failure" }] }, @@ -445,9 +454,9 @@ it("waits for token-owned processes to exit after one cleanup signal", async () }); await vi.runAllTimersAsync(); - const report = await result; + const outcome = await result; - expect(report.owned_process_group).toEqual({ + expect(outcome.report.owned_process_group).toEqual({ state: "cleaned", reason: null, }); @@ -475,7 +484,7 @@ it("retains unrelated withheld processes through cleanup, capture, and JSON", as }), removeTemporaryRoot: async () => undefined, }; - const report = await releaseProcessResources({ + const outcome = await releaseProcessResources({ timers: new Set(), terminal: { pid: 321 }, renderer: undefined, @@ -487,7 +496,7 @@ it("retains unrelated withheld processes through cleanup, capture, and JSON", as { pid: 900, reason: "platform_binary_environment_withheld" }, { pid: 901, reason: "foreign uid: EINVAL" }, ]; - expect(report.owned_process_group).toEqual({ + expect(outcome.report.owned_process_group).toEqual({ state: "cleaned", reason: null, unverified_processes: unverified, @@ -495,7 +504,7 @@ it("retains unrelated withheld processes through cleanup, capture, and JSON", as const original = emptyProcessCapture(); const { cleanup: previousCleanup, ...pending } = original; for (const input of [pending, original]) { - const resolved = resolveProcessResult(input, undefined, report); + const resolved = resolveProcessResult(input, undefined, outcome); const serialized: unknown = JSON.parse(JSON.stringify(resolved)); const capture = parseProcessCapture(serialized); expect(capture.cleanup).toEqual({ @@ -549,9 +558,9 @@ it("retains the last ownership failure when the verification grace expires", asy }); await vi.runAllTimersAsync(); - const report = await result; + const outcome = await result; - expect(report.owned_process_group).toEqual({ + expect(outcome.report.owned_process_group).toEqual({ state: "unverified", reason: "owned process remained after identity recheck", }); @@ -580,7 +589,7 @@ it("passes sampled detached groups into cleanup even when their token is unknown removeTemporaryRoot: async () => undefined, }; - const report = await releaseProcessResources({ + const outcome = await releaseProcessResources({ timers: new Set(), terminal: { pid: 321 }, renderer: undefined, @@ -591,7 +600,7 @@ it("passes sampled detached groups into cleanup even when their token is unknown }); expect(sampledGroups).toEqual([654]); - expect(report.owned_process_group).toEqual({ + expect(outcome.report.owned_process_group).toEqual({ state: "unverified", reason: "sampled group has no verifiable run token", }); @@ -632,7 +641,11 @@ it("retains observations and both causes when process cleanup is unverifiable", let error: ProcessCaptureError | undefined; try { - resolveProcessResult(capture, executionFailure, cleanup); + resolveProcessResult( + capture, + executionFailure, + historicalCleanupOutcome(cleanup), + ); } catch (cause: unknown) { if (!(cause instanceof ProcessCaptureError)) throw cause; error = cause; @@ -702,7 +715,11 @@ it("projects execution and cleanup failures when capture never completed", () => let error: ProcessCaptureError | undefined; try { - resolveProcessResult(undefined, executionFailure, cleanup); + resolveProcessResult( + undefined, + executionFailure, + historicalCleanupOutcome(cleanup), + ); } catch (cause: unknown) { if (!(cause instanceof ProcessCaptureError)) throw cause; error = cause; @@ -737,7 +754,11 @@ it("retains completed observations when finalization fails after clean cleanup", let error: ProcessCaptureError | undefined; try { - resolveProcessResult(capture, executionFailure, cleanup); + resolveProcessResult( + capture, + executionFailure, + historicalCleanupOutcome(cleanup), + ); } catch (cause: unknown) { if (!(cause instanceof ProcessCaptureError)) throw cause; error = cause; @@ -790,7 +811,11 @@ it("preserves cancellation while projecting observations and successful cleanup" }; let error: ProcessCaptureError | undefined; try { - resolveProcessResult(capture, processCaptureCancelled(), cleanup); + resolveProcessResult( + capture, + processCaptureCancelled(), + historicalCleanupOutcome(cleanup), + ); } catch (cause: unknown) { if (!(cause instanceof ProcessCaptureError)) throw cause; error = cause; diff --git a/src/process/capture/ProcessHarness.ts b/src/process/capture/ProcessHarness.ts index 9b10d1a0cb..baa951f960 100644 --- a/src/process/capture/ProcessHarness.ts +++ b/src/process/capture/ProcessHarness.ts @@ -201,16 +201,16 @@ const cleanupFailedStartup = async (options: { temporaryRoot: options.temporaryRoot, ...(options.cleanupHost === undefined ? {} : { host: options.cleanupHost }), }; - const cleanup = + const cleanupOutcome = options.resourceScope === undefined ? await releaseProcessResources(cleanupOptions) : await options.resourceScope.release(cleanupOptions); - const cleanupFailure = cleanupReportFailure(cleanup); + const cleanupFailure = cleanupReportFailure(cleanupOutcome.report); if (cleanupFailure !== undefined) { resolveProcessResult( undefined, options.cause, - cleanup, + cleanupOutcome, options.observations, { scenario: options.scenario, @@ -401,10 +401,11 @@ const finishProcessRun = async (options: { temporaryRoot: options.temporaryRoot, ...(options.cleanupHost === undefined ? {} : { host: options.cleanupHost }), }; - const cleanup = + const cleanupOutcome = options.resourceScope === undefined ? await releaseProcessResources(cleanupOptions) : await options.resourceScope.release(cleanupOptions); + const cleanup = cleanupOutcome.report; let { capture, executionFailure } = options; if (executionFailure === undefined && options.signal?.aborted) executionFailure = processCaptureCancelled(options.signal.reason); @@ -455,7 +456,7 @@ const finishProcessRun = async (options: { return resolveProcessResult( cleanupFailure === undefined ? verifiedCapture : capture, executionFailure, - cleanup, + cleanupOutcome, options.observations, partialContext, ); diff --git a/src/process/capture/TerminalRenderer.test.ts b/src/process/capture/TerminalRenderer.test.ts index e3f653f235..1d1ff04a1f 100644 --- a/src/process/capture/TerminalRenderer.test.ts +++ b/src/process/capture/TerminalRenderer.test.ts @@ -168,7 +168,7 @@ it.each(["write", "resize"] as const)( runId: "failed-observation", temporaryRoot, }); - expect(cleanup.terminal_renderer).toEqual({ + expect(cleanup.report.terminal_renderer).toEqual({ state: "cleaned", reason: null, }); @@ -213,7 +213,7 @@ it("retains disposal failures for a later cleanup retry", async () => { runId: "failed-disposal", temporaryRoot, }); - expect(cleanup.terminal_renderer).toEqual({ + expect(cleanup.report.terminal_renderer).toEqual({ state: "failed", reason: failure.message, }); diff --git a/src/server/registerCloseLifecycleTool.ts b/src/server/registerCloseLifecycleTool.ts index 1c7edcec0c..39d29c8d0a 100644 --- a/src/server/registerCloseLifecycleTool.ts +++ b/src/server/registerCloseLifecycleTool.ts @@ -2,14 +2,27 @@ import { reportLifecycleEnd, reportLifecycleStart, } from "./lifecycleProgress.js"; -import { ok } from "../domain/result.js"; -import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { err, ok } from "../domain/result.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; +import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; import type { Result } from "../domain/result.js"; import { mcpProgressReporter } from "./mcpProgress.js"; import type { LifecycleToolRegistration } from "./registerSessionTools.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; +/** Expected lifecycle failures keep the same typed result when a child rejects. */ +const closeResult = async ( + close: () => Promise>, +): Promise> => { + try { + return await close(); + } catch (cause: unknown) { + if (cause instanceof AnalysisError) return err(cause); + throw cause; + } +}; + /** Register provider cleanup and optional pre-close snapshot persistence. */ export const registerCloseLifecycleTool = ({ server, @@ -22,8 +35,8 @@ export const registerCloseLifecycleTool = ({ closeSession: () => Promise>, ): Promise> => { const [sessionClose, resourceClose] = await Promise.allSettled([ - closeSession(), - closeProcessResources(), + closeResult(closeSession), + closeResult(closeProcessResources), ]); if (sessionClose.status === "rejected") { if (resourceClose.status === "rejected") @@ -48,9 +61,11 @@ export const registerCloseLifecycleTool = ({ const resources = resourceClose.value; if (!closed.ok) { if (!resources.ok) - logger.warn( - { error: resources.error.message }, - "Process capture cleanup failed while closing the session", + return err( + analysisErrorWithCleanupFailure( + closed.error, + resources.error, + ).bindCleanupDependencies([closed.error, resources.error]), ); return closed; } diff --git a/src/server/registerSessionTools.ts b/src/server/registerSessionTools.ts index d93ddcebf3..dedc3ff590 100644 --- a/src/server/registerSessionTools.ts +++ b/src/server/registerSessionTools.ts @@ -15,7 +15,7 @@ import { ProcessCaptureError } from "../process/capture/ProcessCaptureError.js"; import { toolContract } from "../contracts/toolContracts.js"; import { binaryTargetSummary } from "../application/binary/BinaryTargetSummary.js"; import { UnknownRegistryError } from "../domain/unknownRegistryError.js"; -import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; import type { Evidence } from "../domain/evidence.js"; import type { ProcessCapture } from "../domain/process/processCaptureParsing.js"; import { err, ok, type Result } from "../domain/result.js"; @@ -309,6 +309,7 @@ export const registerSessionTools = ( await processResources.retryCleanup(); return ok(null); } catch (cause: unknown) { + if (cause instanceof AnalysisError) return err(cause); return err( new ProcessCaptureError( "process capture cleanup remains incomplete", diff --git a/tests/boundary/mcp/processCleanupRetryMcp.test.ts b/tests/boundary/mcp/processCleanupRetryMcp.test.ts new file mode 100644 index 0000000000..d808e764f9 --- /dev/null +++ b/tests/boundary/mcp/processCleanupRetryMcp.test.ts @@ -0,0 +1,240 @@ +import { access, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { expect, it, vi } from "vitest"; + +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { AnalysisInputError } from "../../../src/domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { err } from "../../../src/domain/result.js"; +import { ProcessCaptureError } from "../../../src/process/capture/ProcessCaptureError.js"; +import { + ProcessCaptureResourceScope, + type ProcessCaptureCleanupHost, + type ProcessCaptureResourceCleanupOptions, +} from "../../../src/process/capture/ProcessCaptureLifecycle.js"; +import { silentLogger } from "../../../src/logger.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { ToolResultDelivery } from "../../../src/server/toolResult.js"; + +const createTemporaryRoot = (): Promise => + mkdtemp(join(tmpdir(), "rea-mcp-process-cleanup-retry-")); + +const cleanupOptions = ( + root: string, + removeTemporaryRoot: (path: string) => Promise, +): ProcessCaptureResourceCleanupOptions => { + const host: ProcessCaptureCleanupHost = { + platform: process.platform, + cleanupProcessGroup: async () => ({ cleaned: true, signaled: false }), + verifyTokenOwnedProcesses: async () => ({ + cleaned: true, + signaled: false, + }), + removeTemporaryRoot, + }; + return { + timers: new Set(), + terminal: undefined, + renderer: undefined, + runId: "mcp-process-cleanup-retry", + temporaryRoot: root, + host, + }; +}; + +const connectSession = async () => { + const session = createTestBinarySession(() => { + throw new Error("close_binary must not open a binary provider"); + }); + const server = createServer( + { kind: "session", session }, + { logger: silentLogger }, + ); + const client = new Client({ name: "process-cleanup-retry", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + return { client, server, session }; +}; + +const closeSession = async ( + resources: Awaited>, +): Promise => { + await resources.client.close(); + await resources.server.close(); + const closed = await resources.session.close(); + if (!closed.ok) throw closed.error; +}; + +const captureCleanupRetry = (root: string, isDenied: () => boolean) => { + const nativeRetry = ProcessCaptureResourceScope.prototype.retryCleanup; + let seeded = false; + let removalAttempts = 0; + let retryFailure: unknown; + const retry = vi + .spyOn(ProcessCaptureResourceScope.prototype, "retryCleanup") + .mockImplementation(async function ( + this: ProcessCaptureResourceScope, + ): Promise { + if (!seeded) { + seeded = true; + await this.release( + cleanupOptions(root, async (path) => { + if (path !== root) + throw new Error("Cleanup seam received an unexpected root"); + removalAttempts += 1; + if (isDenied()) + throw new Error("controlled process-root removal denial"); + await rm(path, { recursive: true, force: true }); + }), + ); + } + try { + await nativeRetry.call(this); + } catch (cause: unknown) { + retryFailure ??= cause; + throw cause; + } + }); + return { + retry, + get removalAttempts() { + return removalAttempts; + }, + get retryFailure() { + return retryFailure; + }, + }; +}; + +it("projects the retained process root through close_binary and resolves its retry", async () => { + const root = await createTemporaryRoot(); + let denyRemoval = true; + const fault = captureCleanupRetry(root, () => denyRemoval); + const resources = await connectSession(); + const callClose = () => + resources.client.callTool({ name: "close_binary", arguments: {} }); + + try { + const failed = parseMcpToolError(await callClose()); + const firstWire = JSON.stringify(failed); + expect(failed.error).toMatchObject({ + code: "cleanup_incomplete", + details: { resources: ["temporary_root"] }, + }); + expect(fault.removalAttempts).toBe(2); + expect(fault.retryFailure).toBeInstanceOf(ProcessCaptureError); + if (!(fault.retryFailure instanceof ProcessCaptureError)) + throw new Error("Expected the process scope's typed retry error"); + const retryError = fault.retryFailure; + expect(retryError.cleanupIncomplete).toBe(true); + expect(retryError.cleanupResources).toEqual(["temporary_root"]); + + denyRemoval = false; + const recovered = await callClose(); + expect(recovered.isError, JSON.stringify(recovered.content)).not.toBe(true); + expect(fault.removalAttempts).toBe(3); + expect(retryError.cleanupIncomplete).toBe(false); + expect(retryError.cleanupResources).toEqual([]); + expect(projectAnalysisError(retryError).code).toBe( + "process_capture_failed", + ); + expect(JSON.stringify(failed)).toBe(firstWire); + await expect(access(root)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + denyRemoval = false; + fault.retry.mockRestore(); + await closeSession(resources); + await rm(root, { recursive: true, force: true }); + } +}); + +it.each(["returned", "thrown", "synchronous"] as const)( + "composes a %s session close failure with the exact pending process root", + async (kind) => { + const root = await createTemporaryRoot(); + let denyRemoval = true; + const fault = captureCleanupRetry(root, () => denyRemoval); + const resources = await connectSession(); + const primary = new AnalysisInputError("close_binary", undefined, [ + { + path: ["session"], + reason: "invalid_value", + message: "primary session-close diagnostic", + }, + ]); + const closeSpy = vi.spyOn(resources.session, "close"); + if (kind === "returned") closeSpy.mockResolvedValue(err(primary)); + else if (kind === "thrown") closeSpy.mockRejectedValue(primary); + else + closeSpy.mockImplementation(() => { + throw primary; + }); + const callClose = () => + resources.client.callTool({ name: "close_binary", arguments: {} }); + const delivery = vi.spyOn(ToolResultDelivery.prototype, "toCallToolResult"); + + try { + const failed = parseMcpToolError(await callClose()); + const firstWire = JSON.stringify(failed); + const delivered = delivery.mock.calls.at(-1)?.[0]; + if (delivered === undefined || delivered.ok) + throw new Error("Expected the composed lifecycle error at delivery"); + const composed = delivered.error; + expect(failed.error).toMatchObject({ + code: "cleanup_incomplete", + details: { + resources: ["temporary_root"], + diagnostics: { + primary_error: { + details: { + issues: [ + { + path: ["session"], + reason: "invalid_value", + message: "primary session-close diagnostic", + }, + ], + }, + }, + }, + }, + }); + expect(fault.removalAttempts).toBe(2); + expect(fault.retryFailure).toBeInstanceOf(ProcessCaptureError); + if (!(fault.retryFailure instanceof ProcessCaptureError)) + throw new Error("Expected the process scope's typed retry error"); + const retryError = fault.retryFailure; + expect(retryError.cleanupIncomplete).toBe(true); + expect(composed.cleanupIncomplete).toBe(true); + + denyRemoval = false; + const recovered = parseMcpToolError(await callClose()); + expect(recovered.error).toMatchObject({ code: "invalid_request" }); + expect(retryError.cleanupIncomplete).toBe(false); + expect(composed.cleanupIncomplete).toBe(false); + expect(projectAnalysisError(composed)).toMatchObject({ + code: "invalid_request", + details: { issues: primary.issues }, + }); + expect(projectAnalysisError(retryError).code).toBe( + "process_capture_failed", + ); + expect(JSON.stringify(failed)).toBe(firstWire); + expect(primary.cleanupIncomplete).toBe(false); + await expect(access(root)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + denyRemoval = false; + closeSpy.mockRestore(); + delivery.mockRestore(); + fault.retry.mockRestore(); + await closeSession(resources); + await rm(root, { recursive: true, force: true }); + } + }, +); diff --git a/tests/boundary/mcp/toolResult.test.ts b/tests/boundary/mcp/toolResult.test.ts index 700419e9ba..ed3a6bfa3a 100644 --- a/tests/boundary/mcp/toolResult.test.ts +++ b/tests/boundary/mcp/toolResult.test.ts @@ -7,7 +7,10 @@ import { } from "@modelcontextprotocol/server"; import { ProcessCaptureError } from "../../../src/process/capture/ProcessCaptureError.js"; -import { resolveProcessResult } from "../../../src/process/capture/ProcessCaptureLifecycle.js"; +import { + resolveProcessResult, + type ProcessCaptureCleanupOutcome, +} from "../../../src/process/capture/ProcessCaptureLifecycle.js"; import { toolContract } from "../../../src/contracts/toolContracts.js"; import type { ToolContract } from "../../../src/contracts/toolContractTypes.js"; import { err } from "../../../src/domain/result.js"; @@ -23,13 +26,19 @@ import type { JsonValue } from "../../../src/domain/jsonValue.js"; import { evidenceResultOf } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; import { toolRegistrationOptions } from "../../../src/server/toolRegistrationOptions.js"; -import type { IncompleteProcessCaptureObservations } from "../../../src/domain/process/processCapture.js"; +import type { + IncompleteProcessCaptureObservations, + ProcessCaptureCleanupReport, +} from "../../../src/domain/process/processCapture.js"; const delivery = new ToolResultDelivery(STDIO_DEFAULT_MAX_BUFFER_SIZE); const UNTRUSTED_TARGET_CONTENT = { provenance: { untrusted_content: true, source: "analysis_target" }, }; const emptyFilesystemCoverage = emptyProcessCapture().truncation_details.filesystem_before; +const historicalCleanupOutcome = ( + report: ProcessCaptureCleanupReport, +): ProcessCaptureCleanupOutcome => ({ report, cleanupLease: undefined }); const captureWithObservedOutput = () => accountFullyObservedProcessCapture({ @@ -80,7 +89,7 @@ describe("completed partial process capture MCP projection", () => { resolveProcessResult( capture, new Error("final filesystem snapshot failed"), - cleanup, + historicalCleanupOutcome(cleanup), ); } catch (cause: unknown) { if (!(cause instanceof ProcessCaptureError)) throw cause; @@ -128,7 +137,11 @@ describe("completed partial process capture MCP projection", () => { let failure: ProcessCaptureError | undefined; try { - resolveProcessResult(capture, executionFailure, cleanup); + resolveProcessResult( + capture, + executionFailure, + historicalCleanupOutcome(cleanup), + ); } catch (cause: unknown) { if (!(cause instanceof ProcessCaptureError)) throw cause; failure = cause; @@ -267,7 +280,7 @@ describe("incomplete partial process observations MCP projection", () => { resolveProcessResult( undefined, new Error("capture failed before completion"), - cleanup, + historicalCleanupOutcome(cleanup), observations, { scenario, rootPid }, ); diff --git a/tests/boundary/process/processCleanupRecoveryCli.test.ts b/tests/boundary/process/processCleanupRecoveryCli.test.ts new file mode 100644 index 0000000000..eb4722cf80 --- /dev/null +++ b/tests/boundary/process/processCleanupRecoveryCli.test.ts @@ -0,0 +1,170 @@ +import { access, readFile, rm, writeFile } from "node:fs/promises"; +import { execFile } from "node:child_process"; +import { join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; + +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { itWithCaptureCapability } from "./processCaptureCapability.js"; + +const execFileAsync = promisify(execFile); +const CLI_TIMEOUT_MS = 30_000; + +const cleanupPreload = ` +import fs from "node:fs"; +import { basename, resolve } from "node:path"; +import { syncBuiltinESMExports } from "node:module"; + +const cliEntry = process.env.REA_TEST_CLI_ENTRY; +if (cliEntry !== undefined && resolve(process.argv[1] ?? "") === cliEntry) { + const markerPath = process.env.REA_TEST_CLEANUP_MARKER; + const originalMkdtemp = fs.promises.mkdtemp; + const originalRm = fs.promises.rm; + let captureRoot; + let removalAttempts = 0; + let released = false; + const record = () => { + if (markerPath !== undefined) { + fs.writeFileSync(markerPath, JSON.stringify({ + root: captureRoot, + removalAttempts, + released, + })); + } + }; + + fs.promises.mkdtemp = async function (prefix, ...options) { + const root = await originalMkdtemp.call(this, prefix, ...options); + if (basename(String(prefix)) === "rea-process-") { + captureRoot = resolve(root); + record(); + } + return root; + }; + fs.promises.rm = async function (path, ...options) { + if (captureRoot === undefined || resolve(String(path)) !== captureRoot) + return originalRm.call(this, path, ...options); + removalAttempts += 1; + record(); + if (removalAttempts === 1) + throw new Error("controlled first temporary-root removal denial"); + const result = await originalRm.call(this, path, ...options); + released = true; + record(); + return result; + }; + syncBuiltinESMExports(); +} +`; + +interface CleanupMarker { + readonly root: string; + readonly removalAttempts: number; + readonly released: boolean; +} + +const readCleanupMarker = async (path: string): Promise => { + const contents = await readFile(path, "utf8"); + const parsed: unknown = JSON.parse(contents); + if (typeof parsed !== "object" || parsed === null) + throw new Error("Cleanup preload did not write its marker object"); + const candidate = parsed as Partial; + if ( + typeof candidate.root !== "string" || + typeof candidate.removalAttempts !== "number" || + typeof candidate.released !== "boolean" + ) + throw new Error("Cleanup preload marker is missing removal results"); + return { + root: candidate.root, + removalAttempts: candidate.removalAttempts, + released: candidate.released, + }; +}; + +itWithCaptureCapability( + "retries the failed capture root before projecting the completed CLI failure", + async () => { + const fixtureRoot = await createTestTempDirectory( + "rea-process-cleanup-recovery-cli-", + ); + const scenarioPath = join(fixtureRoot, "scenario.json"); + const preloadPath = join(fixtureRoot, "cleanup-preload.mjs"); + const markerPath = join(fixtureRoot, "cleanup-marker.json"); + await writeFile( + scenarioPath, + JSON.stringify({ + executable: process.execPath, + arguments: ["-e", "process.stdout.write('capture completed\\n')"], + working_directory: fixtureRoot, + }), + ); + await writeFile(preloadPath, cleanupPreload); + + let captureRoot: string | undefined; + try { + const cliEntry = resolve("scripts/rea.mjs"); + const result = await execFileAsync( + process.execPath, + [cliEntry, "capture-process", scenarioPath, "--json"], + { + cwd: fixtureRoot, + env: { + ...process.env, + NODE_OPTIONS: `--import=${pathToFileURL(preloadPath).href}`, + REA_TEST_CLI_ENTRY: cliEntry, + REA_TEST_CLEANUP_MARKER: markerPath, + }, + timeout: CLI_TIMEOUT_MS, + }, + ).then( + ({ stdout, stderr }) => ({ exitCode: 0, stdout, stderr }), + (cause: unknown) => { + if (!(cause instanceof Error)) throw cause; + const failure = cause as Error & { + readonly code?: number | string; + readonly stdout?: string; + readonly stderr?: string; + }; + return { + exitCode: failure.code, + stdout: failure.stdout ?? "", + stderr: failure.stderr ?? "", + }; + }, + ); + const marker = await readCleanupMarker(markerPath); + captureRoot = marker.root; + expect(result.exitCode, result.stderr || result.stdout).toBe(1); + expect(marker.removalAttempts).toBe(2); + expect(marker.released).toBe(true); + const output: unknown = JSON.parse(result.stdout); + expect(output).toMatchObject({ + error: "Process command failed", + code: "process_capture_failed", + details: { + cleanup_report: { + temporary_root: { + state: "failed", + reason: "controlled first temporary-root removal denial", + }, + }, + partial_observation: { + capture: { + exit: { code: 0, reason: "exited" }, + }, + }, + }, + }); + await expect(access(captureRoot)).rejects.toMatchObject({ + code: "ENOENT", + }); + } finally { + if (captureRoot !== undefined) + await rm(captureRoot, { recursive: true, force: true }); + } + }, + CLI_TIMEOUT_MS, +); diff --git a/tests/boundary/process/processCleanupRetryOwnership.test.ts b/tests/boundary/process/processCleanupRetryOwnership.test.ts new file mode 100644 index 0000000000..0c8a4877b6 --- /dev/null +++ b/tests/boundary/process/processCleanupRetryOwnership.test.ts @@ -0,0 +1,346 @@ +import { mkdtemp, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setImmediate } from "node:timers/promises"; +import { expect, it } from "vitest"; +import { createDeferred } from "../../fixtures/binarySession.js"; + +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { ProcessCaptureError } from "../../../src/process/capture/ProcessCaptureError.js"; +import { + ProcessCaptureResourceScope, + resolveProcessResult, + type ProcessCaptureCleanupHost, + type ProcessCaptureResourceCleanupOptions, +} from "../../../src/process/capture/ProcessCaptureLifecycle.js"; + +const cleanupOptions = ( + temporaryRoot: string, + removeTemporaryRoot: (path: string) => Promise, +): ProcessCaptureResourceCleanupOptions => { + const host: ProcessCaptureCleanupHost = { + platform: process.platform, + cleanupProcessGroup: async () => ({ cleaned: true, signaled: false }), + verifyTokenOwnedProcesses: async () => ({ + cleaned: true, + signaled: false, + }), + removeTemporaryRoot, + }; + return { + timers: new Set(), + terminal: undefined, + renderer: undefined, + runId: "cleanup-retry-boundary", + temporaryRoot, + host, + }; +}; + +const createRoot = (): Promise => + mkdtemp(join(tmpdir(), "rea-process-cleanup-retry-")); + +const expectCaptureError = (operation: () => unknown): ProcessCaptureError => { + try { + operation(); + } catch (cause: unknown) { + if (cause instanceof ProcessCaptureError) return cause; + throw cause; + } + throw new Error("Expected process capture failure"); +}; + +it("resolves the origin and finalization copy after the exact root retry", async () => { + const scope = new ProcessCaptureResourceScope(); + const root = await createRoot(); + const primaryCause = new Error("capture failed before root cleanup"); + const primary = new ProcessCaptureError("capture failed", { + cause: primaryCause, + reason: "cleanup_incomplete", + cleanup: { + reason: "temporary root removal failed", + resources: ["temporary_root"], + }, + }); + let denyRemoval = true; + const release = await scope.release( + cleanupOptions(root, async (path) => { + if (denyRemoval) throw new Error("controlled root removal denial"); + await rm(path, { recursive: true }); + }), + ); + + try { + if (release.cleanupLease === undefined) + throw new Error("Expected the failed root's exact cleanup lease"); + release.cleanupLease.bindCleanupError(primary); + const reported = expectCaptureError(() => + resolveProcessResult(undefined, primary, release), + ); + const historicalReport = reported.cleanupReport; + expect(reported).not.toBe(primary); + expect(primary.cleanupIncomplete).toBe(true); + expect(reported.cleanupIncomplete).toBe(true); + expect(projectAnalysisError(reported).code).toBe("cleanup_incomplete"); + expect(reported.cause).toBe(primaryCause); + expect(historicalReport?.temporary_root).toEqual({ + state: "failed", + reason: "controlled root removal denial", + }); + + denyRemoval = false; + await scope.retryCleanup(); + await scope.close(); + + expect(primary.cleanupIncomplete).toBe(false); + expect(reported.cleanupIncomplete).toBe(false); + expect(projectAnalysisError(reported).code).not.toBe("cleanup_incomplete"); + expect(reported.cause).toBe(primaryCause); + expect(reported.cleanupReport).toBe(historicalReport); + expect(reported.cleanupReport?.temporary_root).toEqual({ + state: "failed", + reason: "controlled root removal denial", + }); + await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + denyRemoval = false; + await scope.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +it("keeps mixed root errors and retry aggregates bound to only unresolved roots", async () => { + const scope = new ProcessCaptureResourceScope(); + const roots = [await createRoot(), await createRoot()] as const; + const denied = new Set(roots); + const outcomes = new Map>>(); + + try { + for (const root of roots) { + outcomes.set( + root, + await scope.release( + cleanupOptions(root, async (path) => { + if (denied.has(path)) + throw new Error(`controlled removal denial: ${path}`); + await rm(path, { recursive: true }); + }), + ), + ); + } + const firstOutcome = outcomes.get(roots[0]); + const secondOutcome = outcomes.get(roots[1]); + if (firstOutcome === undefined || secondOutcome === undefined) + throw new Error("Expected both retained cleanup outcomes"); + const first = expectCaptureError(() => + resolveProcessResult( + undefined, + new ProcessCaptureError("first primary failure"), + firstOutcome, + ), + ); + const second = expectCaptureError(() => + resolveProcessResult( + undefined, + new ProcessCaptureError("second primary failure"), + secondOutcome, + ), + ); + expect(first.cleanupIncomplete).toBe(true); + expect(second.cleanupIncomplete).toBe(true); + + const bothUnresolved: unknown = await scope + .retryCleanup() + .catch((cause: unknown) => cause); + expect(bothUnresolved).toBeInstanceOf(ProcessCaptureError); + if (!(bothUnresolved instanceof ProcessCaptureError)) + throw new Error("Expected both owners in a typed retry aggregate"); + expect(bothUnresolved.cleanupIncomplete).toBe(true); + expect(bothUnresolved.cause).toBeInstanceOf(AggregateError); + if (!(bothUnresolved.cause instanceof AggregateError)) + throw new Error("Expected retained child retry failures"); + expect(bothUnresolved.cause.errors).toHaveLength(2); + + denied.delete(roots[0]); + const retryFailure: unknown = await scope + .retryCleanup() + .catch((cause: unknown) => cause); + expect(retryFailure).toBeInstanceOf(ProcessCaptureError); + if (!(retryFailure instanceof ProcessCaptureError)) + throw new Error("Expected a typed retry aggregate"); + expect(first.cleanupIncomplete).toBe(false); + expect(second.cleanupIncomplete).toBe(true); + expect(bothUnresolved.cleanupIncomplete).toBe(true); + expect(retryFailure.cleanupIncomplete).toBe(true); + expect(retryFailure.cleanupResources).toEqual(["temporary_root"]); + + denied.clear(); + await scope.retryCleanup(); + await scope.close(); + expect(second.cleanupIncomplete).toBe(false); + expect(retryFailure.cleanupIncomplete).toBe(false); + expect(bothUnresolved.cleanupIncomplete).toBe(false); + expect(first.cleanupReport?.temporary_root.state).toBe("failed"); + expect(second.cleanupReport?.temporary_root.state).toBe("failed"); + for (const root of roots) + await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + denied.clear(); + await scope.close(); + await Promise.all( + roots.map((root) => rm(root, { recursive: true, force: true })), + ); + } +}); + +it("joins an already-started release when shutdown begins", async () => { + const scope = new ProcessCaptureResourceScope(); + const root = await createRoot(); + const entered = createDeferred(); + const unblock = createDeferred(); + let attempts = 0; + const releasing = scope.release( + cleanupOptions(root, async (path) => { + attempts += 1; + entered.resolve(); + await unblock.promise; + await rm(path, { recursive: true }); + }), + ); + await entered.promise; + let closed = false; + const closing = scope.close().then(() => { + closed = true; + }); + try { + await setImmediate(); + expect(closed).toBe(false); + await expect(stat(root)).resolves.toBeDefined(); + } finally { + unblock.resolve(); + await Promise.all([releasing, closing]); + await rm(root, { recursive: true, force: true }); + } + expect(attempts).toBe(1); + expect(closed).toBe(true); + await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it("binds a historical failed report after its owner already retried", async () => { + const scope = new ProcessCaptureResourceScope(); + const root = await createRoot(); + let denyRemoval = true; + const release = await scope.release( + cleanupOptions(root, async (path) => { + if (denyRemoval) throw new Error("controlled root removal denial"); + await rm(path, { recursive: true }); + }), + ); + + try { + denyRemoval = false; + await scope.retryCleanup(); + await scope.close(); + + const reported = expectCaptureError(() => + resolveProcessResult( + undefined, + new ProcessCaptureError("primary failure after release"), + release, + ), + ); + expect(reported.cleanupIncomplete).toBe(false); + expect(projectAnalysisError(reported).code).not.toBe("cleanup_incomplete"); + expect(reported.cleanupReport?.temporary_root).toMatchObject({ + state: "failed", + }); + await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + denyRemoval = false; + await scope.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +it("blocks scope admission on owner-bound retry failure and resumes after release", async () => { + const scope = new ProcessCaptureResourceScope(); + const root = await createRoot(); + let denyRemoval = true; + const release = await scope.release( + cleanupOptions(root, async (path) => { + if (denyRemoval) throw new Error("controlled root removal denial"); + await rm(path, { recursive: true }); + }), + ); + const origin = expectCaptureError(() => + resolveProcessResult( + undefined, + new ProcessCaptureError("primary failure"), + release, + ), + ); + + try { + let admitted = false; + const retryFailure: unknown = await scope + .run(async () => { + admitted = true; + }) + .catch((cause: unknown) => cause); + expect(admitted).toBe(false); + expect(retryFailure).toBeInstanceOf(ProcessCaptureError); + if (!(retryFailure instanceof ProcessCaptureError)) + throw new Error("Expected a typed owner-bound retry failure"); + expect(retryFailure.cleanupIncomplete).toBe(true); + expect(retryFailure.cleanupResources).toEqual(["temporary_root"]); + expect(origin.cleanupIncomplete).toBe(true); + + denyRemoval = false; + await expect( + scope.run(async () => { + admitted = true; + return "admitted after cleanup"; + }), + ).resolves.toBe("admitted after cleanup"); + await scope.close(); + expect(admitted).toBe(true); + expect(origin.cleanupIncomplete).toBe(false); + expect(retryFailure.cleanupIncomplete).toBe(false); + await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + denyRemoval = false; + await scope.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +it("keeps historical report edits separate from cleanup authority", async () => { + const scope = new ProcessCaptureResourceScope(); + const root = await createRoot(); + let denied = true; + let attempts = 0; + try { + const outcome = await scope.release( + cleanupOptions(root, async (path) => { + attempts += 1; + if (denied) throw new Error("controlled root removal denial"); + await rm(path, { recursive: true }); + }), + ); + Reflect.set(outcome.report.temporary_root, "state", "cleaned"); + await expect(scope.retryCleanup()).rejects.toBeInstanceOf( + ProcessCaptureError, + ); + expect(attempts).toBe(2); + expect(outcome.cleanupLease?.released).toBe(false); + await expect(stat(root)).resolves.toBeDefined(); + denied = false; + await scope.close(); + expect(attempts).toBe(3); + await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + denied = false; + await scope.close(); + await rm(root, { recursive: true, force: true }); + } +});