mirror of
https://github.com/morluto/rea
synced 2026-10-25 22:08:11 -04:00
fix(process): preserve cleanup ownership through retries and close
Carry historical cleanup reports with the exact lease that can confirm release. Bind originating capture errors and typed retry aggregates to those owners while preserving primary failures and completed observations. Keep those bindings through MCP close and compose mixed session/process cleanup failures. Start both close operations even when a child throws a typed failure synchronously. Fixes #5144 Fixes #5163 Fixes #5166
This commit is contained in:
@@ -450,6 +450,13 @@ and process residual unknowns. Successful cleanup verifies the owned group;
|
||||
it does not attribute those unrelated processes. Related or otherwise unexplained
|
||||
unreadable processes still prevent successful cleanup.
|
||||
|
||||
Failed cleanup retains its exact resources for the owning session to retry.
|
||||
The CLI retries before projecting its error; MCP retries pending cleanup before
|
||||
admitting another capture and during session cleanup. Once every attributed
|
||||
owner confirms release, the live error status clears `cleanup_incomplete`.
|
||||
The original failure, observations and `cleanup_report` retain the facts from
|
||||
the earlier attempt, including any cleanup failure recorded then.
|
||||
|
||||
On macOS, changing a Node process's `process.title` can make its run token
|
||||
unreadable. Node documents that [setting the title overwrites argv memory](https://nodejs.org/download/release/v24.18.0/docs/api/process.html#processtitle).
|
||||
With the pinned toolchain, a live child retained a readable start identity
|
||||
|
||||
@@ -6,13 +6,21 @@ import {
|
||||
normalizeCaptureFailure,
|
||||
ProcessCaptureError,
|
||||
} from "./ProcessCaptureError.js";
|
||||
import { resolveProcessResult } from "./ProcessCaptureLifecycle.js";
|
||||
import {
|
||||
resolveProcessResult,
|
||||
type ProcessCaptureCleanupOutcome,
|
||||
} from "./ProcessCaptureLifecycle.js";
|
||||
import type { ProcessCaptureCleanupReport } from "../../domain/process/processCapture.js";
|
||||
|
||||
const cleanupReport = {
|
||||
owned_process_group: { state: "not_required" as const, reason: null },
|
||||
terminal_renderer: { state: "not_required" as const, reason: null },
|
||||
temporary_root: { state: "cleaned" as const, reason: null },
|
||||
};
|
||||
const historicalCleanupOutcome = (
|
||||
report: ProcessCaptureCleanupReport,
|
||||
): ProcessCaptureCleanupOutcome => ({ report, cleanupLease: undefined });
|
||||
|
||||
const fixture = () => {
|
||||
const cleanup = new ProviderCleanupError(
|
||||
"process-ownership",
|
||||
@@ -70,7 +78,11 @@ it("retains compiler facts and cleanup dependency through capture finalization",
|
||||
const { cleanup, error } = fixture();
|
||||
let finalized: unknown;
|
||||
try {
|
||||
resolveProcessResult(undefined, error, cleanupReport);
|
||||
resolveProcessResult(
|
||||
undefined,
|
||||
error,
|
||||
historicalCleanupOutcome(cleanupReport),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
finalized = cause;
|
||||
}
|
||||
@@ -98,13 +110,17 @@ it("keeps later capture cleanup unresolved after compiler cleanup succeeds", ()
|
||||
const { cleanup, error } = fixture();
|
||||
let finalized: unknown;
|
||||
try {
|
||||
resolveProcessResult(undefined, error, {
|
||||
...cleanupReport,
|
||||
temporary_root: {
|
||||
state: "failed",
|
||||
reason: "capture root removal denied",
|
||||
},
|
||||
});
|
||||
resolveProcessResult(
|
||||
undefined,
|
||||
error,
|
||||
historicalCleanupOutcome({
|
||||
...cleanupReport,
|
||||
temporary_root: {
|
||||
state: "failed",
|
||||
reason: "capture root removal denied",
|
||||
},
|
||||
}),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
finalized = cause;
|
||||
}
|
||||
|
||||
@@ -5,6 +5,10 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { setImmediate, setTimeout as delay } from "node:timers/promises";
|
||||
import type { IPty } from "@lydell/node-pty";
|
||||
import {
|
||||
AnalysisCleanupLease,
|
||||
AnalysisCleanupLeaseGroup,
|
||||
} from "../../artifacts/AnalysisCleanupLease.js";
|
||||
|
||||
import type { ProcessCapture } from "../../domain/process/processCaptureParsing.js";
|
||||
import type { ProcessScenario } from "../../domain/process/processScenario.js";
|
||||
@@ -1095,8 +1099,16 @@ export interface ProcessCaptureResourceCleanupOptions {
|
||||
readonly host?: ProcessCaptureCleanupHost;
|
||||
}
|
||||
|
||||
/** Historical cleanup facts and the exact authority that can confirm recovery. */
|
||||
export interface ProcessCaptureCleanupOutcome {
|
||||
readonly report: ProcessCaptureCleanupReport;
|
||||
/** Undefined means that only historical facts, not an owner, are available. */
|
||||
readonly cleanupLease: AnalysisCleanupLease | undefined;
|
||||
}
|
||||
|
||||
/** One capture's exact resource owner, with completion tracked per resource. */
|
||||
class ProcessCaptureResourceOwner {
|
||||
readonly cleanupLease: AnalysisCleanupLease;
|
||||
readonly #host: ProcessCaptureCleanupHost;
|
||||
#terminalRenderer:
|
||||
| ProcessCaptureCleanupReport["terminal_renderer"]
|
||||
@@ -1112,14 +1124,27 @@ class ProcessCaptureResourceOwner {
|
||||
this.#terminalRenderer = { state: "not_required", reason: null };
|
||||
if (options.terminal === undefined)
|
||||
this.#ownedProcessGroup = { state: "not_required", reason: null };
|
||||
this.cleanupLease = new AnalysisCleanupLease(async () => {
|
||||
const report = await this.close();
|
||||
const failure = cleanupReportFailure(report);
|
||||
if (failure === undefined) return;
|
||||
const error = new ProcessCaptureError(failure, {
|
||||
reason: "cleanup_incomplete",
|
||||
cleanup: { reason: failure, resources: this.cleanupResources },
|
||||
cleanupReport: report,
|
||||
});
|
||||
new AnalysisCleanupLeaseGroup(error, [this.cleanupLease]);
|
||||
throw error;
|
||||
}, this);
|
||||
}
|
||||
|
||||
async close(): Promise<ProcessCaptureCleanupReport> {
|
||||
for (const timer of this.options.timers) timer.cancel();
|
||||
await this.#releaseRenderer();
|
||||
await this.#releaseOwnedProcessGroup();
|
||||
await this.#releaseTemporaryRoot();
|
||||
return {
|
||||
get cleanupResources(): readonly string[] {
|
||||
return unresolvedCleanupResources(this.report);
|
||||
}
|
||||
|
||||
get report(): ProcessCaptureCleanupReport {
|
||||
// Reports are observations; consumers cannot edit the owner's completion state.
|
||||
return structuredClone({
|
||||
owned_process_group: this.#ownedProcessGroup ?? {
|
||||
state: "unverified",
|
||||
reason: "owned process cleanup was not attempted",
|
||||
@@ -1132,7 +1157,29 @@ class ProcessCaptureResourceOwner {
|
||||
state: "unverified",
|
||||
reason: "temporary root cleanup was not attempted",
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async close(): Promise<ProcessCaptureCleanupReport> {
|
||||
for (const timer of this.options.timers) timer.cancel();
|
||||
await this.#releaseRenderer();
|
||||
await this.#releaseOwnedProcessGroup();
|
||||
await this.#releaseTemporaryRoot();
|
||||
return this.report;
|
||||
}
|
||||
|
||||
async release(): Promise<ProcessCaptureCleanupOutcome> {
|
||||
try {
|
||||
await this.cleanupLease.release();
|
||||
return { report: this.report, cleanupLease: this.cleanupLease };
|
||||
} catch (cause: unknown) {
|
||||
if (
|
||||
!(cause instanceof ProcessCaptureError) ||
|
||||
cause.cleanupReport === undefined
|
||||
)
|
||||
throw cause;
|
||||
return { report: cause.cleanupReport, cleanupLease: this.cleanupLease };
|
||||
}
|
||||
}
|
||||
|
||||
async #releaseRenderer(): Promise<void> {
|
||||
@@ -1205,12 +1252,12 @@ const cleanupFailureMessage = (cause: unknown, fallback: string): string =>
|
||||
/** Release one capture's resources, retaining the exact owner for scoped retry. */
|
||||
export const releaseProcessResources = async (
|
||||
options: ProcessCaptureResourceCleanupOptions,
|
||||
): Promise<ProcessCaptureCleanupReport> =>
|
||||
new ProcessCaptureResourceOwner(options).close();
|
||||
): Promise<ProcessCaptureCleanupOutcome> =>
|
||||
new ProcessCaptureResourceOwner(options).release();
|
||||
|
||||
/** Run-local owner collection for resources whose first cleanup did not finish. */
|
||||
export class ProcessCaptureResourceScope {
|
||||
readonly #pending = new Set<ProcessCaptureResourceOwner>();
|
||||
readonly #pending = new Set<AnalysisCleanupLease>();
|
||||
readonly #active = new Set<Promise<unknown>>();
|
||||
#draining: Promise<void> | undefined;
|
||||
#closed = false;
|
||||
@@ -1237,13 +1284,12 @@ export class ProcessCaptureResourceScope {
|
||||
|
||||
async release(
|
||||
options: ProcessCaptureResourceCleanupOptions,
|
||||
): Promise<ProcessCaptureCleanupReport> {
|
||||
): Promise<ProcessCaptureCleanupOutcome> {
|
||||
const owner = new ProcessCaptureResourceOwner(options);
|
||||
const report = await owner.close();
|
||||
if (cleanupReportFailure(report) !== undefined) {
|
||||
this.#pending.add(owner);
|
||||
}
|
||||
return report;
|
||||
this.#pending.add(owner.cleanupLease);
|
||||
const outcome = await owner.release();
|
||||
if (owner.cleanupLease.released) this.#pending.delete(owner.cleanupLease);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
/** Drain active captures, then retry their retained resources without closing admission. */
|
||||
@@ -1263,35 +1309,64 @@ export class ProcessCaptureResourceScope {
|
||||
|
||||
async #drainAndRetry(): Promise<void> {
|
||||
while (this.#active.size > 0) await Promise.allSettled(this.#active);
|
||||
const failures: string[] = [];
|
||||
for (const owner of this.#pending) {
|
||||
const failures: {
|
||||
readonly lease: AnalysisCleanupLease;
|
||||
readonly cause: unknown;
|
||||
}[] = [];
|
||||
for (const lease of this.#pending) {
|
||||
try {
|
||||
const report = await owner.close();
|
||||
if (cleanupReportFailure(report) === undefined)
|
||||
this.#pending.delete(owner);
|
||||
else
|
||||
failures.push(cleanupReportFailure(report) ?? "cleanup incomplete");
|
||||
await lease.release();
|
||||
this.#pending.delete(lease);
|
||||
} catch (cause: unknown) {
|
||||
failures.push(
|
||||
cause instanceof Error ? cause.message || cause.name : String(cause),
|
||||
);
|
||||
failures.push({ lease, cause });
|
||||
}
|
||||
}
|
||||
if (failures.length > 0)
|
||||
throw new AggregateError(
|
||||
failures,
|
||||
`Process capture cleanup remains incomplete: ${failures.join("; ")}`,
|
||||
);
|
||||
if (failures.length === 0) return;
|
||||
const reason = failures
|
||||
.map(({ cause }) => cleanupFailureMessage(cause, "cleanup incomplete"))
|
||||
.join("; ");
|
||||
const error = new ProcessCaptureError(
|
||||
`Process capture cleanup remains incomplete: ${reason}`,
|
||||
{
|
||||
cause: new AggregateError(
|
||||
failures.map(({ cause }) => cause),
|
||||
"Process capture resources remain owned",
|
||||
),
|
||||
reason: "cleanup_incomplete",
|
||||
cleanup: {
|
||||
reason,
|
||||
resources: [
|
||||
...new Set(
|
||||
failures.flatMap(({ lease }) => lease.cleanupResources ?? []),
|
||||
),
|
||||
],
|
||||
},
|
||||
},
|
||||
);
|
||||
new AnalysisCleanupLeaseGroup(
|
||||
error,
|
||||
failures.map(({ lease }) => lease),
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const unresolvedCleanupOutcomes = (
|
||||
report: ProcessCaptureCleanupReport,
|
||||
): [string, ProcessCaptureCleanupReport[keyof ProcessCaptureCleanupReport]][] =>
|
||||
Object.entries(report).filter(
|
||||
([, outcome]) =>
|
||||
outcome.state === "failed" || outcome.state === "unverified",
|
||||
);
|
||||
|
||||
const unresolvedCleanupResources = (
|
||||
report: ProcessCaptureCleanupReport,
|
||||
): string[] => unresolvedCleanupOutcomes(report).map(([resource]) => resource);
|
||||
|
||||
export const cleanupReportFailure = (
|
||||
report: ProcessCaptureCleanupReport,
|
||||
): string | undefined => {
|
||||
const failures = Object.entries(report).filter(
|
||||
([, outcome]) =>
|
||||
outcome.state === "failed" || outcome.state === "unverified",
|
||||
);
|
||||
const failures = unresolvedCleanupOutcomes(report);
|
||||
return failures.length === 0
|
||||
? undefined
|
||||
: failures
|
||||
@@ -1374,13 +1449,14 @@ export interface TerminalCaptureRetention {
|
||||
export const resolveProcessResult = (
|
||||
capture: PendingProcessCapture | ProcessCapture | undefined,
|
||||
executionFailure: unknown,
|
||||
cleanup: ProcessCaptureCleanupReport,
|
||||
cleanupOutcome: ProcessCaptureCleanupOutcome,
|
||||
observations?: IncompleteProcessCaptureObservations,
|
||||
partialContext?: {
|
||||
readonly scenario: ProcessScenario;
|
||||
readonly rootPid?: number;
|
||||
},
|
||||
): ProcessCapture => {
|
||||
const cleanup = cleanupOutcome.report;
|
||||
const cleanupFailure = cleanupReportFailure(cleanup);
|
||||
const executionFailureReason =
|
||||
describeProcessCaptureExecutionFailure(executionFailure);
|
||||
@@ -1429,12 +1505,7 @@ export const resolveProcessResult = (
|
||||
}
|
||||
}
|
||||
if (cleanupFailure !== undefined) {
|
||||
const cleanupResources = Object.entries(cleanup)
|
||||
.filter(
|
||||
([, outcome]) =>
|
||||
outcome.state === "failed" || outcome.state === "unverified",
|
||||
)
|
||||
.map(([resource]) => resource);
|
||||
const cleanupResources = unresolvedCleanupResources(cleanup);
|
||||
const cleanupError = new ProcessCaptureError(cleanupFailure, {
|
||||
cause: executionFailure,
|
||||
reason: "cleanup_incomplete",
|
||||
@@ -1445,6 +1516,10 @@ export const resolveProcessResult = (
|
||||
...(partialObservation === undefined ? {} : { partialObservation }),
|
||||
cleanupReport: cleanup,
|
||||
});
|
||||
if (cleanupOutcome.cleanupLease !== undefined)
|
||||
new AnalysisCleanupLeaseGroup(cleanupError, [
|
||||
cleanupOutcome.cleanupLease,
|
||||
]);
|
||||
if (executionFailure instanceof ProcessCaptureError)
|
||||
throw executionFailure.withFinalization({
|
||||
...(executionFailureReason === undefined
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
createProcessCaptureObservationBuffer,
|
||||
prepareProcessCapture,
|
||||
resolveProcessResult,
|
||||
type ProcessCaptureCleanupOutcome,
|
||||
type ProcessCaptureCleanupHost,
|
||||
} from "./ProcessCaptureLifecycle.js";
|
||||
import { DarwinProcessOwnershipInspectionError } from "../DarwinProcessRunTokenReader.js";
|
||||
@@ -32,6 +33,7 @@ import { DarwinProcessOwnershipInspectionError } from "../DarwinProcessRunTokenR
|
||||
import {
|
||||
partialProcessCaptureObservationSchema,
|
||||
processCaptureSchema,
|
||||
type ProcessCaptureCleanupReport,
|
||||
} from "../../domain/process/processCapture.js";
|
||||
const emptyFilesystemCoverage =
|
||||
emptyProcessCapture().truncation_details.filesystem_before;
|
||||
@@ -40,6 +42,10 @@ const incompleteFilesystemCoverage = {
|
||||
enumeration_truncated: true,
|
||||
enumeration_reasons: ["identity_changed"],
|
||||
} satisfies typeof emptyFilesystemCoverage;
|
||||
const historicalCleanupOutcome = (
|
||||
report: ProcessCaptureCleanupReport,
|
||||
): ProcessCaptureCleanupOutcome => ({ report, cleanupLease: undefined });
|
||||
|
||||
it("retains cancellation and cleanup facts when ownership baseline inspection aborts", async () => {
|
||||
const controller = new AbortController();
|
||||
const reason = new Error("ownership baseline caller cancellation");
|
||||
@@ -194,7 +200,7 @@ it("projects complete after-snapshot metadata when a later capture stage fails",
|
||||
resolveProcessResult(
|
||||
undefined,
|
||||
new Error("capture manifest failed after final snapshot"),
|
||||
cleanup,
|
||||
historicalCleanupOutcome(cleanup),
|
||||
observations,
|
||||
{ scenario },
|
||||
);
|
||||
@@ -318,7 +324,7 @@ it("keeps empty cleanup exception messages actionable in the report", async () =
|
||||
throw new Error("");
|
||||
},
|
||||
};
|
||||
const report = await releaseProcessResources({
|
||||
const outcome = await releaseProcessResources({
|
||||
timers: new Set(),
|
||||
terminal: undefined,
|
||||
renderer: undefined,
|
||||
@@ -327,7 +333,10 @@ it("keeps empty cleanup exception messages actionable in the report", async () =
|
||||
host,
|
||||
});
|
||||
|
||||
expect(report.temporary_root).toEqual({ state: "failed", reason: "Error" });
|
||||
expect(outcome.report.temporary_root).toEqual({
|
||||
state: "failed",
|
||||
reason: "Error",
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["termination", "verification"] as const)(
|
||||
@@ -365,16 +374,16 @@ it.each(["termination", "verification"] as const)(
|
||||
});
|
||||
|
||||
expect(removeTemporaryRoot).not.toHaveBeenCalled();
|
||||
expect(cleanup.temporary_root).toEqual({
|
||||
expect(cleanup.report.temporary_root).toEqual({
|
||||
state: "unverified",
|
||||
reason: `temporary root retained because owned process cleanup is failed: ${cleanupFailure.message}`,
|
||||
});
|
||||
expect(cleanup.owned_process_group).toMatchObject({
|
||||
expect(cleanup.report.owned_process_group).toMatchObject({
|
||||
state: "failed",
|
||||
reason: cleanupFailure.message,
|
||||
});
|
||||
if (stage === "verification")
|
||||
expect(cleanup.owned_process_group.unverified_processes).toEqual([
|
||||
expect(cleanup.report.owned_process_group.unverified_processes).toEqual([
|
||||
{ pid: 900, reason: "environment unavailable" },
|
||||
]);
|
||||
|
||||
@@ -397,7 +406,7 @@ it.each(["termination", "verification"] as const)(
|
||||
).toMatchObject({
|
||||
code: "cleanup_incomplete",
|
||||
details: {
|
||||
cleanup_report: cleanup,
|
||||
cleanup_report: cleanup.report,
|
||||
execution_failure: "capture snapshot failed",
|
||||
partial_observation: {
|
||||
capture: { frames: [{ data: "collected before failure" }] },
|
||||
@@ -445,9 +454,9 @@ it("waits for token-owned processes to exit after one cleanup signal", async ()
|
||||
});
|
||||
|
||||
await vi.runAllTimersAsync();
|
||||
const report = await result;
|
||||
const outcome = await result;
|
||||
|
||||
expect(report.owned_process_group).toEqual({
|
||||
expect(outcome.report.owned_process_group).toEqual({
|
||||
state: "cleaned",
|
||||
reason: null,
|
||||
});
|
||||
@@ -475,7 +484,7 @@ it("retains unrelated withheld processes through cleanup, capture, and JSON", as
|
||||
}),
|
||||
removeTemporaryRoot: async () => undefined,
|
||||
};
|
||||
const report = await releaseProcessResources({
|
||||
const outcome = await releaseProcessResources({
|
||||
timers: new Set(),
|
||||
terminal: { pid: 321 },
|
||||
renderer: undefined,
|
||||
@@ -487,7 +496,7 @@ it("retains unrelated withheld processes through cleanup, capture, and JSON", as
|
||||
{ pid: 900, reason: "platform_binary_environment_withheld" },
|
||||
{ pid: 901, reason: "foreign uid: EINVAL" },
|
||||
];
|
||||
expect(report.owned_process_group).toEqual({
|
||||
expect(outcome.report.owned_process_group).toEqual({
|
||||
state: "cleaned",
|
||||
reason: null,
|
||||
unverified_processes: unverified,
|
||||
@@ -495,7 +504,7 @@ it("retains unrelated withheld processes through cleanup, capture, and JSON", as
|
||||
const original = emptyProcessCapture();
|
||||
const { cleanup: previousCleanup, ...pending } = original;
|
||||
for (const input of [pending, original]) {
|
||||
const resolved = resolveProcessResult(input, undefined, report);
|
||||
const resolved = resolveProcessResult(input, undefined, outcome);
|
||||
const serialized: unknown = JSON.parse(JSON.stringify(resolved));
|
||||
const capture = parseProcessCapture(serialized);
|
||||
expect(capture.cleanup).toEqual({
|
||||
@@ -549,9 +558,9 @@ it("retains the last ownership failure when the verification grace expires", asy
|
||||
});
|
||||
|
||||
await vi.runAllTimersAsync();
|
||||
const report = await result;
|
||||
const outcome = await result;
|
||||
|
||||
expect(report.owned_process_group).toEqual({
|
||||
expect(outcome.report.owned_process_group).toEqual({
|
||||
state: "unverified",
|
||||
reason: "owned process remained after identity recheck",
|
||||
});
|
||||
@@ -580,7 +589,7 @@ it("passes sampled detached groups into cleanup even when their token is unknown
|
||||
removeTemporaryRoot: async () => undefined,
|
||||
};
|
||||
|
||||
const report = await releaseProcessResources({
|
||||
const outcome = await releaseProcessResources({
|
||||
timers: new Set(),
|
||||
terminal: { pid: 321 },
|
||||
renderer: undefined,
|
||||
@@ -591,7 +600,7 @@ it("passes sampled detached groups into cleanup even when their token is unknown
|
||||
});
|
||||
|
||||
expect(sampledGroups).toEqual([654]);
|
||||
expect(report.owned_process_group).toEqual({
|
||||
expect(outcome.report.owned_process_group).toEqual({
|
||||
state: "unverified",
|
||||
reason: "sampled group has no verifiable run token",
|
||||
});
|
||||
@@ -632,7 +641,11 @@ it("retains observations and both causes when process cleanup is unverifiable",
|
||||
|
||||
let error: ProcessCaptureError | undefined;
|
||||
try {
|
||||
resolveProcessResult(capture, executionFailure, cleanup);
|
||||
resolveProcessResult(
|
||||
capture,
|
||||
executionFailure,
|
||||
historicalCleanupOutcome(cleanup),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
if (!(cause instanceof ProcessCaptureError)) throw cause;
|
||||
error = cause;
|
||||
@@ -702,7 +715,11 @@ it("projects execution and cleanup failures when capture never completed", () =>
|
||||
|
||||
let error: ProcessCaptureError | undefined;
|
||||
try {
|
||||
resolveProcessResult(undefined, executionFailure, cleanup);
|
||||
resolveProcessResult(
|
||||
undefined,
|
||||
executionFailure,
|
||||
historicalCleanupOutcome(cleanup),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
if (!(cause instanceof ProcessCaptureError)) throw cause;
|
||||
error = cause;
|
||||
@@ -737,7 +754,11 @@ it("retains completed observations when finalization fails after clean cleanup",
|
||||
|
||||
let error: ProcessCaptureError | undefined;
|
||||
try {
|
||||
resolveProcessResult(capture, executionFailure, cleanup);
|
||||
resolveProcessResult(
|
||||
capture,
|
||||
executionFailure,
|
||||
historicalCleanupOutcome(cleanup),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
if (!(cause instanceof ProcessCaptureError)) throw cause;
|
||||
error = cause;
|
||||
@@ -790,7 +811,11 @@ it("preserves cancellation while projecting observations and successful cleanup"
|
||||
};
|
||||
let error: ProcessCaptureError | undefined;
|
||||
try {
|
||||
resolveProcessResult(capture, processCaptureCancelled(), cleanup);
|
||||
resolveProcessResult(
|
||||
capture,
|
||||
processCaptureCancelled(),
|
||||
historicalCleanupOutcome(cleanup),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
if (!(cause instanceof ProcessCaptureError)) throw cause;
|
||||
error = cause;
|
||||
|
||||
@@ -201,16 +201,16 @@ const cleanupFailedStartup = async (options: {
|
||||
temporaryRoot: options.temporaryRoot,
|
||||
...(options.cleanupHost === undefined ? {} : { host: options.cleanupHost }),
|
||||
};
|
||||
const cleanup =
|
||||
const cleanupOutcome =
|
||||
options.resourceScope === undefined
|
||||
? await releaseProcessResources(cleanupOptions)
|
||||
: await options.resourceScope.release(cleanupOptions);
|
||||
const cleanupFailure = cleanupReportFailure(cleanup);
|
||||
const cleanupFailure = cleanupReportFailure(cleanupOutcome.report);
|
||||
if (cleanupFailure !== undefined) {
|
||||
resolveProcessResult(
|
||||
undefined,
|
||||
options.cause,
|
||||
cleanup,
|
||||
cleanupOutcome,
|
||||
options.observations,
|
||||
{
|
||||
scenario: options.scenario,
|
||||
@@ -401,10 +401,11 @@ const finishProcessRun = async (options: {
|
||||
temporaryRoot: options.temporaryRoot,
|
||||
...(options.cleanupHost === undefined ? {} : { host: options.cleanupHost }),
|
||||
};
|
||||
const cleanup =
|
||||
const cleanupOutcome =
|
||||
options.resourceScope === undefined
|
||||
? await releaseProcessResources(cleanupOptions)
|
||||
: await options.resourceScope.release(cleanupOptions);
|
||||
const cleanup = cleanupOutcome.report;
|
||||
let { capture, executionFailure } = options;
|
||||
if (executionFailure === undefined && options.signal?.aborted)
|
||||
executionFailure = processCaptureCancelled(options.signal.reason);
|
||||
@@ -455,7 +456,7 @@ const finishProcessRun = async (options: {
|
||||
return resolveProcessResult(
|
||||
cleanupFailure === undefined ? verifiedCapture : capture,
|
||||
executionFailure,
|
||||
cleanup,
|
||||
cleanupOutcome,
|
||||
options.observations,
|
||||
partialContext,
|
||||
);
|
||||
|
||||
@@ -168,7 +168,7 @@ it.each(["write", "resize"] as const)(
|
||||
runId: "failed-observation",
|
||||
temporaryRoot,
|
||||
});
|
||||
expect(cleanup.terminal_renderer).toEqual({
|
||||
expect(cleanup.report.terminal_renderer).toEqual({
|
||||
state: "cleaned",
|
||||
reason: null,
|
||||
});
|
||||
@@ -213,7 +213,7 @@ it("retains disposal failures for a later cleanup retry", async () => {
|
||||
runId: "failed-disposal",
|
||||
temporaryRoot,
|
||||
});
|
||||
expect(cleanup.terminal_renderer).toEqual({
|
||||
expect(cleanup.report.terminal_renderer).toEqual({
|
||||
state: "failed",
|
||||
reason: failure.message,
|
||||
});
|
||||
|
||||
@@ -2,14 +2,27 @@ import {
|
||||
reportLifecycleEnd,
|
||||
reportLifecycleStart,
|
||||
} from "./lifecycleProgress.js";
|
||||
import { ok } from "../domain/result.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { err, ok } from "../domain/result.js";
|
||||
import { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js";
|
||||
import type { Result } from "../domain/result.js";
|
||||
import { mcpProgressReporter } from "./mcpProgress.js";
|
||||
import type { LifecycleToolRegistration } from "./registerSessionTools.js";
|
||||
import { logToolExecution } from "./toolLogging.js";
|
||||
import { toolRegistrationOptions } from "./toolRegistrationOptions.js";
|
||||
|
||||
/** Expected lifecycle failures keep the same typed result when a child rejects. */
|
||||
const closeResult = async <Value>(
|
||||
close: () => Promise<Result<Value, AnalysisError>>,
|
||||
): Promise<Result<Value, AnalysisError>> => {
|
||||
try {
|
||||
return await close();
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof AnalysisError) return err(cause);
|
||||
throw cause;
|
||||
}
|
||||
};
|
||||
|
||||
/** Register provider cleanup and optional pre-close snapshot persistence. */
|
||||
export const registerCloseLifecycleTool = ({
|
||||
server,
|
||||
@@ -22,8 +35,8 @@ export const registerCloseLifecycleTool = ({
|
||||
closeSession: () => Promise<Result<Value, AnalysisError>>,
|
||||
): Promise<Result<Value, AnalysisError>> => {
|
||||
const [sessionClose, resourceClose] = await Promise.allSettled([
|
||||
closeSession(),
|
||||
closeProcessResources(),
|
||||
closeResult(closeSession),
|
||||
closeResult(closeProcessResources),
|
||||
]);
|
||||
if (sessionClose.status === "rejected") {
|
||||
if (resourceClose.status === "rejected")
|
||||
@@ -48,9 +61,11 @@ export const registerCloseLifecycleTool = ({
|
||||
const resources = resourceClose.value;
|
||||
if (!closed.ok) {
|
||||
if (!resources.ok)
|
||||
logger.warn(
|
||||
{ error: resources.error.message },
|
||||
"Process capture cleanup failed while closing the session",
|
||||
return err(
|
||||
analysisErrorWithCleanupFailure(
|
||||
closed.error,
|
||||
resources.error,
|
||||
).bindCleanupDependencies([closed.error, resources.error]),
|
||||
);
|
||||
return closed;
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ import { ProcessCaptureError } from "../process/capture/ProcessCaptureError.js";
|
||||
import { toolContract } from "../contracts/toolContracts.js";
|
||||
import { binaryTargetSummary } from "../application/binary/BinaryTargetSummary.js";
|
||||
import { UnknownRegistryError } from "../domain/unknownRegistryError.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { Evidence } from "../domain/evidence.js";
|
||||
import type { ProcessCapture } from "../domain/process/processCaptureParsing.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
@@ -309,6 +309,7 @@ export const registerSessionTools = (
|
||||
await processResources.retryCleanup();
|
||||
return ok(null);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof AnalysisError) return err(cause);
|
||||
return err(
|
||||
new ProcessCaptureError(
|
||||
"process capture cleanup remains incomplete",
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
import { access, mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
|
||||
import { expect, it, vi } from "vitest";
|
||||
|
||||
import { createTestBinarySession } from "../../fixtures/binarySession.js";
|
||||
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
|
||||
import { AnalysisInputError } from "../../../src/domain/analysisErrorCore.js";
|
||||
import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js";
|
||||
import { err } from "../../../src/domain/result.js";
|
||||
import { ProcessCaptureError } from "../../../src/process/capture/ProcessCaptureError.js";
|
||||
import {
|
||||
ProcessCaptureResourceScope,
|
||||
type ProcessCaptureCleanupHost,
|
||||
type ProcessCaptureResourceCleanupOptions,
|
||||
} from "../../../src/process/capture/ProcessCaptureLifecycle.js";
|
||||
import { silentLogger } from "../../../src/logger.js";
|
||||
import { createServer } from "../../../src/server/createServer.js";
|
||||
import { ToolResultDelivery } from "../../../src/server/toolResult.js";
|
||||
|
||||
const createTemporaryRoot = (): Promise<string> =>
|
||||
mkdtemp(join(tmpdir(), "rea-mcp-process-cleanup-retry-"));
|
||||
|
||||
const cleanupOptions = (
|
||||
root: string,
|
||||
removeTemporaryRoot: (path: string) => Promise<void>,
|
||||
): ProcessCaptureResourceCleanupOptions => {
|
||||
const host: ProcessCaptureCleanupHost = {
|
||||
platform: process.platform,
|
||||
cleanupProcessGroup: async () => ({ cleaned: true, signaled: false }),
|
||||
verifyTokenOwnedProcesses: async () => ({
|
||||
cleaned: true,
|
||||
signaled: false,
|
||||
}),
|
||||
removeTemporaryRoot,
|
||||
};
|
||||
return {
|
||||
timers: new Set(),
|
||||
terminal: undefined,
|
||||
renderer: undefined,
|
||||
runId: "mcp-process-cleanup-retry",
|
||||
temporaryRoot: root,
|
||||
host,
|
||||
};
|
||||
};
|
||||
|
||||
const connectSession = async () => {
|
||||
const session = createTestBinarySession(() => {
|
||||
throw new Error("close_binary must not open a binary provider");
|
||||
});
|
||||
const server = createServer(
|
||||
{ kind: "session", session },
|
||||
{ logger: silentLogger },
|
||||
);
|
||||
const client = new Client({ name: "process-cleanup-retry", version: "1" });
|
||||
const [clientTransport, serverTransport] =
|
||||
InMemoryTransport.createLinkedPair();
|
||||
await server.connect(serverTransport);
|
||||
await client.connect(clientTransport);
|
||||
return { client, server, session };
|
||||
};
|
||||
|
||||
const closeSession = async (
|
||||
resources: Awaited<ReturnType<typeof connectSession>>,
|
||||
): Promise<void> => {
|
||||
await resources.client.close();
|
||||
await resources.server.close();
|
||||
const closed = await resources.session.close();
|
||||
if (!closed.ok) throw closed.error;
|
||||
};
|
||||
|
||||
const captureCleanupRetry = (root: string, isDenied: () => boolean) => {
|
||||
const nativeRetry = ProcessCaptureResourceScope.prototype.retryCleanup;
|
||||
let seeded = false;
|
||||
let removalAttempts = 0;
|
||||
let retryFailure: unknown;
|
||||
const retry = vi
|
||||
.spyOn(ProcessCaptureResourceScope.prototype, "retryCleanup")
|
||||
.mockImplementation(async function (
|
||||
this: ProcessCaptureResourceScope,
|
||||
): Promise<void> {
|
||||
if (!seeded) {
|
||||
seeded = true;
|
||||
await this.release(
|
||||
cleanupOptions(root, async (path) => {
|
||||
if (path !== root)
|
||||
throw new Error("Cleanup seam received an unexpected root");
|
||||
removalAttempts += 1;
|
||||
if (isDenied())
|
||||
throw new Error("controlled process-root removal denial");
|
||||
await rm(path, { recursive: true, force: true });
|
||||
}),
|
||||
);
|
||||
}
|
||||
try {
|
||||
await nativeRetry.call(this);
|
||||
} catch (cause: unknown) {
|
||||
retryFailure ??= cause;
|
||||
throw cause;
|
||||
}
|
||||
});
|
||||
return {
|
||||
retry,
|
||||
get removalAttempts() {
|
||||
return removalAttempts;
|
||||
},
|
||||
get retryFailure() {
|
||||
return retryFailure;
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
it("projects the retained process root through close_binary and resolves its retry", async () => {
|
||||
const root = await createTemporaryRoot();
|
||||
let denyRemoval = true;
|
||||
const fault = captureCleanupRetry(root, () => denyRemoval);
|
||||
const resources = await connectSession();
|
||||
const callClose = () =>
|
||||
resources.client.callTool({ name: "close_binary", arguments: {} });
|
||||
|
||||
try {
|
||||
const failed = parseMcpToolError(await callClose());
|
||||
const firstWire = JSON.stringify(failed);
|
||||
expect(failed.error).toMatchObject({
|
||||
code: "cleanup_incomplete",
|
||||
details: { resources: ["temporary_root"] },
|
||||
});
|
||||
expect(fault.removalAttempts).toBe(2);
|
||||
expect(fault.retryFailure).toBeInstanceOf(ProcessCaptureError);
|
||||
if (!(fault.retryFailure instanceof ProcessCaptureError))
|
||||
throw new Error("Expected the process scope's typed retry error");
|
||||
const retryError = fault.retryFailure;
|
||||
expect(retryError.cleanupIncomplete).toBe(true);
|
||||
expect(retryError.cleanupResources).toEqual(["temporary_root"]);
|
||||
|
||||
denyRemoval = false;
|
||||
const recovered = await callClose();
|
||||
expect(recovered.isError, JSON.stringify(recovered.content)).not.toBe(true);
|
||||
expect(fault.removalAttempts).toBe(3);
|
||||
expect(retryError.cleanupIncomplete).toBe(false);
|
||||
expect(retryError.cleanupResources).toEqual([]);
|
||||
expect(projectAnalysisError(retryError).code).toBe(
|
||||
"process_capture_failed",
|
||||
);
|
||||
expect(JSON.stringify(failed)).toBe(firstWire);
|
||||
await expect(access(root)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} finally {
|
||||
denyRemoval = false;
|
||||
fault.retry.mockRestore();
|
||||
await closeSession(resources);
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["returned", "thrown", "synchronous"] as const)(
|
||||
"composes a %s session close failure with the exact pending process root",
|
||||
async (kind) => {
|
||||
const root = await createTemporaryRoot();
|
||||
let denyRemoval = true;
|
||||
const fault = captureCleanupRetry(root, () => denyRemoval);
|
||||
const resources = await connectSession();
|
||||
const primary = new AnalysisInputError("close_binary", undefined, [
|
||||
{
|
||||
path: ["session"],
|
||||
reason: "invalid_value",
|
||||
message: "primary session-close diagnostic",
|
||||
},
|
||||
]);
|
||||
const closeSpy = vi.spyOn(resources.session, "close");
|
||||
if (kind === "returned") closeSpy.mockResolvedValue(err(primary));
|
||||
else if (kind === "thrown") closeSpy.mockRejectedValue(primary);
|
||||
else
|
||||
closeSpy.mockImplementation(() => {
|
||||
throw primary;
|
||||
});
|
||||
const callClose = () =>
|
||||
resources.client.callTool({ name: "close_binary", arguments: {} });
|
||||
const delivery = vi.spyOn(ToolResultDelivery.prototype, "toCallToolResult");
|
||||
|
||||
try {
|
||||
const failed = parseMcpToolError(await callClose());
|
||||
const firstWire = JSON.stringify(failed);
|
||||
const delivered = delivery.mock.calls.at(-1)?.[0];
|
||||
if (delivered === undefined || delivered.ok)
|
||||
throw new Error("Expected the composed lifecycle error at delivery");
|
||||
const composed = delivered.error;
|
||||
expect(failed.error).toMatchObject({
|
||||
code: "cleanup_incomplete",
|
||||
details: {
|
||||
resources: ["temporary_root"],
|
||||
diagnostics: {
|
||||
primary_error: {
|
||||
details: {
|
||||
issues: [
|
||||
{
|
||||
path: ["session"],
|
||||
reason: "invalid_value",
|
||||
message: "primary session-close diagnostic",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(fault.removalAttempts).toBe(2);
|
||||
expect(fault.retryFailure).toBeInstanceOf(ProcessCaptureError);
|
||||
if (!(fault.retryFailure instanceof ProcessCaptureError))
|
||||
throw new Error("Expected the process scope's typed retry error");
|
||||
const retryError = fault.retryFailure;
|
||||
expect(retryError.cleanupIncomplete).toBe(true);
|
||||
expect(composed.cleanupIncomplete).toBe(true);
|
||||
|
||||
denyRemoval = false;
|
||||
const recovered = parseMcpToolError(await callClose());
|
||||
expect(recovered.error).toMatchObject({ code: "invalid_request" });
|
||||
expect(retryError.cleanupIncomplete).toBe(false);
|
||||
expect(composed.cleanupIncomplete).toBe(false);
|
||||
expect(projectAnalysisError(composed)).toMatchObject({
|
||||
code: "invalid_request",
|
||||
details: { issues: primary.issues },
|
||||
});
|
||||
expect(projectAnalysisError(retryError).code).toBe(
|
||||
"process_capture_failed",
|
||||
);
|
||||
expect(JSON.stringify(failed)).toBe(firstWire);
|
||||
expect(primary.cleanupIncomplete).toBe(false);
|
||||
await expect(access(root)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} finally {
|
||||
denyRemoval = false;
|
||||
closeSpy.mockRestore();
|
||||
delivery.mockRestore();
|
||||
fault.retry.mockRestore();
|
||||
await closeSession(resources);
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
@@ -7,7 +7,10 @@ import {
|
||||
} from "@modelcontextprotocol/server";
|
||||
|
||||
import { ProcessCaptureError } from "../../../src/process/capture/ProcessCaptureError.js";
|
||||
import { resolveProcessResult } from "../../../src/process/capture/ProcessCaptureLifecycle.js";
|
||||
import {
|
||||
resolveProcessResult,
|
||||
type ProcessCaptureCleanupOutcome,
|
||||
} from "../../../src/process/capture/ProcessCaptureLifecycle.js";
|
||||
import { toolContract } from "../../../src/contracts/toolContracts.js";
|
||||
import type { ToolContract } from "../../../src/contracts/toolContractTypes.js";
|
||||
import { err } from "../../../src/domain/result.js";
|
||||
@@ -23,13 +26,19 @@ import type { JsonValue } from "../../../src/domain/jsonValue.js";
|
||||
import { evidenceResultOf } from "../../../src/contracts/toolOutputSchemaPrimitives.js";
|
||||
import { toolRegistrationOptions } from "../../../src/server/toolRegistrationOptions.js";
|
||||
|
||||
import type { IncompleteProcessCaptureObservations } from "../../../src/domain/process/processCapture.js";
|
||||
import type {
|
||||
IncompleteProcessCaptureObservations,
|
||||
ProcessCaptureCleanupReport,
|
||||
} from "../../../src/domain/process/processCapture.js";
|
||||
const delivery = new ToolResultDelivery(STDIO_DEFAULT_MAX_BUFFER_SIZE);
|
||||
const UNTRUSTED_TARGET_CONTENT = {
|
||||
provenance: { untrusted_content: true, source: "analysis_target" },
|
||||
};
|
||||
const emptyFilesystemCoverage =
|
||||
emptyProcessCapture().truncation_details.filesystem_before;
|
||||
const historicalCleanupOutcome = (
|
||||
report: ProcessCaptureCleanupReport,
|
||||
): ProcessCaptureCleanupOutcome => ({ report, cleanupLease: undefined });
|
||||
|
||||
const captureWithObservedOutput = () =>
|
||||
accountFullyObservedProcessCapture({
|
||||
@@ -80,7 +89,7 @@ describe("completed partial process capture MCP projection", () => {
|
||||
resolveProcessResult(
|
||||
capture,
|
||||
new Error("final filesystem snapshot failed"),
|
||||
cleanup,
|
||||
historicalCleanupOutcome(cleanup),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
if (!(cause instanceof ProcessCaptureError)) throw cause;
|
||||
@@ -128,7 +137,11 @@ describe("completed partial process capture MCP projection", () => {
|
||||
|
||||
let failure: ProcessCaptureError | undefined;
|
||||
try {
|
||||
resolveProcessResult(capture, executionFailure, cleanup);
|
||||
resolveProcessResult(
|
||||
capture,
|
||||
executionFailure,
|
||||
historicalCleanupOutcome(cleanup),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
if (!(cause instanceof ProcessCaptureError)) throw cause;
|
||||
failure = cause;
|
||||
@@ -267,7 +280,7 @@ describe("incomplete partial process observations MCP projection", () => {
|
||||
resolveProcessResult(
|
||||
undefined,
|
||||
new Error("capture failed before completion"),
|
||||
cleanup,
|
||||
historicalCleanupOutcome(cleanup),
|
||||
observations,
|
||||
{ scenario, rootPid },
|
||||
);
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
import { access, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { execFile } from "node:child_process";
|
||||
import { join, resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { expect } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { itWithCaptureCapability } from "./processCaptureCapability.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const CLI_TIMEOUT_MS = 30_000;
|
||||
|
||||
const cleanupPreload = `
|
||||
import fs from "node:fs";
|
||||
import { basename, resolve } from "node:path";
|
||||
import { syncBuiltinESMExports } from "node:module";
|
||||
|
||||
const cliEntry = process.env.REA_TEST_CLI_ENTRY;
|
||||
if (cliEntry !== undefined && resolve(process.argv[1] ?? "") === cliEntry) {
|
||||
const markerPath = process.env.REA_TEST_CLEANUP_MARKER;
|
||||
const originalMkdtemp = fs.promises.mkdtemp;
|
||||
const originalRm = fs.promises.rm;
|
||||
let captureRoot;
|
||||
let removalAttempts = 0;
|
||||
let released = false;
|
||||
const record = () => {
|
||||
if (markerPath !== undefined) {
|
||||
fs.writeFileSync(markerPath, JSON.stringify({
|
||||
root: captureRoot,
|
||||
removalAttempts,
|
||||
released,
|
||||
}));
|
||||
}
|
||||
};
|
||||
|
||||
fs.promises.mkdtemp = async function (prefix, ...options) {
|
||||
const root = await originalMkdtemp.call(this, prefix, ...options);
|
||||
if (basename(String(prefix)) === "rea-process-") {
|
||||
captureRoot = resolve(root);
|
||||
record();
|
||||
}
|
||||
return root;
|
||||
};
|
||||
fs.promises.rm = async function (path, ...options) {
|
||||
if (captureRoot === undefined || resolve(String(path)) !== captureRoot)
|
||||
return originalRm.call(this, path, ...options);
|
||||
removalAttempts += 1;
|
||||
record();
|
||||
if (removalAttempts === 1)
|
||||
throw new Error("controlled first temporary-root removal denial");
|
||||
const result = await originalRm.call(this, path, ...options);
|
||||
released = true;
|
||||
record();
|
||||
return result;
|
||||
};
|
||||
syncBuiltinESMExports();
|
||||
}
|
||||
`;
|
||||
|
||||
interface CleanupMarker {
|
||||
readonly root: string;
|
||||
readonly removalAttempts: number;
|
||||
readonly released: boolean;
|
||||
}
|
||||
|
||||
const readCleanupMarker = async (path: string): Promise<CleanupMarker> => {
|
||||
const contents = await readFile(path, "utf8");
|
||||
const parsed: unknown = JSON.parse(contents);
|
||||
if (typeof parsed !== "object" || parsed === null)
|
||||
throw new Error("Cleanup preload did not write its marker object");
|
||||
const candidate = parsed as Partial<CleanupMarker>;
|
||||
if (
|
||||
typeof candidate.root !== "string" ||
|
||||
typeof candidate.removalAttempts !== "number" ||
|
||||
typeof candidate.released !== "boolean"
|
||||
)
|
||||
throw new Error("Cleanup preload marker is missing removal results");
|
||||
return {
|
||||
root: candidate.root,
|
||||
removalAttempts: candidate.removalAttempts,
|
||||
released: candidate.released,
|
||||
};
|
||||
};
|
||||
|
||||
itWithCaptureCapability(
|
||||
"retries the failed capture root before projecting the completed CLI failure",
|
||||
async () => {
|
||||
const fixtureRoot = await createTestTempDirectory(
|
||||
"rea-process-cleanup-recovery-cli-",
|
||||
);
|
||||
const scenarioPath = join(fixtureRoot, "scenario.json");
|
||||
const preloadPath = join(fixtureRoot, "cleanup-preload.mjs");
|
||||
const markerPath = join(fixtureRoot, "cleanup-marker.json");
|
||||
await writeFile(
|
||||
scenarioPath,
|
||||
JSON.stringify({
|
||||
executable: process.execPath,
|
||||
arguments: ["-e", "process.stdout.write('capture completed\\n')"],
|
||||
working_directory: fixtureRoot,
|
||||
}),
|
||||
);
|
||||
await writeFile(preloadPath, cleanupPreload);
|
||||
|
||||
let captureRoot: string | undefined;
|
||||
try {
|
||||
const cliEntry = resolve("scripts/rea.mjs");
|
||||
const result = await execFileAsync(
|
||||
process.execPath,
|
||||
[cliEntry, "capture-process", scenarioPath, "--json"],
|
||||
{
|
||||
cwd: fixtureRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
NODE_OPTIONS: `--import=${pathToFileURL(preloadPath).href}`,
|
||||
REA_TEST_CLI_ENTRY: cliEntry,
|
||||
REA_TEST_CLEANUP_MARKER: markerPath,
|
||||
},
|
||||
timeout: CLI_TIMEOUT_MS,
|
||||
},
|
||||
).then(
|
||||
({ stdout, stderr }) => ({ exitCode: 0, stdout, stderr }),
|
||||
(cause: unknown) => {
|
||||
if (!(cause instanceof Error)) throw cause;
|
||||
const failure = cause as Error & {
|
||||
readonly code?: number | string;
|
||||
readonly stdout?: string;
|
||||
readonly stderr?: string;
|
||||
};
|
||||
return {
|
||||
exitCode: failure.code,
|
||||
stdout: failure.stdout ?? "",
|
||||
stderr: failure.stderr ?? "",
|
||||
};
|
||||
},
|
||||
);
|
||||
const marker = await readCleanupMarker(markerPath);
|
||||
captureRoot = marker.root;
|
||||
expect(result.exitCode, result.stderr || result.stdout).toBe(1);
|
||||
expect(marker.removalAttempts).toBe(2);
|
||||
expect(marker.released).toBe(true);
|
||||
const output: unknown = JSON.parse(result.stdout);
|
||||
expect(output).toMatchObject({
|
||||
error: "Process command failed",
|
||||
code: "process_capture_failed",
|
||||
details: {
|
||||
cleanup_report: {
|
||||
temporary_root: {
|
||||
state: "failed",
|
||||
reason: "controlled first temporary-root removal denial",
|
||||
},
|
||||
},
|
||||
partial_observation: {
|
||||
capture: {
|
||||
exit: { code: 0, reason: "exited" },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
await expect(access(captureRoot)).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
} finally {
|
||||
if (captureRoot !== undefined)
|
||||
await rm(captureRoot, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
CLI_TIMEOUT_MS,
|
||||
);
|
||||
@@ -0,0 +1,346 @@
|
||||
import { mkdtemp, rm, stat } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { setImmediate } from "node:timers/promises";
|
||||
import { expect, it } from "vitest";
|
||||
import { createDeferred } from "../../fixtures/binarySession.js";
|
||||
|
||||
import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js";
|
||||
import { ProcessCaptureError } from "../../../src/process/capture/ProcessCaptureError.js";
|
||||
import {
|
||||
ProcessCaptureResourceScope,
|
||||
resolveProcessResult,
|
||||
type ProcessCaptureCleanupHost,
|
||||
type ProcessCaptureResourceCleanupOptions,
|
||||
} from "../../../src/process/capture/ProcessCaptureLifecycle.js";
|
||||
|
||||
const cleanupOptions = (
|
||||
temporaryRoot: string,
|
||||
removeTemporaryRoot: (path: string) => Promise<void>,
|
||||
): ProcessCaptureResourceCleanupOptions => {
|
||||
const host: ProcessCaptureCleanupHost = {
|
||||
platform: process.platform,
|
||||
cleanupProcessGroup: async () => ({ cleaned: true, signaled: false }),
|
||||
verifyTokenOwnedProcesses: async () => ({
|
||||
cleaned: true,
|
||||
signaled: false,
|
||||
}),
|
||||
removeTemporaryRoot,
|
||||
};
|
||||
return {
|
||||
timers: new Set(),
|
||||
terminal: undefined,
|
||||
renderer: undefined,
|
||||
runId: "cleanup-retry-boundary",
|
||||
temporaryRoot,
|
||||
host,
|
||||
};
|
||||
};
|
||||
|
||||
const createRoot = (): Promise<string> =>
|
||||
mkdtemp(join(tmpdir(), "rea-process-cleanup-retry-"));
|
||||
|
||||
const expectCaptureError = (operation: () => unknown): ProcessCaptureError => {
|
||||
try {
|
||||
operation();
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof ProcessCaptureError) return cause;
|
||||
throw cause;
|
||||
}
|
||||
throw new Error("Expected process capture failure");
|
||||
};
|
||||
|
||||
it("resolves the origin and finalization copy after the exact root retry", async () => {
|
||||
const scope = new ProcessCaptureResourceScope();
|
||||
const root = await createRoot();
|
||||
const primaryCause = new Error("capture failed before root cleanup");
|
||||
const primary = new ProcessCaptureError("capture failed", {
|
||||
cause: primaryCause,
|
||||
reason: "cleanup_incomplete",
|
||||
cleanup: {
|
||||
reason: "temporary root removal failed",
|
||||
resources: ["temporary_root"],
|
||||
},
|
||||
});
|
||||
let denyRemoval = true;
|
||||
const release = await scope.release(
|
||||
cleanupOptions(root, async (path) => {
|
||||
if (denyRemoval) throw new Error("controlled root removal denial");
|
||||
await rm(path, { recursive: true });
|
||||
}),
|
||||
);
|
||||
|
||||
try {
|
||||
if (release.cleanupLease === undefined)
|
||||
throw new Error("Expected the failed root's exact cleanup lease");
|
||||
release.cleanupLease.bindCleanupError(primary);
|
||||
const reported = expectCaptureError(() =>
|
||||
resolveProcessResult(undefined, primary, release),
|
||||
);
|
||||
const historicalReport = reported.cleanupReport;
|
||||
expect(reported).not.toBe(primary);
|
||||
expect(primary.cleanupIncomplete).toBe(true);
|
||||
expect(reported.cleanupIncomplete).toBe(true);
|
||||
expect(projectAnalysisError(reported).code).toBe("cleanup_incomplete");
|
||||
expect(reported.cause).toBe(primaryCause);
|
||||
expect(historicalReport?.temporary_root).toEqual({
|
||||
state: "failed",
|
||||
reason: "controlled root removal denial",
|
||||
});
|
||||
|
||||
denyRemoval = false;
|
||||
await scope.retryCleanup();
|
||||
await scope.close();
|
||||
|
||||
expect(primary.cleanupIncomplete).toBe(false);
|
||||
expect(reported.cleanupIncomplete).toBe(false);
|
||||
expect(projectAnalysisError(reported).code).not.toBe("cleanup_incomplete");
|
||||
expect(reported.cause).toBe(primaryCause);
|
||||
expect(reported.cleanupReport).toBe(historicalReport);
|
||||
expect(reported.cleanupReport?.temporary_root).toEqual({
|
||||
state: "failed",
|
||||
reason: "controlled root removal denial",
|
||||
});
|
||||
await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} finally {
|
||||
denyRemoval = false;
|
||||
await scope.close();
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps mixed root errors and retry aggregates bound to only unresolved roots", async () => {
|
||||
const scope = new ProcessCaptureResourceScope();
|
||||
const roots = [await createRoot(), await createRoot()] as const;
|
||||
const denied = new Set(roots);
|
||||
const outcomes = new Map<string, Awaited<ReturnType<typeof scope.release>>>();
|
||||
|
||||
try {
|
||||
for (const root of roots) {
|
||||
outcomes.set(
|
||||
root,
|
||||
await scope.release(
|
||||
cleanupOptions(root, async (path) => {
|
||||
if (denied.has(path))
|
||||
throw new Error(`controlled removal denial: ${path}`);
|
||||
await rm(path, { recursive: true });
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
const firstOutcome = outcomes.get(roots[0]);
|
||||
const secondOutcome = outcomes.get(roots[1]);
|
||||
if (firstOutcome === undefined || secondOutcome === undefined)
|
||||
throw new Error("Expected both retained cleanup outcomes");
|
||||
const first = expectCaptureError(() =>
|
||||
resolveProcessResult(
|
||||
undefined,
|
||||
new ProcessCaptureError("first primary failure"),
|
||||
firstOutcome,
|
||||
),
|
||||
);
|
||||
const second = expectCaptureError(() =>
|
||||
resolveProcessResult(
|
||||
undefined,
|
||||
new ProcessCaptureError("second primary failure"),
|
||||
secondOutcome,
|
||||
),
|
||||
);
|
||||
expect(first.cleanupIncomplete).toBe(true);
|
||||
expect(second.cleanupIncomplete).toBe(true);
|
||||
|
||||
const bothUnresolved: unknown = await scope
|
||||
.retryCleanup()
|
||||
.catch((cause: unknown) => cause);
|
||||
expect(bothUnresolved).toBeInstanceOf(ProcessCaptureError);
|
||||
if (!(bothUnresolved instanceof ProcessCaptureError))
|
||||
throw new Error("Expected both owners in a typed retry aggregate");
|
||||
expect(bothUnresolved.cleanupIncomplete).toBe(true);
|
||||
expect(bothUnresolved.cause).toBeInstanceOf(AggregateError);
|
||||
if (!(bothUnresolved.cause instanceof AggregateError))
|
||||
throw new Error("Expected retained child retry failures");
|
||||
expect(bothUnresolved.cause.errors).toHaveLength(2);
|
||||
|
||||
denied.delete(roots[0]);
|
||||
const retryFailure: unknown = await scope
|
||||
.retryCleanup()
|
||||
.catch((cause: unknown) => cause);
|
||||
expect(retryFailure).toBeInstanceOf(ProcessCaptureError);
|
||||
if (!(retryFailure instanceof ProcessCaptureError))
|
||||
throw new Error("Expected a typed retry aggregate");
|
||||
expect(first.cleanupIncomplete).toBe(false);
|
||||
expect(second.cleanupIncomplete).toBe(true);
|
||||
expect(bothUnresolved.cleanupIncomplete).toBe(true);
|
||||
expect(retryFailure.cleanupIncomplete).toBe(true);
|
||||
expect(retryFailure.cleanupResources).toEqual(["temporary_root"]);
|
||||
|
||||
denied.clear();
|
||||
await scope.retryCleanup();
|
||||
await scope.close();
|
||||
expect(second.cleanupIncomplete).toBe(false);
|
||||
expect(retryFailure.cleanupIncomplete).toBe(false);
|
||||
expect(bothUnresolved.cleanupIncomplete).toBe(false);
|
||||
expect(first.cleanupReport?.temporary_root.state).toBe("failed");
|
||||
expect(second.cleanupReport?.temporary_root.state).toBe("failed");
|
||||
for (const root of roots)
|
||||
await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} finally {
|
||||
denied.clear();
|
||||
await scope.close();
|
||||
await Promise.all(
|
||||
roots.map((root) => rm(root, { recursive: true, force: true })),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("joins an already-started release when shutdown begins", async () => {
|
||||
const scope = new ProcessCaptureResourceScope();
|
||||
const root = await createRoot();
|
||||
const entered = createDeferred<void>();
|
||||
const unblock = createDeferred<void>();
|
||||
let attempts = 0;
|
||||
const releasing = scope.release(
|
||||
cleanupOptions(root, async (path) => {
|
||||
attempts += 1;
|
||||
entered.resolve();
|
||||
await unblock.promise;
|
||||
await rm(path, { recursive: true });
|
||||
}),
|
||||
);
|
||||
await entered.promise;
|
||||
let closed = false;
|
||||
const closing = scope.close().then(() => {
|
||||
closed = true;
|
||||
});
|
||||
try {
|
||||
await setImmediate();
|
||||
expect(closed).toBe(false);
|
||||
await expect(stat(root)).resolves.toBeDefined();
|
||||
} finally {
|
||||
unblock.resolve();
|
||||
await Promise.all([releasing, closing]);
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
expect(attempts).toBe(1);
|
||||
expect(closed).toBe(true);
|
||||
await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
|
||||
it("binds a historical failed report after its owner already retried", async () => {
|
||||
const scope = new ProcessCaptureResourceScope();
|
||||
const root = await createRoot();
|
||||
let denyRemoval = true;
|
||||
const release = await scope.release(
|
||||
cleanupOptions(root, async (path) => {
|
||||
if (denyRemoval) throw new Error("controlled root removal denial");
|
||||
await rm(path, { recursive: true });
|
||||
}),
|
||||
);
|
||||
|
||||
try {
|
||||
denyRemoval = false;
|
||||
await scope.retryCleanup();
|
||||
await scope.close();
|
||||
|
||||
const reported = expectCaptureError(() =>
|
||||
resolveProcessResult(
|
||||
undefined,
|
||||
new ProcessCaptureError("primary failure after release"),
|
||||
release,
|
||||
),
|
||||
);
|
||||
expect(reported.cleanupIncomplete).toBe(false);
|
||||
expect(projectAnalysisError(reported).code).not.toBe("cleanup_incomplete");
|
||||
expect(reported.cleanupReport?.temporary_root).toMatchObject({
|
||||
state: "failed",
|
||||
});
|
||||
await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} finally {
|
||||
denyRemoval = false;
|
||||
await scope.close();
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("blocks scope admission on owner-bound retry failure and resumes after release", async () => {
|
||||
const scope = new ProcessCaptureResourceScope();
|
||||
const root = await createRoot();
|
||||
let denyRemoval = true;
|
||||
const release = await scope.release(
|
||||
cleanupOptions(root, async (path) => {
|
||||
if (denyRemoval) throw new Error("controlled root removal denial");
|
||||
await rm(path, { recursive: true });
|
||||
}),
|
||||
);
|
||||
const origin = expectCaptureError(() =>
|
||||
resolveProcessResult(
|
||||
undefined,
|
||||
new ProcessCaptureError("primary failure"),
|
||||
release,
|
||||
),
|
||||
);
|
||||
|
||||
try {
|
||||
let admitted = false;
|
||||
const retryFailure: unknown = await scope
|
||||
.run(async () => {
|
||||
admitted = true;
|
||||
})
|
||||
.catch((cause: unknown) => cause);
|
||||
expect(admitted).toBe(false);
|
||||
expect(retryFailure).toBeInstanceOf(ProcessCaptureError);
|
||||
if (!(retryFailure instanceof ProcessCaptureError))
|
||||
throw new Error("Expected a typed owner-bound retry failure");
|
||||
expect(retryFailure.cleanupIncomplete).toBe(true);
|
||||
expect(retryFailure.cleanupResources).toEqual(["temporary_root"]);
|
||||
expect(origin.cleanupIncomplete).toBe(true);
|
||||
|
||||
denyRemoval = false;
|
||||
await expect(
|
||||
scope.run(async () => {
|
||||
admitted = true;
|
||||
return "admitted after cleanup";
|
||||
}),
|
||||
).resolves.toBe("admitted after cleanup");
|
||||
await scope.close();
|
||||
expect(admitted).toBe(true);
|
||||
expect(origin.cleanupIncomplete).toBe(false);
|
||||
expect(retryFailure.cleanupIncomplete).toBe(false);
|
||||
await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} finally {
|
||||
denyRemoval = false;
|
||||
await scope.close();
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps historical report edits separate from cleanup authority", async () => {
|
||||
const scope = new ProcessCaptureResourceScope();
|
||||
const root = await createRoot();
|
||||
let denied = true;
|
||||
let attempts = 0;
|
||||
try {
|
||||
const outcome = await scope.release(
|
||||
cleanupOptions(root, async (path) => {
|
||||
attempts += 1;
|
||||
if (denied) throw new Error("controlled root removal denial");
|
||||
await rm(path, { recursive: true });
|
||||
}),
|
||||
);
|
||||
Reflect.set(outcome.report.temporary_root, "state", "cleaned");
|
||||
await expect(scope.retryCleanup()).rejects.toBeInstanceOf(
|
||||
ProcessCaptureError,
|
||||
);
|
||||
expect(attempts).toBe(2);
|
||||
expect(outcome.cleanupLease?.released).toBe(false);
|
||||
await expect(stat(root)).resolves.toBeDefined();
|
||||
denied = false;
|
||||
await scope.close();
|
||||
expect(attempts).toBe(3);
|
||||
await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} finally {
|
||||
denied = false;
|
||||
await scope.close();
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user