mirror of
https://github.com/morluto/rea
synced 2026-10-25 22:08:11 -04:00
fix(verifiers): share command ownership and preserve stdin
Move command and retained resource lifetimes into a shared verifier owner, keeping package workspace release as a separate dependency. Migrate Pi and Hermes verification to that owner while preserving open pipes, explicit EOF, diagnostics, and retained labs. Require positive writable finalization before accepting EOF delivery. Cover premature close, opaque failures, late ownership delivery, mixed cleanup outcomes, exact retry, and real descendant cleanup with authored process boundaries.
This commit is contained in:
morluto
@@ -1,73 +1,27 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
import {
|
||||
ArtifactResourceScope,
|
||||
type ArtifactResourceOwner,
|
||||
} from "../../src/artifacts/ArtifactResourceScope.js";
|
||||
import { AnalysisCleanupLease } from "../../src/artifacts/AnalysisCleanupLease.js";
|
||||
import type { RetainedResourceScope } from "../../src/application/AnalysisProvider.js";
|
||||
import {
|
||||
prepareOwnedCommand,
|
||||
runOwnedCommand,
|
||||
} from "../../src/process/OwnedCommand.js";
|
||||
import {
|
||||
spawnOwnedProviderProcess,
|
||||
type OwnedProviderProcessSpawnOptions,
|
||||
type ProviderProcessSnapshot,
|
||||
type SpawnedOwnedProviderProcess,
|
||||
} from "../../src/process/ProviderProcess.js";
|
||||
import { retainOwnedCommandCleanup } from "../../src/providers/OwnedCommandCleanup.js";
|
||||
import { WindowsOwnedProcess } from "../../src/windows/WindowsOwnedProcess.js";
|
||||
createVerifierCommands,
|
||||
VerifierCommandExitError,
|
||||
type VerifierCommands,
|
||||
type VerifierCommandsOptions,
|
||||
type VerifierCommandOptions,
|
||||
type VerifierCommandOutput,
|
||||
type VerifierCleanupOwner,
|
||||
} from "./verifier-command-owner.mjs";
|
||||
import { npmInvocation } from "./npm-invocation.mjs";
|
||||
import { createWindowsOwnedCommand } from "./windows-owned-command.mjs";
|
||||
|
||||
export interface PackageVerifierWorkspace {
|
||||
readonly path: string;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
interface PackageCommandContext {
|
||||
readonly signal?: AbortSignal | undefined;
|
||||
readonly cwd?: string | undefined;
|
||||
readonly env?: NodeJS.ProcessEnv | undefined;
|
||||
readonly timeout?: number | undefined;
|
||||
readonly windowsHide?: boolean | undefined;
|
||||
}
|
||||
|
||||
export type PackageCommandOptions = PackageCommandContext &
|
||||
(
|
||||
| {
|
||||
/** execFile-compatible capacity for each output stream. */
|
||||
readonly maxBuffer?: number | undefined;
|
||||
readonly diagnosticBytes?: never;
|
||||
}
|
||||
| {
|
||||
/** Canonical command capacity across both streams. */
|
||||
readonly diagnosticBytes: number;
|
||||
readonly maxBuffer?: never;
|
||||
}
|
||||
);
|
||||
|
||||
export interface PackageCommandOutput {
|
||||
readonly stdout: string;
|
||||
readonly stderr: string;
|
||||
}
|
||||
|
||||
export interface PackageCleanupOwner {
|
||||
close(): Promise<void>;
|
||||
readonly cleanupResources?: readonly string[] | undefined;
|
||||
}
|
||||
|
||||
export interface PackageVerifierCommands {
|
||||
exec(
|
||||
command: string,
|
||||
arguments_: readonly string[],
|
||||
options?: PackageCommandOptions,
|
||||
): Promise<PackageCommandOutput>;
|
||||
export interface PackageVerifierCommands extends Pick<
|
||||
VerifierCommands,
|
||||
"exec"
|
||||
> {
|
||||
execNpm(
|
||||
arguments_: readonly string[],
|
||||
options?: PackageCommandOptions,
|
||||
): Promise<PackageCommandOutput>;
|
||||
options?: VerifierCommandOptions,
|
||||
): Promise<VerifierCommandOutput>;
|
||||
run(
|
||||
command: string,
|
||||
arguments_: readonly string[],
|
||||
@@ -78,29 +32,10 @@ export interface PackageVerifierCommands {
|
||||
arguments_: readonly string[],
|
||||
environment?: NodeJS.ProcessEnv,
|
||||
): Promise<{ readonly stdout: string; readonly status: number }>;
|
||||
adopt(owner: PackageCleanupOwner, resource?: string): void;
|
||||
adopt(owner: VerifierCleanupOwner, resource?: string): void;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
/** An ordinary nonzero exit whose process and output cleanup both completed. */
|
||||
export class PackageCommandExitError extends Error {
|
||||
readonly stdout: string;
|
||||
readonly stderr: string;
|
||||
constructor(
|
||||
readonly command: string,
|
||||
readonly arguments_: readonly string[],
|
||||
readonly code: number,
|
||||
readonly snapshot: ProviderProcessSnapshot,
|
||||
) {
|
||||
super(
|
||||
`Command failed: ${[command, ...arguments_].join(" ")}\n${snapshot.stderr.text}`,
|
||||
);
|
||||
this.name = "PackageCommandExitError";
|
||||
this.stdout = snapshot.stdout.text;
|
||||
this.stderr = snapshot.stderr.text;
|
||||
}
|
||||
}
|
||||
|
||||
/** The same command/workspace capability remains available after failed close. */
|
||||
export class PackageVerifierCleanupFailure extends Error {
|
||||
readonly path: string;
|
||||
@@ -118,142 +53,27 @@ export class PackageVerifierCleanupFailure extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
const launchPackageCommand = async (
|
||||
input: OwnedProviderProcessSpawnOptions,
|
||||
): Promise<SpawnedOwnedProviderProcess> => {
|
||||
if (process.platform !== "win32") return spawnOwnedProviderProcess(input);
|
||||
input.signal?.throwIfAborted();
|
||||
const child = createWindowsOwnedCommand({
|
||||
WindowsOwnedProcess,
|
||||
command: input.command,
|
||||
arguments: input.arguments,
|
||||
cwd: input.cwd,
|
||||
environment: {
|
||||
...input.hostEnvironment,
|
||||
...input.env,
|
||||
REA_PROCESS_RUN_ID: input.runId,
|
||||
},
|
||||
});
|
||||
// These labels describe the launch; cleanup authority is the exact native job.
|
||||
return {
|
||||
process: child,
|
||||
ownership: {
|
||||
runId: input.runId,
|
||||
leaderPid: child.pid,
|
||||
processGroupId: child.pid,
|
||||
expectedParentPid: process.pid,
|
||||
},
|
||||
cleanup: () => child.cleanup(),
|
||||
};
|
||||
};
|
||||
|
||||
const commandLimits = (options: PackageCommandOptions) => {
|
||||
const timeout = options.timeout ?? 0;
|
||||
if (!Number.isSafeInteger(timeout) || timeout < 0)
|
||||
throw new RangeError(
|
||||
"Package command timeout must be a nonnegative integer",
|
||||
);
|
||||
if (options.maxBuffer !== undefined && options.diagnosticBytes !== undefined)
|
||||
throw new TypeError("Select a per-stream or aggregate command capacity");
|
||||
const capacity = options.diagnosticBytes ?? options.maxBuffer ?? 1024 * 1024;
|
||||
if (typeof capacity !== "number" || Number.isNaN(capacity) || capacity < 0)
|
||||
throw new RangeError("Package command output capacity must be nonnegative");
|
||||
if (process.platform === "win32" && options.windowsHide === false)
|
||||
throw new Error(
|
||||
"Owned package commands require hidden Windows process launch",
|
||||
);
|
||||
return {
|
||||
timeoutMs: timeout === 0 ? null : timeout,
|
||||
diagnosticBytes:
|
||||
options.diagnosticBytes === undefined
|
||||
? { stdout: capacity, stderr: capacity }
|
||||
: capacity,
|
||||
acceptNonZeroExit: true,
|
||||
};
|
||||
};
|
||||
|
||||
class PackageCommands implements PackageVerifierCommands {
|
||||
readonly #resources = new ArtifactResourceScope();
|
||||
readonly #controller = new AbortController();
|
||||
readonly #adopted = new WeakMap<PackageCleanupOwner, ArtifactResourceOwner>();
|
||||
readonly #resourceOwner: RetainedResourceScope;
|
||||
#closed = false;
|
||||
#workspaceReleaseStarted = false;
|
||||
#closing: Promise<void> | undefined;
|
||||
|
||||
constructor(
|
||||
readonly workspace: PackageVerifierWorkspace,
|
||||
readonly launcher: (
|
||||
input: OwnedProviderProcessSpawnOptions,
|
||||
) => Promise<SpawnedOwnedProviderProcess>,
|
||||
) {
|
||||
this.#resourceOwner = {
|
||||
retain: (owner, resource) => this.adopt(owner, resource),
|
||||
run: (work) => this.#resources.run(work),
|
||||
};
|
||||
}
|
||||
private readonly commands: VerifierCommands,
|
||||
) {}
|
||||
|
||||
exec(
|
||||
command: string,
|
||||
arguments_: readonly string[],
|
||||
options: PackageCommandOptions = {},
|
||||
): Promise<PackageCommandOutput> {
|
||||
if (this.#closed)
|
||||
return Promise.reject(new Error("Package command owner is closed"));
|
||||
return this.#resources.run(async () => {
|
||||
try {
|
||||
const limits = commandLimits(options);
|
||||
const signal =
|
||||
options.signal === undefined
|
||||
? this.#controller.signal
|
||||
: AbortSignal.any([this.#controller.signal, options.signal]);
|
||||
// Match execFile's command deadline while keeping native ownership
|
||||
// preparation independently bounded and inside this resource lifetime.
|
||||
await prepareOwnedCommand(signal);
|
||||
const snapshot = await runOwnedCommand(
|
||||
{
|
||||
command,
|
||||
arguments: arguments_,
|
||||
cwd: options.cwd ?? process.cwd(),
|
||||
hostEnvironment: options.env ?? process.env,
|
||||
expectedCommand: null,
|
||||
runId: `rea-package-command-${randomUUID()}`,
|
||||
sweepTokenOwnedProcesses: true,
|
||||
},
|
||||
limits,
|
||||
{
|
||||
signal,
|
||||
launcher: this.launcher,
|
||||
},
|
||||
);
|
||||
const output = {
|
||||
stdout: snapshot.stdout.text,
|
||||
stderr: snapshot.stderr.text,
|
||||
};
|
||||
if (typeof snapshot.exitCode === "number" && snapshot.exitCode !== 0)
|
||||
throw new PackageCommandExitError(
|
||||
command,
|
||||
arguments_,
|
||||
snapshot.exitCode,
|
||||
snapshot,
|
||||
);
|
||||
return output;
|
||||
} catch (cause: unknown) {
|
||||
retainOwnedCommandCleanup(
|
||||
cause,
|
||||
this.#resourceOwner,
|
||||
"package-verifier",
|
||||
command,
|
||||
);
|
||||
throw cause;
|
||||
}
|
||||
});
|
||||
options: VerifierCommandOptions = {},
|
||||
): Promise<VerifierCommandOutput> {
|
||||
return this.commands.exec(command, arguments_, options);
|
||||
}
|
||||
|
||||
execNpm(
|
||||
arguments_: readonly string[],
|
||||
options: PackageCommandOptions = {},
|
||||
): Promise<PackageCommandOutput> {
|
||||
options: VerifierCommandOptions = {},
|
||||
): Promise<VerifierCommandOutput> {
|
||||
const invocation = npmInvocation(arguments_, options.env ?? process.env);
|
||||
return this.exec(invocation.command, invocation.arguments, options);
|
||||
}
|
||||
@@ -277,30 +97,19 @@ class PackageCommands implements PackageVerifierCommands {
|
||||
status: 0,
|
||||
};
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof PackageCommandExitError && cause.code === 1)
|
||||
return { stdout: cause.stdout, status: 1 };
|
||||
if (cause instanceof VerifierCommandExitError && cause.code === 1)
|
||||
return { stdout: cause.snapshot.stdout.text, status: 1 };
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
|
||||
adopt(owner: PackageCleanupOwner, resource = this.workspace.path): void {
|
||||
adopt(owner: VerifierCleanupOwner, resource = this.workspace.path): void {
|
||||
if (this.#workspaceReleaseStarted)
|
||||
throw new Error("Package workspace release has started");
|
||||
let retained = this.#adopted.get(owner);
|
||||
if (retained === undefined) {
|
||||
retained = {
|
||||
kind: "lease",
|
||||
resource,
|
||||
lease: AnalysisCleanupLease.forCloseOwner(owner, resource),
|
||||
};
|
||||
this.#adopted.set(owner, retained);
|
||||
}
|
||||
this.#resources.retain(retained);
|
||||
this.commands.adopt(owner, resource);
|
||||
}
|
||||
|
||||
close(): Promise<void> {
|
||||
this.#closed = true;
|
||||
this.#controller.abort(new Error("Package command owner closed"));
|
||||
if (this.#closing !== undefined) return this.#closing;
|
||||
let attempt: Promise<void>;
|
||||
attempt = this.#release().catch((cause: unknown) => {
|
||||
@@ -312,21 +121,21 @@ class PackageCommands implements PackageVerifierCommands {
|
||||
}
|
||||
|
||||
async #release(): Promise<void> {
|
||||
await this.#resources.close();
|
||||
await this.commands.close();
|
||||
this.#workspaceReleaseStarted = true;
|
||||
await this.workspace.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** Allocate one command lifetime whose workspace depends on completed cleanup. */
|
||||
/** Remove the package workspace only after its shared command owner releases. */
|
||||
export async function createPackageVerifierCommands({
|
||||
workspace,
|
||||
launcher = launchPackageCommand,
|
||||
}: {
|
||||
launcher,
|
||||
}: VerifierCommandsOptions & {
|
||||
readonly workspace: PackageVerifierWorkspace;
|
||||
readonly launcher?: (
|
||||
input: OwnedProviderProcessSpawnOptions,
|
||||
) => Promise<SpawnedOwnedProviderProcess>;
|
||||
}): Promise<PackageVerifierCommands> {
|
||||
return new PackageCommands(workspace, launcher);
|
||||
const commands = await createVerifierCommands(
|
||||
launcher === undefined ? {} : { launcher },
|
||||
);
|
||||
return new PackageCommands(workspace, commands);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
import type { RetainedResourceScope } from "../../src/application/AnalysisProvider.js";
|
||||
import { AnalysisCleanupLease } from "../../src/artifacts/AnalysisCleanupLease.js";
|
||||
import {
|
||||
ArtifactResourceScope,
|
||||
type ArtifactResourceOwner,
|
||||
} from "../../src/artifacts/ArtifactResourceScope.js";
|
||||
import {
|
||||
prepareOwnedCommand,
|
||||
runOwnedCommand,
|
||||
} from "../../src/process/OwnedCommand.js";
|
||||
import {
|
||||
spawnOwnedProviderProcess,
|
||||
type OwnedProviderProcessSpawnOptions,
|
||||
type ProviderProcessSnapshot,
|
||||
type SpawnedOwnedProviderProcess,
|
||||
} from "../../src/process/ProviderProcess.js";
|
||||
import { retainOwnedCommandCleanup } from "../../src/providers/OwnedCommandCleanup.js";
|
||||
import { WindowsOwnedProcess } from "../../src/windows/WindowsOwnedProcess.js";
|
||||
import { createWindowsOwnedCommand } from "./windows-owned-command.mjs";
|
||||
|
||||
interface VerifierCommandContext {
|
||||
readonly signal?: AbortSignal | undefined;
|
||||
readonly cwd?: string | undefined;
|
||||
readonly env?: NodeJS.ProcessEnv | undefined;
|
||||
readonly timeout?: number | undefined;
|
||||
readonly windowsHide?: boolean | undefined;
|
||||
/** Acquire a real owned stdin pipe; closed-pipe sends EOF before awaiting output. */
|
||||
readonly stdin?: "pipe" | "closed-pipe";
|
||||
}
|
||||
|
||||
export type VerifierCommandOptions = VerifierCommandContext &
|
||||
(
|
||||
| {
|
||||
/** execFile-compatible capacity for each output stream. */
|
||||
readonly maxBuffer?: number | undefined;
|
||||
readonly diagnosticBytes?: never;
|
||||
}
|
||||
| {
|
||||
/** Canonical command capacity across both streams. */
|
||||
readonly diagnosticBytes: number;
|
||||
readonly maxBuffer?: never;
|
||||
}
|
||||
);
|
||||
|
||||
export interface VerifierCommandOutput {
|
||||
readonly stdout: string;
|
||||
readonly stderr: string;
|
||||
}
|
||||
|
||||
export interface VerifierCleanupOwner {
|
||||
close(): Promise<void>;
|
||||
readonly cleanupResources?: readonly string[] | undefined;
|
||||
}
|
||||
|
||||
/** Command and adopted-resource ownership without filesystem removal authority. */
|
||||
export interface VerifierCommands {
|
||||
exec(
|
||||
command: string,
|
||||
arguments_: readonly string[],
|
||||
options?: VerifierCommandOptions,
|
||||
): Promise<VerifierCommandOutput>;
|
||||
adopt(owner: VerifierCleanupOwner, resource: string): void;
|
||||
readonly cleanupResources: readonly string[];
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface VerifierCommandsOptions {
|
||||
readonly launcher?: (
|
||||
input: OwnedProviderProcessSpawnOptions,
|
||||
) => Promise<SpawnedOwnedProviderProcess>;
|
||||
}
|
||||
|
||||
/** An ordinary nonzero exit whose process and output cleanup both completed. */
|
||||
export class VerifierCommandExitError extends Error {
|
||||
constructor(
|
||||
readonly command: string,
|
||||
readonly arguments_: readonly string[],
|
||||
readonly code: number,
|
||||
readonly snapshot: ProviderProcessSnapshot,
|
||||
) {
|
||||
super(
|
||||
`Command failed: ${[command, ...arguments_].join(" ")}\n${snapshot.stderr.text}`,
|
||||
);
|
||||
this.name = "VerifierCommandExitError";
|
||||
}
|
||||
}
|
||||
|
||||
const launchVerifierCommand = async (
|
||||
input: OwnedProviderProcessSpawnOptions,
|
||||
): Promise<SpawnedOwnedProviderProcess> => {
|
||||
if (process.platform !== "win32") return spawnOwnedProviderProcess(input);
|
||||
input.signal?.throwIfAborted();
|
||||
const child = createWindowsOwnedCommand({
|
||||
WindowsOwnedProcess,
|
||||
command: input.command,
|
||||
arguments: input.arguments,
|
||||
cwd: input.cwd,
|
||||
environment: {
|
||||
...input.hostEnvironment,
|
||||
...input.env,
|
||||
REA_PROCESS_RUN_ID: input.runId,
|
||||
},
|
||||
});
|
||||
// These labels describe the launch; cleanup authority is the exact native job.
|
||||
return {
|
||||
process: child,
|
||||
ownership: {
|
||||
runId: input.runId,
|
||||
leaderPid: child.pid,
|
||||
processGroupId: child.pid,
|
||||
expectedParentPid: process.pid,
|
||||
},
|
||||
cleanup: () => child.cleanup(),
|
||||
};
|
||||
};
|
||||
|
||||
const commandLimits = (options: VerifierCommandOptions) => {
|
||||
if (
|
||||
options.stdin !== undefined &&
|
||||
options.stdin !== "pipe" &&
|
||||
options.stdin !== "closed-pipe"
|
||||
)
|
||||
throw new TypeError(
|
||||
"Verifier command stdin must be pipe or closed-pipe when supplied",
|
||||
);
|
||||
const timeout = options.timeout ?? 0;
|
||||
if (!Number.isSafeInteger(timeout) || timeout < 0)
|
||||
throw new RangeError(
|
||||
"Verifier command timeout must be a nonnegative integer",
|
||||
);
|
||||
if (options.maxBuffer !== undefined && options.diagnosticBytes !== undefined)
|
||||
throw new TypeError("Select a per-stream or aggregate command capacity");
|
||||
const capacity = options.diagnosticBytes ?? options.maxBuffer ?? 1024 * 1024;
|
||||
if (typeof capacity !== "number" || Number.isNaN(capacity) || capacity < 0)
|
||||
throw new RangeError(
|
||||
"Verifier command output capacity must be nonnegative",
|
||||
);
|
||||
if (process.platform === "win32" && options.windowsHide === false)
|
||||
throw new Error(
|
||||
"Owned verifier commands require hidden Windows process launch",
|
||||
);
|
||||
return {
|
||||
timeoutMs: timeout === 0 ? null : timeout,
|
||||
diagnosticBytes:
|
||||
options.diagnosticBytes === undefined
|
||||
? { stdout: capacity, stderr: capacity }
|
||||
: capacity,
|
||||
acceptNonZeroExit: true,
|
||||
...(options.stdin === undefined ? {} : { stdin: options.stdin }),
|
||||
};
|
||||
};
|
||||
|
||||
class VerifierCommandOwner implements VerifierCommands {
|
||||
readonly #resources = new ArtifactResourceScope();
|
||||
readonly #controller = new AbortController();
|
||||
readonly #adopted = new WeakMap<
|
||||
VerifierCleanupOwner,
|
||||
ArtifactResourceOwner
|
||||
>();
|
||||
readonly #resourceOwner: RetainedResourceScope;
|
||||
#closed = false;
|
||||
#released = false;
|
||||
#closing: Promise<void> | undefined;
|
||||
|
||||
constructor(
|
||||
private readonly launcher: (
|
||||
input: OwnedProviderProcessSpawnOptions,
|
||||
) => Promise<SpawnedOwnedProviderProcess>,
|
||||
) {
|
||||
this.#resourceOwner = {
|
||||
retain: (owner, resource) => this.adopt(owner, resource),
|
||||
run: (work) => this.#resources.run(work),
|
||||
};
|
||||
}
|
||||
|
||||
get cleanupResources(): readonly string[] {
|
||||
return this.#resources.cleanupResources;
|
||||
}
|
||||
|
||||
exec(
|
||||
command: string,
|
||||
arguments_: readonly string[],
|
||||
options: VerifierCommandOptions = {},
|
||||
): Promise<VerifierCommandOutput> {
|
||||
if (this.#closed)
|
||||
return Promise.reject(new Error("Verifier command owner is closed"));
|
||||
return this.#resources.run(async () => {
|
||||
try {
|
||||
const limits = commandLimits(options);
|
||||
const signal =
|
||||
options.signal === undefined
|
||||
? this.#controller.signal
|
||||
: AbortSignal.any([this.#controller.signal, options.signal]);
|
||||
// Independently bounded native preparation stays outside the command's
|
||||
// deadline and inside this admitted resource lifetime.
|
||||
await prepareOwnedCommand(signal);
|
||||
const snapshot = await runOwnedCommand(
|
||||
{
|
||||
command,
|
||||
arguments: arguments_,
|
||||
cwd: options.cwd ?? process.cwd(),
|
||||
hostEnvironment: options.env ?? process.env,
|
||||
expectedCommand: null,
|
||||
runId: `rea-verifier-command-${randomUUID()}`,
|
||||
sweepTokenOwnedProcesses: true,
|
||||
},
|
||||
limits,
|
||||
{ signal, launcher: this.launcher },
|
||||
);
|
||||
if (typeof snapshot.exitCode === "number" && snapshot.exitCode !== 0)
|
||||
throw new VerifierCommandExitError(
|
||||
command,
|
||||
arguments_,
|
||||
snapshot.exitCode,
|
||||
snapshot,
|
||||
);
|
||||
return {
|
||||
stdout: snapshot.stdout.text,
|
||||
stderr: snapshot.stderr.text,
|
||||
};
|
||||
} catch (cause: unknown) {
|
||||
retainOwnedCommandCleanup(
|
||||
cause,
|
||||
this.#resourceOwner,
|
||||
"verifier",
|
||||
command,
|
||||
);
|
||||
throw cause;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
adopt(owner: VerifierCleanupOwner, resource: string): void {
|
||||
if (this.#released)
|
||||
throw new Error("Verifier command resource release has completed");
|
||||
let retained = this.#adopted.get(owner);
|
||||
if (retained === undefined) {
|
||||
retained = {
|
||||
kind: "lease",
|
||||
resource,
|
||||
lease: AnalysisCleanupLease.forCloseOwner(owner, resource),
|
||||
};
|
||||
this.#adopted.set(owner, retained);
|
||||
}
|
||||
// An admitted command can transfer a late preparation/cleanup capability
|
||||
// while close is joining it. Acquisition must precede this scope's release.
|
||||
this.#resources.retain(retained);
|
||||
}
|
||||
|
||||
close(): Promise<void> {
|
||||
this.#closed = true;
|
||||
this.#controller.abort(new Error("Verifier command owner closed"));
|
||||
if (this.#closing !== undefined) return this.#closing;
|
||||
let attempt: Promise<void>;
|
||||
attempt = this.#resources
|
||||
.close()
|
||||
.then(() => {
|
||||
this.#released = true;
|
||||
})
|
||||
.catch((cause: unknown) => {
|
||||
if (this.#closing === attempt) this.#closing = undefined;
|
||||
throw cause;
|
||||
});
|
||||
this.#closing = attempt;
|
||||
return attempt;
|
||||
}
|
||||
}
|
||||
|
||||
/** Allocate command ownership without acquiring or deleting an evidence root. */
|
||||
export async function createVerifierCommands({
|
||||
launcher = launchVerifierCommand,
|
||||
}: VerifierCommandsOptions = {}): Promise<VerifierCommands> {
|
||||
return new VerifierCommandOwner(launcher);
|
||||
}
|
||||
@@ -2,7 +2,7 @@ import { lstat, open } from "node:fs/promises";
|
||||
|
||||
import { ArtifactResourceScope } from "../../src/artifacts/ArtifactResourceScope.js";
|
||||
import { OwnedFileHandle } from "../../src/filesystem/OwnedFileHandle.js";
|
||||
import type { PackageVerifierCommands } from "./package-command-owner.mjs";
|
||||
import type { VerifierCommands } from "./verifier-command-owner.mjs";
|
||||
import { withVerifierCleanup } from "./verifier-cleanup.mjs";
|
||||
|
||||
export type VerifierFileSnapshotEntry =
|
||||
@@ -19,7 +19,7 @@ export type VerifierFileSnapshotEntry =
|
||||
|
||||
/** Capture source values while the command owner retains every in-flight file. */
|
||||
export async function captureVerifierFileSnapshot(
|
||||
commands: Pick<PackageVerifierCommands, "adopt">,
|
||||
commands: Pick<VerifierCommands, "adopt">,
|
||||
paths: readonly string[],
|
||||
): Promise<readonly VerifierFileSnapshotEntry[]> {
|
||||
const resources = new ArtifactResourceScope();
|
||||
@@ -61,7 +61,7 @@ export async function captureVerifierFileSnapshot(
|
||||
|
||||
/** Compare the same selected paths against their completed baseline facts. */
|
||||
export async function assertVerifierFilesUnchanged(
|
||||
commands: Pick<PackageVerifierCommands, "adopt">,
|
||||
commands: Pick<VerifierCommands, "adopt">,
|
||||
snapshot: readonly VerifierFileSnapshotEntry[],
|
||||
label: string,
|
||||
): Promise<void> {
|
||||
|
||||
@@ -19,16 +19,16 @@ import { ProviderStartupDeadline } from "../../src/process/ProviderDeadline.js";
|
||||
import { processCaptureOwnershipUnavailableReason } from "../../src/process/capture/ProcessCaptureCapability.js";
|
||||
import { retainOwnedCommandCleanup } from "../../src/providers/OwnedCommandCleanup.js";
|
||||
import type {
|
||||
PackageCleanupOwner,
|
||||
PackageVerifierCommands,
|
||||
} from "./package-command-owner.mjs";
|
||||
VerifierCleanupOwner,
|
||||
VerifierCommands,
|
||||
} from "./verifier-command-owner.mjs";
|
||||
import {
|
||||
closeVerifierResources,
|
||||
withVerifierCleanup,
|
||||
} from "./verifier-cleanup.mjs";
|
||||
|
||||
export interface VerifierTerminalOptions {
|
||||
readonly commands: Pick<PackageVerifierCommands, "adopt">;
|
||||
readonly commands: Pick<VerifierCommands, "adopt">;
|
||||
readonly command: string;
|
||||
readonly args: readonly string[];
|
||||
readonly cwd: string;
|
||||
@@ -79,7 +79,7 @@ export class VerifierTerminalCleanupFailure extends Error {
|
||||
readonly reason = "cleanup_incomplete";
|
||||
|
||||
constructor(
|
||||
readonly cleanupOwner: PackageCleanupOwner,
|
||||
readonly cleanupOwner: VerifierCleanupOwner,
|
||||
readonly output: string,
|
||||
message: string,
|
||||
options?: ErrorOptions,
|
||||
@@ -102,7 +102,7 @@ const decodedPrefix = (text: string, capacity: number): string => {
|
||||
return text.slice(0, end);
|
||||
};
|
||||
|
||||
class VerifierTerminalOwner implements PackageCleanupOwner {
|
||||
class VerifierTerminalOwner implements VerifierCleanupOwner {
|
||||
readonly #controller = new AbortController();
|
||||
readonly #runId = randomUUID();
|
||||
readonly #evidence = new ProcessOwnershipEvidenceOwner();
|
||||
@@ -409,7 +409,7 @@ export async function runOwnedVerifierTerminal(
|
||||
"Verifier terminal output capacity must be a non-negative safe integer",
|
||||
);
|
||||
const owner = new VerifierTerminalOwner(options, dependencies);
|
||||
options.commands.adopt(owner);
|
||||
options.commands.adopt(owner, `verifier terminal ${options.command}`);
|
||||
owner.start();
|
||||
let release: Promise<void> | undefined;
|
||||
const finish = (): Promise<void> => (release ??= owner.release());
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
export { createVerifierCommands } from "./verifier-command-owner.mjs";
|
||||
export type {
|
||||
VerifierCommands,
|
||||
VerifierCommandsOptions,
|
||||
VerifierCommandOptions,
|
||||
VerifierCommandOutput,
|
||||
VerifierCleanupOwner,
|
||||
VerifierCommandExitError,
|
||||
} from "./verifier-command-owner.mjs";
|
||||
@@ -0,0 +1,13 @@
|
||||
let implementation;
|
||||
|
||||
/** Load the development command owner and its canonical process source graph. */
|
||||
export async function createVerifierCommands(options = {}) {
|
||||
implementation ??= import("tsx/esm/api").then(({ tsImport }) =>
|
||||
tsImport(
|
||||
new URL("./verifier-command-owner.mts", import.meta.url).href,
|
||||
import.meta.url,
|
||||
),
|
||||
);
|
||||
const module = await implementation;
|
||||
return module.createVerifierCommands(options);
|
||||
}
|
||||
@@ -2,7 +2,4 @@ export { createPackageVerifierCommands } from "./package-command-owner.mjs";
|
||||
export type {
|
||||
PackageVerifierCommands,
|
||||
PackageVerifierWorkspace,
|
||||
PackageCommandOptions,
|
||||
PackageCommandOutput,
|
||||
PackageCleanupOwner,
|
||||
} from "./package-command-owner.mjs";
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { PackageVerifierCommands } from "./verify-package-commands.mjs";
|
||||
import type { VerifierCommands } from "./verifier-command-owner.mjs";
|
||||
|
||||
export interface SetupTerminalOptions {
|
||||
readonly commands: Pick<PackageVerifierCommands, "adopt">;
|
||||
readonly commands: Pick<VerifierCommands, "adopt">;
|
||||
readonly command: string;
|
||||
readonly environment: NodeJS.ProcessEnv;
|
||||
readonly root: string;
|
||||
|
||||
@@ -8,7 +8,8 @@ import {
|
||||
TOOL_CONTRACTS,
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
import { exec } from "./lib/verify-package-core.mjs";
|
||||
import { createVerifierCommands } from "./lib/verify-commands.mjs";
|
||||
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
||||
import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
|
||||
@@ -43,86 +44,9 @@ const skillPath = join(
|
||||
"SKILL.md",
|
||||
);
|
||||
const verifier = createVerifierRun();
|
||||
await Promise.all([
|
||||
mkdir(account),
|
||||
mkdir(profile, { recursive: true }),
|
||||
...(stickyHermes ? [mkdir(selectedProfile, { recursive: true })] : []),
|
||||
mkdir(target, { recursive: true }),
|
||||
]);
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
const environment = {
|
||||
PATH: process.env.PATH ?? "",
|
||||
HOME: account,
|
||||
USERPROFILE: account,
|
||||
[client === "pi" ? "PI_CODING_AGENT_DIR" : "HERMES_HOME"]: profile,
|
||||
PI_OFFLINE: "1",
|
||||
REA_PROCESS_RUN_ID: verifier.run_id,
|
||||
};
|
||||
const configPath = join(
|
||||
selectedProfile,
|
||||
client === "pi" ? "mcp.json" : "config.yaml",
|
||||
);
|
||||
if (stickyHermes) await writeFile(join(profile, "active_profile"), "coder\n");
|
||||
const original =
|
||||
client === "pi"
|
||||
? JSON.stringify({
|
||||
mcpServers: { other: { command: "unrelated-server", enabled: false } },
|
||||
})
|
||||
: "# Keep caller settings.\nmcp_servers:\n other:\n command: unrelated-server\n enabled: false\n";
|
||||
await writeFile(configPath, original);
|
||||
const clientVersion = (
|
||||
await exec(command, ["--version"], { env: environment, timeout: 30_000 })
|
||||
).stdout.trim();
|
||||
for (const args of [
|
||||
["setup", "--client", client, "--dry-run"],
|
||||
["setup", "--client", client, "--yes"],
|
||||
["doctor", "--client", client, "--skill"],
|
||||
]) {
|
||||
const result = await exec(
|
||||
process.execPath,
|
||||
[join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],
|
||||
{ env: environment, cwd: workspace, timeout: 60_000 },
|
||||
);
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
if (args[0] === "doctor") assert.equal(parsed.healthy, true);
|
||||
else
|
||||
assert.equal(
|
||||
parsed.status,
|
||||
args.includes("--dry-run") ? "planned" : "ready",
|
||||
);
|
||||
await writeFile(
|
||||
join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`),
|
||||
result.stdout,
|
||||
);
|
||||
}
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
const installedConfiguration = await readFile(configPath, "utf8");
|
||||
assert(installedConfiguration.includes("unrelated-server"));
|
||||
if (client === "hermes")
|
||||
assert(installedConfiguration.includes("# Keep caller settings."));
|
||||
const repeated = await exec(
|
||||
process.execPath,
|
||||
[
|
||||
join(runtimeRoot, "scripts/rea.mjs"),
|
||||
"setup",
|
||||
"--client",
|
||||
client,
|
||||
"--yes",
|
||||
"--json",
|
||||
],
|
||||
{ env: environment, cwd: workspace, timeout: 60_000 },
|
||||
);
|
||||
assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []);
|
||||
assert.equal(await readFile(configPath, "utf8"), installedConfiguration);
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
assert(
|
||||
(await readFile(skillPath, "utf8")).includes(
|
||||
"name: reverse-engineer-anything",
|
||||
),
|
||||
);
|
||||
let clientVersion = null;
|
||||
let fixture;
|
||||
let passed = false;
|
||||
let evidenceSeen = false;
|
||||
let skillLoaded = false;
|
||||
let skillRequested = false;
|
||||
@@ -132,331 +56,460 @@ let describeRequested = false;
|
||||
let finalSeen = false;
|
||||
let catalog;
|
||||
let artifactDigest;
|
||||
const extractEvidence = (value) => {
|
||||
if (typeof value === "string") {
|
||||
if (
|
||||
value.startsWith("<untrusted_tool_result ") &&
|
||||
value.endsWith("</untrusted_tool_result>")
|
||||
) {
|
||||
// Hermes wraps external MCP content with its own instruction banner.
|
||||
// Decode only that observed envelope, retaining the original request artifact.
|
||||
const start = value.indexOf("\n\n");
|
||||
if (start >= 0)
|
||||
return extractEvidence(
|
||||
value
|
||||
.slice(start + 2, value.lastIndexOf("</untrusted_tool_result>"))
|
||||
.trim(),
|
||||
);
|
||||
}
|
||||
try {
|
||||
return extractEvidence(JSON.parse(value));
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
if (
|
||||
typeof value.evidence_id === "string" &&
|
||||
value.normalized_result?.graph?.nodes
|
||||
)
|
||||
return value;
|
||||
for (const child of Object.values(value)) {
|
||||
const found = extractEvidence(child);
|
||||
if (found) return found;
|
||||
}
|
||||
}
|
||||
};
|
||||
const validateEvidence = async (value) => {
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(value);
|
||||
await assertJavaScriptApplicationInputPaths(evidence, target);
|
||||
const shape = evidence.normalized_result.graph.nodes
|
||||
.flatMap((n) => n.observations)
|
||||
.map((o) => o.properties)
|
||||
.find(
|
||||
(p) =>
|
||||
p.semantic_role === "export-return-shapes" &&
|
||||
p.exported_name === "result",
|
||||
const commands = await createVerifierCommands();
|
||||
await withVerifierCleanup(async () => {
|
||||
await withVerifierCleanup(async () => {
|
||||
await Promise.all([
|
||||
mkdir(account),
|
||||
mkdir(profile, { recursive: true }),
|
||||
...(stickyHermes ? [mkdir(selectedProfile, { recursive: true })] : []),
|
||||
mkdir(target, { recursive: true }),
|
||||
]);
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
assert(shape?.return_shape_coverage?.projection_complete);
|
||||
const fields = shape.static_return_shapes.flatMap((s) => s.fields);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) =>
|
||||
f.path === "/clientCompatibility" &&
|
||||
f.state === "literal" &&
|
||||
f.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) => f.path === "/count" && f.state === "literal" && f.value === 7,
|
||||
),
|
||||
);
|
||||
const serialized = JSON.stringify(evidence);
|
||||
await writeFile(join(lab, "evidence.json"), serialized);
|
||||
artifactDigest = createHash("sha256").update(serialized).digest("hex");
|
||||
evidenceSeen = true;
|
||||
};
|
||||
const fixture = await createOpenAiModelFixture({
|
||||
directory: lab,
|
||||
prefix: `${client}-${mode}`,
|
||||
onRequest: async ({ body, tools, requestIndex }) => {
|
||||
assert(
|
||||
requestIndex <= 6,
|
||||
"Client did not complete the bounded fixture workflow.",
|
||||
const environment = {
|
||||
PATH: process.env.PATH ?? "",
|
||||
HOME: account,
|
||||
USERPROFILE: account,
|
||||
[client === "pi" ? "PI_CODING_AGENT_DIR" : "HERMES_HOME"]: profile,
|
||||
PI_OFFLINE: "1",
|
||||
REA_PROCESS_RUN_ID: verifier.run_id,
|
||||
};
|
||||
const configPath = join(
|
||||
selectedProfile,
|
||||
client === "pi" ? "mcp.json" : "config.yaml",
|
||||
);
|
||||
const messages = body.messages ?? [];
|
||||
const toolMessages = messages.filter((m) => m.role === "tool");
|
||||
const serializedTools = JSON.stringify(toolMessages);
|
||||
const call = (name, args) => ({
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: [
|
||||
if (stickyHermes)
|
||||
await writeFile(join(profile, "active_profile"), "coder\n");
|
||||
const original =
|
||||
client === "pi"
|
||||
? JSON.stringify({
|
||||
mcpServers: {
|
||||
other: { command: "unrelated-server", enabled: false },
|
||||
},
|
||||
})
|
||||
: "# Keep caller settings.\nmcp_servers:\n other:\n command: unrelated-server\n enabled: false\n";
|
||||
await writeFile(configPath, original);
|
||||
clientVersion = (
|
||||
await commands.exec(command, ["--version"], {
|
||||
env: environment,
|
||||
timeout: 30_000,
|
||||
stdin: "pipe",
|
||||
})
|
||||
).stdout.trim();
|
||||
for (const args of [
|
||||
["setup", "--client", client, "--dry-run"],
|
||||
["setup", "--client", client, "--yes"],
|
||||
["doctor", "--client", client, "--skill"],
|
||||
]) {
|
||||
const result = await commands.exec(
|
||||
process.execPath,
|
||||
[join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],
|
||||
{
|
||||
id: `call_rea_${requestIndex}`,
|
||||
type: "function",
|
||||
function: { name, arguments: JSON.stringify(args) },
|
||||
env: environment,
|
||||
cwd: workspace,
|
||||
timeout: 60_000,
|
||||
stdin: "pipe",
|
||||
},
|
||||
],
|
||||
});
|
||||
if (
|
||||
client === "pi" &&
|
||||
callRequested &&
|
||||
serializedTools.includes("REA_CLIENT_FIXTURE")
|
||||
) {
|
||||
const fact = toolMessages
|
||||
.flatMap((m) => {
|
||||
const text =
|
||||
typeof m.content === "string"
|
||||
? m.content
|
||||
: JSON.stringify(m.content);
|
||||
const marker = text.indexOf('{"REA_CLIENT_FIXTURE":true');
|
||||
if (marker < 0) return [];
|
||||
const end = text.indexOf("}\n", marker);
|
||||
try {
|
||||
return [
|
||||
JSON.parse(text.slice(marker, end < 0 ? undefined : end + 1)),
|
||||
];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
})
|
||||
.find((v) => Array.isArray(v.catalog));
|
||||
assert(
|
||||
fact,
|
||||
"Pi codemode must return its actual catalog and skill-read receipt.",
|
||||
);
|
||||
catalog = fact.catalog;
|
||||
skillLoaded = fact.skill_loaded === true;
|
||||
await validateEvidence(JSON.parse(await readFile(artifactPath, "utf8")));
|
||||
} else if (client === "hermes") {
|
||||
const search = tools.find((t) => t.name === "tool_search");
|
||||
const exposed = tools
|
||||
.filter((t) => t.name.startsWith("mcp__rea__"))
|
||||
.map((t) => t.name.slice(10));
|
||||
const deferred = [
|
||||
...(search?.description ?? "").matchAll(/^- mcp__rea__(\w+):/gmu),
|
||||
].map((m) => m[1]);
|
||||
const native = [...new Set([...exposed, ...deferred])];
|
||||
// Hermes advertises MCP prompt utilities beside server tools.
|
||||
if (native.length)
|
||||
catalog = native.filter(
|
||||
(name) => !["get_prompt", "list_prompts"].includes(name),
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
if (args[0] === "doctor") assert.equal(parsed.healthy, true);
|
||||
else
|
||||
assert.equal(
|
||||
parsed.status,
|
||||
args.includes("--dry-run") ? "planned" : "ready",
|
||||
);
|
||||
skillLoaded ||=
|
||||
serializedTools.includes(skillPath) &&
|
||||
serializedTools.includes("reverse-engineer-anything");
|
||||
for (const message of toolMessages) {
|
||||
const evidence = extractEvidence(message.content);
|
||||
if (evidence) await validateEvidence(evidence);
|
||||
await writeFile(
|
||||
join(
|
||||
lab,
|
||||
`${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`,
|
||||
),
|
||||
result.stdout,
|
||||
);
|
||||
}
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
const installedConfiguration = await readFile(configPath, "utf8");
|
||||
assert(installedConfiguration.includes("unrelated-server"));
|
||||
if (client === "hermes")
|
||||
assert(installedConfiguration.includes("# Keep caller settings."));
|
||||
const repeated = await commands.exec(
|
||||
process.execPath,
|
||||
[
|
||||
join(runtimeRoot, "scripts/rea.mjs"),
|
||||
"setup",
|
||||
"--client",
|
||||
client,
|
||||
"--yes",
|
||||
"--json",
|
||||
],
|
||||
{
|
||||
env: environment,
|
||||
cwd: workspace,
|
||||
timeout: 60_000,
|
||||
stdin: "pipe",
|
||||
},
|
||||
);
|
||||
assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []);
|
||||
assert.equal(await readFile(configPath, "utf8"), installedConfiguration);
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
assert(
|
||||
(await readFile(skillPath, "utf8")).includes(
|
||||
"name: reverse-engineer-anything",
|
||||
),
|
||||
);
|
||||
const extractEvidence = (value) => {
|
||||
if (typeof value === "string") {
|
||||
if (
|
||||
value.startsWith("<untrusted_tool_result ") &&
|
||||
value.endsWith("</untrusted_tool_result>")
|
||||
) {
|
||||
// Hermes wraps external MCP content with its own instruction banner.
|
||||
// Decode only that observed envelope, retaining the original request artifact.
|
||||
const start = value.indexOf("\n\n");
|
||||
if (start >= 0)
|
||||
return extractEvidence(
|
||||
value
|
||||
.slice(start + 2, value.lastIndexOf("</untrusted_tool_result>"))
|
||||
.trim(),
|
||||
);
|
||||
}
|
||||
try {
|
||||
return extractEvidence(JSON.parse(value));
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (mode === "chat" || evidenceSeen) {
|
||||
finalSeen = true;
|
||||
return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
|
||||
}
|
||||
if (client === "pi") {
|
||||
const tool = tools.find((t) => t.name === "codemode");
|
||||
if (value && typeof value === "object") {
|
||||
if (
|
||||
typeof value.evidence_id === "string" &&
|
||||
value.normalized_result?.graph?.nodes
|
||||
)
|
||||
return value;
|
||||
for (const child of Object.values(value)) {
|
||||
const found = extractEvidence(child);
|
||||
if (found) return found;
|
||||
}
|
||||
}
|
||||
};
|
||||
const validateEvidence = async (value) => {
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(value);
|
||||
await assertJavaScriptApplicationInputPaths(evidence, target);
|
||||
const shape = evidence.normalized_result.graph.nodes
|
||||
.flatMap((n) => n.observations)
|
||||
.map((o) => o.properties)
|
||||
.find(
|
||||
(p) =>
|
||||
p.semantic_role === "export-return-shapes" &&
|
||||
p.exported_name === "result",
|
||||
);
|
||||
assert(shape?.return_shape_coverage?.projection_complete);
|
||||
const fields = shape.static_return_shapes.flatMap((s) => s.fields);
|
||||
assert(
|
||||
tool,
|
||||
`Native Pi codemode unavailable: ${tools.map((t) => t.name).join(", ")}`,
|
||||
fields.some(
|
||||
(f) =>
|
||||
f.path === "/clientCompatibility" &&
|
||||
f.state === "literal" &&
|
||||
f.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
JSON.stringify(messages).includes(skillPath),
|
||||
"Pi must advertise the installed REA skill.",
|
||||
fields.some(
|
||||
(f) => f.path === "/count" && f.state === "literal" && f.value === 7,
|
||||
),
|
||||
);
|
||||
callRequested = true;
|
||||
return call(tool.name, {
|
||||
code: `await describeNamespace("rea"); const skill = await tools.read({path:${JSON.stringify(skillPath)}}); const result = await tools.mcp__rea__analyze_javascript_application({input_path:${JSON.stringify(target)},format:"directory"}); if(result.isError) throw new Error(JSON.stringify(result)); const evidence = result.structuredContent ?? JSON.parse(result.content.find(c=>c.type==="text").text); await tools.write({path:${JSON.stringify(artifactPath)},content:JSON.stringify(evidence)}); text({REA_CLIENT_FIXTURE:true,skill_loaded:typeof skill==="string"&&skill.includes("name: reverse-engineer-anything"),catalog:ALL_TOOLS.filter(t=>t.name.startsWith("mcp__rea__")).map(t=>t.name.slice(10))});`,
|
||||
});
|
||||
}
|
||||
if (!skillRequested) {
|
||||
const tool = tools.find((t) => t.name === "skill_view");
|
||||
assert(tool, "Hermes skill_view must be available.");
|
||||
skillRequested = true;
|
||||
return call(tool.name, { name: "reverse-engineer-anything" });
|
||||
}
|
||||
if (!searchRequested) {
|
||||
const tool = tools.find((t) => t.name === "tool_search");
|
||||
assert(tool, "Hermes native deferred tool_search must be available.");
|
||||
searchRequested = true;
|
||||
return call(tool.name, { queries: ["analyze_javascript_application"] });
|
||||
}
|
||||
assert(
|
||||
serializedTools.includes("mcp__rea__analyze_javascript_application"),
|
||||
"Hermes must discover the analysis tool.",
|
||||
);
|
||||
if (!describeRequested) {
|
||||
const tool = tools.find((t) => t.name === "tool_describe");
|
||||
assert(tool, "Hermes must load the actual parameter schema.");
|
||||
describeRequested = true;
|
||||
return call(tool.name, {
|
||||
names: ["mcp__rea__analyze_javascript_application"],
|
||||
});
|
||||
}
|
||||
const tool = tools.find((t) => t.name === "tool_call");
|
||||
assert(tool, "Hermes native deferred tool_call must be available.");
|
||||
callRequested = true;
|
||||
return call(tool.name, {
|
||||
calls: [
|
||||
{
|
||||
name: "mcp__rea__analyze_javascript_application",
|
||||
arguments: { input_path: target, format: "directory" },
|
||||
},
|
||||
],
|
||||
});
|
||||
},
|
||||
});
|
||||
if (client === "pi") {
|
||||
await writeFile(
|
||||
join(profile, "models.json"),
|
||||
JSON.stringify({
|
||||
providers: {
|
||||
"rea-fixture": {
|
||||
baseUrl: fixture.baseUrl,
|
||||
api: "openai-completions",
|
||||
apiKey: "local-fixture-key",
|
||||
models: [
|
||||
const serialized = JSON.stringify(evidence);
|
||||
await writeFile(join(lab, "evidence.json"), serialized);
|
||||
artifactDigest = createHash("sha256").update(serialized).digest("hex");
|
||||
evidenceSeen = true;
|
||||
};
|
||||
fixture = await createOpenAiModelFixture({
|
||||
directory: lab,
|
||||
prefix: `${client}-${mode}`,
|
||||
onRequest: async ({ body, tools, requestIndex }) => {
|
||||
assert(
|
||||
requestIndex <= 6,
|
||||
"Client did not complete the bounded fixture workflow.",
|
||||
);
|
||||
const messages = body.messages ?? [];
|
||||
const toolMessages = messages.filter((m) => m.role === "tool");
|
||||
const serializedTools = JSON.stringify(toolMessages);
|
||||
const call = (name, args) => ({
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: "rea-client-fixture",
|
||||
reasoning: false,
|
||||
contextWindow: 200000,
|
||||
maxTokens: 4096,
|
||||
id: `call_rea_${requestIndex}`,
|
||||
type: "function",
|
||||
function: { name, arguments: JSON.stringify(args) },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
if (
|
||||
client === "pi" &&
|
||||
callRequested &&
|
||||
serializedTools.includes("REA_CLIENT_FIXTURE")
|
||||
) {
|
||||
const fact = toolMessages
|
||||
.flatMap((m) => {
|
||||
const text =
|
||||
typeof m.content === "string"
|
||||
? m.content
|
||||
: JSON.stringify(m.content);
|
||||
const marker = text.indexOf('{"REA_CLIENT_FIXTURE":true');
|
||||
if (marker < 0) return [];
|
||||
const end = text.indexOf("}\n", marker);
|
||||
try {
|
||||
return [
|
||||
JSON.parse(text.slice(marker, end < 0 ? undefined : end + 1)),
|
||||
];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
})
|
||||
.find((v) => Array.isArray(v.catalog));
|
||||
assert(
|
||||
fact,
|
||||
"Pi codemode must return its actual catalog and skill-read receipt.",
|
||||
);
|
||||
catalog = fact.catalog;
|
||||
skillLoaded = fact.skill_loaded === true;
|
||||
await validateEvidence(
|
||||
JSON.parse(await readFile(artifactPath, "utf8")),
|
||||
);
|
||||
} else if (client === "hermes") {
|
||||
const search = tools.find((t) => t.name === "tool_search");
|
||||
const exposed = tools
|
||||
.filter((t) => t.name.startsWith("mcp__rea__"))
|
||||
.map((t) => t.name.slice(10));
|
||||
const deferred = [
|
||||
...(search?.description ?? "").matchAll(/^- mcp__rea__(\w+):/gmu),
|
||||
].map((m) => m[1]);
|
||||
const native = [...new Set([...exposed, ...deferred])];
|
||||
// Hermes advertises MCP prompt utilities beside server tools.
|
||||
if (native.length)
|
||||
catalog = native.filter(
|
||||
(name) => !["get_prompt", "list_prompts"].includes(name),
|
||||
);
|
||||
skillLoaded ||=
|
||||
serializedTools.includes(skillPath) &&
|
||||
serializedTools.includes("reverse-engineer-anything");
|
||||
for (const message of toolMessages) {
|
||||
const evidence = extractEvidence(message.content);
|
||||
if (evidence) await validateEvidence(evidence);
|
||||
}
|
||||
}
|
||||
if (mode === "chat" || evidenceSeen) {
|
||||
finalSeen = true;
|
||||
return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
|
||||
}
|
||||
if (client === "pi") {
|
||||
const tool = tools.find((t) => t.name === "codemode");
|
||||
assert(
|
||||
tool,
|
||||
`Native Pi codemode unavailable: ${tools.map((t) => t.name).join(", ")}`,
|
||||
);
|
||||
assert(
|
||||
JSON.stringify(messages).includes(skillPath),
|
||||
"Pi must advertise the installed REA skill.",
|
||||
);
|
||||
callRequested = true;
|
||||
return call(tool.name, {
|
||||
code: `await describeNamespace("rea"); const skill = await tools.read({path:${JSON.stringify(skillPath)}}); const result = await tools.mcp__rea__analyze_javascript_application({input_path:${JSON.stringify(target)},format:"directory"}); if(result.isError) throw new Error(JSON.stringify(result)); const evidence = result.structuredContent ?? JSON.parse(result.content.find(c=>c.type==="text").text); await tools.write({path:${JSON.stringify(artifactPath)},content:JSON.stringify(evidence)}); text({REA_CLIENT_FIXTURE:true,skill_loaded:typeof skill==="string"&&skill.includes("name: reverse-engineer-anything"),catalog:ALL_TOOLS.filter(t=>t.name.startsWith("mcp__rea__")).map(t=>t.name.slice(10))});`,
|
||||
});
|
||||
}
|
||||
if (!skillRequested) {
|
||||
const tool = tools.find((t) => t.name === "skill_view");
|
||||
assert(tool, "Hermes skill_view must be available.");
|
||||
skillRequested = true;
|
||||
return call(tool.name, { name: "reverse-engineer-anything" });
|
||||
}
|
||||
if (!searchRequested) {
|
||||
const tool = tools.find((t) => t.name === "tool_search");
|
||||
assert(tool, "Hermes native deferred tool_search must be available.");
|
||||
searchRequested = true;
|
||||
return call(tool.name, {
|
||||
queries: ["analyze_javascript_application"],
|
||||
});
|
||||
}
|
||||
assert(
|
||||
serializedTools.includes("mcp__rea__analyze_javascript_application"),
|
||||
"Hermes must discover the analysis tool.",
|
||||
);
|
||||
if (!describeRequested) {
|
||||
const tool = tools.find((t) => t.name === "tool_describe");
|
||||
assert(tool, "Hermes must load the actual parameter schema.");
|
||||
describeRequested = true;
|
||||
return call(tool.name, {
|
||||
names: ["mcp__rea__analyze_javascript_application"],
|
||||
});
|
||||
}
|
||||
const tool = tools.find((t) => t.name === "tool_call");
|
||||
assert(tool, "Hermes native deferred tool_call must be available.");
|
||||
callRequested = true;
|
||||
return call(tool.name, {
|
||||
calls: [
|
||||
{
|
||||
name: "mcp__rea__analyze_javascript_application",
|
||||
arguments: { input_path: target, format: "directory" },
|
||||
},
|
||||
],
|
||||
});
|
||||
},
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
const configured = await readFile(configPath, "utf8");
|
||||
await writeFile(
|
||||
configPath,
|
||||
configured +
|
||||
`\nmodel:\n default: rea-client-fixture\n provider: custom\n base_url: ${fixture.baseUrl}\nagent:\n max_turns: 6\n`,
|
||||
);
|
||||
}
|
||||
const args =
|
||||
client === "pi"
|
||||
? [
|
||||
"--offline",
|
||||
"--provider",
|
||||
"rea-fixture",
|
||||
"--model",
|
||||
"rea-client-fixture",
|
||||
"--no-session",
|
||||
"--no-context-files",
|
||||
"--mode",
|
||||
"json",
|
||||
"-p",
|
||||
mode === "chat"
|
||||
? "Say hello briefly."
|
||||
: `Use REA to analyze ${target}.`,
|
||||
]
|
||||
: [
|
||||
"--oneshot",
|
||||
mode === "chat"
|
||||
? "Say hello briefly."
|
||||
: `Use REA to analyze ${target}.`,
|
||||
"--provider",
|
||||
"custom",
|
||||
"--model",
|
||||
"rea-client-fixture",
|
||||
"--reasoning",
|
||||
"none",
|
||||
];
|
||||
let failure;
|
||||
try {
|
||||
const execution = exec(command, args, {
|
||||
env: {
|
||||
...environment,
|
||||
OPENAI_API_KEY: "local-fixture-key",
|
||||
OPENAI_BASE_URL: fixture.baseUrl,
|
||||
},
|
||||
cwd: workspace,
|
||||
timeout: 120_000,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
});
|
||||
// Pi consumes piped input before its first prompt; an unused open pipe is not EOF.
|
||||
execution.child.stdin.end();
|
||||
const result = await execution;
|
||||
await writeFile(join(lab, "client.stdout"), result.stdout);
|
||||
await writeFile(join(lab, "client.stderr"), result.stderr);
|
||||
assert.equal(fixture.failure, undefined);
|
||||
assert(finalSeen);
|
||||
assert(result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK"));
|
||||
if (mode === "call") {
|
||||
assert(evidenceSeen);
|
||||
assert(
|
||||
skillLoaded,
|
||||
"Native client must actually read the installed skill.",
|
||||
);
|
||||
assert.deepEqual(
|
||||
catalog?.sort(),
|
||||
TOOL_CONTRACTS.map((t) => t.name).sort(),
|
||||
"The complete REA catalog must survive native client discovery.",
|
||||
);
|
||||
}
|
||||
} catch (cause) {
|
||||
failure = cause;
|
||||
if (typeof cause?.stdout === "string")
|
||||
await writeFile(join(lab, "client.stdout"), cause.stdout);
|
||||
if (typeof cause?.stderr === "string")
|
||||
await writeFile(join(lab, "client.stderr"), cause.stderr);
|
||||
} finally {
|
||||
await fixture.close();
|
||||
const receipt = {
|
||||
client,
|
||||
clientVersion,
|
||||
mode,
|
||||
passed: failure === undefined,
|
||||
artifacts: lab,
|
||||
runtimeRoot,
|
||||
host: {
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
node: process.versions.node,
|
||||
},
|
||||
modelFixture:
|
||||
"loopback deterministic OpenAI-compatible endpoint; no live provider claim",
|
||||
skillDiscovery:
|
||||
});
|
||||
if (client === "pi") {
|
||||
await writeFile(
|
||||
join(profile, "models.json"),
|
||||
JSON.stringify({
|
||||
providers: {
|
||||
"rea-fixture": {
|
||||
baseUrl: fixture.baseUrl,
|
||||
api: "openai-completions",
|
||||
apiKey: "local-fixture-key",
|
||||
models: [
|
||||
{
|
||||
id: "rea-client-fixture",
|
||||
reasoning: false,
|
||||
contextWindow: 200000,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
const configured = await readFile(configPath, "utf8");
|
||||
await writeFile(
|
||||
configPath,
|
||||
configured +
|
||||
`\nmodel:\n default: rea-client-fixture\n provider: custom\n base_url: ${fixture.baseUrl}\nagent:\n max_turns: 6\n`,
|
||||
);
|
||||
}
|
||||
const args =
|
||||
client === "pi"
|
||||
? "native HOME shared personal directory; no explicit skill setting"
|
||||
: "native HERMES_HOME personal directory",
|
||||
stickyHermes,
|
||||
catalogSize: catalog?.length,
|
||||
evidenceSeen,
|
||||
skillLoaded,
|
||||
artifactDigest,
|
||||
finalSeen,
|
||||
requests: fixture.requests,
|
||||
probes: fixture.probes,
|
||||
fixtureFailure: fixture.failure,
|
||||
verifier: await completeVerifierRun(verifier),
|
||||
};
|
||||
await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
|
||||
console.log(JSON.stringify(receipt));
|
||||
}
|
||||
if (failure !== undefined) throw failure;
|
||||
? [
|
||||
"--offline",
|
||||
"--provider",
|
||||
"rea-fixture",
|
||||
"--model",
|
||||
"rea-client-fixture",
|
||||
"--no-session",
|
||||
"--no-context-files",
|
||||
"--mode",
|
||||
"json",
|
||||
"-p",
|
||||
mode === "chat"
|
||||
? "Say hello briefly."
|
||||
: `Use REA to analyze ${target}.`,
|
||||
]
|
||||
: [
|
||||
"--oneshot",
|
||||
mode === "chat"
|
||||
? "Say hello briefly."
|
||||
: `Use REA to analyze ${target}.`,
|
||||
"--provider",
|
||||
"custom",
|
||||
"--model",
|
||||
"rea-client-fixture",
|
||||
"--reasoning",
|
||||
"none",
|
||||
];
|
||||
const result = await commands
|
||||
.exec(command, args, {
|
||||
env: {
|
||||
...environment,
|
||||
OPENAI_API_KEY: "local-fixture-key",
|
||||
OPENAI_BASE_URL: fixture.baseUrl,
|
||||
},
|
||||
cwd: workspace,
|
||||
timeout: 120_000,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
// Pi consumes piped input before its first prompt; an unused open pipe is not EOF.
|
||||
stdin: "closed-pipe",
|
||||
})
|
||||
.catch(async (cause) => {
|
||||
const outputWriters = [];
|
||||
if (typeof cause?.snapshot?.stdout?.text === "string")
|
||||
outputWriters.push(() =>
|
||||
writeFile(join(lab, "client.stdout"), cause.snapshot.stdout.text),
|
||||
);
|
||||
if (typeof cause?.snapshot?.stderr?.text === "string")
|
||||
outputWriters.push(() =>
|
||||
writeFile(join(lab, "client.stderr"), cause.snapshot.stderr.text),
|
||||
);
|
||||
return withVerifierCleanup(() => {
|
||||
throw cause;
|
||||
}, outputWriters);
|
||||
});
|
||||
await withVerifierCleanup(
|
||||
() => writeFile(join(lab, "client.stdout"), result.stdout),
|
||||
[() => writeFile(join(lab, "client.stderr"), result.stderr)],
|
||||
);
|
||||
assert.equal(fixture.failure, undefined);
|
||||
assert(finalSeen);
|
||||
assert(result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK"));
|
||||
if (mode === "call") {
|
||||
assert(evidenceSeen);
|
||||
assert(
|
||||
skillLoaded,
|
||||
"Native client must actually read the installed skill.",
|
||||
);
|
||||
assert.deepEqual(
|
||||
catalog?.sort(),
|
||||
TOOL_CONTRACTS.map((t) => t.name).sort(),
|
||||
"The complete REA catalog must survive native client discovery.",
|
||||
);
|
||||
}
|
||||
}, [
|
||||
() => commands.close(),
|
||||
async () => {
|
||||
await fixture?.close();
|
||||
},
|
||||
]);
|
||||
passed = true;
|
||||
}, [
|
||||
async () => {
|
||||
const receipt = {
|
||||
client,
|
||||
clientVersion,
|
||||
mode,
|
||||
passed,
|
||||
artifacts: lab,
|
||||
runtimeRoot,
|
||||
host: {
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
node: process.versions.node,
|
||||
},
|
||||
modelFixture:
|
||||
"loopback deterministic OpenAI-compatible endpoint; no live provider claim",
|
||||
skillDiscovery:
|
||||
client === "pi"
|
||||
? "native HOME shared personal directory; no explicit skill setting"
|
||||
: "native HERMES_HOME personal directory",
|
||||
stickyHermes,
|
||||
catalogSize: catalog?.length,
|
||||
evidenceSeen,
|
||||
skillLoaded,
|
||||
artifactDigest,
|
||||
finalSeen,
|
||||
requests: fixture?.requests,
|
||||
probes: fixture?.probes,
|
||||
fixtureFailure: fixture?.failure,
|
||||
verifier: await completeVerifierRun(verifier),
|
||||
};
|
||||
await withVerifierCleanup(
|
||||
() =>
|
||||
writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)),
|
||||
[
|
||||
() => {
|
||||
console.log(JSON.stringify(receipt));
|
||||
},
|
||||
],
|
||||
);
|
||||
},
|
||||
]);
|
||||
|
||||
Reference in New Issue
Block a user