rea/tests/boundary/process/agentEvaluationCleanup.test.ts

import { chmod, readFile, stat, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";

import { expect, it } from "vitest";
import { z } from "zod";

import { closeVerifierResources } from "../../../scripts/lib/verifier-cleanup.mjs";
import { resolveSwiftcCompileTimeoutMs } from "../../../src/process/DarwinProcessRunTokenReader.js";
import {
  execFileOutput,
  execFileOutputFailure,
} from "../../../src/process/ExecFileOutput.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";

const receiptSchema = z.object({ root: z.string(), codex: z.string() });
const preparationBudgetMs =
  process.platform === "darwin" ? resolveSwiftcCompileTimeoutMs() : 0;

// The executable fixture needs a POSIX shebang. Only the permission-denial
// cases additionally need an ordinary user. No auth copying or model requests run.
for (const { denied, keep } of [
  { denied: false, keep: false },
  { denied: true, keep: false },
  { denied: false, keep: true },
  { denied: true, keep: true },
]) {
  it.skipIf(
    process.platform === "win32" || (denied && process.getuid?.() === 0),
  )(
    `cleans independent evaluation files and retains failures (denied: ${String(denied)}, keep: ${String(keep)})`,
    process.platform === "darwin"
      ? { timeout: Math.min(2_147_483_647, preparationBudgetMs + 60_000) }
      : {},
    async () => {
      const directory = await createTestTempDirectory(
        "rea-agent-eval-cleanup-",
      );
      const receipt = join(directory, "receipt.json");
      const command = join(directory, "codex-fixture.mjs");
      const parent = join(directory, "evaluations");
      await writeFile(
        command,
        `#!${process.execPath}
import { chmodSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
if (process.argv[2] !== "--version") throw new Error("Unexpected model invocation");
const parent = ${JSON.stringify(parent)};
const entries = readdirSync(parent);
if (entries.length !== 1) throw new Error("Expected exactly one owned run");
const root = join(parent, entries[0]);
const codex = join(root, "codex");
for (const child of ["codex", "account", "targets"]) mkdirSync(join(root, child));
writeFileSync(join(codex, "auth.json"), "inert synthetic fixture");
writeFileSync(${JSON.stringify(receipt)}, JSON.stringify({ root, codex }));
if (${String(denied)}) chmodSync(codex, 0o500);
process.stderr.write("synthetic version failure");
process.exitCode = 1;
`,
        { mode: 0o700 },
      );
      let run: { root: string; codex: string } | undefined;
      let primaryError: unknown;
      try {
        let stderr: string;
        try {
          await execFileOutput(
            process.execPath,
            [
              fileURLToPath(
                new URL(
                  "../../../scripts/verify-agent-experience.mjs",
                  import.meta.url,
                ),
              ),
            ],
            {
              env: {
                ...process.env,
                REA_CODEX_CLI: command,
                REA_AGENT_EVAL_ROOT: parent,
                REA_AGENT_EVAL_TIMEOUT_MS: "10000",
                REA_AGENT_EVAL_KEEP_FIXTURES: String(keep),
              },
              timeout: Math.min(2_147_483_647, preparationBudgetMs + 15_000),
              stopSignal: "SIGTERM",
              maxBuffer: 1024 * 1024,
            },
          );
          throw new Error(
            "The synthetic version failure must reject verification",
          );
        } catch (cause) {
          const failure = execFileOutputFailure(cause);
          if (failure === undefined) throw cause;
          expect(failure.code).toBe(1);
          stderr = failure.stderr;
        }
        // Cleanup must preserve the original version diagnostic too.
        expect(stderr).toContain(
          "--version failed with 1: synthetic version failure",
        );
        run = receiptSchema.parse(JSON.parse(await readFile(receipt, "utf8")));
        if (denied) {
          expect.soft(stderr).toContain("Verifier cleanup failed");
          expect(stderr).toContain("EACCES");
          expect(stderr).toContain(join(run.codex, "auth.json"));
          expect(await readFile(join(run.codex, "auth.json"), "utf8")).toBe(
            "inert synthetic fixture",
          );
        } else {
          expect(stderr).not.toContain("Verifier cleanup failed");
          await expect(
            stat(join(run.codex, "auth.json")),
          ).rejects.toMatchObject({
            code: "ENOENT",
          });
        }
        for (const child of ["account", "targets"]) {
          const entry = stat(join(run.root, child));
          if (keep) expect((await entry).isDirectory()).toBe(true);
          else await expect(entry).rejects.toMatchObject({ code: "ENOENT" });
        }
        if (!denied && !keep)
          await expect(stat(run.root)).rejects.toMatchObject({
            code: "ENOENT",
          });
      } catch (cause) {
        primaryError = cause;
        throw cause;
      } finally {
        // Restore this test's deliberately denied subtree before fixture teardown.
        if (denied)
          await closeVerifierResources(
            [
              async () => {
                try {
                  run ??= receiptSchema.parse(
                    JSON.parse(await readFile(receipt, "utf8")),
                  );
                } catch (cause) {
                  // Startup can fail before the executable creates its receipt.
                  if (
                    cause instanceof Error &&
                    "code" in cause &&
                    cause.code === "ENOENT"
                  )
                    return;
                  throw cause;
                }
                await chmod(run.codex, 0o700);
              },
            ],
            primaryError,
          );
      }
    },
  );
}