mirror/rea
1
0
Fork 0
mirror of https://github.com/morluto/rea synced 2026-10-25 22:08:11 -04:00
Code Issues Packages Projects Releases Wiki Activity

test(evaluation): include native preparation in test deadlines (#5627)

Browse Source 
Derive Darwin subprocess test budgets from the bounded ownership compiler budget while preserving short command deadlines. Always join the retained command scope after test failures.

Refs #5094.
This commit is contained in:
gensokyo-dev
2026-10-21 01:10:11 +08:00
committed by GitHub
parent d0eca0517e
commit d98fc3859d
2 changed files with 26 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/boundary/process/agentEvaluationCleanup.test.ts
+7 -1
View File
@@ -6,6 +6,7 @@ import { expect, it } from "vitest";
import { z } from "zod";
import { closeVerifierResources } from "../../../scripts/lib/verifier-cleanup.mjs";
import { resolveSwiftcCompileTimeoutMs } from "../../../src/process/DarwinProcessRunTokenReader.js";
import {
execFileOutput,
execFileOutputFailure,
@@ -13,6 +14,8 @@ import {
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
const receiptSchema = z.object({ root: z.string(), codex: z.string() });
const preparationBudgetMs =
process.platform === "darwin" ? resolveSwiftcCompileTimeoutMs() : 0;
// The executable fixture needs a POSIX shebang. Only the permission-denial
// cases additionally need an ordinary user. No auth copying or model requests run.
@@ -26,6 +29,9 @@ for (const { denied, keep } of [
process.platform === "win32" || (denied && process.getuid?.() === 0),
)(
`cleans independent evaluation files and retains failures (denied: ${String(denied)}, keep: ${String(keep)})`,
process.platform === "darwin"
? { timeout: Math.min(2_147_483_647, preparationBudgetMs + 60_000) }
: {},
async () => {
const directory = await createTestTempDirectory(
"rea-agent-eval-cleanup-",
@@ -76,7 +82,7 @@ process.exitCode = 1;
REA_AGENT_EVAL_TIMEOUT_MS: "10000",
REA_AGENT_EVAL_KEEP_FIXTURES: String(keep),
},
timeout: 15_000,
timeout: Math.min(2_147_483_647, preparationBudgetMs + 15_000),
stopSignal: "SIGTERM",
maxBuffer: 1024 * 1024,
},
tests/boundary/process/agentEvaluationCommands.test.ts
+19 -3
View File
@@ -3,12 +3,17 @@ import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { expect } from "vitest";
import { resolveSwiftcCompileTimeoutMs } from "../../../src/process/DarwinProcessRunTokenReader.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { processTest } from "../../support/process/processFixture.js";
// The driver prepares ownership before exercising its short command budgets.
const preparationBudgetMs =
process.platform === "darwin" ? resolveSwiftcCompileTimeoutMs() : 0;
processTest.skipIf(process.platform === "win32")(
"joins real evaluation command/output cleanup while retaining timeout diagnostics",
{ timeout: 40_000 },
{ timeout: Math.min(2_147_483_647, preparationBudgetMs + 40_000) },
async ({ processes }) => {
const root = await createTestTempDirectory("rea-agent-command-");
const fixture = join(root, "codex-fixture.mjs");
@@ -63,10 +68,16 @@ const { createAgentEvaluationCommands } = await import(${JSON.stringify(
const { OwnedCommandFailure, prepareOwnedCommand } = await import(${JSON.stringify(
new URL("../../../dist/process/OwnedCommand.js", import.meta.url).href,
)});
const { closeVerifierResources } = await import(${JSON.stringify(
new URL("../../../scripts/lib/verifier-cleanup.mjs", import.meta.url)
.href,
)});
await prepareOwnedCommand();
const root = ${JSON.stringify(root)}, fixture = ${JSON.stringify(fixture)};
const commands = createAgentEvaluationCommands({ codex: fixture, evaluationRoot: root,
evaluationEnvironment: process.env, timeoutMs: 1000, repositoryRoot: root });
let primaryError;
try {
for (const name of ["runCodex", "runProcess", "readCommandVersion"]) {
await assert.rejects(() => name === "runCodex" ? commands.runCodex("fixture", []) :
name === "runProcess" ? commands.runProcess(fixture, [], root, 1000) : commands.readCommandVersion(fixture), error => {
@@ -98,12 +109,17 @@ await assert.rejects(() => commands.runProcess(fixture, [], root, 2000), error =
assert.ok(error.snapshot.stderr.bytes <= 32768);
return true;
});
await commands.close();
} catch (cause) {
primaryError = cause;
throw cause;
} finally {
await closeVerifierResources([() => commands.close()], primaryError);
}
console.log(JSON.stringify({ passed: true, timeoutHelpers: 3, controls: 7, childrenClosed: children.every(x => x.closed) }));
`,
);
const result = await processes.run(process.execPath, [driver], {
timeoutMs: 35_000,
timeoutMs: Math.min(2_147_483_647, preparationBudgetMs + 35_000),
});
expect(result.exitCode, result.stderr).toBe(0);
expect(JSON.parse(result.stdout)).toEqual({