From be698d4761bf9ad86e730e40e7a645f2607190b3 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:20:52 +0800 Subject: [PATCH] test: verify assertion contracts and the production evaluation gate --- scripts/lib/agent-evaluation-report-gate.mjs | 18 +++ scripts/verify-agent-experience.mjs | 9 +- src/config/bridgeMaxRequestBytes.test.ts | 14 +- .../sourceMaps/SourceMapDecoder.test.ts | 8 +- .../model-evals/agentEvaluationReport.test.ts | 124 +++++------------- 5 files changed, 69 insertions(+), 104 deletions(-) create mode 100644 scripts/lib/agent-evaluation-report-gate.mjs diff --git a/scripts/lib/agent-evaluation-report-gate.mjs b/scripts/lib/agent-evaluation-report-gate.mjs new file mode 100644 index 0000000000..1ae5c705fa --- /dev/null +++ b/scripts/lib/agent-evaluation-report-gate.mjs @@ -0,0 +1,18 @@ +import { tsImport } from "tsx/esm/api"; + +const { agentEvaluationPassed } = await tsImport( + new URL( + "../../tests/evaluation/model-evals/AgentEvaluationReport.ts", + import.meta.url, + ).href, + import.meta.url, +); + +/** Fail the runner after publishing a report with any rejected scenario. */ +export const assertAgentEvaluationReportPassed = (results) => { + const failed = results.filter((result) => !agentEvaluationPassed(result)); + if (failed.length > 0) + throw new Error( + `Agent routing/workflow/answer checks failed: ${failed.map(({ id }) => id).join(", ")}`, + ); +}; diff --git a/scripts/verify-agent-experience.mjs b/scripts/verify-agent-experience.mjs index a6b0fdc480..a9923e4e61 100644 --- a/scripts/verify-agent-experience.mjs +++ b/scripts/verify-agent-experience.mjs @@ -15,6 +15,7 @@ import { tsImport } from "tsx/esm/api"; import { PRODUCT_IDENTITY } from "../dist/identity.js"; import { createAgentEvaluationCommands } from "./lib/agent-evaluation-commands.mjs"; +import { assertAgentEvaluationReportPassed } from "./lib/agent-evaluation-report-gate.mjs"; import { closeVerifierResources, withVerifierCleanup, @@ -44,7 +45,7 @@ const { ).href, import.meta.url, ); -const { agentEvaluationPassed, summarizeFactualCorrectness } = await tsImport( +const { summarizeFactualCorrectness } = await tsImport( new URL( "../tests/evaluation/model-evals/AgentEvaluationReport.ts", import.meta.url, @@ -413,11 +414,7 @@ await withVerifierCleanup( await writeFile(resolve(reportPath), encoded); } - const failed = results.filter((result) => !agentEvaluationPassed(result)); - if (failed.length > 0) - throw new Error( - `Agent routing/workflow/answer checks failed: ${failed.map(({ id }) => id).join(", ")}`, - ); + assertAgentEvaluationReportPassed(results); }, // Even a deliberately retained fixture directory must never retain auth. [ diff --git a/src/config/bridgeMaxRequestBytes.test.ts b/src/config/bridgeMaxRequestBytes.test.ts index 1d931cc395..1037bb79fb 100644 --- a/src/config/bridgeMaxRequestBytes.test.ts +++ b/src/config/bridgeMaxRequestBytes.test.ts @@ -142,6 +142,16 @@ it.each([ const doctor = parseEnvironment({ [BRIDGE_MAX_REQUEST_BYTES_SETTING]: value, }); - expect(startup.ok).toBe(false); - expect(doctor.ok).toBe(false); + if (startup.ok) throw new Error(`Startup accepted ${JSON.stringify(value)}`); + expect(startup.error._tag).toBe("ConfigurationError"); + expect(startup.error.settings).toContainEqual({ + setting: BRIDGE_MAX_REQUEST_BYTES_SETTING, + constraint: bridgeMaxRequestBytesConstraint, + }); + if (doctor.ok) throw new Error(`Doctor accepted ${JSON.stringify(value)}`); + expect(doctor.error._tag).toBe("ConfigurationError"); + expect(doctor.error.settings).toContainEqual({ + setting: BRIDGE_MAX_REQUEST_BYTES_SETTING, + constraint: bridgeMaxRequestBytesConstraint, + }); }); diff --git a/src/javascript/sourceMaps/SourceMapDecoder.test.ts b/src/javascript/sourceMaps/SourceMapDecoder.test.ts index 9236716b0f..7e08c96d4f 100644 --- a/src/javascript/sourceMaps/SourceMapDecoder.test.ts +++ b/src/javascript/sourceMaps/SourceMapDecoder.test.ts @@ -124,6 +124,7 @@ it("settles every source-map cleanup failure when final close releases the exact prefix: "rea-source-map-final-cleanup-test-", }); let attempts = 0; + let denyCleanup = true; const launchCause = new Error("controlled launch failure"); const decoder = new SourceMapDecoder({ prepareOwnership: async () => {}, @@ -131,7 +132,7 @@ it("settles every source-map cleanup failure when final close releases the exact path: root.path, close: async () => { attempts += 1; - if (attempts <= 2) + if (denyCleanup && attempts <= 2) throw new Error(`runtime cleanup attempt ${String(attempts)} failed`); await root.close(); }, @@ -177,6 +178,7 @@ it("settles every source-map cleanup failure when final close releases the exact "runtime cleanup attempt 2 failed", ]); } finally { + denyCleanup = false; await decoder.close(); } }); @@ -186,6 +188,7 @@ it("attributes cleanup resources to the remaining process and directory owners", prefix: "rea-source-map-mixed-cleanup-test-", }); let denyProcessRelease = true; + let denyRootRelease = true; let rootAttempts = 0; let runId: string | undefined; const decoder = new SourceMapDecoder({ @@ -194,7 +197,7 @@ it("attributes cleanup resources to the remaining process and directory owners", path: root.path, close: async () => { rootAttempts += 1; - if (rootAttempts === 1) + if (denyRootRelease && rootAttempts === 1) throw new Error("controlled directory cleanup failure"); await root.close(); }, @@ -235,6 +238,7 @@ it("attributes cleanup resources to the remaining process and directory owners", await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" }); } finally { denyProcessRelease = false; + denyRootRelease = false; await decoder.close(); } }); diff --git a/tests/evaluation/model-evals/agentEvaluationReport.test.ts b/tests/evaluation/model-evals/agentEvaluationReport.test.ts index d51c59e7ab..8d2f8375cd 100644 --- a/tests/evaluation/model-evals/agentEvaluationReport.test.ts +++ b/tests/evaluation/model-evals/agentEvaluationReport.test.ts @@ -1,7 +1,5 @@ -import { readFile } from "node:fs/promises"; +import { spawnSync } from "node:child_process"; -import { parse } from "@babel/parser"; -import * as t from "@babel/types"; import { describe, expect, it } from "vitest"; import { @@ -22,102 +20,40 @@ const routingSuccess = { }; describe("agent evaluation release gate", () => { - it("rejects incorrect fixture claims even when every text heuristic passes", async () => { - const predicate = await runnerFailurePredicate(); - expect(predicate({ ...routingSuccess, factualCorrectness: "passed" })).toBe( - false, + it("exits unsuccessfully for a rejected report after accepting a passing report", () => { + const passing = { + id: "passing-fixture", + ...routingSuccess, + factualCorrectness: "passed", + }; + const accepted = runReportGate([passing]); + expect(accepted.status).toBe(0); + const rejected = runReportGate([ + passing, + { ...passing, id: "incorrect-fixture", factualCorrectness: "failed" }, + ]); + // Negative control: a runner that lets the failed report pass exits zero. + expect(rejected.status).toBe(1); + expect(rejected.stderr).toContain( + "Agent routing/workflow/answer checks failed: incorrect-fixture", ); - expect( - predicate({ - exitCode: 0, - naturalUse: true, - correctFirstTool: true, - repeatedCallCount: 0, - inputValidationFailureCount: 0, - requiredToolSubsequenceMet: true, - inputTokens: 100, - answerHeuristicsMet: true, - epistemicCuePresent: true, - factualCorrectness: "failed", - }), - ).toBe(true); - }); - - it("selects the results gate inside cleanup without using an unrelated filter", async () => { - const predicate = await runnerFailurePredicate(` - const failed = unrelated.filter(() => true); - await withVerifierCleanup(async () => { - const failed = results.filter(item => !agentEvaluationPassed(item)); - }); - `); - expect(predicate({ ...routingSuccess, factualCorrectness: "passed" })).toBe( - false, - ); - expect(predicate({ ...routingSuccess, factualCorrectness: "failed" })).toBe( - true, - ); - }); - - it("rejects a missing results failure gate", async () => { - await expect( - runnerFailurePredicate("const failed = unrelated.filter(() => false);"), - ).rejects.toThrow("Expected one runner failure gate, found 0"); - }); - - it("rejects ambiguous results failure gates", async () => { - await expect( - runnerFailurePredicate(` - function first() { const failed = results.filter(item => !agentEvaluationPassed(item)); } - function second() { const failed = results.filter(() => true); } - `), - ).rejects.toThrow("Expected one runner failure gate, found 2"); }); }); -async function runnerFailurePredicate( - providedSource?: string, -): Promise<(item: unknown) => unknown> { - const source = - providedSource ?? - (await readFile("scripts/verify-agent-experience.mjs", "utf8")); - const program = parse(source, { sourceType: "module" }).program; - const predicates: { readonly start: number; readonly end: number }[] = []; - t.traverseFast(program, (node) => { - if ( - !t.isVariableDeclarator(node) || - !t.isIdentifier(node.id, { name: "failed" }) - ) - return; - const initializer = node.init; - if ( - !t.isCallExpression(initializer) || - !t.isMemberExpression(initializer.callee) || - initializer.callee.computed || - !t.isIdentifier(initializer.callee.object, { name: "results" }) || - !t.isIdentifier(initializer.callee.property, { name: "filter" }) - ) - return; - const argument = initializer.arguments[0]; - if ( - t.isArrowFunctionExpression(argument) && - typeof argument.start === "number" && - typeof argument.end === "number" - ) - predicates.push({ start: argument.start, end: argument.end }); - }); - const [predicate] = predicates; - if (predicates.length !== 1 || predicate === undefined) - throw new Error( - `Expected one runner failure gate, found ${predicates.length}`, - ); - const factory = new Function( - "agentEvaluationPassed", - `return (${source.slice(predicate.start, predicate.end)});`, +function runReportGate(results: readonly object[]) { + const gateUrl = new URL( + "../../../scripts/lib/agent-evaluation-report-gate.mjs", + import.meta.url, + ); + const script = ` + const { assertAgentEvaluationReportPassed } = await import(${JSON.stringify(gateUrl.href)}); + assertAgentEvaluationReportPassed(JSON.parse(process.argv[1])); + `; + return spawnSync( + process.execPath, + ["--input-type=module", "-e", script, JSON.stringify(results)], + { encoding: "utf8", timeout: 15_000 }, ); - const result: unknown = factory(agentEvaluationPassed); - if (typeof result !== "function") - throw new Error("Runner gate is not callable"); - return (item) => result(item); } describe("scoped factual report", () => {