test: verify assertion contracts and the production evaluation gate

scripts/lib/agent-evaluation-report-gate.mjs

@@ -0,0 +1,18 @@
+import { tsImport } from "tsx/esm/api";
+
+const { agentEvaluationPassed } = await tsImport(
+  new URL(
+    "../../tests/evaluation/model-evals/AgentEvaluationReport.ts",
+    import.meta.url,
+  ).href,
+  import.meta.url,
+);
+
+/** Fail the runner after publishing a report with any rejected scenario. */
+export const assertAgentEvaluationReportPassed = (results) => {
+  const failed = results.filter((result) => !agentEvaluationPassed(result));
+  if (failed.length > 0)
+    throw new Error(
+      `Agent routing/workflow/answer checks failed: ${failed.map(({ id }) => id).join(", ")}`,
+    );
+};

scripts/verify-agent-experience.mjs

@@ -15,6 +15,7 @@ import { tsImport } from "tsx/esm/api";
 
 import { PRODUCT_IDENTITY } from "../dist/identity.js";
 import { createAgentEvaluationCommands } from "./lib/agent-evaluation-commands.mjs";
+import { assertAgentEvaluationReportPassed } from "./lib/agent-evaluation-report-gate.mjs";
 import {
   closeVerifierResources,
   withVerifierCleanup,
@@ -44,7 +45,7 @@ const {
   ).href,
   import.meta.url,
 );
-const { agentEvaluationPassed, summarizeFactualCorrectness } = await tsImport(
+const { summarizeFactualCorrectness } = await tsImport(
   new URL(
     "../tests/evaluation/model-evals/AgentEvaluationReport.ts",
     import.meta.url,
@@ -413,11 +414,7 @@ await withVerifierCleanup(
       await writeFile(resolve(reportPath), encoded);
     }
 
-    const failed = results.filter((result) => !agentEvaluationPassed(result));
-    if (failed.length > 0)
-      throw new Error(
-        `Agent routing/workflow/answer checks failed: ${failed.map(({ id }) => id).join(", ")}`,
-      );
+    assertAgentEvaluationReportPassed(results);
   },
   // Even a deliberately retained fixture directory must never retain auth.
   [

src/config/bridgeMaxRequestBytes.test.ts

@@ -142,6 +142,16 @@ it.each([
   const doctor = parseEnvironment({
     [BRIDGE_MAX_REQUEST_BYTES_SETTING]: value,
   });
-  expect(startup.ok).toBe(false);
-  expect(doctor.ok).toBe(false);
+  if (startup.ok) throw new Error(`Startup accepted ${JSON.stringify(value)}`);
+  expect(startup.error._tag).toBe("ConfigurationError");
+  expect(startup.error.settings).toContainEqual({
+    setting: BRIDGE_MAX_REQUEST_BYTES_SETTING,
+    constraint: bridgeMaxRequestBytesConstraint,
+  });
+  if (doctor.ok) throw new Error(`Doctor accepted ${JSON.stringify(value)}`);
+  expect(doctor.error._tag).toBe("ConfigurationError");
+  expect(doctor.error.settings).toContainEqual({
+    setting: BRIDGE_MAX_REQUEST_BYTES_SETTING,
+    constraint: bridgeMaxRequestBytesConstraint,
+  });
 });

src/javascript/sourceMaps/SourceMapDecoder.test.ts

@@ -124,6 +124,7 @@ it("settles every source-map cleanup failure when final close releases the exact
     prefix: "rea-source-map-final-cleanup-test-",
   });
   let attempts = 0;
+  let denyCleanup = true;
   const launchCause = new Error("controlled launch failure");
   const decoder = new SourceMapDecoder({
     prepareOwnership: async () => {},
@@ -131,7 +132,7 @@ it("settles every source-map cleanup failure when final close releases the exact
       path: root.path,
       close: async () => {
         attempts += 1;
-        if (attempts <= 2)
+        if (denyCleanup && attempts <= 2)
           throw new Error(`runtime cleanup attempt ${String(attempts)} failed`);
         await root.close();
       },
@@ -177,6 +178,7 @@ it("settles every source-map cleanup failure when final close releases the exact
       "runtime cleanup attempt 2 failed",
     ]);
   } finally {
+    denyCleanup = false;
     await decoder.close();
   }
 });
@@ -186,6 +188,7 @@ it("attributes cleanup resources to the remaining process and directory owners",
     prefix: "rea-source-map-mixed-cleanup-test-",
   });
   let denyProcessRelease = true;
+  let denyRootRelease = true;
   let rootAttempts = 0;
   let runId: string | undefined;
   const decoder = new SourceMapDecoder({
@@ -194,7 +197,7 @@ it("attributes cleanup resources to the remaining process and directory owners",
       path: root.path,
       close: async () => {
         rootAttempts += 1;
-        if (rootAttempts === 1)
+        if (denyRootRelease && rootAttempts === 1)
           throw new Error("controlled directory cleanup failure");
         await root.close();
       },
@@ -235,6 +238,7 @@ it("attributes cleanup resources to the remaining process and directory owners",
     await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" });
   } finally {
     denyProcessRelease = false;
+    denyRootRelease = false;
     await decoder.close();
   }
 });

tests/evaluation/model-evals/agentEvaluationReport.test.ts

@@ -1,7 +1,5 @@
-import { readFile } from "node:fs/promises";
+import { spawnSync } from "node:child_process";
 
-import { parse } from "@babel/parser";
-import * as t from "@babel/types";
 import { describe, expect, it } from "vitest";
 
 import {
@@ -22,102 +20,40 @@ const routingSuccess = {
 };
 
 describe("agent evaluation release gate", () => {
-  it("rejects incorrect fixture claims even when every text heuristic passes", async () => {
-    const predicate = await runnerFailurePredicate();
-    expect(predicate({ ...routingSuccess, factualCorrectness: "passed" })).toBe(
-      false,
+  it("exits unsuccessfully for a rejected report after accepting a passing report", () => {
+    const passing = {
+      id: "passing-fixture",
+      ...routingSuccess,
+      factualCorrectness: "passed",
+    };
+    const accepted = runReportGate([passing]);
+    expect(accepted.status).toBe(0);
+    const rejected = runReportGate([
+      passing,
+      { ...passing, id: "incorrect-fixture", factualCorrectness: "failed" },
+    ]);
+    // Negative control: a runner that lets the failed report pass exits zero.
+    expect(rejected.status).toBe(1);
+    expect(rejected.stderr).toContain(
+      "Agent routing/workflow/answer checks failed: incorrect-fixture",
     );
-    expect(
-      predicate({
-        exitCode: 0,
-        naturalUse: true,
-        correctFirstTool: true,
-        repeatedCallCount: 0,
-        inputValidationFailureCount: 0,
-        requiredToolSubsequenceMet: true,
-        inputTokens: 100,
-        answerHeuristicsMet: true,
-        epistemicCuePresent: true,
-        factualCorrectness: "failed",
-      }),
-    ).toBe(true);
-  });
-
-  it("selects the results gate inside cleanup without using an unrelated filter", async () => {
-    const predicate = await runnerFailurePredicate(`
-      const failed = unrelated.filter(() => true);
-      await withVerifierCleanup(async () => {
-        const failed = results.filter(item => !agentEvaluationPassed(item));
-      });
-    `);
-    expect(predicate({ ...routingSuccess, factualCorrectness: "passed" })).toBe(
-      false,
-    );
-    expect(predicate({ ...routingSuccess, factualCorrectness: "failed" })).toBe(
-      true,
-    );
-  });
-
-  it("rejects a missing results failure gate", async () => {
-    await expect(
-      runnerFailurePredicate("const failed = unrelated.filter(() => false);"),
-    ).rejects.toThrow("Expected one runner failure gate, found 0");
-  });
-
-  it("rejects ambiguous results failure gates", async () => {
-    await expect(
-      runnerFailurePredicate(`
-        function first() { const failed = results.filter(item => !agentEvaluationPassed(item)); }
-        function second() { const failed = results.filter(() => true); }
-      `),
-    ).rejects.toThrow("Expected one runner failure gate, found 2");
   });
 });
 
-async function runnerFailurePredicate(
-  providedSource?: string,
-): Promise<(item: unknown) => unknown> {
-  const source =
-    providedSource ??
-    (await readFile("scripts/verify-agent-experience.mjs", "utf8"));
-  const program = parse(source, { sourceType: "module" }).program;
-  const predicates: { readonly start: number; readonly end: number }[] = [];
-  t.traverseFast(program, (node) => {
-    if (
-      !t.isVariableDeclarator(node) ||
-      !t.isIdentifier(node.id, { name: "failed" })
-    )
-      return;
-    const initializer = node.init;
-    if (
-      !t.isCallExpression(initializer) ||
-      !t.isMemberExpression(initializer.callee) ||
-      initializer.callee.computed ||
-      !t.isIdentifier(initializer.callee.object, { name: "results" }) ||
-      !t.isIdentifier(initializer.callee.property, { name: "filter" })
-    )
-      return;
-    const argument = initializer.arguments[0];
-    if (
-      t.isArrowFunctionExpression(argument) &&
-      typeof argument.start === "number" &&
-      typeof argument.end === "number"
-    )
-      predicates.push({ start: argument.start, end: argument.end });
-  });
-  const [predicate] = predicates;
-  if (predicates.length !== 1 || predicate === undefined)
-    throw new Error(
-      `Expected one runner failure gate, found ${predicates.length}`,
-    );
-  const factory = new Function(
-    "agentEvaluationPassed",
-    `return (${source.slice(predicate.start, predicate.end)});`,
+function runReportGate(results: readonly object[]) {
+  const gateUrl = new URL(
+    "../../../scripts/lib/agent-evaluation-report-gate.mjs",
+    import.meta.url,
+  );
+  const script = `
+    const { assertAgentEvaluationReportPassed } = await import(${JSON.stringify(gateUrl.href)});
+    assertAgentEvaluationReportPassed(JSON.parse(process.argv[1]));
+  `;
+  return spawnSync(
+    process.execPath,
+    ["--input-type=module", "-e", script, JSON.stringify(results)],
+    { encoding: "utf8", timeout: 15_000 },
   );
-  const result: unknown = factory(agentEvaluationPassed);
-  if (typeof result !== "function")
-    throw new Error("Runner gate is not callable");
-  return (item) => result(item);
 }
 
 describe("scoped factual report", () => {