fix(ci): validate artifact MCP replies against named schemas (#6021)

Bind artifact lifecycle and analysis calls to their advertised named SDK
output schemas, sharing discovery per connection and rebinding on reconnect.
Reject schema overrides and ambiguous catalogs, preserve request options,
and retain primary failures while attempting all owned cleanup.

Add real-stdio boundary regressions for malformed output, discovery reuse,
reconnection, schema ambiguity, override rejection and timeout preservation.

Assisted-by: Reimu (gensokyo-skills:reimu-incident-triage)
Assisted-by: Kogasa (gensokyo-skills:kogasa-surprise-testing)

docs/testing.md

@@ -256,6 +256,22 @@ pass that definition through the SDK's `toolDefinition` call option. This keeps
 the selected output-schema validation without eagerly compiling unrelated
 catalog schemas. Re-read definitions if the server advertises a changed list.
 
+Artifact verification uses `artifactMcpCall` for both lifecycle and analysis
+calls. A fresh SDK client has no held tool listing, so a bare `callTool` can
+skip output-schema validation. The helper discovers actual advertised
+definitions, rejects missing or ambiguous schemas, and validates the requested
+tool without compiling unrelated catalog schemas. REA registers process-static
+tool contracts; bindings share one discovery per negotiated connection and are
+discarded on reconnect or failed discovery. Keep raw artifact calls on this
+shared path and retain explicit request options such as timeouts.
+Caller-supplied tool definitions are rejected: the helper owns selection from
+the advertised catalog so a stale or permissive schema cannot bypass it.
+
+The regression fixture changes only `canonical_path` in a real stdio response
+to a number. The old helper accepts that response; the named schema must reject
+it. This client-boundary fault injection does not establish native Apple
+behavior, which remains covered by the macOS verification lane.
+
 Report unavailable authority as a named skip. Await owned cleanup before a
 verifier reports success. Native process inspectors are released at Vitest
 worker teardown; fork termination does not run Node exit hooks, and per-file

scripts/lib/artifact-e2e.mjs

@@ -4,6 +4,67 @@ import { promisify } from "node:util";
 import { fileURLToPath } from "node:url";
 import { Client } from "@modelcontextprotocol/client";
 import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
+import { closeVerifierResources } from "./verifier-cleanup.mjs";
+
+// REA registers process-static tool contracts. Bind their parsed definitions to
+// the SDK's negotiated connection identity, including when a Client reconnects.
+const toolBindings = new WeakMap();
+
+const advertisedTool = async (client, name) => {
+  const connection = client.getServerVersion();
+  assert.ok(connection, "MCP must be initialized before binding tool schemas");
+  let binding = toolBindings.get(client);
+  if (binding?.connection !== connection) {
+    binding = {
+      connection,
+      definitions: client.listTools().then(({ tools }) => {
+        const definitions = new Map();
+        for (const tool of tools) {
+          assert.ok(
+            !definitions.has(tool.name),
+            `Duplicate MCP tool: ${tool.name}`,
+          );
+          definitions.set(tool.name, tool);
+        }
+        return definitions;
+      }),
+    };
+    toolBindings.set(client, binding);
+  }
+  let definitions;
+  try {
+    definitions = await binding.definitions;
+    assert.equal(
+      client.getServerVersion(),
+      connection,
+      "MCP connection changed while binding tool schemas",
+    );
+  } catch (cause) {
+    if (toolBindings.get(client) === binding) toolBindings.delete(client);
+    throw cause;
+  }
+  const tool = definitions.get(name);
+  assert.ok(tool, `MCP catalog did not advertise ${name}`);
+  return tool;
+};
+
+/** Call an advertised tool with SDK output validation of its actual schema. */
+export async function artifactMcpCall(client, parameters, options = {}) {
+  options.signal?.throwIfAborted();
+  assert.equal(
+    options.toolDefinition,
+    undefined,
+    "Artifact verification owns the advertised schema; pass request options only",
+  );
+  const toolDefinition = await advertisedTool(client, parameters.name);
+  assert.notEqual(
+    toolDefinition.outputSchema,
+    undefined,
+    `MCP tool omitted its output schema: ${parameters.name}`,
+  );
+  options.signal?.throwIfAborted();
+  return client.callTool(parameters, { ...options, toolDefinition });
+}
 
 const environment = () => {
   const compileCache =
@@ -66,9 +127,10 @@ const withOpenedArtifactMcp = async (target, openTool, verify) => {
     stderr: "pipe",
   });
   const client = new Client({ name: "artifact-real-e2e", version: "1" });
+  let failure;
   try {
     await client.connect(transport);
-    const opened = await client.callTool({
+    const opened = await artifactMcpCall(client, {
       name: openTool,
       arguments: { path: target },
     });
@@ -76,9 +138,15 @@ const withOpenedArtifactMcp = async (target, openTool, verify) => {
     if (openTool === "open_bundle_resources")
       assert.equal(opened.structuredContent?.result?.caller_path, target);
     await verify(client);
+  } catch (cause) {
+    failure = cause;
+    throw cause;
   } finally {
-    await client.close();
-    await transport.close();
+    toolBindings.delete(client);
+    await closeVerifierResources(
+      [() => client.close(), () => transport.close()],
+      failure,
+    );
   }
 };
 
@@ -94,7 +162,10 @@ export async function withBundleResourcesMcp(target, verify) {
 
 /** Require a successful structured MCP tool result rather than its text rendering. */
 export async function artifactMcpResult(client, name, arguments_ = {}) {
-  const result = await client.callTool({ name, arguments: arguments_ });
+  const result = await artifactMcpCall(client, {
+    name,
+    arguments: arguments_,
+  });
   assert.notEqual(result.isError, true, JSON.stringify(result));
   assert.ok(
     result.structuredContent?.normalized_result,

scripts/verify-native-calls.mjs

@@ -10,6 +10,7 @@ import { promisify } from "node:util";
 import {
   artifactCli,
   artifactMcpResult,
+  artifactMcpCall,
   withArtifactMcp,
 } from "./lib/artifact-e2e.mjs";
 
@@ -225,7 +226,8 @@ const checkHardenedRuntime = async (directory, binary) => {
   assert.equal(denied.code, "capability_unavailable");
   assert.match(denied.details.reason, /^debugger-attach-denied: /u);
   await withArtifactMcp(blocked, async (client) => {
-    const deniedMcp = await client.callTool(
+    const deniedMcp = await artifactMcpCall(
+      client,
       {
         name: "observe_native_calls",
         arguments: INPUT,

scripts/verify/apple/asset-catalog.mjs

@@ -9,6 +9,7 @@ import { promisify } from "node:util";
 import {
   artifactCli,
   artifactMcpResult,
+  artifactMcpCall,
   withArtifactMcp,
   withBundleResourcesMcp,
 } from "../../lib/artifact-e2e.mjs";
@@ -97,7 +98,7 @@ try {
       offset: first.next_offset,
     });
     assert.deepEqual([...first.records, ...next.records], cli.records);
-    const invalid = await client.callTool({
+    const invalid = await artifactMcpCall(client, {
       name: "inspect_asset_catalog",
       arguments: { offset: -1 },
     });

scripts/verify/apple/keyed-archive.mjs

@@ -9,6 +9,7 @@ import { createHash } from "node:crypto";
 import {
   artifactCli,
   artifactMcpResult,
+  artifactMcpCall,
   withArtifactMcp,
 } from "../../lib/artifact-e2e.mjs";
 import {
@@ -53,7 +54,7 @@ try {
     });
     assert.deepEqual(page.objects, [graph.objects[1]]);
     assert.equal(page.next_offset, 2);
-    const invalid = await client.callTool({
+    const invalid = await artifactMcpCall(client, {
       name: "inspect_keyed_archive",
       arguments: { path: "../other.plist" },
     });

scripts/verify/apple/macho-occurrences.mjs

@@ -5,6 +5,7 @@ import { relative, sep } from "node:path";
 import { parseArtifactInventoryEvidence } from "../../../dist/domain/artifactInventoryEvidence.js";
 import {
   artifactCliEvidence,
+  artifactMcpCall,
   withArtifactMcp,
 } from "../../lib/artifact-e2e.mjs";
 
@@ -53,7 +54,7 @@ export async function verifyMachOOccurrences(app, executable) {
       assert.equal(node.sha256, direct.manifest.root_sha256);
       assert.equal(node.format, root.artifact_format);
       await withArtifactMcp(app, async (client) => {
-        const reply = await client.callTool({
+        const reply = await artifactMcpCall(client, {
           name: "inspect_artifact",
           arguments: {},
         });

scripts/verify/apple/macos-dylib-e2e.mjs

@@ -7,6 +7,7 @@ import { promisify } from "node:util";
 import {
   artifactCliEvidence,
   artifactMcpResult,
+  artifactMcpCall,
   withArtifactMcp,
 } from "../../lib/artifact-e2e.mjs";
 
@@ -180,7 +181,7 @@ export async function verifyDylibResolution(app) {
       await artifactMcpResult(client, "trace_dylib_resolution"),
       trace,
     );
-    const invalid = await client.callTool({
+    const invalid = await artifactMcpCall(client, {
       name: "trace_dylib_resolution",
       arguments: { roots: ["../outside"] },
     });

scripts/verify/apple/native-provenance.mjs

@@ -5,6 +5,7 @@ import { readFile } from "node:fs/promises";
 import { toolContract } from "../../../dist/contracts/toolContracts.js";
 import {
   artifactCliEvidence,
+  artifactMcpCall,
   withArtifactMcp,
   withBundleResourcesMcp,
 } from "../../lib/artifact-e2e.mjs";
@@ -27,7 +28,7 @@ export async function verifyNativeProvenance(app, executable) {
   assert.equal(resourcePlistCli.subject, null);
   assert.equal(resourcePlistCli.raw_result, null);
   await withBundleResourcesMcp(app, async (client) => {
-    const response = await client.callTool({
+    const response = await artifactMcpCall(client, {
       name: "inspect_plist",
       arguments: {},
     });
@@ -58,13 +59,10 @@ export async function verifyNativeProvenance(app, executable) {
   assert.equal(binaryPlistCli.raw_result, null);
   assert.equal(binaryPlistCli.normalized_result.resource_root, undefined);
   await withArtifactMcp(app, async (client) => {
-    const definitions = new Map(
-      (await client.listTools()).tools.map((tool) => [tool.name, tool]),
-    );
-    const plistResponse = await client.callTool(
-      { name: "inspect_plist", arguments: {} },
-      { toolDefinition: definitions.get("inspect_plist") },
-    );
+    const plistResponse = await artifactMcpCall(client, {
+      name: "inspect_plist",
+      arguments: {},
+    });
     assert.notEqual(plistResponse.isError, true, JSON.stringify(plistResponse));
     const binaryPlistMcp = toolContract("inspect_plist").outputSchema.parse(
       plistResponse.structuredContent,
@@ -98,13 +96,10 @@ export async function verifyNativeProvenance(app, executable) {
           arguments_,
         ),
       );
-      const response = await client.callTool(
-        {
-          name: operation,
-          arguments: operation === "demangle_swift" ? { symbols } : {},
-        },
-        { toolDefinition: definitions.get(operation) },
-      );
+      const response = await artifactMcpCall(client, {
+        name: operation,
+        arguments: operation === "demangle_swift" ? { symbols } : {},
+      });
       assert.notEqual(response.isError, true, JSON.stringify(response));
       const mcp = toolContract(operation).outputSchema.parse(
         response.structuredContent,

tests/boundary/process/artifactMcpVerifierOutput.test.ts

@@ -0,0 +1,126 @@
+import { execFile } from "node:child_process";
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+import { expect, it } from "vitest";
+import { z } from "zod";
+
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+
+const exec = promisify(execFile);
+const observationSchema = z.strictObject({
+  accepted: z.boolean(),
+  catalogRequests: z.number().int().nonnegative(),
+  corruptedReplies: z.number().int().nonnegative(),
+  toolCalls: z.number().int().nonnegative(),
+  toolTimeouts: z.array(z.number().nullable()),
+  error: z
+    .strictObject({
+      name: z.string(),
+      message: z.string(),
+      code: z.union([z.number(), z.string()]).nullable(),
+    })
+    .nullable(),
+});
+
+async function probe(mode: string, name = "Fixture.app") {
+  const root = await createTestTempDirectory("rea-artifact-mcp-schema-");
+  const target = join(root, name);
+  await mkdir(join(target, "Contents", "Resources"), { recursive: true });
+  await writeFile(
+    join(target, "Contents", "Info.plist"),
+    '<?xml version="1.0"?><plist version="1.0"><dict><key>CFBundleIdentifier</key><string>dev.rea.schema-probe</string></dict></plist>',
+  );
+  const { stdout } = await exec(
+    process.execPath,
+    [
+      fileURLToPath(
+        new URL("../../fixtures/artifactMcpVerifierProbe.mjs", import.meta.url),
+      ),
+      mode,
+      target,
+    ],
+    // Retain cleanup time within the existing 30s test budget.
+    { timeout: 25_000 },
+  );
+  return observationSchema.parse(JSON.parse(stdout));
+}
+
+it("validates the real resource admission reply against its advertised schema", async () => {
+  expect(await probe("healthy", "Fixture.app")).toEqual({
+    accepted: true,
+    catalogRequests: 1,
+    corruptedReplies: 0,
+    toolCalls: 1,
+    toolTimeouts: [null],
+    error: null,
+  });
+});
+
+it.each(["Fixture.app", "Fixture with spaces 雨.app"])(
+  "rejects a corrupted canonical path in a real admission reply: %s",
+  async (name) => {
+    const result = await probe("corrupt", name);
+    expect(result.accepted).toBe(false);
+    expect(result.catalogRequests).toBe(1);
+    expect(result.corruptedReplies).toBe(1);
+    expect(result.error).toMatchObject({
+      name: "ProtocolError",
+      code: -32602,
+      message: expect.stringMatching(/output schema.*canonical_path/u),
+    });
+  },
+);
+
+it.each(["reuse", "concurrent"])(
+  "shares one schema discovery within one MCP connection: %s",
+  async (mode) => {
+    const result = await probe(mode);
+    expect(result.accepted).toBe(true);
+    expect(result.catalogRequests).toBe(1);
+    expect(result.toolCalls).toBe(mode === "reuse" ? 3 : 2);
+  },
+);
+
+it("binds the new advertised schema after reconnecting the same client", async () => {
+  const result = await probe("reconnect");
+  expect(result.accepted).toBe(false);
+  expect(result.catalogRequests).toBe(2);
+  expect(result.toolCalls).toBe(2);
+  expect(result.error).toMatchObject({
+    code: -32602,
+    message: expect.stringContaining("__rea_second_connection"),
+  });
+});
+
+it("rejects an omitted output schema before calling the advertised tool", async () => {
+  const result = await probe("missing-schema");
+  expect(result.accepted).toBe(false);
+  expect(result.toolCalls).toBe(0);
+  expect(result.error?.message).toMatch(/omitted its output schema/u);
+});
+
+it("rejects a supplied schema override before dispatching that call", async () => {
+  const result = await probe("override");
+  expect(result.accepted).toBe(false);
+  expect(result.toolCalls).toBe(1);
+  expect(result.error?.message).toMatch(/owns the advertised schema/u);
+});
+
+it("preserves explicit request timeouts on schema-bound calls", async () => {
+  const result = await probe("request-options");
+  expect(result.accepted).toBe(true);
+  expect(result.catalogRequests).toBe(1);
+  expect(result.toolTimeouts).toEqual([null, 12_345]);
+});
+
+it.each(["duplicate-first", "duplicate-last"])(
+  "rejects ambiguous named schemas independently of catalog order: %s",
+  async (mode) => {
+    const result = await probe(mode);
+    expect(result.accepted).toBe(false);
+    expect(result.toolCalls).toBe(0);
+    expect(result.error?.message).toMatch(/Duplicate MCP tool/u);
+  },
+);

tests/fixtures/artifactMcpVerifierProbe.mjs

@@ -0,0 +1,209 @@
+import assert from "node:assert/strict";
+import { fileURLToPath } from "node:url";
+import { Client } from "@modelcontextprotocol/client";
+import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
+
+import * as artifact from "../../scripts/lib/artifact-e2e.mjs";
+import { closeVerifierResources } from "../../scripts/lib/verifier-cleanup.mjs";
+
+const [mode, target] = process.argv.slice(2);
+assert.ok(
+  [
+    "healthy",
+    "corrupt",
+    "reuse",
+    "concurrent",
+    "reconnect",
+    "missing-schema",
+    "duplicate-first",
+    "duplicate-last",
+    "override",
+    "request-options",
+  ].includes(mode),
+);
+assert.equal(typeof target, "string");
+
+// Isolate the fault to this child client. The server, resource admission and
+// advertised schema are real; alter only the reply before tool validation.
+const request = Client.prototype.request;
+let catalogRequests = 0;
+let corruptedReplies = 0;
+let toolCalls = 0;
+let connection = 0;
+const toolTimeouts = [];
+Client.prototype.request = async function (input, options) {
+  if (input.method === "tools/list") catalogRequests++;
+  if (input.method === "tools/call") {
+    toolCalls++;
+    toolTimeouts.push(options?.timeout ?? null);
+  }
+  const result = await request.call(this, input, options);
+  if (
+    mode === "override" &&
+    input.method === "tools/call" &&
+    input.params.name === "binary_session"
+  ) {
+    corruptedReplies++;
+    return { ...result, structuredContent: { result: 777 } };
+  }
+  if (input.method === "tools/list") {
+    if (mode === "missing-schema")
+      return {
+        ...result,
+        tools: result.tools.map((tool) => {
+          if (tool.name !== "open_bundle_resources") return tool;
+          const { outputSchema: _schema, ...withoutSchema } = tool;
+          return withoutSchema;
+        }),
+      };
+    if (mode === "duplicate-first" || mode === "duplicate-last") {
+      const listed = result.tools.find(
+        (tool) => tool.name === "open_bundle_resources",
+      );
+      assert.ok(listed);
+      const duplicate = {
+        ...listed,
+        outputSchema: { type: "object", required: ["__rea_wrong_schema"] },
+      };
+      return {
+        ...result,
+        tools:
+          mode === "duplicate-first"
+            ? [duplicate, ...result.tools]
+            : [...result.tools, duplicate],
+      };
+    }
+    if (mode === "reconnect" && connection === 2)
+      return {
+        ...result,
+        tools: result.tools.map((tool) =>
+          tool.name === "binary_session"
+            ? {
+                ...tool,
+                outputSchema: {
+                  type: "object",
+                  required: ["__rea_second_connection"],
+                },
+              }
+            : tool,
+        ),
+      };
+  }
+  if (
+    mode !== "corrupt" ||
+    input.method !== "tools/call" ||
+    input.params.name !== "open_bundle_resources"
+  )
+    return result;
+  assert.equal(
+    typeof result.structuredContent?.result?.canonical_path,
+    "string",
+    "The real reply omitted the field selected for fault injection",
+  );
+  corruptedReplies++;
+  return {
+    ...result,
+    structuredContent: {
+      ...result.structuredContent,
+      result: { ...result.structuredContent.result, canonical_path: 777 },
+    },
+  };
+};
+
+const status = async (client) => {
+  const result = await artifact.artifactMcpCall(client, {
+    name: "binary_session",
+    arguments: {},
+  });
+  assert.notEqual(result.isError, true, JSON.stringify(result));
+};
+
+async function standaloneConnections() {
+  const client = new Client({ name: "artifact-binding-probe", version: "1" });
+  let firstVersion;
+  for (
+    connection = 1;
+    connection <= (mode === "reconnect" ? 2 : 1);
+    connection++
+  ) {
+    const transport = new StdioClientTransport({
+      command: process.execPath,
+      args: [
+        fileURLToPath(new URL("../../scripts/rea.mjs", import.meta.url)),
+        "mcp",
+      ],
+      env: {
+        PATH: process.env.PATH ?? "/usr/bin:/bin",
+        REA_LOG_LEVEL: "silent",
+      },
+      stderr: "pipe",
+    });
+    let failure;
+    try {
+      await client.connect(transport);
+      const version = client.getServerVersion();
+      assert.ok(version);
+      if (firstVersion !== undefined) {
+        assert.notEqual(version, firstVersion);
+        assert.equal(version.name, firstVersion.name);
+        assert.equal(version.version, firstVersion.version);
+      }
+      firstVersion = version;
+      if (mode === "concurrent")
+        await Promise.all([status(client), status(client)]);
+      else await status(client);
+    } catch (cause) {
+      failure = cause;
+      throw cause;
+    } finally {
+      await closeVerifierResources(
+        [() => client.close(), () => transport.close()],
+        failure,
+      );
+    }
+  }
+}
+
+let error;
+try {
+  if (mode === "concurrent" || mode === "reconnect")
+    await standaloneConnections();
+  else
+    await artifact.withBundleResourcesMcp(target, async (client) => {
+      if (mode === "reuse") await Promise.all([status(client), status(client)]);
+      if (mode === "override")
+        await artifact.artifactMcpCall(
+          client,
+          { name: "binary_session", arguments: {} },
+          {
+            toolDefinition: {
+              name: "binary_session",
+              inputSchema: { type: "object" },
+              outputSchema: { type: "object" },
+            },
+          },
+        );
+      if (mode === "request-options")
+        await artifact.artifactMcpCall(
+          client,
+          { name: "binary_session", arguments: {} },
+          { timeout: 12_345 },
+        );
+    });
+} catch (cause) {
+  error = {
+    name: cause instanceof Error ? cause.name : "unknown",
+    message: cause instanceof Error ? cause.message : String(cause),
+    code: cause?.code ?? null,
+  };
+}
+process.stdout.write(
+  JSON.stringify({
+    accepted: error === undefined,
+    catalogRequests,
+    corruptedReplies,
+    toolCalls,
+    toolTimeouts,
+    error: error ?? null,
+  }) + "\n",
+);