From 59094307719bdd65199d7b56320e183fd6ef6074 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:13:55 +0800 Subject: [PATCH] perf(test): share SDK catalog compilation across deferred clients --- docs/testing.md | 12 ++ .../mcp/catalogSchemaValidator.test.ts | 144 ++++++++++++++++++ tests/boundary/mcp/deferredRuntime.test.ts | 19 ++- tests/fixtures/catalogSchemaValidator.ts | 39 +++++ 4 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 tests/boundary/mcp/catalogSchemaValidator.test.ts create mode 100644 tests/fixtures/catalogSchemaValidator.ts diff --git a/docs/testing.md b/docs/testing.md index 946db07bf1..58ad768755 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -331,6 +331,18 @@ pass that definition through the SDK's `toolDefinition` call option. This keeps the selected output-schema validation without eagerly compiling unrelated catalog schemas. Re-read definitions if the server advertises a changed list. +The deferred-runtime discovery matrix retains independent SDK clients and real +CLI children. Its file-owned `catalogSchemaValidator` fixture shares the SDK's +default AJV compiler only for full canonical documents from the expected +catalog. The catalog's schema identifiers bound the retained compiled set; +changed documents, other dialects and unknown schemas use isolated default SDK +compilers, so an old identifier cannot select an obsolete validator. The +provider is released at file teardown. Every tool result still follows the +SDK's output validation. This reuse can remove compilation across fresh +clients; the SDK already reuses compiled identifiers within a client. Compare +the complete matrix and peak process-family memory before claiming a benefit, +since shared validators remain live between cases. + Artifact verification uses `artifactMcpCall` for both lifecycle and analysis calls. A fresh SDK client has no held tool listing, so a bare `callTool` can skip output-schema validation. The helper discovers actual advertised diff --git a/tests/boundary/mcp/catalogSchemaValidator.test.ts b/tests/boundary/mcp/catalogSchemaValidator.test.ts new file mode 100644 index 0000000000..5b34db6657 --- /dev/null +++ b/tests/boundary/mcp/catalogSchemaValidator.test.ts @@ -0,0 +1,144 @@ +import { + Client, + InMemoryTransport, + type JsonSchemaType, +} from "@modelcontextprotocol/client"; +import { Server } from "@modelcontextprotocol/server"; +import { expect, it } from "vitest"; +import { createCatalogSchemaValidator } from "../../fixtures/catalogSchemaValidator.js"; + +const integerSchema: JsonSchemaType = { + $schema: "https://json-schema.org/draft/2020-12/schema", + $id: "urn:rea:test:output", + type: "object", + properties: { value: { type: "integer" } }, + required: ["value"], + additionalProperties: false, +}; + +it("validates reused documents without freezing or changing caller data", () => { + const schema = structuredClone(integerSchema); + const validator = createCatalogSchemaValidator([schema]); + for (const document of [schema, structuredClone(schema)]) { + const validate = validator.getValidator(document); + const input = { value: 7 }; + expect(validate(input)).toEqual({ + valid: true, + data: input, + errorMessage: undefined, + }); + expect(validate({ value: "7" }).valid).toBe(false); + expect(validate({ value: 7, extra: true }).valid).toBe(false); + expect(input).toEqual({ value: 7 }); + } + expect(schema).toEqual(integerSchema); + expect(Object.isFrozen(schema)).toBe(false); +}); + +it("isolates changed and unknown schemas even when their id was compiled", () => { + const schema = structuredClone(integerSchema); + const validator = createCatalogSchemaValidator([schema]); + expect(validator.getValidator(schema)({ value: 7 }).valid).toBe(true); + + schema.properties = { value: { type: "string", minLength: 2 } }; + const changed = validator.getValidator(schema); + expect(changed({ value: "ok" }).valid).toBe(true); + expect(changed({ value: 7 }).valid).toBe(false); + expect(changed({ value: "x" }).valid).toBe(false); + expect(validator.getValidator(integerSchema)({ value: 7 }).valid).toBe(true); + expect(validator.getValidator(integerSchema)({ value: "ok" }).valid).toBe( + false, + ); + + const withoutId: JsonSchemaType = { type: "string", format: "email" }; + expect(validator.getValidator(withoutId)("valid@example.com").valid).toBe( + true, + ); + expect(validator.getValidator(withoutId)("invalid").valid).toBe(false); + expect(() => + validator.getValidator({ ...integerSchema, pattern: "[" }), + ).toThrow(); + expect(validator.getValidator(integerSchema)({ value: 7 }).valid).toBe(true); +}); + +it("preserves dialect dispatch when a document changes only its dialect", () => { + const schema: JsonSchemaType = { + $schema: "https://json-schema.org/draft/2020-12/schema", + $id: "urn:rea:test:dialect", + type: "array", + prefixItems: [{ type: "integer" }], + items: { type: "string" }, + }; + const validator = createCatalogSchemaValidator([schema]); + expect(validator.getValidator(schema)([7]).valid).toBe(true); + expect(validator.getValidator(schema)(["seven"]).valid).toBe(false); + const draft7 = { + ...schema, + $schema: "http://json-schema.org/draft-07/schema#", + }; + expect(validator.getValidator(draft7)([7]).valid).toBe(false); + expect(validator.getValidator(draft7)(["seven"]).valid).toBe(true); +}); + +it("refuses an expected catalog that aliases different documents by id", () => { + expect(() => + createCatalogSchemaValidator([ + integerSchema, + { ...integerSchema, required: [] }, + ]), + ).toThrow("reuse an id for different documents"); +}); + +it("keeps SDK output validation active across fresh clients and schema changes", async () => { + const validator = createCatalogSchemaValidator([integerSchema]); + for (const { schema, valid } of [ + { schema: integerSchema, valid: { value: 7 } }, + { schema: structuredClone(integerSchema), valid: { value: 7 } }, + { + schema: { + ...integerSchema, + properties: { value: { type: "string" as const } }, + }, + valid: { value: "seven" }, + }, + ]) { + let payload: Record = valid; + const server = new Server( + { name: "output-validation-control", version: "1" }, + { capabilities: { tools: {} } }, + ); + server.setRequestHandler("tools/list", async () => ({ + tools: [ + { + name: "checked", + inputSchema: { type: "object" }, + outputSchema: schema, + }, + ], + })); + server.setRequestHandler("tools/call", async () => ({ + content: [], + structuredContent: payload, + })); + const client = new Client( + { name: "validated-client", version: "1" }, + { jsonSchemaValidator: validator }, + ); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + await client.listTools(); + expect( + (await client.callTool({ name: "checked" })).structuredContent, + ).toEqual(payload); + payload = { unexpected: true }; + await expect(client.callTool({ name: "checked" })).rejects.toThrow( + "Structured content does not match the tool's output schema", + ); + } finally { + await Promise.allSettled([client.close(), server.close()]); + } + } +}); diff --git a/tests/boundary/mcp/deferredRuntime.test.ts b/tests/boundary/mcp/deferredRuntime.test.ts index 3718f20650..37a0ea799f 100644 --- a/tests/boundary/mcp/deferredRuntime.test.ts +++ b/tests/boundary/mcp/deferredRuntime.test.ts @@ -1,4 +1,4 @@ -import { Client } from "@modelcontextprotocol/client"; +import { Client, type jsonSchemaValidator } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { spawnSync } from "node:child_process"; import { @@ -10,14 +10,22 @@ import { writeFile, } from "node:fs/promises"; import { resolve, join } from "node:path"; -import { expect, it, onTestFinished } from "vitest"; +import { afterAll, expect, it, onTestFinished } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { GENERATED_MCP_TOOL_CATALOG } from "../../fixtures/mcpToolCatalog.js"; +import { createCatalogSchemaValidator } from "../../fixtures/catalogSchemaValidator.js"; import { PROMPT_CONTRACTS } from "../../../src/contracts/promptContracts.js"; import { evidenceSchema } from "../../../src/domain/evidence.js"; import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; import { SERVER_START_FAILED } from "../../../src/main/messages.js"; +// Independent SDK sessions still validate every result. Only their immutable +// catalog compilation is shared, and released after this fixture file finishes. +let outputValidator: jsonSchemaValidator | undefined; +afterAll(() => { + outputValidator = undefined; +}); + const connect = async ( mode: "legacy" | "auto", environment: Record = {}, @@ -41,7 +49,12 @@ const connect = async ( }); const client = new Client( { name: "deferred-runtime-boundary", version: "1" }, - { versionNegotiation: { mode } }, + { + versionNegotiation: { mode }, + jsonSchemaValidator: (outputValidator ??= createCatalogSchemaValidator( + GENERATED_MCP_TOOL_CATALOG.map(({ outputSchema }) => outputSchema), + )), + }, ); onTestFinished(async () => { await client.close(); diff --git a/tests/fixtures/catalogSchemaValidator.ts b/tests/fixtures/catalogSchemaValidator.ts new file mode 100644 index 0000000000..8c599d6e93 --- /dev/null +++ b/tests/fixtures/catalogSchemaValidator.ts @@ -0,0 +1,39 @@ +import type { + JsonSchemaType, + jsonSchemaValidator, +} from "@modelcontextprotocol/client"; +import { AjvJsonSchemaValidator } from "@modelcontextprotocol/client/validators/ajv"; +import { canonicalJson } from "../../src/domain/comparisonSemantics.js"; + +/** + * Share the SDK's default compiler for this fixture's finite catalog only. + * Full documents (including dialect) admit reuse; the SDK owns compilation + * and its $id cache. Other documents get an isolated default SDK compiler so + * a changed schema cannot inherit an earlier validator for the same $id. + * The caller owns this provider's lifetime, including the compiled catalog. + */ +export const createCatalogSchemaValidator = ( + schemas: readonly Readonly>[], +): jsonSchemaValidator => { + const documents = new Map(); + for (const schema of schemas) { + if (typeof schema.$id !== "string") continue; + const document = canonicalJson(schema, "Fixture output schema"); + const previous = documents.get(schema.$id); + if (previous !== undefined && previous !== document) + throw new TypeError( + "Fixture output schemas reuse an id for different documents", + ); + documents.set(schema.$id, document); + } + const expected = new Set(documents.values()); + const compiler = new AjvJsonSchemaValidator(); + return { + getValidator(schema: JsonSchemaType) { + const delegate = expected.has(canonicalJson(schema, "Tool output schema")) + ? compiler + : new AjvJsonSchemaValidator(); + return delegate.getValidator(schema); + }, + }; +};