diff --git a/scripts/verify-claude-client.mjs b/scripts/verify-claude-client.mjs index d652400705..4824b60b21 100644 --- a/scripts/verify-claude-client.mjs +++ b/scripts/verify-claude-client.mjs @@ -18,7 +18,10 @@ import { import { createVerifierCommands } from "./lib/verify-commands.mjs"; import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; -import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; +import { + CLIENT_JAVASCRIPT_FIXTURE_SOURCE, + assertClientJavaScriptFixtureEvidence, +} from "./verify/clients/javascript-fixture.mjs"; import { expandMcpSessionStatus, sortVerifierNames, @@ -90,10 +93,7 @@ await withVerifierCleanup(async () => { } const target = join(lab, "fixture α测试"); await mkdir(target); - await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', - ); + await writeFile(join(target, "app.js"), CLIENT_JAVASCRIPT_FIXTURE_SOURCE); const schemaValidator = new Ajv2020({ strict: false, validateFormats: false, @@ -249,33 +249,9 @@ await withVerifierCleanup(async () => { value.normalized_result?.graph?.nodes?.length > 0, ); if (observed) { - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(observed); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((n) => n.observations) - .map((o) => o.properties) - .find( - (p) => - p.semantic_role === "export-return-shapes" && - p.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((s) => s.fields); - assert( - fields.some( - (f) => - f.path === "/clientCompatibility" && - f.state === "literal" && - f.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (f) => - f.path === "/count" && f.state === "literal" && f.value === 7, - ), + const { evidence } = await assertClientJavaScriptFixtureEvidence( + observed, + target, ); const bytes = JSON.stringify(evidence); artifactDigest = createHash("sha256").update(bytes).digest("hex"); diff --git a/scripts/verify-commandcode-client.mjs b/scripts/verify-commandcode-client.mjs index 8819f4fc45..e88fa3ee9f 100644 --- a/scripts/verify-commandcode-client.mjs +++ b/scripts/verify-commandcode-client.mjs @@ -5,14 +5,14 @@ import { homedir } from "node:os"; import { isAbsolute, join, relative } from "node:path"; import { fileURLToPath } from "node:url"; import Ajv2020 from "ajv/dist/2020.js"; -import { - TOOL_CONTRACTS, - toolContract, -} from "../dist/contracts/toolContracts.js"; +import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js"; import { createVerifierCommands } from "./lib/verify-commands.mjs"; import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; -import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; +import { + CLIENT_JAVASCRIPT_FIXTURE_SOURCE, + assertClientJavaScriptFixtureEvidence, +} from "./verify/clients/javascript-fixture.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -80,10 +80,7 @@ await withVerifierCleanup(async () => { timeout: 30_000, stdin: "pipe", }); - await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', - ); + await writeFile(join(target, "app.js"), CLIENT_JAVASCRIPT_FIXTURE_SOURCE); const environment = { PATH: process.env.PATH, NODE_OPTIONS: process.env.NODE_OPTIONS, @@ -246,43 +243,7 @@ await withVerifierCleanup(async () => { format: "directory", }); } - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(JSON.parse(last)); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shapes = evidence.normalized_result.graph.nodes - .flatMap((node) => node.observations) - .map((observation) => observation.properties) - .filter( - (properties) => - properties.semantic_role === "export-return-shapes" && - properties.exported_name === "result", - ); - assert( - shapes.length > 0 && - shapes.every( - (shape) => shape.return_shape_coverage?.projection_complete, - ), - ); - const fields = shapes.flatMap((shape) => - shape.static_return_shapes.flatMap((item) => item.fields), - ); - assert( - fields.some( - (field) => - field.path === "/clientCompatibility" && - field.state === "literal" && - field.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (field) => - field.path === "/count" && - field.state === "literal" && - field.value === 7, - ), - ); + await assertClientJavaScriptFixtureEvidence(JSON.parse(last), target); evidenceBytes = Buffer.byteLength(last); await writeFile(join(lab, "native-evidence.json"), last); evidenceSeen = true; diff --git a/scripts/verify-copilot-client.mjs b/scripts/verify-copilot-client.mjs index 4225f87561..71b08fb7a5 100644 --- a/scripts/verify-copilot-client.mjs +++ b/scripts/verify-copilot-client.mjs @@ -15,7 +15,10 @@ import { compactPresentationForMcpInputSchemaProfile } from "../dist/config/mcpI import { createVerifierCommands } from "./lib/verify-commands.mjs"; import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; -import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; +import { + CLIENT_JAVASCRIPT_FIXTURE_SOURCE, + assertClientJavaScriptFixtureEvidence, +} from "./verify/clients/javascript-fixture.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -108,10 +111,7 @@ await withVerifierCleanup(async () => { } const target = join(lab, "fixture α测试"); await mkdir(target); - await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', - ); + await writeFile(join(target, "app.js"), CLIENT_JAVASCRIPT_FIXTURE_SOURCE); const schemaValidator = new Ajv2020({ strict: false, validateFormats: false, @@ -206,33 +206,9 @@ await withVerifierCleanup(async () => { containsText(body.messages, skillDirectory); const observed = extractEvidence(replies); if (observed) { - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(observed); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((n) => n.observations) - .map((o) => o.properties) - .find( - (p) => - p.semantic_role === "export-return-shapes" && - p.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((s) => s.fields); - assert( - fields.some( - (f) => - f.path === "/clientCompatibility" && - f.state === "literal" && - f.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (f) => - f.path === "/count" && f.state === "literal" && f.value === 7, - ), + const { evidence } = await assertClientJavaScriptFixtureEvidence( + observed, + target, ); const bytes = JSON.stringify(evidence); artifactDigest = createHash("sha256").update(bytes).digest("hex"); diff --git a/scripts/verify-deepseek-client.mjs b/scripts/verify-deepseek-client.mjs index 9f5a3bbc1e..5964eb6301 100644 --- a/scripts/verify-deepseek-client.mjs +++ b/scripts/verify-deepseek-client.mjs @@ -6,15 +6,15 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { Ajv2020 } from "ajv/dist/2020.js"; import { parse as parseYaml } from "yaml"; -import { - TOOL_CONTRACTS, - toolContract, -} from "../dist/contracts/toolContracts.js"; +import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js"; import { pathExists } from "./lib/verify-package-core.mjs"; import { createVerifierCommands } from "./lib/verify-commands.mjs"; import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; -import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; +import { + CLIENT_JAVASCRIPT_FIXTURE_SOURCE, + assertClientJavaScriptFixtureEvidence, +} from "./verify/clients/javascript-fixture.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -78,10 +78,7 @@ await withVerifierCleanup(async () => { stdin: "pipe", }, ); - await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', - ); + await writeFile(join(target, "app.js"), CLIENT_JAVASCRIPT_FIXTURE_SOURCE); clientVersion = ( await commands.exec(command, ["--version"], { env: environment, @@ -197,37 +194,9 @@ await withVerifierCleanup(async () => { for (const message of toolMessages) { const found = extractEvidence(message.content); if (found === undefined) continue; - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(found); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((node) => node.observations) - .map((observation) => observation.properties) - .find( - (properties) => - properties.semantic_role === "export-return-shapes" && - properties.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap( - (entry) => entry.fields, - ); - assert( - fields.some( - (field) => - field.path === "/clientCompatibility" && - field.state === "literal" && - field.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (field) => - field.path === "/count" && - field.state === "literal" && - field.value === 7, - ), + const { evidence } = await assertClientJavaScriptFixtureEvidence( + found, + target, ); const text = JSON.stringify(evidence); await writeFile(join(lab, "evidence.json"), text); diff --git a/scripts/verify-gemini-client.mjs b/scripts/verify-gemini-client.mjs index eec9f69ce6..9e99a69a62 100644 --- a/scripts/verify-gemini-client.mjs +++ b/scripts/verify-gemini-client.mjs @@ -17,7 +17,10 @@ import { import { createVerifierCommands } from "./lib/verify-commands.mjs"; import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; -import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; +import { + CLIENT_JAVASCRIPT_FIXTURE_SOURCE, + assertClientJavaScriptFixtureEvidence, +} from "./verify/clients/javascript-fixture.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createGeminiModelFixture } from "./verify/clients/gemini-model-fixture.mjs"; @@ -188,10 +191,7 @@ await withVerifierCleanup(async () => { ); const target = join(lab, "fixture α测试"); await mkdir(target); - await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', - ); + await writeFile(join(target, "app.js"), CLIENT_JAVASCRIPT_FIXTURE_SOURCE); const schemaValidator = new Ajv2020({ strict: false, validateFormats: false, @@ -329,33 +329,9 @@ await withVerifierCleanup(async () => { ); const observed = extractEvidence(responses); if (observed) { - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(observed); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((n) => n.observations) - .map((o) => o.properties) - .find( - (p) => - p.semantic_role === "export-return-shapes" && - p.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((s) => s.fields); - assert( - fields.some( - (f) => - f.path === "/clientCompatibility" && - f.state === "literal" && - f.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (f) => - f.path === "/count" && f.state === "literal" && f.value === 7, - ), + const { evidence } = await assertClientJavaScriptFixtureEvidence( + observed, + target, ); const bytes = JSON.stringify(evidence); artifactDigest = createHash("sha256").update(bytes).digest("hex"); diff --git a/scripts/verify-grok-client.mjs b/scripts/verify-grok-client.mjs index 67ec83a200..753c8ae998 100644 --- a/scripts/verify-grok-client.mjs +++ b/scripts/verify-grok-client.mjs @@ -5,14 +5,14 @@ import { homedir, tmpdir } from "node:os"; import { isAbsolute, join, relative } from "node:path"; import { fileURLToPath } from "node:url"; import Ajv2020 from "ajv/dist/2020.js"; -import { - TOOL_CONTRACTS, - toolContract, -} from "../dist/contracts/toolContracts.js"; +import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js"; import { createVerifierCommands } from "./lib/verify-commands.mjs"; import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; -import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; +import { + CLIENT_JAVASCRIPT_FIXTURE_SOURCE, + assertClientJavaScriptFixtureEvidence, +} from "./verify/clients/javascript-fixture.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -95,7 +95,7 @@ await withVerifierCleanup(async () => { await mkdir(target); await writeFile( join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n' + + CLIENT_JAVASCRIPT_FIXTURE_SOURCE + // A real analysis result large enough to exercise native MCP offloading. Array.from( { length: 60 }, @@ -135,42 +135,9 @@ await withVerifierCleanup(async () => { }; }; const verifyEvidence = async (bytes) => { - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(JSON.parse(bytes.toString("utf8"))); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shapes = evidence.normalized_result.graph.nodes - .flatMap((node) => node.observations) - .map((observation) => observation.properties) - .filter( - (properties) => - properties.semantic_role === "export-return-shapes" && - properties.exported_name === "result", - ); - assert( - shapes.length > 0 && - shapes.every( - (shape) => shape.return_shape_coverage?.projection_complete, - ), - ); - const fields = shapes.flatMap((shape) => - shape.static_return_shapes.flatMap((item) => item.fields), - ); - assert( - fields.some( - (field) => - field.path === "/clientCompatibility" && - field.state === "literal" && - field.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (field) => - field.path === "/count" && - field.state === "literal" && - field.value === 7, - ), + const { evidence, shapes } = await assertClientJavaScriptFixtureEvidence( + JSON.parse(bytes.toString("utf8")), + target, ); artifactDigest = createHash("sha256").update(bytes).digest("hex"); artifactBytes = bytes.length; diff --git a/scripts/verify-native-agent-client.mjs b/scripts/verify-native-agent-client.mjs index b99337d651..e4e62b3756 100644 --- a/scripts/verify-native-agent-client.mjs +++ b/scripts/verify-native-agent-client.mjs @@ -4,14 +4,14 @@ import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; -import { - TOOL_CONTRACTS, - toolContract, -} from "../dist/contracts/toolContracts.js"; +import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js"; import { createVerifierCommands } from "./lib/verify-commands.mjs"; import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; -import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; +import { + CLIENT_JAVASCRIPT_FIXTURE_SOURCE, + assertClientJavaScriptFixtureEvidence, +} from "./verify/clients/javascript-fixture.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -66,10 +66,7 @@ await withVerifierCleanup(async () => { ...(stickyHermes ? [mkdir(selectedProfile, { recursive: true })] : []), mkdir(target, { recursive: true }), ]); - await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', - ); + await writeFile(join(target, "app.js"), CLIENT_JAVASCRIPT_FIXTURE_SOURCE); const environment = { PATH: process.env.PATH ?? "", HOME: account, @@ -195,32 +192,9 @@ await withVerifierCleanup(async () => { } }; const validateEvidence = async (value) => { - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(value); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((n) => n.observations) - .map((o) => o.properties) - .find( - (p) => - p.semantic_role === "export-return-shapes" && - p.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((s) => s.fields); - assert( - fields.some( - (f) => - f.path === "/clientCompatibility" && - f.state === "literal" && - f.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (f) => f.path === "/count" && f.state === "literal" && f.value === 7, - ), + const { evidence } = await assertClientJavaScriptFixtureEvidence( + value, + target, ); const serialized = JSON.stringify(evidence); await writeFile(join(lab, "evidence.json"), serialized); diff --git a/scripts/verify-omp-client.mjs b/scripts/verify-omp-client.mjs index 973e85c47b..bb2599e828 100644 --- a/scripts/verify-omp-client.mjs +++ b/scripts/verify-omp-client.mjs @@ -4,14 +4,14 @@ import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { homedir, tmpdir } from "node:os"; import { join, relative } from "node:path"; import { fileURLToPath } from "node:url"; -import { - TOOL_CONTRACTS, - toolContract, -} from "../dist/contracts/toolContracts.js"; +import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js"; import { createVerifierCommands } from "./lib/verify-commands.mjs"; import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; -import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; +import { + CLIENT_JAVASCRIPT_FIXTURE_SOURCE, + assertClientJavaScriptFixtureEvidence, +} from "./verify/clients/javascript-fixture.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -85,10 +85,7 @@ await withVerifierCleanup(async () => { }; const target = join(lab, "fixture α测试"); await mkdir(target); - await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', - ); + await writeFile(join(target, "app.js"), CLIENT_JAVASCRIPT_FIXTURE_SOURCE); const analysisDevice = "xd://mcp__rea_analyze_javascript_application"; const containsText = (value, text) => { if (typeof value === "string") return value.includes(text); @@ -171,37 +168,9 @@ await withVerifierCleanup(async () => { evidenceReply, "OMP must deliver the full analysis result to the next model turn", ); - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(JSON.parse(evidenceReply.content)); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((node) => node.observations) - .map((observation) => observation.properties) - .find( - (properties) => - properties.semantic_role === "export-return-shapes" && - properties.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap( - (item) => item.fields, - ); - assert( - fields.some( - (field) => - field.path === "/clientCompatibility" && - field.state === "literal" && - field.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (field) => - field.path === "/count" && - field.state === "literal" && - field.value === 7, - ), + const { evidence } = await assertClientJavaScriptFixtureEvidence( + JSON.parse(evidenceReply.content), + target, ); const bytes = JSON.stringify(evidence); artifactDigest = createHash("sha256").update(bytes).digest("hex"); diff --git a/scripts/verify-opencode-client.mjs b/scripts/verify-opencode-client.mjs index 6ea41ef3bc..2273130bfc 100644 --- a/scripts/verify-opencode-client.mjs +++ b/scripts/verify-opencode-client.mjs @@ -5,14 +5,14 @@ import { tmpdir } from "node:os"; import { isAbsolute, join, relative, sep } from "node:path"; import { fileURLToPath } from "node:url"; import Ajv2020 from "ajv/dist/2020.js"; -import { - TOOL_CONTRACTS, - toolContract, -} from "../dist/contracts/toolContracts.js"; +import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js"; import { createVerifierCommands } from "./lib/verify-commands.mjs"; import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; -import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; +import { + CLIENT_JAVASCRIPT_FIXTURE_SOURCE, + assertClientJavaScriptFixtureEvidence, +} from "./verify/clients/javascript-fixture.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -108,10 +108,7 @@ await withVerifierCleanup(async () => { } const target = join(lab, "fixture α测试"); await mkdir(target); - await writeFile( - join(target, "app.js"), - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', - ); + await writeFile(join(target, "app.js"), CLIENT_JAVASCRIPT_FIXTURE_SOURCE); const schemaValidator = new Ajv2020({ strict: false, validateFormats: false, @@ -183,33 +180,9 @@ await withVerifierCleanup(async () => { ); const observed = extractEvidence(replies); if (observed) { - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(observed); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((n) => n.observations) - .map((o) => o.properties) - .find( - (p) => - p.semantic_role === "export-return-shapes" && - p.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((s) => s.fields); - assert( - fields.some( - (f) => - f.path === "/clientCompatibility" && - f.state === "literal" && - f.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (f) => - f.path === "/count" && f.state === "literal" && f.value === 7, - ), + const { evidence } = await assertClientJavaScriptFixtureEvidence( + observed, + target, ); const bytes = JSON.stringify(evidence); artifactDigest = createHash("sha256").update(bytes).digest("hex"); diff --git a/scripts/verify-qwen-client.mjs b/scripts/verify-qwen-client.mjs index 1acb14f572..0d05f41ca8 100644 --- a/scripts/verify-qwen-client.mjs +++ b/scripts/verify-qwen-client.mjs @@ -15,12 +15,12 @@ import { createHash } from "node:crypto"; import { createVerifierCommands } from "./lib/verify-commands.mjs"; import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs"; import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; -import { assertJavaScriptApplicationInputPaths } from "./lib/javascript-application-paths.mjs"; -import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { - TOOL_CONTRACTS, - toolContract, -} from "../dist/contracts/toolContracts.js"; + CLIENT_JAVASCRIPT_FIXTURE_SOURCE, + assertClientJavaScriptFixtureEvidence, +} from "./verify/clients/javascript-fixture.mjs"; +import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; +import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js"; const repo = fileURLToPath(new URL("..", import.meta.url)); const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo; @@ -191,10 +191,7 @@ await withVerifierCleanup(async () => { } const target = join(lab, "fixture α测试"); await mkdir(target, { recursive: true }); - await writeFile( - `${target}/app.js`, - 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', - ); + await writeFile(`${target}/app.js`, CLIENT_JAVASCRIPT_FIXTURE_SOURCE); const extractEvidence = (value) => { if (typeof value === "string") { try { @@ -257,35 +254,11 @@ await withVerifierCleanup(async () => { assert(artifactPath); const artifact = await readFile(artifactPath); artifactDigest = createHash("sha256").update(artifact).digest("hex"); - const evidence = toolContract( - "analyze_javascript_application", - ).outputSchema.parse(JSON.parse(artifact.toString("utf8"))); + const { evidence } = await assertClientJavaScriptFixtureEvidence( + JSON.parse(artifact.toString("utf8")), + target, + ); assert(extractEvidence(evidence)); - await assertJavaScriptApplicationInputPaths(evidence, target); - const shape = evidence.normalized_result.graph.nodes - .flatMap((n) => n.observations) - .map((o) => o.properties) - .find( - (p) => - p.semantic_role === "export-return-shapes" && - p.exported_name === "result", - ); - assert(shape?.return_shape_coverage?.projection_complete); - const fields = shape.static_return_shapes.flatMap((s) => s.fields); - assert( - fields.some( - (f) => - f.path === "/clientCompatibility" && - f.state === "literal" && - f.value === "REA_CLIENT_FIXTURE", - ), - ); - assert( - fields.some( - (f) => - f.path === "/count" && f.state === "literal" && f.value === 7, - ), - ); evidenceSeen = true; await writeFile( `${lab}/${client}-${mode}-evidence.json`, diff --git a/scripts/verify/clients/javascript-fixture.d.mts b/scripts/verify/clients/javascript-fixture.d.mts new file mode 100644 index 0000000000..286cf01c7d --- /dev/null +++ b/scripts/verify/clients/javascript-fixture.d.mts @@ -0,0 +1,21 @@ +import type { z } from "zod"; +import type { JavaScriptApplicationAnalysisResult } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import type { toolContract } from "../../../src/contracts/toolContracts.js"; + +type FixtureContract = ReturnType< + typeof toolContract<"analyze_javascript_application"> +>; +type FixtureEvidence = Omit< + z.output, + "normalized_result" +> & { + readonly normalized_result: JavaScriptApplicationAnalysisResult; +}; +type FixtureProperties = + FixtureEvidence["normalized_result"]["graph"]["nodes"][number]["observations"][number]["properties"]; + +export const CLIENT_JAVASCRIPT_FIXTURE_SOURCE: string; +export function assertClientJavaScriptFixtureEvidence( + value: unknown, + inputPath: string, +): Promise<{ evidence: FixtureEvidence; shapes: FixtureProperties[] }>; diff --git a/scripts/verify/clients/javascript-fixture.mjs b/scripts/verify/clients/javascript-fixture.mjs new file mode 100644 index 0000000000..588ce7832d --- /dev/null +++ b/scripts/verify/clients/javascript-fixture.mjs @@ -0,0 +1,56 @@ +import assert from "node:assert/strict"; +import { toolContract } from "../../../dist/contracts/toolContracts.js"; +import { assertJavaScriptApplicationInputPaths } from "../../lib/javascript-application-paths.mjs"; + +export const CLIENT_JAVASCRIPT_FIXTURE_SOURCE = + 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n'; + +/** Validate the same analyst result across native client transports. */ +export const assertClientJavaScriptFixtureEvidence = async ( + value, + inputPath, +) => { + const evidence = toolContract( + "analyze_javascript_application", + ).outputSchema.parse(value); + assert( + Array.isArray(evidence.normalized_result.graph?.nodes), + "Client must return the full application graph", + ); + await assertJavaScriptApplicationInputPaths(evidence, inputPath); + const shapes = evidence.normalized_result.graph.nodes + .flatMap((node) => node.observations) + .map((observation) => observation.properties) + .filter( + (properties) => + properties.semantic_role === "export-return-shapes" && + properties.exported_name === "result", + ); + assert(shapes.length > 0, "Client must return the fixture's result export"); + assert( + shapes.every((shape) => shape.return_shape_coverage?.projection_complete), + "Every fixture result observation must have complete return-shape coverage", + ); + for (const shape of shapes) { + const fields = shape.static_return_shapes.flatMap((item) => item.fields); + assert( + fields.some( + (field) => + field.path === "/clientCompatibility" && + field.state === "literal" && + field.value === "REA_CLIENT_FIXTURE", + ), + "Client must return the fixture's compatibility literal", + ); + assert( + fields.some( + (field) => + field.path === "/count" && + field.state === "literal" && + field.value === 7, + ), + "Client must return the fixture's count literal", + ); + } + return { evidence, shapes }; +}; diff --git a/tests/boundary/javascript/clientFixtureEvidence.test.ts b/tests/boundary/javascript/clientFixtureEvidence.test.ts new file mode 100644 index 0000000000..746cc42638 --- /dev/null +++ b/tests/boundary/javascript/clientFixtureEvidence.test.ts @@ -0,0 +1,140 @@ +import assert from "node:assert/strict"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { assertClientJavaScriptFixtureEvidence } from "../../../scripts/verify/clients/javascript-fixture.mjs"; +import { + createJavaScriptApplicationGraph, + createJavaScriptApplicationNode, +} from "../../../src/domain/javascript/javascriptApplicationGraph.js"; +import { createJavaScriptSemanticGraph } from "../../../src/domain/javascript/javascriptSemanticGraph.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { analyzeJavaScriptApplication } from "../../support/javascriptApplicationScope.js"; + +it("checks native client fixture Evidence, provenance, and every result observation", async () => { + const target = await createTestTempDirectory("rea-client-evidence-"); + // Independent source exercises the verifier's expectations through production + // analysis instead of manufacturing a copy of its graph representation. + await writeFile( + join(target, "result.js"), + 'export function result() { return { count: 7, clientCompatibility: "REA_CLIENT_FIXTURE" }; }', + ); + const result = await analyzeJavaScriptApplication({ + input_path: target, + format: "directory", + }); + if (!result.ok) throw result.error; + const validated = await assertClientJavaScriptFixtureEvidence( + result.value, + target, + ); + const evidence = validated.evidence; + expect(evidence.evidence_id).toBe(result.value.evidence_id); + expect(validated.shapes.length).toBeGreaterThan(0); + + await expect( + assertClientJavaScriptFixtureEvidence({}, target), + ).rejects.toThrow(); + await expect( + assertClientJavaScriptFixtureEvidence( + { + ...evidence, + parameters: { ...evidence.parameters, input_path: target + "/other" }, + }, + target, + ), + ).rejects.toThrow("caller's input_path"); + await expect( + assertClientJavaScriptFixtureEvidence( + { + ...evidence, + subject: { ...evidence.subject, local_path: target + "/other" }, + }, + target, + ), + ).rejects.toThrow("resolved artifact"); + await expect( + assertClientJavaScriptFixtureEvidence( + { + ...evidence, + normalized_result: { + ...evidence.normalized_result, + input_path: target + "/other", + }, + }, + target, + ), + ).rejects.toThrow("resolved artifact"); + + const withDuplicate = (properties: Record) => { + const { graph_id: _graphId, ...originalGraph } = + evidence.normalized_result.graph; + const graph = createJavaScriptApplicationGraph({ + ...originalGraph, + nodes: originalGraph.nodes.map((node) => { + const observation = node.observations.find( + (item) => + item.properties.semantic_role === "export-return-shapes" && + item.properties.exported_name === "result", + ); + if (observation === undefined) return node; + const observations = node.observations.map( + ({ observation_id: _id, identifier_strategy: _strategy, ...value }) => + value, + ); + return createJavaScriptApplicationNode({ + kind: node.kind, + identity: node.identity, + observations: [ + ...observations, + { + label: observation.label, + source_map_reference: observation.source_map_reference, + properties, + evidence: observation.evidence, + }, + ], + }); + }), + }); + const { graph_id: _semanticGraphId, ...semantic } = + evidence.normalized_result.semantic_graph; + return { + ...evidence, + normalized_result: { + ...evidence.normalized_result, + graph, + semantic_graph: createJavaScriptSemanticGraph({ + ...semantic, + application_graph_id: graph.graph_id, + }), + }, + }; + }; + const shape = validated.shapes[0]!; + const coverage = shape.return_shape_coverage; + assert(coverage !== null && typeof coverage === "object"); + const incomplete = withDuplicate({ + ...shape, + return_shape_coverage: { + ...coverage, + projection_complete: false, + }, + }); + // Establish schema-valid fixtures so rejection proves the client assertion. + toolContract("analyze_javascript_application").outputSchema.parse(incomplete); + await expect( + assertClientJavaScriptFixtureEvidence(incomplete, target), + ).rejects.toThrow("complete return-shape coverage"); + + const missingLiteral = withDuplicate({ ...shape, static_return_shapes: [] }); + toolContract("analyze_javascript_application").outputSchema.parse( + missingLiteral, + ); + await expect( + assertClientJavaScriptFixtureEvidence(missingLiteral, target), + ).rejects.toThrow("compatibility literal"); +});