From 95c8f57aa2eafab4c8d7d43482b33ceefaf96b16 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 19 Oct 2026 08:02:03 +0800 Subject: [PATCH] fix(investigation): scope automatic unknowns to result Evidence Construct automatic unknown identity once at the application boundary. Validate the original question before adding immutable Evidence scope, and preserve each result's supporting link through CLI and MCP producers. Apply the shared policy to derived investigations, managed comparisons and verification, composed workflow residuals, and process-capture residuals. This prevents registry conflicts and silent loss of later result links. Keep explicit creation/update semantics and exact Evidence replays intact. Cover all affected producer families with public MCP regressions, preserve snapshot provenance distinctions, and retain blank-question/error handling. Remove the superseded single-result call-path test. Fixes #3333 Co-authored-by: Dragolone <160818051+Dragolone@users.noreply.github.com> --- docs/mcp-contracts.md | 6 + src/application/WorkflowEvidence.test.ts | 47 +++- src/application/WorkflowEvidence.ts | 7 +- .../investigation/WorkflowUnknown.ts | 29 ++ .../registerInvestigationTools/helpers.ts | 8 +- .../compareManagedMembers.ts | 18 +- .../verifyManagedNativeBoundaries.ts | 18 +- src/server/registerSessionTools.ts | 7 +- tests/boundary/mcp/ghidraNativeApiMcp.test.ts | 122 +++++++++ .../mcp/investigationAnalysisMcp.test.ts | 34 +-- .../managedWorkflowUnknownIdentityMcp.test.ts | 257 ++++++++++++++++++ .../processResidualUnknownIdentityMcp.test.ts | 72 +++++ .../mcp/workflowUnknownIdentityMcp.test.ts | 248 +++++++++++++++++ .../process/processSessionMcp.test.ts | 6 +- 14 files changed, 822 insertions(+), 57 deletions(-) create mode 100644 src/application/investigation/WorkflowUnknown.ts create mode 100644 tests/boundary/mcp/managedWorkflowUnknownIdentityMcp.test.ts create mode 100644 tests/boundary/mcp/processResidualUnknownIdentityMcp.test.ts create mode 100644 tests/boundary/mcp/workflowUnknownIdentityMcp.test.ts diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index a4de713cb5..840e7f241e 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -132,6 +132,12 @@ is null. `total` counts all matching heads; each page preserves complete records in stable unknown ID order. Filters apply before paging. Pages reflect current session state, so keep the unknown ledger unchanged while traversing it. +Automatically retained workflow questions include their immutable result +Evidence ID. Distinct results retain independent questions and source links; +repeating the same result does not create another unknown. Workflow Evidence +keeps the original residual question text. Explicit `record_unknown` requests +keep caller-selected identity and reject duplicate creation. + For a portable exported bundle, the CLI shares this projection: `rea inspect-evidence-bundle '{"path":"/tmp/evidence.json","detail":"summary","filters":{"operation":"analyze_function"}}' --format json`. The CLI validates and materializes that local bundle under its existing memory diff --git a/src/application/WorkflowEvidence.test.ts b/src/application/WorkflowEvidence.test.ts index d62522fccc..d2346411fc 100644 --- a/src/application/WorkflowEvidence.test.ts +++ b/src/application/WorkflowEvidence.test.ts @@ -212,7 +212,7 @@ describe("workflow residual question retention", () => { expect(recordWorkflowUnknowns(input).ok).toBe(true); expect(records.listUnknowns()).toMatchObject([ { - question: result.residual_unknowns[0], + question: `${result.residual_unknowns[0]} (Evidence ${evidence.evidence_id})`, domain: "native-api", supporting_evidence_ids: [evidence.evidence_id], required_authority: "shipped-artifact", @@ -220,6 +220,47 @@ describe("workflow residual question retention", () => { ]); }); + it("retains one identical residual question per distinct workflow Evidence", () => { + const records = new InvestigationRecords(); + const result = { + residual_unknowns: ["Which provider resolves this boundary?"], + }; + const evidence = ["0x1000", "0x2000"].map((procedure) => + createWorkflowEvidence({ + target: undefined, + operation: "inspect_native_api", + parameters: { procedure }, + result, + upstreamProfile: undefined, + upstreamLimitations: [], + }), + ); + expect(evidence[0]?.evidence_id).not.toBe(evidence[1]?.evidence_id); + + for (const item of evidence) { + expect(records.recordEvidence(item).ok).toBe(true); + const input = { + name: "inspect_native_api", + result, + evidenceId: item.evidence_id, + recordUnknown: ( + unknown: Parameters[0], + ) => records.recordUnknown(unknown), + }; + expect(recordWorkflowUnknowns(input).ok).toBe(true); + expect(recordWorkflowUnknowns(input).ok).toBe(true); + } + + expect(records.listUnknowns()).toHaveLength(2); + expect( + records + .listUnknowns() + .map(({ supporting_evidence_ids }) => supporting_evidence_ids), + ).toEqual( + expect.arrayContaining(evidence.map(({ evidence_id }) => [evidence_id])), + ); + }); + it.each([ ["spaces", " Which boundary? "], ["line feed and tab", "\nWhich boundary?\t"], @@ -248,7 +289,7 @@ describe("workflow residual question retention", () => { }).ok, ).toBe(true); expect(records.listUnknowns()).toMatchObject([ - { question: "Which boundary?" }, + { question: `Which boundary? (Evidence ${evidence.evidence_id})` }, ]); expect(evidence.normalized_result).toEqual(result); }, @@ -328,7 +369,7 @@ it.each([ }).ok, ).toBe(true); expect(records.listUnknowns()).toMatchObject([ - { question: "Which boundary?" }, + { question: `Which boundary? (Evidence ${evidence.evidence_id})` }, ]); expect(records.evidenceById(evidence.evidence_id)).toEqual(evidence); expect(evidence.normalized_result).toMatchObject({ diff --git a/src/application/WorkflowEvidence.ts b/src/application/WorkflowEvidence.ts index 599a5de290..f707ae8092 100644 --- a/src/application/WorkflowEvidence.ts +++ b/src/application/WorkflowEvidence.ts @@ -23,6 +23,7 @@ import { import type { JsonValue } from "../domain/jsonValue.js"; import type { UnknownRegistryPort } from "./investigation/InvestigationRecordPort.js"; import { UnknownRegistryError } from "../domain/unknownRegistryError.js"; +import { workflowUnknownInput } from "./investigation/WorkflowUnknown.js"; import { renderSourceLimitations, sourceLimitationKey, @@ -180,12 +181,10 @@ export const recordWorkflowUnknowns = ({ return { ok: true, value: null }; for (const question of result.residual_unknowns) { if (typeof question !== "string") continue; - const recorded = recordUnknown({ + const input = workflowUnknownInput(evidenceId, { question, severity: "medium", domain: name === "inspect_native_api" ? "native-api" : "control-flow", - supporting_evidence_ids: [evidenceId], - contradicting_evidence_ids: [], required_authority: "shipped-artifact", required_confidence: "observed", required_environment: null, @@ -200,6 +199,8 @@ export const recordWorkflowUnknowns = ({ ], relationships: [], }); + if (!input.ok) return input; + const recorded = recordUnknown(input.value); if ( !recorded.ok && !( diff --git a/src/application/investigation/WorkflowUnknown.ts b/src/application/investigation/WorkflowUnknown.ts new file mode 100644 index 0000000000..9d36595b14 --- /dev/null +++ b/src/application/investigation/WorkflowUnknown.ts @@ -0,0 +1,29 @@ +import { + recordUnknownInputSchema, + type RecordUnknownInput, +} from "../../domain/residualUnknown.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { UnknownRegistryError } from "../../domain/unknownRegistryError.js"; + +/** Scope an automatic question to its immutable result before registry mutation. */ +export const workflowUnknownInput = ( + evidenceId: string, + input: Omit< + RecordUnknownInput, + "supporting_evidence_ids" | "contradicting_evidence_ids" + >, +): Result => { + const question = recordUnknownInputSchema.shape.question.safeParse( + input.question, + ); + if (!question.success) + return err( + new UnknownRegistryError("invalid-transition", { cause: question.error }), + ); + return ok({ + ...input, + question: `${question.data} (Evidence ${evidenceId})`, + supporting_evidence_ids: [evidenceId], + contradicting_evidence_ids: [], + }); +}; diff --git a/src/server/registerInvestigationTools/helpers.ts b/src/server/registerInvestigationTools/helpers.ts index 557f96225f..556225d55a 100644 --- a/src/server/registerInvestigationTools/helpers.ts +++ b/src/server/registerInvestigationTools/helpers.ts @@ -4,6 +4,7 @@ import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import { err, ok, type Result } from "../../domain/result.js"; import { recordDerivedEvidence } from "../recordDerivedEvidence.js"; +import { workflowUnknownInput } from "../../application/investigation/WorkflowUnknown.js"; import type { WorkflowUnknownInput } from "./types.js"; import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; @@ -64,16 +65,17 @@ export const recordWorkflowEvidence = ( if (!unresolved) { return recordDerivedEvidence(session, evidence, undefined); } - return recordDerivedEvidence(session, evidence, { + const unknown = workflowUnknownInput(evidence.evidence_id, { question: input.question, severity: "high", domain: input.domain, - supporting_evidence_ids: [evidence.evidence_id], - contradicting_evidence_ids: [], required_authority: input.requiredAuthority, required_confidence: input.requiredConfidence, required_environment: null, recommended_probes: [...input.probes], relationships: [], }); + return unknown.ok + ? recordDerivedEvidence(session, evidence, unknown.value) + : unknown; }; diff --git a/src/server/registerManagedWorkflowTools/compareManagedMembers.ts b/src/server/registerManagedWorkflowTools/compareManagedMembers.ts index cea75f4574..6313211c41 100644 --- a/src/server/registerManagedWorkflowTools/compareManagedMembers.ts +++ b/src/server/registerManagedWorkflowTools/compareManagedMembers.ts @@ -1,3 +1,4 @@ +import { workflowUnknownInput } from "../../application/investigation/WorkflowUnknown.js"; import type { EvidenceMcpServer } from "../EvidenceMcpServer.js"; import { runAdmittedToolOperation } from "../admittedToolOperation.js"; import { recordSessionEvidenceSources } from "../sessionEvidence.js"; @@ -64,14 +65,12 @@ export const registerCompareManagedMembers = ( result.value.normalized_result, ); const unknown = comparison.summary.unknown > 0; - const output = unknown - ? options.recordEvidenceWithUnknown?.(result.value, { + const unknownInput = unknown + ? workflowUnknownInput(result.value.evidence_id, { question: "Which managed members remain unmatched or ambiguous across these versions?", severity: "medium", domain: "managed-member-comparison", - supporting_evidence_ids: [result.value.evidence_id], - contradicting_evidence_ids: [], required_authority: "shipped-artifact", required_confidence: "observed", required_environment: null, @@ -84,7 +83,16 @@ export const registerCompareManagedMembers = ( ], relationships: [], }) - : options.recordEvidence?.(result.value); + : undefined; + const output = + unknownInput === undefined + ? options.recordEvidence?.(result.value) + : unknownInput.ok + ? options.recordEvidenceWithUnknown?.( + result.value, + unknownInput.value, + ) + : unknownInput; return server.delivery.toEvidenceToolResult( result.value, compareContract, diff --git a/src/server/registerManagedWorkflowTools/verifyManagedNativeBoundaries.ts b/src/server/registerManagedWorkflowTools/verifyManagedNativeBoundaries.ts index 02559c680c..5bea7f9521 100644 --- a/src/server/registerManagedWorkflowTools/verifyManagedNativeBoundaries.ts +++ b/src/server/registerManagedWorkflowTools/verifyManagedNativeBoundaries.ts @@ -1,3 +1,4 @@ +import { workflowUnknownInput } from "../../application/investigation/WorkflowUnknown.js"; import type { EvidenceMcpServer } from "../EvidenceMcpServer.js"; import { runAdmittedToolOperation } from "../admittedToolOperation.js"; import { recordSessionEvidenceSources } from "../sessionEvidence.js"; @@ -83,14 +84,12 @@ export const registerVerifyManagedNativeBoundaries = ( verification.summary.unresolved > 0 || verification.summary.contradicted > 0 || verification.summary.native_body_unresolved > 0; - const output = unknown - ? options.recordEvidenceWithUnknown?.(result.value, { + const unknownInput = unknown + ? workflowUnknownInput(result.value.evidence_id, { question: "Which managed/native boundaries remain unresolved or contradicted by the supplied native Evidence?", severity: "medium", domain: "managed-native-verification", - supporting_evidence_ids: [result.value.evidence_id], - contradicting_evidence_ids: [], required_authority: "shipped-artifact", required_confidence: "observed", required_environment: null, @@ -108,7 +107,16 @@ export const registerVerifyManagedNativeBoundaries = ( ], relationships: [], }) - : options.recordEvidence?.(result.value); + : undefined; + const output = + unknownInput === undefined + ? options.recordEvidence?.(result.value) + : unknownInput.ok + ? options.recordEvidenceWithUnknown?.( + result.value, + unknownInput.value, + ) + : unknownInput; return server.delivery.toEvidenceToolResult( result.value, nativeVerificationContract, diff --git a/src/server/registerSessionTools.ts b/src/server/registerSessionTools.ts index 7c992112d3..d93ddcebf3 100644 --- a/src/server/registerSessionTools.ts +++ b/src/server/registerSessionTools.ts @@ -1,5 +1,6 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { workflowUnknownInput } from "../application/investigation/WorkflowUnknown.js"; import { getNavigationContext, @@ -48,12 +49,10 @@ const recordProcessResidualUnknowns = ( residuals: ProcessCapture["residual_unknowns"], ): Result => { for (const residual of residuals) { - const unknown = session.recordUnknown({ + const input = workflowUnknownInput(evidence.evidence_id, { question: `Was ${residual.scope} behavior fully observed during capture?`, severity: "medium", domain: `process-${residual.scope}`, - supporting_evidence_ids: [evidence.evidence_id], - contradicting_evidence_ids: [], required_authority: "controlled-replay", required_confidence: "observed", required_environment: evidence.environment, @@ -66,6 +65,8 @@ const recordProcessResidualUnknowns = ( ], relationships: [], }); + if (!input.ok) return input; + const unknown = session.recordUnknown(input.value); if ( !unknown.ok && !( diff --git a/tests/boundary/mcp/ghidraNativeApiMcp.test.ts b/tests/boundary/mcp/ghidraNativeApiMcp.test.ts index 2988d1a179..0a31f33baf 100644 --- a/tests/boundary/mcp/ghidraNativeApiMcp.test.ts +++ b/tests/boundary/mcp/ghidraNativeApiMcp.test.ts @@ -9,7 +9,11 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; import { nativeLoadImageSchema } from "../../../src/domain/native/nativeLoadImage.js"; +import { nativeApiInspectionResultSchema } from "../../../src/domain/native/nativeApiBoundary.js"; +import { functionDossierSchema } from "../../../src/domain/hopperValues.js"; import { ok } from "../../../src/domain/result.js"; +import { ghidraFunctionDossier } from "../../../src/domain/ghidraValues.fixture.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; it.each([ { state: "final", failed: [] }, @@ -129,3 +133,121 @@ describe("Ghidra MCP native API evidence", () => { } }, 30_000); }); + +it("retains unavailable Ghidra native API observations across provenance and exact replay", async () => { + const residual = "The controlled provider omitted native API boundary facts."; + // This wire fixture is controlled and never launches Ghidra. + const harness = await connectGhidraMcp( + "ghidra-native-api-unavailable-identity", + async (operation, input) => { + if (operation !== "analyze_function") + throw new Error(`Unexpected controlled Ghidra operation: ${operation}`); + const procedure = String(input.procedure); + const base = functionDossierSchema.parse(ghidraFunctionDossier()); + const address = procedure === "0x401000" ? procedure : "0x401100"; + return ok( + jsonValueSchema.parse({ + value: functionDossierSchema.parse({ + ...base, + procedure: { + ...base.procedure, + address, + name: address === "0x401000" ? "fixture_main" : "fixture_other", + body: { + ...base.procedure.body, + ranges: [ + { + start: address, + end: address === "0x401000" ? "0x401005" : "0x401105", + }, + ], + }, + }, + native_api: { + available: false, + reason: residual, + residual_unknowns: [residual], + }, + }), + limitations: [], + }), + ); + }, + ); + const { mcp, session } = harness; + try { + const inspect = async (procedure: string) => { + const reply = await mcp.callTool({ + name: "inspect_native_api", + arguments: { procedure }, + }); + expect(reply.isError, JSON.stringify(reply.content)).not.toBe(true); + const evidence = sessionEvidence(session, reply.structuredContent); + const result = nativeApiInspectionResultSchema.parse( + evidence.normalized_result, + ); + expect(result).toMatchObject({ + procedure: { + address: procedure, + }, + boundary: { + available: false, + reason: residual, + residual_unknowns: [residual], + }, + residual_unknowns: [residual], + }); + return evidence; + }; + + const first = await inspect("0x401000"); + const replay = await inspect("0x401000"); + expect(replay.normalized_result).toEqual(first.normalized_result); + expect(replay.evidence_id).not.toBe(first.evidence_id); + expect(replay.limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining("Loaded from a local REA analysis snapshot"), + ]), + ); + const warmBundle = session.exportEvidenceBundle(); + const warmReplay = await inspect("0x401000"); + expect(warmReplay.evidence_id).toBe(replay.evidence_id); + expect(session.exportEvidenceBundle()).toEqual(warmBundle); + + const second = await inspect("0x401100"); + expect(second.evidence_id).not.toBe(first.evidence_id); + expect(session.evidenceById(first.evidence_id)).toEqual(first); + expect(session.evidenceById(replay.evidence_id)).toEqual(replay); + + const listed = await mcp.callTool({ + name: "list_unknowns", + arguments: { domain: "native-api" }, + }); + expect(listed.isError, JSON.stringify(listed.content)).not.toBe(true); + const unknownPage = toolContract("list_unknowns").outputSchema.parse( + listed.structuredContent, + ).result; + expect(unknownPage.total).toBe(3); + expect(unknownPage.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + question: expect.stringContaining(first.evidence_id), + supporting_evidence_ids: [first.evidence_id], + }), + expect.objectContaining({ + question: expect.stringContaining(replay.evidence_id), + supporting_evidence_ids: [replay.evidence_id], + }), + expect.objectContaining({ + question: expect.stringContaining(second.evidence_id), + supporting_evidence_ids: [second.evidence_id], + }), + ]), + ); + expect( + unknownPage.items.every(({ question }) => question.includes(residual)), + ).toBe(true); + } finally { + await harness.close(); + } +}, 30_000); diff --git a/tests/boundary/mcp/investigationAnalysisMcp.test.ts b/tests/boundary/mcp/investigationAnalysisMcp.test.ts index d35cc8a357..6c2420c232 100644 --- a/tests/boundary/mcp/investigationAnalysisMcp.test.ts +++ b/tests/boundary/mcp/investigationAnalysisMcp.test.ts @@ -56,8 +56,7 @@ it("aggregates comparison Evidence and records an approved runtime gap", async ( }); expect(session.listUnknowns({ domain: "changed-behavior" })).toMatchObject([ { - question: - "Did both versions behave the same under a complete controlled replay?", + question: `Did both versions behave the same under a complete controlled replay? (Evidence ${evidence.evidence_id})`, }, ]); } finally { @@ -406,8 +405,7 @@ it.each([ else expect(session.listUnknowns({ domain: "call-path" })).toMatchObject([ { - question: - "Can the requested call path be established from complete analysis?", + question: `Can the requested call path be established from complete analysis? (Evidence ${evidence.evidence_id})`, supporting_evidence_ids: [evidence.evidence_id], }, ]); @@ -417,32 +415,6 @@ it.each([ }, ); -it("records a safe unresolved call-path question", async () => { - const { session, server, client } = await connected(); - const dossier = dossierWithCallees(["0x2000"]); - expect(session.recordEvidence(dossier).ok).toBe(true); - try { - const response = await client.callTool({ - name: "build_call_path", - arguments: { - ...INVESTIGATION_EXAMPLES.build_call_path, - functions: [dossier], - start: { address: "0x1000" }, - goal: { address: "0x3000" }, - }, - }); - expect(response.isError, JSON.stringify(response)).not.toBe(true); - expect(session.listUnknowns({ domain: "call-path" })).toMatchObject([ - { - question: - "Can the requested call path be established from complete analysis?", - }, - ]); - } finally { - await close(session, server, client); - } -}); - it("records an explicit non-causal static/runtime hypothesis", async () => { const { session, server, client } = await connected(); for (const evidence of [ @@ -586,7 +558,7 @@ it("cannot omit a session-owned active unknown from reconstruction input", async session.listUnknowns({ domain: "reconstruction-verification" }), ).toContainEqual( expect.objectContaining({ - question: "Does the reconstruction satisfy every declared claim?", + question: `Does the reconstruction satisfy every declared claim? (Evidence ${evidence.evidence_id})`, }), ); } finally { diff --git a/tests/boundary/mcp/managedWorkflowUnknownIdentityMcp.test.ts b/tests/boundary/mcp/managedWorkflowUnknownIdentityMcp.test.ts new file mode 100644 index 0000000000..56b044b78f --- /dev/null +++ b/tests/boundary/mcp/managedWorkflowUnknownIdentityMcp.test.ts @@ -0,0 +1,257 @@ +import { expect, it, onTestFinished } from "vitest"; + +import { + MANAGED_NATIVE_VERIFICATION_EXAMPLE, + MANAGED_RECONSTRUCTION_IMPORT_EXAMPLE, +} from "../../../src/contracts/managed/managedWorkflowExamples.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; +import { + managedMemberInspectionSchema, + managedNativeBoundaryInspectionSchema, +} from "../../../src/domain/managed/managedArtifact.js"; +import { managedMemberComparisonResultSchema } from "../../../src/domain/managed/managedMemberComparison.js"; +import { managedNativeVerificationResultSchema } from "../../../src/domain/managed/managedNativeVerificationSchemas.js"; +import { inspectMachoSchema } from "../../../src/domain/native/nativeInspection.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; + +const emptyExportEvidence = () => { + const original = MANAGED_NATIVE_VERIFICATION_EXAMPLE.native_observations[0]; + if (original === undefined || original.subject === null) + throw new Error("Managed verification example is missing native Evidence"); + const result = inspectMachoSchema.parse(original.normalized_result); + return createEvidence( + { + path: "/fixtures/empty-exports.dylib", + sha256: "8".repeat(64), + format: original.subject.format, + ...(original.subject.architecture === null + ? {} + : { architecture: original.subject.architecture }), + }, + original.provider, + { + operation: "inspect_macho", + parameters: original.parameters, + result: jsonValueSchema.parse({ + ...result, + exports: { + items: [], + total: 0, + exhaustive: true, + limitations: [], + }, + }), + rawResult: null, + limitations: [], + }, + ); +}; + +const secondDeclaration = () => { + const original = MANAGED_NATIVE_VERIFICATION_EXAMPLE.managed_boundaries; + const result = managedNativeBoundaryInspectionSchema.parse( + original.normalized_result, + ); + const pinvokes = result.pinvoke_imports; + if (pinvokes.length !== 1) + throw new Error("Managed verification example must contain one P/Invoke"); + const pinvoke = pinvokes[0]; + if (pinvoke === undefined) + throw new Error("Managed verification example has invalid P/Invoke data"); + return createEvidence(undefined, original.provider, { + operation: "inspect_managed_native_boundaries", + parameters: original.parameters, + result: jsonValueSchema.parse({ + ...result, + pinvoke_imports: [ + { + ...pinvoke, + token: "0x1c000002", + member_token: "0x06000002", + member_name: "OpenNativeAgain", + import_name: "open_native_again", + }, + ], + }), + rawResult: null, + limitations: original.limitations, + }); +}; + +const memberEvidence = ( + id: string, + digest: string, + methodName: string | null, + partial: boolean, +) => { + const original = MANAGED_RECONSTRUCTION_IMPORT_EXAMPLE.static_members; + const result = managedMemberInspectionSchema.parse( + original.normalized_result, + ); + const method = result.methods[0]; + if (method === undefined) + throw new Error("Managed reconstruction example is missing a method"); + const normalized = managedMemberInspectionSchema.parse({ + ...result, + artifact: { + ...result.artifact, + path: `/fixtures/${id}.dll`, + sha256: digest, + }, + identity_scope: { + ...result.identity_scope, + requires_artifact_sha256: digest, + }, + methods: methodName === null ? [] : [{ ...method, name: methodName }], + metadata: partial + ? { ...result.metadata, status: "partial" } + : result.metadata, + coverage: partial + ? { + state: "partial", + issues: [ + { + code: "invalid-tables", + scope: "MethodDef", + offset: null, + detail: "The method inventory is incomplete in this fixture.", + }, + ], + } + : result.coverage, + }); + return createEvidence( + { path: `/fixtures/${id}.dll`, sha256: digest, format: "pe" }, + original.provider, + { + operation: "inspect_managed_members", + parameters: original.parameters, + result: jsonValueSchema.parse(normalized), + rawResult: null, + limitations: normalized.limitations, + }, + ); +}; + +it("records distinct unresolved managed boundary checks and makes exact replay idempotent", async () => { + const harness = await createApplicationMcpHarness({ + session: createTestBinarySession(() => { + throw new Error("Managed/native verification must not launch a provider"); + }), + }); + onTestFinished(harness.close); + const { client, session } = harness; + const managed = MANAGED_NATIVE_VERIFICATION_EXAMPLE.managed_boundaries; + const native = emptyExportEvidence(); + + const verify = async (boundaries: typeof managed) => { + const response = await client.callTool({ + name: "verify_managed_native_boundaries", + arguments: { + managed_boundaries: boundaries, + native_observations: [native], + }, + }); + expect(response.isError, JSON.stringify(response.content)).not.toBe(true); + const evidence = parseEvidence(response.structuredContent); + const result = managedNativeVerificationResultSchema.parse( + evidence.normalized_result, + ); + expect(result.summary).toMatchObject({ unresolved: 1 }); + expect(result.pinvoke_imports).toEqual([ + expect.objectContaining({ + status: "unresolved", + basis: "no-native-candidate", + }), + ]); + return evidence; + }; + + const first = await verify(managed); + const firstBundle = session.exportEvidenceBundle(); + const replay = await verify(managed); + expect(replay.evidence_id).toBe(first.evidence_id); + expect(session.exportEvidenceBundle()).toEqual(firstBundle); + expect(session.listUnknowns()).toHaveLength(1); + + const second = await verify(secondDeclaration()); + expect(second.evidence_id).not.toBe(first.evidence_id); + const unknowns = await client.callTool({ + name: "list_unknowns", + arguments: { domain: "managed-native-verification" }, + }); + expect(unknowns.isError, JSON.stringify(unknowns.content)).not.toBe(true); + expect(unknowns.structuredContent).toMatchObject({ + result: { + total: 2, + items: expect.arrayContaining([ + expect.objectContaining({ + supporting_evidence_ids: [first.evidence_id], + }), + expect.objectContaining({ + supporting_evidence_ids: [second.evidence_id], + }), + ]), + }, + }); +}); + +it("retains distinct unresolved managed member comparisons and exact replay", async () => { + const harness = await createApplicationMcpHarness({ + session: createTestBinarySession(() => { + throw new Error("Managed member comparison must not launch a provider"); + }), + }); + onTestFinished(harness.close); + const { client, session } = harness; + const right = memberEvidence("members-partial", "a".repeat(64), null, true); + + const compare = async (left: ReturnType) => { + const response = await client.callTool({ + name: "compare_managed_members", + arguments: { left, right }, + }); + expect(response.isError, JSON.stringify(response.content)).not.toBe(true); + const evidence = parseEvidence(response.structuredContent); + const result = managedMemberComparisonResultSchema.parse( + evidence.normalized_result, + ); + expect(result.summary.unknown).toBeGreaterThan(0); + return evidence; + }; + + const first = await compare( + memberEvidence("members-first", "b".repeat(64), "Main", false), + ); + const firstBundle = session.exportEvidenceBundle(); + const replay = await compare( + memberEvidence("members-first", "b".repeat(64), "Main", false), + ); + expect(replay.evidence_id).toBe(first.evidence_id); + expect(session.exportEvidenceBundle()).toEqual(firstBundle); + + const second = await compare( + memberEvidence("members-second", "c".repeat(64), "Open", false), + ); + expect(second.evidence_id).not.toBe(first.evidence_id); + const unknowns = await client.callTool({ + name: "list_unknowns", + arguments: { domain: "managed-member-comparison" }, + }); + expect(unknowns.isError, JSON.stringify(unknowns.content)).not.toBe(true); + expect(unknowns.structuredContent).toMatchObject({ + result: { + total: 2, + items: expect.arrayContaining([ + expect.objectContaining({ + supporting_evidence_ids: [first.evidence_id], + }), + expect.objectContaining({ + supporting_evidence_ids: [second.evidence_id], + }), + ]), + }, + }); +}); diff --git a/tests/boundary/mcp/processResidualUnknownIdentityMcp.test.ts b/tests/boundary/mcp/processResidualUnknownIdentityMcp.test.ts new file mode 100644 index 0000000000..48d9b84765 --- /dev/null +++ b/tests/boundary/mcp/processResidualUnknownIdentityMcp.test.ts @@ -0,0 +1,72 @@ +import { fileURLToPath } from "node:url"; + +import { expect } from "vitest"; + +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { silentLogger } from "../../../src/logger.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { itWithCaptureCapability } from "../process/processCaptureCapability.js"; + +const partialProcessFixture = fileURLToPath( + new URL("../../fixtures/processFidelity.mjs", import.meta.url), +); + +itWithCaptureCapability( + "retains residuals from distinct process captures with the same missing scope", + async () => { + const harness = await createApplicationMcpHarness({ + session: createTestBinarySession(() => { + throw new Error("Process capture must not launch an analysis provider"); + }), + serverOptions: { logger: silentLogger }, + }); + const { client } = harness; + try { + const captures = []; + for (const settleMs of [50, 75]) { + const response = await client.callTool({ + name: "capture_process_scenario", + arguments: { + executable: process.execPath, + arguments: [partialProcessFixture, "partial"], + settle_ms: settleMs, + }, + }); + expect(response.isError, JSON.stringify(response.content)).not.toBe( + true, + ); + captures.push(parseEvidence(response.structuredContent)); + } + expect(captures[0]?.evidence_id).not.toBe(captures[1]?.evidence_id); + for (const capture of captures) + expect(capture.normalized_result).toMatchObject({ + residual_unknowns: expect.arrayContaining([ + expect.objectContaining({ scope: "network" }), + ]), + }); + + const unknowns = await client.callTool({ + name: "list_unknowns", + arguments: { domain: "process-network" }, + }); + expect(unknowns.isError, JSON.stringify(unknowns.content)).not.toBe(true); + expect(unknowns.structuredContent).toMatchObject({ + result: { + total: 2, + items: expect.arrayContaining( + captures.map((capture) => + expect.objectContaining({ + question: `Was network behavior fully observed during capture? (Evidence ${capture.evidence_id})`, + supporting_evidence_ids: [capture.evidence_id], + }), + ), + ), + }, + }); + } finally { + await harness.close(); + } + }, + 30_000, +); diff --git a/tests/boundary/mcp/workflowUnknownIdentityMcp.test.ts b/tests/boundary/mcp/workflowUnknownIdentityMcp.test.ts new file mode 100644 index 0000000000..d563d571ca --- /dev/null +++ b/tests/boundary/mcp/workflowUnknownIdentityMcp.test.ts @@ -0,0 +1,248 @@ +import type { Client } from "@modelcontextprotocol/client"; +import { expect, it } from "vitest"; + +import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { functionDossierSchema } from "../../../src/domain/hopperValues.js"; +import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; + +const withClient = async ( + run: ( + client: Client, + session: ReturnType, + ) => Promise, +): Promise => { + const session = createTestBinarySession(() => { + throw new Error("This workflow must not launch an analysis provider"); + }); + const harness = await createApplicationMcpHarness({ session }); + try { + await run(harness.client, session); + } finally { + await harness.close(); + } +}; + +const pathSource = () => { + const template = FUNCTION_COMPARISON_EXAMPLE.left; + if (template.subject === null) + throw new Error("Call-path fixtures must be artifact-bound"); + const dossier = functionDossierSchema.parse(template.normalized_result); + return createEvidence( + { + path: `${template.subject.local_path}/workflow-unknown-path`, + sha256: template.subject.digest.sha256, + format: template.subject.format, + }, + template.provider, + { + operation: "analyze_function", + parameters: { + ...template.parameters, + observation: "workflow-unknown-path", + }, + result: jsonValueSchema.parse({ + ...dossier, + callees: [ + { + address: "0x2000", + name: "missing-helper", + classification: null, + body: { available: false, reason: "No body was observed." }, + }, + ], + }), + confidence: template.confidence, + authority: template.authority, + relationshipCoverage: { status: "complete" }, + }, + ); +}; + +const incompleteRight = (suffix: string) => { + const template = FUNCTION_COMPARISON_EXAMPLE.right; + if (template.subject === null) + throw new Error("Comparison fixtures must be artifact-bound"); + return createEvidence( + { + path: `${template.subject.local_path}/${suffix}`, + sha256: template.subject.digest.sha256, + format: template.subject.format, + }, + template.provider, + { + operation: "analyze_function", + parameters: { ...template.parameters, observation: suffix }, + result: jsonValueSchema.parse( + functionDossierSchema.parse( + FUNCTION_COMPARISON_EXAMPLE.left.normalized_result, + ), + ), + confidence: template.confidence, + authority: template.authority, + relationshipCoverage: { + status: "unknown", + reason: "The fixture omits call relationship coverage.", + }, + }, + ); +}; + +const compareIncomplete = async (client: Client, suffix: string) => { + const response = await client.callTool({ + name: "compare_functions", + arguments: { + left: FUNCTION_COMPARISON_EXAMPLE.left, + right: incompleteRight(suffix), + }, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const comparison = parseEvidence(response.structuredContent); + expect(comparison.normalized_result).toMatchObject({ status: "unknown" }); + return comparison; +}; + +const verifyCallClaim = async ( + client: Client, + comparison: ReturnType, +) => + client.callTool({ + name: "verify_reconstruction", + arguments: { + specification: { + name: "Same unresolved call claim", + claims: [ + { + kind: "structural-function", + claim_id: "call-coverage", + title: "Call coverage is complete", + comparison_evidence_id: comparison.evidence_id, + dimension: "calls", + }, + ], + }, + }, + }); + +const assertUnknownEvidence = ( + response: Awaited>, +) => { + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect( + parseEvidence(response.structuredContent).normalized_result, + ).toMatchObject({ status: "unknown" }); +}; + +it("retains separate unresolved call-path workflows in one session", async () => { + await withClient(async (client, session) => { + const source = pathSource(); + expect(session.recordEvidence(source).ok).toBe(true); + const responses = []; + for (const goal of ["0x3000", "0x4000", "0x3000"]) + responses.push( + await client.callTool({ + name: "build_call_path", + arguments: { + functions: [source], + start: { address: "0x1000" }, + goal: { address: goal }, + }, + }), + ); + responses.forEach(assertUnknownEvidence); + const ids = responses.map( + (response) => parseEvidence(response.structuredContent).evidence_id, + ); + expect(ids[2]).toBe(ids[0]); + expect(ids[1]).not.toBe(ids[0]); + expect( + session + .listUnknowns({ domain: "call-path" }) + .map(({ supporting_evidence_ids }) => supporting_evidence_ids), + ).toEqual(expect.arrayContaining([...new Set(ids)].map((id) => [id]))); + expect( + session + .listUnknowns({ domain: "call-path" }) + .map(({ question }) => question) + .sort(), + ).toEqual( + [...new Set(ids)] + .map( + (id) => + `Can the requested call path be established from complete analysis? (Evidence ${id})`, + ) + .sort(), + ); + const unknowns = await client.callTool({ + name: "list_unknowns", + arguments: { domain: "call-path" }, + }); + expect(unknowns.structuredContent).toMatchObject({ + result: { total: 2, items: expect.any(Array) }, + }); + }); +}); + +it("retains separate unresolved reconstruction verifications with identical claim names", async () => { + await withClient(async (client, session) => { + const firstComparison = await compareIncomplete( + client, + "verification-unknown-a", + ); + const secondComparison = await compareIncomplete( + client, + "verification-unknown-b", + ); + const responses = [ + await verifyCallClaim(client, firstComparison), + await verifyCallClaim(client, secondComparison), + await verifyCallClaim(client, firstComparison), + ]; + responses.forEach(assertUnknownEvidence); + const ids = responses.map( + (response) => parseEvidence(response.structuredContent).evidence_id, + ); + expect(ids[2]).toBe(ids[0]); + expect(ids[1]).not.toBe(ids[0]); + expect( + session + .listUnknowns({ domain: "reconstruction-verification" }) + .map(({ supporting_evidence_ids }) => supporting_evidence_ids), + ).toEqual(expect.arrayContaining([...new Set(ids)].map((id) => [id]))); + const unknowns = await client.callTool({ + name: "list_unknowns", + arguments: { domain: "reconstruction-verification" }, + }); + expect(unknowns.structuredContent).toMatchObject({ + result: { total: 2, items: expect.any(Array) }, + }); + }); +}); + +it("continues rejecting duplicate explicit record_unknown identities", async () => { + await withClient(async (client) => { + const input = { + question: "Does this explicitly submitted question remain unique?", + severity: "medium", + domain: "manual-duplicate-control", + required_authority: null, + required_confidence: "derived", + required_environment: null, + recommended_probes: [], + relationships: [], + }; + const first = await client.callTool({ + name: "record_unknown", + arguments: input, + }); + expect(first.isError).not.toBe(true); + const duplicate = await client.callTool({ + name: "record_unknown", + arguments: input, + }); + expect(parseMcpToolError(duplicate).error.code).toBe("revision_conflict"); + }); +}); diff --git a/tests/boundary/process/processSessionMcp.test.ts b/tests/boundary/process/processSessionMcp.test.ts index 41e2c25856..a26f422e6d 100644 --- a/tests/boundary/process/processSessionMcp.test.ts +++ b/tests/boundary/process/processSessionMcp.test.ts @@ -199,9 +199,7 @@ itWithCaptureCapability( captured.structuredContent, ), ).toBe(true); - expect( - contract.outputSchema.safeParse(captured.structuredContent).success, - ).toBe(true); + const evidence = contract.outputSchema.parse(captured.structuredContent); const listedUnknowns = await mcp.callTool({ name: "list_unknowns", arguments: {}, @@ -224,7 +222,7 @@ itWithCaptureCapability( .parse(listedUnknowns.structuredContent).result.items; expect(listed).toContainEqual( expect.objectContaining({ - question: "Was network behavior fully observed during capture?", + question: `Was network behavior fully observed during capture? (Evidence ${evidence.evidence_id})`, domain: "process-network", }), );