diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index a4de713cb5..840e7f241e 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -132,6 +132,12 @@ is null. `total` counts all matching heads; each page preserves complete records in stable unknown ID order. Filters apply before paging. Pages reflect current session state, so keep the unknown ledger unchanged while traversing it. +Automatically retained workflow questions include their immutable result +Evidence ID. Distinct results retain independent questions and source links; +repeating the same result does not create another unknown. Workflow Evidence +keeps the original residual question text. Explicit `record_unknown` requests +keep caller-selected identity and reject duplicate creation. + For a portable exported bundle, the CLI shares this projection: `rea inspect-evidence-bundle '{"path":"/tmp/evidence.json","detail":"summary","filters":{"operation":"analyze_function"}}' --format json`. The CLI validates and materializes that local bundle under its existing memory diff --git a/src/application/WorkflowEvidence.test.ts b/src/application/WorkflowEvidence.test.ts index d62522fccc..d2346411fc 100644 --- a/src/application/WorkflowEvidence.test.ts +++ b/src/application/WorkflowEvidence.test.ts @@ -212,7 +212,7 @@ describe("workflow residual question retention", () => { expect(recordWorkflowUnknowns(input).ok).toBe(true); expect(records.listUnknowns()).toMatchObject([ { - question: result.residual_unknowns[0], + question: `${result.residual_unknowns[0]} (Evidence ${evidence.evidence_id})`, domain: "native-api", supporting_evidence_ids: [evidence.evidence_id], required_authority: "shipped-artifact", @@ -220,6 +220,47 @@ describe("workflow residual question retention", () => { ]); }); + it("retains one identical residual question per distinct workflow Evidence", () => { + const records = new InvestigationRecords(); + const result = { + residual_unknowns: ["Which provider resolves this boundary?"], + }; + const evidence = ["0x1000", "0x2000"].map((procedure) => + createWorkflowEvidence({ + target: undefined, + operation: "inspect_native_api", + parameters: { procedure }, + result, + upstreamProfile: undefined, + upstreamLimitations: [], + }), + ); + expect(evidence[0]?.evidence_id).not.toBe(evidence[1]?.evidence_id); + + for (const item of evidence) { + expect(records.recordEvidence(item).ok).toBe(true); + const input = { + name: "inspect_native_api", + result, + evidenceId: item.evidence_id, + recordUnknown: ( + unknown: Parameters[0], + ) => records.recordUnknown(unknown), + }; + expect(recordWorkflowUnknowns(input).ok).toBe(true); + expect(recordWorkflowUnknowns(input).ok).toBe(true); + } + + expect(records.listUnknowns()).toHaveLength(2); + expect( + records + .listUnknowns() + .map(({ supporting_evidence_ids }) => supporting_evidence_ids), + ).toEqual( + expect.arrayContaining(evidence.map(({ evidence_id }) => [evidence_id])), + ); + }); + it.each([ ["spaces", " Which boundary? "], ["line feed and tab", "\nWhich boundary?\t"], @@ -248,7 +289,7 @@ describe("workflow residual question retention", () => { }).ok, ).toBe(true); expect(records.listUnknowns()).toMatchObject([ - { question: "Which boundary?" }, + { question: `Which boundary? (Evidence ${evidence.evidence_id})` }, ]); expect(evidence.normalized_result).toEqual(result); }, @@ -328,7 +369,7 @@ it.each([ }).ok, ).toBe(true); expect(records.listUnknowns()).toMatchObject([ - { question: "Which boundary?" }, + { question: `Which boundary? (Evidence ${evidence.evidence_id})` }, ]); expect(records.evidenceById(evidence.evidence_id)).toEqual(evidence); expect(evidence.normalized_result).toMatchObject({ diff --git a/src/application/WorkflowEvidence.ts b/src/application/WorkflowEvidence.ts index 599a5de290..f707ae8092 100644 --- a/src/application/WorkflowEvidence.ts +++ b/src/application/WorkflowEvidence.ts @@ -23,6 +23,7 @@ import { import type { JsonValue } from "../domain/jsonValue.js"; import type { UnknownRegistryPort } from "./investigation/InvestigationRecordPort.js"; import { UnknownRegistryError } from "../domain/unknownRegistryError.js"; +import { workflowUnknownInput } from "./investigation/WorkflowUnknown.js"; import { renderSourceLimitations, sourceLimitationKey, @@ -180,12 +181,10 @@ export const recordWorkflowUnknowns = ({ return { ok: true, value: null }; for (const question of result.residual_unknowns) { if (typeof question !== "string") continue; - const recorded = recordUnknown({ + const input = workflowUnknownInput(evidenceId, { question, severity: "medium", domain: name === "inspect_native_api" ? "native-api" : "control-flow", - supporting_evidence_ids: [evidenceId], - contradicting_evidence_ids: [], required_authority: "shipped-artifact", required_confidence: "observed", required_environment: null, @@ -200,6 +199,8 @@ export const recordWorkflowUnknowns = ({ ], relationships: [], }); + if (!input.ok) return input; + const recorded = recordUnknown(input.value); if ( !recorded.ok && !( diff --git a/src/application/investigation/WorkflowUnknown.ts b/src/application/investigation/WorkflowUnknown.ts new file mode 100644 index 0000000000..9d36595b14 --- /dev/null +++ b/src/application/investigation/WorkflowUnknown.ts @@ -0,0 +1,29 @@ +import { + recordUnknownInputSchema, + type RecordUnknownInput, +} from "../../domain/residualUnknown.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { UnknownRegistryError } from "../../domain/unknownRegistryError.js"; + +/** Scope an automatic question to its immutable result before registry mutation. */ +export const workflowUnknownInput = ( + evidenceId: string, + input: Omit< + RecordUnknownInput, + "supporting_evidence_ids" | "contradicting_evidence_ids" + >, +): Result => { + const question = recordUnknownInputSchema.shape.question.safeParse( + input.question, + ); + if (!question.success) + return err( + new UnknownRegistryError("invalid-transition", { cause: question.error }), + ); + return ok({ + ...input, + question: `${question.data} (Evidence ${evidenceId})`, + supporting_evidence_ids: [evidenceId], + contradicting_evidence_ids: [], + }); +}; diff --git a/src/server/registerInvestigationTools/helpers.ts b/src/server/registerInvestigationTools/helpers.ts index 557f96225f..556225d55a 100644 --- a/src/server/registerInvestigationTools/helpers.ts +++ b/src/server/registerInvestigationTools/helpers.ts @@ -4,6 +4,7 @@ import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import { err, ok, type Result } from "../../domain/result.js"; import { recordDerivedEvidence } from "../recordDerivedEvidence.js"; +import { workflowUnknownInput } from "../../application/investigation/WorkflowUnknown.js"; import type { WorkflowUnknownInput } from "./types.js"; import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; @@ -64,16 +65,17 @@ export const recordWorkflowEvidence = ( if (!unresolved) { return recordDerivedEvidence(session, evidence, undefined); } - return recordDerivedEvidence(session, evidence, { + const unknown = workflowUnknownInput(evidence.evidence_id, { question: input.question, severity: "high", domain: input.domain, - supporting_evidence_ids: [evidence.evidence_id], - contradicting_evidence_ids: [], required_authority: input.requiredAuthority, required_confidence: input.requiredConfidence, required_environment: null, recommended_probes: [...input.probes], relationships: [], }); + return unknown.ok + ? recordDerivedEvidence(session, evidence, unknown.value) + : unknown; }; diff --git a/src/server/registerManagedWorkflowTools/compareManagedMembers.ts b/src/server/registerManagedWorkflowTools/compareManagedMembers.ts index cea75f4574..6313211c41 100644 --- a/src/server/registerManagedWorkflowTools/compareManagedMembers.ts +++ b/src/server/registerManagedWorkflowTools/compareManagedMembers.ts @@ -1,3 +1,4 @@ +import { workflowUnknownInput } from "../../application/investigation/WorkflowUnknown.js"; import type { EvidenceMcpServer } from "../EvidenceMcpServer.js"; import { runAdmittedToolOperation } from "../admittedToolOperation.js"; import { recordSessionEvidenceSources } from "../sessionEvidence.js"; @@ -64,14 +65,12 @@ export const registerCompareManagedMembers = ( result.value.normalized_result, ); const unknown = comparison.summary.unknown > 0; - const output = unknown - ? options.recordEvidenceWithUnknown?.(result.value, { + const unknownInput = unknown + ? workflowUnknownInput(result.value.evidence_id, { question: "Which managed members remain unmatched or ambiguous across these versions?", severity: "medium", domain: "managed-member-comparison", - supporting_evidence_ids: [result.value.evidence_id], - contradicting_evidence_ids: [], required_authority: "shipped-artifact", required_confidence: "observed", required_environment: null, @@ -84,7 +83,16 @@ export const registerCompareManagedMembers = ( ], relationships: [], }) - : options.recordEvidence?.(result.value); + : undefined; + const output = + unknownInput === undefined + ? options.recordEvidence?.(result.value) + : unknownInput.ok + ? options.recordEvidenceWithUnknown?.( + result.value, + unknownInput.value, + ) + : unknownInput; return server.delivery.toEvidenceToolResult( result.value, compareContract, diff --git a/src/server/registerManagedWorkflowTools/verifyManagedNativeBoundaries.ts b/src/server/registerManagedWorkflowTools/verifyManagedNativeBoundaries.ts index 02559c680c..5bea7f9521 100644 --- a/src/server/registerManagedWorkflowTools/verifyManagedNativeBoundaries.ts +++ b/src/server/registerManagedWorkflowTools/verifyManagedNativeBoundaries.ts @@ -1,3 +1,4 @@ +import { workflowUnknownInput } from "../../application/investigation/WorkflowUnknown.js"; import type { EvidenceMcpServer } from "../EvidenceMcpServer.js"; import { runAdmittedToolOperation } from "../admittedToolOperation.js"; import { recordSessionEvidenceSources } from "../sessionEvidence.js"; @@ -83,14 +84,12 @@ export const registerVerifyManagedNativeBoundaries = ( verification.summary.unresolved > 0 || verification.summary.contradicted > 0 || verification.summary.native_body_unresolved > 0; - const output = unknown - ? options.recordEvidenceWithUnknown?.(result.value, { + const unknownInput = unknown + ? workflowUnknownInput(result.value.evidence_id, { question: "Which managed/native boundaries remain unresolved or contradicted by the supplied native Evidence?", severity: "medium", domain: "managed-native-verification", - supporting_evidence_ids: [result.value.evidence_id], - contradicting_evidence_ids: [], required_authority: "shipped-artifact", required_confidence: "observed", required_environment: null, @@ -108,7 +107,16 @@ export const registerVerifyManagedNativeBoundaries = ( ], relationships: [], }) - : options.recordEvidence?.(result.value); + : undefined; + const output = + unknownInput === undefined + ? options.recordEvidence?.(result.value) + : unknownInput.ok + ? options.recordEvidenceWithUnknown?.( + result.value, + unknownInput.value, + ) + : unknownInput; return server.delivery.toEvidenceToolResult( result.value, nativeVerificationContract, diff --git a/src/server/registerSessionTools.ts b/src/server/registerSessionTools.ts index 7c992112d3..d93ddcebf3 100644 --- a/src/server/registerSessionTools.ts +++ b/src/server/registerSessionTools.ts @@ -1,5 +1,6 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { workflowUnknownInput } from "../application/investigation/WorkflowUnknown.js"; import { getNavigationContext, @@ -48,12 +49,10 @@ const recordProcessResidualUnknowns = ( residuals: ProcessCapture["residual_unknowns"], ): Result => { for (const residual of residuals) { - const unknown = session.recordUnknown({ + const input = workflowUnknownInput(evidence.evidence_id, { question: `Was ${residual.scope} behavior fully observed during capture?`, severity: "medium", domain: `process-${residual.scope}`, - supporting_evidence_ids: [evidence.evidence_id], - contradicting_evidence_ids: [], required_authority: "controlled-replay", required_confidence: "observed", required_environment: evidence.environment, @@ -66,6 +65,8 @@ const recordProcessResidualUnknowns = ( ], relationships: [], }); + if (!input.ok) return input; + const unknown = session.recordUnknown(input.value); if ( !unknown.ok && !( diff --git a/tests/boundary/mcp/ghidraNativeApiMcp.test.ts b/tests/boundary/mcp/ghidraNativeApiMcp.test.ts index 2988d1a179..0a31f33baf 100644 --- a/tests/boundary/mcp/ghidraNativeApiMcp.test.ts +++ b/tests/boundary/mcp/ghidraNativeApiMcp.test.ts @@ -9,7 +9,11 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; import { nativeLoadImageSchema } from "../../../src/domain/native/nativeLoadImage.js"; +import { nativeApiInspectionResultSchema } from "../../../src/domain/native/nativeApiBoundary.js"; +import { functionDossierSchema } from "../../../src/domain/hopperValues.js"; import { ok } from "../../../src/domain/result.js"; +import { ghidraFunctionDossier } from "../../../src/domain/ghidraValues.fixture.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; it.each([ { state: "final", failed: [] }, @@ -129,3 +133,121 @@ describe("Ghidra MCP native API evidence", () => { } }, 30_000); }); + +it("retains unavailable Ghidra native API observations across provenance and exact replay", async () => { + const residual = "The controlled provider omitted native API boundary facts."; + // This wire fixture is controlled and never launches Ghidra. + const harness = await connectGhidraMcp( + "ghidra-native-api-unavailable-identity", + async (operation, input) => { + if (operation !== "analyze_function") + throw new Error(`Unexpected controlled Ghidra operation: ${operation}`); + const procedure = String(input.procedure); + const base = functionDossierSchema.parse(ghidraFunctionDossier()); + const address = procedure === "0x401000" ? procedure : "0x401100"; + return ok( + jsonValueSchema.parse({ + value: functionDossierSchema.parse({ + ...base, + procedure: { + ...base.procedure, + address, + name: address === "0x401000" ? "fixture_main" : "fixture_other", + body: { + ...base.procedure.body, + ranges: [ + { + start: address, + end: address === "0x401000" ? "0x401005" : "0x401105", + }, + ], + }, + }, + native_api: { + available: false, + reason: residual, + residual_unknowns: [residual], + }, + }), + limitations: [], + }), + ); + }, + ); + const { mcp, session } = harness; + try { + const inspect = async (procedure: string) => { + const reply = await mcp.callTool({ + name: "inspect_native_api", + arguments: { procedure }, + }); + expect(reply.isError, JSON.stringify(reply.content)).not.toBe(true); + const evidence = sessionEvidence(session, reply.structuredContent); + const result = nativeApiInspectionResultSchema.parse( + evidence.normalized_result, + ); + expect(result).toMatchObject({ + procedure: { + address: procedure, + }, + boundary: { + available: false, + reason: residual, + residual_unknowns: [residual], + }, + residual_unknowns: [residual], + }); + return evidence; + }; + + const first = await inspect("0x401000"); + const replay = await inspect("0x401000"); + expect(replay.normalized_result).toEqual(first.normalized_result); + expect(replay.evidence_id).not.toBe(first.evidence_id); + expect(replay.limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining("Loaded from a local REA analysis snapshot"), + ]), + ); + const warmBundle = session.exportEvidenceBundle(); + const warmReplay = await inspect("0x401000"); + expect(warmReplay.evidence_id).toBe(replay.evidence_id); + expect(session.exportEvidenceBundle()).toEqual(warmBundle); + + const second = await inspect("0x401100"); + expect(second.evidence_id).not.toBe(first.evidence_id); + expect(session.evidenceById(first.evidence_id)).toEqual(first); + expect(session.evidenceById(replay.evidence_id)).toEqual(replay); + + const listed = await mcp.callTool({ + name: "list_unknowns", + arguments: { domain: "native-api" }, + }); + expect(listed.isError, JSON.stringify(listed.content)).not.toBe(true); + const unknownPage = toolContract("list_unknowns").outputSchema.parse( + listed.structuredContent, + ).result; + expect(unknownPage.total).toBe(3); + expect(unknownPage.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + question: expect.stringContaining(first.evidence_id), + supporting_evidence_ids: [first.evidence_id], + }), + expect.objectContaining({ + question: expect.stringContaining(replay.evidence_id), + supporting_evidence_ids: [replay.evidence_id], + }), + expect.objectContaining({ + question: expect.stringContaining(second.evidence_id), + supporting_evidence_ids: [second.evidence_id], + }), + ]), + ); + expect( + unknownPage.items.every(({ question }) => question.includes(residual)), + ).toBe(true); + } finally { + await harness.close(); + } +}, 30_000); diff --git a/tests/boundary/mcp/investigationAnalysisMcp.test.ts b/tests/boundary/mcp/investigationAnalysisMcp.test.ts index d35cc8a357..6c2420c232 100644 --- a/tests/boundary/mcp/investigationAnalysisMcp.test.ts +++ b/tests/boundary/mcp/investigationAnalysisMcp.test.ts @@ -56,8 +56,7 @@ it("aggregates comparison Evidence and records an approved runtime gap", async ( }); expect(session.listUnknowns({ domain: "changed-behavior" })).toMatchObject([ { - question: - "Did both versions behave the same under a complete controlled replay?", + question: `Did both versions behave the same under a complete controlled replay? (Evidence ${evidence.evidence_id})`, }, ]); } finally { @@ -406,8 +405,7 @@ it.each([ else expect(session.listUnknowns({ domain: "call-path" })).toMatchObject([ { - question: - "Can the requested call path be established from complete analysis?", + question: `Can the requested call path be established from complete analysis? (Evidence ${evidence.evidence_id})`, supporting_evidence_ids: [evidence.evidence_id], }, ]); @@ -417,32 +415,6 @@ it.each([ }, ); -it("records a safe unresolved call-path question", async () => { - const { session, server, client } = await connected(); - const dossier = dossierWithCallees(["0x2000"]); - expect(session.recordEvidence(dossier).ok).toBe(true); - try { - const response = await client.callTool({ - name: "build_call_path", - arguments: { - ...INVESTIGATION_EXAMPLES.build_call_path, - functions: [dossier], - start: { address: "0x1000" }, - goal: { address: "0x3000" }, - }, - }); - expect(response.isError, JSON.stringify(response)).not.toBe(true); - expect(session.listUnknowns({ domain: "call-path" })).toMatchObject([ - { - question: - "Can the requested call path be established from complete analysis?", - }, - ]); - } finally { - await close(session, server, client); - } -}); - it("records an explicit non-causal static/runtime hypothesis", async () => { const { session, server, client } = await connected(); for (const evidence of [ @@ -586,7 +558,7 @@ it("cannot omit a session-owned active unknown from reconstruction input", async session.listUnknowns({ domain: "reconstruction-verification" }), ).toContainEqual( expect.objectContaining({ - question: "Does the reconstruction satisfy every declared claim?", + question: `Does the reconstruction satisfy every declared claim? (Evidence ${evidence.evidence_id})`, }), ); } finally { diff --git a/tests/boundary/mcp/managedWorkflowUnknownIdentityMcp.test.ts b/tests/boundary/mcp/managedWorkflowUnknownIdentityMcp.test.ts new file mode 100644 index 0000000000..56b044b78f --- /dev/null +++ b/tests/boundary/mcp/managedWorkflowUnknownIdentityMcp.test.ts @@ -0,0 +1,257 @@ +import { expect, it, onTestFinished } from "vitest"; + +import { + MANAGED_NATIVE_VERIFICATION_EXAMPLE, + MANAGED_RECONSTRUCTION_IMPORT_EXAMPLE, +} from "../../../src/contracts/managed/managedWorkflowExamples.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; +import { + managedMemberInspectionSchema, + managedNativeBoundaryInspectionSchema, +} from "../../../src/domain/managed/managedArtifact.js"; +import { managedMemberComparisonResultSchema } from "../../../src/domain/managed/managedMemberComparison.js"; +import { managedNativeVerificationResultSchema } from "../../../src/domain/managed/managedNativeVerificationSchemas.js"; +import { inspectMachoSchema } from "../../../src/domain/native/nativeInspection.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; + +const emptyExportEvidence = () => { + const original = MANAGED_NATIVE_VERIFICATION_EXAMPLE.native_observations[0]; + if (original === undefined || original.subject === null) + throw new Error("Managed verification example is missing native Evidence"); + const result = inspectMachoSchema.parse(original.normalized_result); + return createEvidence( + { + path: "/fixtures/empty-exports.dylib", + sha256: "8".repeat(64), + format: original.subject.format, + ...(original.subject.architecture === null + ? {} + : { architecture: original.subject.architecture }), + }, + original.provider, + { + operation: "inspect_macho", + parameters: original.parameters, + result: jsonValueSchema.parse({ + ...result, + exports: { + items: [], + total: 0, + exhaustive: true, + limitations: [], + }, + }), + rawResult: null, + limitations: [], + }, + ); +}; + +const secondDeclaration = () => { + const original = MANAGED_NATIVE_VERIFICATION_EXAMPLE.managed_boundaries; + const result = managedNativeBoundaryInspectionSchema.parse( + original.normalized_result, + ); + const pinvokes = result.pinvoke_imports; + if (pinvokes.length !== 1) + throw new Error("Managed verification example must contain one P/Invoke"); + const pinvoke = pinvokes[0]; + if (pinvoke === undefined) + throw new Error("Managed verification example has invalid P/Invoke data"); + return createEvidence(undefined, original.provider, { + operation: "inspect_managed_native_boundaries", + parameters: original.parameters, + result: jsonValueSchema.parse({ + ...result, + pinvoke_imports: [ + { + ...pinvoke, + token: "0x1c000002", + member_token: "0x06000002", + member_name: "OpenNativeAgain", + import_name: "open_native_again", + }, + ], + }), + rawResult: null, + limitations: original.limitations, + }); +}; + +const memberEvidence = ( + id: string, + digest: string, + methodName: string | null, + partial: boolean, +) => { + const original = MANAGED_RECONSTRUCTION_IMPORT_EXAMPLE.static_members; + const result = managedMemberInspectionSchema.parse( + original.normalized_result, + ); + const method = result.methods[0]; + if (method === undefined) + throw new Error("Managed reconstruction example is missing a method"); + const normalized = managedMemberInspectionSchema.parse({ + ...result, + artifact: { + ...result.artifact, + path: `/fixtures/${id}.dll`, + sha256: digest, + }, + identity_scope: { + ...result.identity_scope, + requires_artifact_sha256: digest, + }, + methods: methodName === null ? [] : [{ ...method, name: methodName }], + metadata: partial + ? { ...result.metadata, status: "partial" } + : result.metadata, + coverage: partial + ? { + state: "partial", + issues: [ + { + code: "invalid-tables", + scope: "MethodDef", + offset: null, + detail: "The method inventory is incomplete in this fixture.", + }, + ], + } + : result.coverage, + }); + return createEvidence( + { path: `/fixtures/${id}.dll`, sha256: digest, format: "pe" }, + original.provider, + { + operation: "inspect_managed_members", + parameters: original.parameters, + result: jsonValueSchema.parse(normalized), + rawResult: null, + limitations: normalized.limitations, + }, + ); +}; + +it("records distinct unresolved managed boundary checks and makes exact replay idempotent", async () => { + const harness = await createApplicationMcpHarness({ + session: createTestBinarySession(() => { + throw new Error("Managed/native verification must not launch a provider"); + }), + }); + onTestFinished(harness.close); + const { client, session } = harness; + const managed = MANAGED_NATIVE_VERIFICATION_EXAMPLE.managed_boundaries; + const native = emptyExportEvidence(); + + const verify = async (boundaries: typeof managed) => { + const response = await client.callTool({ + name: "verify_managed_native_boundaries", + arguments: { + managed_boundaries: boundaries, + native_observations: [native], + }, + }); + expect(response.isError, JSON.stringify(response.content)).not.toBe(true); + const evidence = parseEvidence(response.structuredContent); + const result = managedNativeVerificationResultSchema.parse( + evidence.normalized_result, + ); + expect(result.summary).toMatchObject({ unresolved: 1 }); + expect(result.pinvoke_imports).toEqual([ + expect.objectContaining({ + status: "unresolved", + basis: "no-native-candidate", + }), + ]); + return evidence; + }; + + const first = await verify(managed); + const firstBundle = session.exportEvidenceBundle(); + const replay = await verify(managed); + expect(replay.evidence_id).toBe(first.evidence_id); + expect(session.exportEvidenceBundle()).toEqual(firstBundle); + expect(session.listUnknowns()).toHaveLength(1); + + const second = await verify(secondDeclaration()); + expect(second.evidence_id).not.toBe(first.evidence_id); + const unknowns = await client.callTool({ + name: "list_unknowns", + arguments: { domain: "managed-native-verification" }, + }); + expect(unknowns.isError, JSON.stringify(unknowns.content)).not.toBe(true); + expect(unknowns.structuredContent).toMatchObject({ + result: { + total: 2, + items: expect.arrayContaining([ + expect.objectContaining({ + supporting_evidence_ids: [first.evidence_id], + }), + expect.objectContaining({ + supporting_evidence_ids: [second.evidence_id], + }), + ]), + }, + }); +}); + +it("retains distinct unresolved managed member comparisons and exact replay", async () => { + const harness = await createApplicationMcpHarness({ + session: createTestBinarySession(() => { + throw new Error("Managed member comparison must not launch a provider"); + }), + }); + onTestFinished(harness.close); + const { client, session } = harness; + const right = memberEvidence("members-partial", "a".repeat(64), null, true); + + const compare = async (left: ReturnType) => { + const response = await client.callTool({ + name: "compare_managed_members", + arguments: { left, right }, + }); + expect(response.isError, JSON.stringify(response.content)).not.toBe(true); + const evidence = parseEvidence(response.structuredContent); + const result = managedMemberComparisonResultSchema.parse( + evidence.normalized_result, + ); + expect(result.summary.unknown).toBeGreaterThan(0); + return evidence; + }; + + const first = await compare( + memberEvidence("members-first", "b".repeat(64), "Main", false), + ); + const firstBundle = session.exportEvidenceBundle(); + const replay = await compare( + memberEvidence("members-first", "b".repeat(64), "Main", false), + ); + expect(replay.evidence_id).toBe(first.evidence_id); + expect(session.exportEvidenceBundle()).toEqual(firstBundle); + + const second = await compare( + memberEvidence("members-second", "c".repeat(64), "Open", false), + ); + expect(second.evidence_id).not.toBe(first.evidence_id); + const unknowns = await client.callTool({ + name: "list_unknowns", + arguments: { domain: "managed-member-comparison" }, + }); + expect(unknowns.isError, JSON.stringify(unknowns.content)).not.toBe(true); + expect(unknowns.structuredContent).toMatchObject({ + result: { + total: 2, + items: expect.arrayContaining([ + expect.objectContaining({ + supporting_evidence_ids: [first.evidence_id], + }), + expect.objectContaining({ + supporting_evidence_ids: [second.evidence_id], + }), + ]), + }, + }); +}); diff --git a/tests/boundary/mcp/processResidualUnknownIdentityMcp.test.ts b/tests/boundary/mcp/processResidualUnknownIdentityMcp.test.ts new file mode 100644 index 0000000000..48d9b84765 --- /dev/null +++ b/tests/boundary/mcp/processResidualUnknownIdentityMcp.test.ts @@ -0,0 +1,72 @@ +import { fileURLToPath } from "node:url"; + +import { expect } from "vitest"; + +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { silentLogger } from "../../../src/logger.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { itWithCaptureCapability } from "../process/processCaptureCapability.js"; + +const partialProcessFixture = fileURLToPath( + new URL("../../fixtures/processFidelity.mjs", import.meta.url), +); + +itWithCaptureCapability( + "retains residuals from distinct process captures with the same missing scope", + async () => { + const harness = await createApplicationMcpHarness({ + session: createTestBinarySession(() => { + throw new Error("Process capture must not launch an analysis provider"); + }), + serverOptions: { logger: silentLogger }, + }); + const { client } = harness; + try { + const captures = []; + for (const settleMs of [50, 75]) { + const response = await client.callTool({ + name: "capture_process_scenario", + arguments: { + executable: process.execPath, + arguments: [partialProcessFixture, "partial"], + settle_ms: settleMs, + }, + }); + expect(response.isError, JSON.stringify(response.content)).not.toBe( + true, + ); + captures.push(parseEvidence(response.structuredContent)); + } + expect(captures[0]?.evidence_id).not.toBe(captures[1]?.evidence_id); + for (const capture of captures) + expect(capture.normalized_result).toMatchObject({ + residual_unknowns: expect.arrayContaining([ + expect.objectContaining({ scope: "network" }), + ]), + }); + + const unknowns = await client.callTool({ + name: "list_unknowns", + arguments: { domain: "process-network" }, + }); + expect(unknowns.isError, JSON.stringify(unknowns.content)).not.toBe(true); + expect(unknowns.structuredContent).toMatchObject({ + result: { + total: 2, + items: expect.arrayContaining( + captures.map((capture) => + expect.objectContaining({ + question: `Was network behavior fully observed during capture? (Evidence ${capture.evidence_id})`, + supporting_evidence_ids: [capture.evidence_id], + }), + ), + ), + }, + }); + } finally { + await harness.close(); + } + }, + 30_000, +); diff --git a/tests/boundary/mcp/workflowUnknownIdentityMcp.test.ts b/tests/boundary/mcp/workflowUnknownIdentityMcp.test.ts new file mode 100644 index 0000000000..d563d571ca --- /dev/null +++ b/tests/boundary/mcp/workflowUnknownIdentityMcp.test.ts @@ -0,0 +1,248 @@ +import type { Client } from "@modelcontextprotocol/client"; +import { expect, it } from "vitest"; + +import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { functionDossierSchema } from "../../../src/domain/hopperValues.js"; +import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; + +const withClient = async ( + run: ( + client: Client, + session: ReturnType, + ) => Promise, +): Promise => { + const session = createTestBinarySession(() => { + throw new Error("This workflow must not launch an analysis provider"); + }); + const harness = await createApplicationMcpHarness({ session }); + try { + await run(harness.client, session); + } finally { + await harness.close(); + } +}; + +const pathSource = () => { + const template = FUNCTION_COMPARISON_EXAMPLE.left; + if (template.subject === null) + throw new Error("Call-path fixtures must be artifact-bound"); + const dossier = functionDossierSchema.parse(template.normalized_result); + return createEvidence( + { + path: `${template.subject.local_path}/workflow-unknown-path`, + sha256: template.subject.digest.sha256, + format: template.subject.format, + }, + template.provider, + { + operation: "analyze_function", + parameters: { + ...template.parameters, + observation: "workflow-unknown-path", + }, + result: jsonValueSchema.parse({ + ...dossier, + callees: [ + { + address: "0x2000", + name: "missing-helper", + classification: null, + body: { available: false, reason: "No body was observed." }, + }, + ], + }), + confidence: template.confidence, + authority: template.authority, + relationshipCoverage: { status: "complete" }, + }, + ); +}; + +const incompleteRight = (suffix: string) => { + const template = FUNCTION_COMPARISON_EXAMPLE.right; + if (template.subject === null) + throw new Error("Comparison fixtures must be artifact-bound"); + return createEvidence( + { + path: `${template.subject.local_path}/${suffix}`, + sha256: template.subject.digest.sha256, + format: template.subject.format, + }, + template.provider, + { + operation: "analyze_function", + parameters: { ...template.parameters, observation: suffix }, + result: jsonValueSchema.parse( + functionDossierSchema.parse( + FUNCTION_COMPARISON_EXAMPLE.left.normalized_result, + ), + ), + confidence: template.confidence, + authority: template.authority, + relationshipCoverage: { + status: "unknown", + reason: "The fixture omits call relationship coverage.", + }, + }, + ); +}; + +const compareIncomplete = async (client: Client, suffix: string) => { + const response = await client.callTool({ + name: "compare_functions", + arguments: { + left: FUNCTION_COMPARISON_EXAMPLE.left, + right: incompleteRight(suffix), + }, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const comparison = parseEvidence(response.structuredContent); + expect(comparison.normalized_result).toMatchObject({ status: "unknown" }); + return comparison; +}; + +const verifyCallClaim = async ( + client: Client, + comparison: ReturnType, +) => + client.callTool({ + name: "verify_reconstruction", + arguments: { + specification: { + name: "Same unresolved call claim", + claims: [ + { + kind: "structural-function", + claim_id: "call-coverage", + title: "Call coverage is complete", + comparison_evidence_id: comparison.evidence_id, + dimension: "calls", + }, + ], + }, + }, + }); + +const assertUnknownEvidence = ( + response: Awaited>, +) => { + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect( + parseEvidence(response.structuredContent).normalized_result, + ).toMatchObject({ status: "unknown" }); +}; + +it("retains separate unresolved call-path workflows in one session", async () => { + await withClient(async (client, session) => { + const source = pathSource(); + expect(session.recordEvidence(source).ok).toBe(true); + const responses = []; + for (const goal of ["0x3000", "0x4000", "0x3000"]) + responses.push( + await client.callTool({ + name: "build_call_path", + arguments: { + functions: [source], + start: { address: "0x1000" }, + goal: { address: goal }, + }, + }), + ); + responses.forEach(assertUnknownEvidence); + const ids = responses.map( + (response) => parseEvidence(response.structuredContent).evidence_id, + ); + expect(ids[2]).toBe(ids[0]); + expect(ids[1]).not.toBe(ids[0]); + expect( + session + .listUnknowns({ domain: "call-path" }) + .map(({ supporting_evidence_ids }) => supporting_evidence_ids), + ).toEqual(expect.arrayContaining([...new Set(ids)].map((id) => [id]))); + expect( + session + .listUnknowns({ domain: "call-path" }) + .map(({ question }) => question) + .sort(), + ).toEqual( + [...new Set(ids)] + .map( + (id) => + `Can the requested call path be established from complete analysis? (Evidence ${id})`, + ) + .sort(), + ); + const unknowns = await client.callTool({ + name: "list_unknowns", + arguments: { domain: "call-path" }, + }); + expect(unknowns.structuredContent).toMatchObject({ + result: { total: 2, items: expect.any(Array) }, + }); + }); +}); + +it("retains separate unresolved reconstruction verifications with identical claim names", async () => { + await withClient(async (client, session) => { + const firstComparison = await compareIncomplete( + client, + "verification-unknown-a", + ); + const secondComparison = await compareIncomplete( + client, + "verification-unknown-b", + ); + const responses = [ + await verifyCallClaim(client, firstComparison), + await verifyCallClaim(client, secondComparison), + await verifyCallClaim(client, firstComparison), + ]; + responses.forEach(assertUnknownEvidence); + const ids = responses.map( + (response) => parseEvidence(response.structuredContent).evidence_id, + ); + expect(ids[2]).toBe(ids[0]); + expect(ids[1]).not.toBe(ids[0]); + expect( + session + .listUnknowns({ domain: "reconstruction-verification" }) + .map(({ supporting_evidence_ids }) => supporting_evidence_ids), + ).toEqual(expect.arrayContaining([...new Set(ids)].map((id) => [id]))); + const unknowns = await client.callTool({ + name: "list_unknowns", + arguments: { domain: "reconstruction-verification" }, + }); + expect(unknowns.structuredContent).toMatchObject({ + result: { total: 2, items: expect.any(Array) }, + }); + }); +}); + +it("continues rejecting duplicate explicit record_unknown identities", async () => { + await withClient(async (client) => { + const input = { + question: "Does this explicitly submitted question remain unique?", + severity: "medium", + domain: "manual-duplicate-control", + required_authority: null, + required_confidence: "derived", + required_environment: null, + recommended_probes: [], + relationships: [], + }; + const first = await client.callTool({ + name: "record_unknown", + arguments: input, + }); + expect(first.isError).not.toBe(true); + const duplicate = await client.callTool({ + name: "record_unknown", + arguments: input, + }); + expect(parseMcpToolError(duplicate).error.code).toBe("revision_conflict"); + }); +}); diff --git a/tests/boundary/process/processSessionMcp.test.ts b/tests/boundary/process/processSessionMcp.test.ts index 41e2c25856..a26f422e6d 100644 --- a/tests/boundary/process/processSessionMcp.test.ts +++ b/tests/boundary/process/processSessionMcp.test.ts @@ -199,9 +199,7 @@ itWithCaptureCapability( captured.structuredContent, ), ).toBe(true); - expect( - contract.outputSchema.safeParse(captured.structuredContent).success, - ).toBe(true); + const evidence = contract.outputSchema.parse(captured.structuredContent); const listedUnknowns = await mcp.callTool({ name: "list_unknowns", arguments: {}, @@ -224,7 +222,7 @@ itWithCaptureCapability( .parse(listedUnknowns.structuredContent).result.items; expect(listed).toContainEqual( expect.objectContaining({ - question: "Was network behavior fully observed during capture?", + question: `Was network behavior fully observed during capture? (Evidence ${evidence.evidence_id})`, domain: "process-network", }), );