mirror of
https://github.com/morluto/rea
synced 2026-10-26 06:16:17 -04:00
fix(investigation): scope automatic unknowns to result Evidence
Construct automatic unknown identity once at the application boundary. Validate the original question before adding immutable Evidence scope, and preserve each result's supporting link through CLI and MCP producers. Apply the shared policy to derived investigations, managed comparisons and verification, composed workflow residuals, and process-capture residuals. This prevents registry conflicts and silent loss of later result links. Keep explicit creation/update semantics and exact Evidence replays intact. Cover all affected producer families with public MCP regressions, preserve snapshot provenance distinctions, and retain blank-question/error handling. Remove the superseded single-result call-path test. Fixes #3333 Co-authored-by: Dragolone <160818051+Dragolone@users.noreply.github.com>
This commit is contained in:
@@ -132,6 +132,12 @@ is null. `total` counts all matching heads; each page preserves complete records
|
||||
in stable unknown ID order. Filters apply before paging. Pages reflect current
|
||||
session state, so keep the unknown ledger unchanged while traversing it.
|
||||
|
||||
Automatically retained workflow questions include their immutable result
|
||||
Evidence ID. Distinct results retain independent questions and source links;
|
||||
repeating the same result does not create another unknown. Workflow Evidence
|
||||
keeps the original residual question text. Explicit `record_unknown` requests
|
||||
keep caller-selected identity and reject duplicate creation.
|
||||
|
||||
For a portable exported bundle, the CLI shares this projection:
|
||||
`rea inspect-evidence-bundle '{"path":"/tmp/evidence.json","detail":"summary","filters":{"operation":"analyze_function"}}' --format json`.
|
||||
The CLI validates and materializes that local bundle under its existing memory
|
||||
|
||||
@@ -212,7 +212,7 @@ describe("workflow residual question retention", () => {
|
||||
expect(recordWorkflowUnknowns(input).ok).toBe(true);
|
||||
expect(records.listUnknowns()).toMatchObject([
|
||||
{
|
||||
question: result.residual_unknowns[0],
|
||||
question: `${result.residual_unknowns[0]} (Evidence ${evidence.evidence_id})`,
|
||||
domain: "native-api",
|
||||
supporting_evidence_ids: [evidence.evidence_id],
|
||||
required_authority: "shipped-artifact",
|
||||
@@ -220,6 +220,47 @@ describe("workflow residual question retention", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("retains one identical residual question per distinct workflow Evidence", () => {
|
||||
const records = new InvestigationRecords();
|
||||
const result = {
|
||||
residual_unknowns: ["Which provider resolves this boundary?"],
|
||||
};
|
||||
const evidence = ["0x1000", "0x2000"].map((procedure) =>
|
||||
createWorkflowEvidence({
|
||||
target: undefined,
|
||||
operation: "inspect_native_api",
|
||||
parameters: { procedure },
|
||||
result,
|
||||
upstreamProfile: undefined,
|
||||
upstreamLimitations: [],
|
||||
}),
|
||||
);
|
||||
expect(evidence[0]?.evidence_id).not.toBe(evidence[1]?.evidence_id);
|
||||
|
||||
for (const item of evidence) {
|
||||
expect(records.recordEvidence(item).ok).toBe(true);
|
||||
const input = {
|
||||
name: "inspect_native_api",
|
||||
result,
|
||||
evidenceId: item.evidence_id,
|
||||
recordUnknown: (
|
||||
unknown: Parameters<InvestigationRecords["recordUnknown"]>[0],
|
||||
) => records.recordUnknown(unknown),
|
||||
};
|
||||
expect(recordWorkflowUnknowns(input).ok).toBe(true);
|
||||
expect(recordWorkflowUnknowns(input).ok).toBe(true);
|
||||
}
|
||||
|
||||
expect(records.listUnknowns()).toHaveLength(2);
|
||||
expect(
|
||||
records
|
||||
.listUnknowns()
|
||||
.map(({ supporting_evidence_ids }) => supporting_evidence_ids),
|
||||
).toEqual(
|
||||
expect.arrayContaining(evidence.map(({ evidence_id }) => [evidence_id])),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["spaces", " Which boundary? "],
|
||||
["line feed and tab", "\nWhich boundary?\t"],
|
||||
@@ -248,7 +289,7 @@ describe("workflow residual question retention", () => {
|
||||
}).ok,
|
||||
).toBe(true);
|
||||
expect(records.listUnknowns()).toMatchObject([
|
||||
{ question: "Which boundary?" },
|
||||
{ question: `Which boundary? (Evidence ${evidence.evidence_id})` },
|
||||
]);
|
||||
expect(evidence.normalized_result).toEqual(result);
|
||||
},
|
||||
@@ -328,7 +369,7 @@ it.each([
|
||||
}).ok,
|
||||
).toBe(true);
|
||||
expect(records.listUnknowns()).toMatchObject([
|
||||
{ question: "Which boundary?" },
|
||||
{ question: `Which boundary? (Evidence ${evidence.evidence_id})` },
|
||||
]);
|
||||
expect(records.evidenceById(evidence.evidence_id)).toEqual(evidence);
|
||||
expect(evidence.normalized_result).toMatchObject({
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import type { UnknownRegistryPort } from "./investigation/InvestigationRecordPort.js";
|
||||
import { UnknownRegistryError } from "../domain/unknownRegistryError.js";
|
||||
import { workflowUnknownInput } from "./investigation/WorkflowUnknown.js";
|
||||
import {
|
||||
renderSourceLimitations,
|
||||
sourceLimitationKey,
|
||||
@@ -180,12 +181,10 @@ export const recordWorkflowUnknowns = ({
|
||||
return { ok: true, value: null };
|
||||
for (const question of result.residual_unknowns) {
|
||||
if (typeof question !== "string") continue;
|
||||
const recorded = recordUnknown({
|
||||
const input = workflowUnknownInput(evidenceId, {
|
||||
question,
|
||||
severity: "medium",
|
||||
domain: name === "inspect_native_api" ? "native-api" : "control-flow",
|
||||
supporting_evidence_ids: [evidenceId],
|
||||
contradicting_evidence_ids: [],
|
||||
required_authority: "shipped-artifact",
|
||||
required_confidence: "observed",
|
||||
required_environment: null,
|
||||
@@ -200,6 +199,8 @@ export const recordWorkflowUnknowns = ({
|
||||
],
|
||||
relationships: [],
|
||||
});
|
||||
if (!input.ok) return input;
|
||||
const recorded = recordUnknown(input.value);
|
||||
if (
|
||||
!recorded.ok &&
|
||||
!(
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import {
|
||||
recordUnknownInputSchema,
|
||||
type RecordUnknownInput,
|
||||
} from "../../domain/residualUnknown.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import { UnknownRegistryError } from "../../domain/unknownRegistryError.js";
|
||||
|
||||
/** Scope an automatic question to its immutable result before registry mutation. */
|
||||
export const workflowUnknownInput = (
|
||||
evidenceId: string,
|
||||
input: Omit<
|
||||
RecordUnknownInput,
|
||||
"supporting_evidence_ids" | "contradicting_evidence_ids"
|
||||
>,
|
||||
): Result<RecordUnknownInput, UnknownRegistryError> => {
|
||||
const question = recordUnknownInputSchema.shape.question.safeParse(
|
||||
input.question,
|
||||
);
|
||||
if (!question.success)
|
||||
return err(
|
||||
new UnknownRegistryError("invalid-transition", { cause: question.error }),
|
||||
);
|
||||
return ok({
|
||||
...input,
|
||||
question: `${question.data} (Evidence ${evidenceId})`,
|
||||
supporting_evidence_ids: [evidenceId],
|
||||
contradicting_evidence_ids: [],
|
||||
});
|
||||
};
|
||||
@@ -4,6 +4,7 @@ import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import { recordDerivedEvidence } from "../recordDerivedEvidence.js";
|
||||
import { workflowUnknownInput } from "../../application/investigation/WorkflowUnknown.js";
|
||||
import type { WorkflowUnknownInput } from "./types.js";
|
||||
import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";
|
||||
|
||||
@@ -64,16 +65,17 @@ export const recordWorkflowEvidence = (
|
||||
if (!unresolved) {
|
||||
return recordDerivedEvidence(session, evidence, undefined);
|
||||
}
|
||||
return recordDerivedEvidence(session, evidence, {
|
||||
const unknown = workflowUnknownInput(evidence.evidence_id, {
|
||||
question: input.question,
|
||||
severity: "high",
|
||||
domain: input.domain,
|
||||
supporting_evidence_ids: [evidence.evidence_id],
|
||||
contradicting_evidence_ids: [],
|
||||
required_authority: input.requiredAuthority,
|
||||
required_confidence: input.requiredConfidence,
|
||||
required_environment: null,
|
||||
recommended_probes: [...input.probes],
|
||||
relationships: [],
|
||||
});
|
||||
return unknown.ok
|
||||
? recordDerivedEvidence(session, evidence, unknown.value)
|
||||
: unknown;
|
||||
};
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { workflowUnknownInput } from "../../application/investigation/WorkflowUnknown.js";
|
||||
import type { EvidenceMcpServer } from "../EvidenceMcpServer.js";
|
||||
import { runAdmittedToolOperation } from "../admittedToolOperation.js";
|
||||
import { recordSessionEvidenceSources } from "../sessionEvidence.js";
|
||||
@@ -64,14 +65,12 @@ export const registerCompareManagedMembers = (
|
||||
result.value.normalized_result,
|
||||
);
|
||||
const unknown = comparison.summary.unknown > 0;
|
||||
const output = unknown
|
||||
? options.recordEvidenceWithUnknown?.(result.value, {
|
||||
const unknownInput = unknown
|
||||
? workflowUnknownInput(result.value.evidence_id, {
|
||||
question:
|
||||
"Which managed members remain unmatched or ambiguous across these versions?",
|
||||
severity: "medium",
|
||||
domain: "managed-member-comparison",
|
||||
supporting_evidence_ids: [result.value.evidence_id],
|
||||
contradicting_evidence_ids: [],
|
||||
required_authority: "shipped-artifact",
|
||||
required_confidence: "observed",
|
||||
required_environment: null,
|
||||
@@ -84,7 +83,16 @@ export const registerCompareManagedMembers = (
|
||||
],
|
||||
relationships: [],
|
||||
})
|
||||
: options.recordEvidence?.(result.value);
|
||||
: undefined;
|
||||
const output =
|
||||
unknownInput === undefined
|
||||
? options.recordEvidence?.(result.value)
|
||||
: unknownInput.ok
|
||||
? options.recordEvidenceWithUnknown?.(
|
||||
result.value,
|
||||
unknownInput.value,
|
||||
)
|
||||
: unknownInput;
|
||||
return server.delivery.toEvidenceToolResult(
|
||||
result.value,
|
||||
compareContract,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { workflowUnknownInput } from "../../application/investigation/WorkflowUnknown.js";
|
||||
import type { EvidenceMcpServer } from "../EvidenceMcpServer.js";
|
||||
import { runAdmittedToolOperation } from "../admittedToolOperation.js";
|
||||
import { recordSessionEvidenceSources } from "../sessionEvidence.js";
|
||||
@@ -83,14 +84,12 @@ export const registerVerifyManagedNativeBoundaries = (
|
||||
verification.summary.unresolved > 0 ||
|
||||
verification.summary.contradicted > 0 ||
|
||||
verification.summary.native_body_unresolved > 0;
|
||||
const output = unknown
|
||||
? options.recordEvidenceWithUnknown?.(result.value, {
|
||||
const unknownInput = unknown
|
||||
? workflowUnknownInput(result.value.evidence_id, {
|
||||
question:
|
||||
"Which managed/native boundaries remain unresolved or contradicted by the supplied native Evidence?",
|
||||
severity: "medium",
|
||||
domain: "managed-native-verification",
|
||||
supporting_evidence_ids: [result.value.evidence_id],
|
||||
contradicting_evidence_ids: [],
|
||||
required_authority: "shipped-artifact",
|
||||
required_confidence: "observed",
|
||||
required_environment: null,
|
||||
@@ -108,7 +107,16 @@ export const registerVerifyManagedNativeBoundaries = (
|
||||
],
|
||||
relationships: [],
|
||||
})
|
||||
: options.recordEvidence?.(result.value);
|
||||
: undefined;
|
||||
const output =
|
||||
unknownInput === undefined
|
||||
? options.recordEvidence?.(result.value)
|
||||
: unknownInput.ok
|
||||
? options.recordEvidenceWithUnknown?.(
|
||||
result.value,
|
||||
unknownInput.value,
|
||||
)
|
||||
: unknownInput;
|
||||
return server.delivery.toEvidenceToolResult(
|
||||
result.value,
|
||||
nativeVerificationContract,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { EvidenceMcpServer } from "./EvidenceMcpServer.js";
|
||||
import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
|
||||
import { workflowUnknownInput } from "../application/investigation/WorkflowUnknown.js";
|
||||
|
||||
import {
|
||||
getNavigationContext,
|
||||
@@ -48,12 +49,10 @@ const recordProcessResidualUnknowns = (
|
||||
residuals: ProcessCapture["residual_unknowns"],
|
||||
): Result<null, AnalysisError> => {
|
||||
for (const residual of residuals) {
|
||||
const unknown = session.recordUnknown({
|
||||
const input = workflowUnknownInput(evidence.evidence_id, {
|
||||
question: `Was ${residual.scope} behavior fully observed during capture?`,
|
||||
severity: "medium",
|
||||
domain: `process-${residual.scope}`,
|
||||
supporting_evidence_ids: [evidence.evidence_id],
|
||||
contradicting_evidence_ids: [],
|
||||
required_authority: "controlled-replay",
|
||||
required_confidence: "observed",
|
||||
required_environment: evidence.environment,
|
||||
@@ -66,6 +65,8 @@ const recordProcessResidualUnknowns = (
|
||||
],
|
||||
relationships: [],
|
||||
});
|
||||
if (!input.ok) return input;
|
||||
const unknown = session.recordUnknown(input.value);
|
||||
if (
|
||||
!unknown.ok &&
|
||||
!(
|
||||
|
||||
@@ -9,7 +9,11 @@ import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
|
||||
import { nativeLoadImageSchema } from "../../../src/domain/native/nativeLoadImage.js";
|
||||
import { nativeApiInspectionResultSchema } from "../../../src/domain/native/nativeApiBoundary.js";
|
||||
import { functionDossierSchema } from "../../../src/domain/hopperValues.js";
|
||||
import { ok } from "../../../src/domain/result.js";
|
||||
import { ghidraFunctionDossier } from "../../../src/domain/ghidraValues.fixture.js";
|
||||
import { toolContract } from "../../../src/contracts/toolContracts.js";
|
||||
|
||||
it.each([
|
||||
{ state: "final", failed: [] },
|
||||
@@ -129,3 +133,121 @@ describe("Ghidra MCP native API evidence", () => {
|
||||
}
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
it("retains unavailable Ghidra native API observations across provenance and exact replay", async () => {
|
||||
const residual = "The controlled provider omitted native API boundary facts.";
|
||||
// This wire fixture is controlled and never launches Ghidra.
|
||||
const harness = await connectGhidraMcp(
|
||||
"ghidra-native-api-unavailable-identity",
|
||||
async (operation, input) => {
|
||||
if (operation !== "analyze_function")
|
||||
throw new Error(`Unexpected controlled Ghidra operation: ${operation}`);
|
||||
const procedure = String(input.procedure);
|
||||
const base = functionDossierSchema.parse(ghidraFunctionDossier());
|
||||
const address = procedure === "0x401000" ? procedure : "0x401100";
|
||||
return ok(
|
||||
jsonValueSchema.parse({
|
||||
value: functionDossierSchema.parse({
|
||||
...base,
|
||||
procedure: {
|
||||
...base.procedure,
|
||||
address,
|
||||
name: address === "0x401000" ? "fixture_main" : "fixture_other",
|
||||
body: {
|
||||
...base.procedure.body,
|
||||
ranges: [
|
||||
{
|
||||
start: address,
|
||||
end: address === "0x401000" ? "0x401005" : "0x401105",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
native_api: {
|
||||
available: false,
|
||||
reason: residual,
|
||||
residual_unknowns: [residual],
|
||||
},
|
||||
}),
|
||||
limitations: [],
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
const { mcp, session } = harness;
|
||||
try {
|
||||
const inspect = async (procedure: string) => {
|
||||
const reply = await mcp.callTool({
|
||||
name: "inspect_native_api",
|
||||
arguments: { procedure },
|
||||
});
|
||||
expect(reply.isError, JSON.stringify(reply.content)).not.toBe(true);
|
||||
const evidence = sessionEvidence(session, reply.structuredContent);
|
||||
const result = nativeApiInspectionResultSchema.parse(
|
||||
evidence.normalized_result,
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
procedure: {
|
||||
address: procedure,
|
||||
},
|
||||
boundary: {
|
||||
available: false,
|
||||
reason: residual,
|
||||
residual_unknowns: [residual],
|
||||
},
|
||||
residual_unknowns: [residual],
|
||||
});
|
||||
return evidence;
|
||||
};
|
||||
|
||||
const first = await inspect("0x401000");
|
||||
const replay = await inspect("0x401000");
|
||||
expect(replay.normalized_result).toEqual(first.normalized_result);
|
||||
expect(replay.evidence_id).not.toBe(first.evidence_id);
|
||||
expect(replay.limitations).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.stringContaining("Loaded from a local REA analysis snapshot"),
|
||||
]),
|
||||
);
|
||||
const warmBundle = session.exportEvidenceBundle();
|
||||
const warmReplay = await inspect("0x401000");
|
||||
expect(warmReplay.evidence_id).toBe(replay.evidence_id);
|
||||
expect(session.exportEvidenceBundle()).toEqual(warmBundle);
|
||||
|
||||
const second = await inspect("0x401100");
|
||||
expect(second.evidence_id).not.toBe(first.evidence_id);
|
||||
expect(session.evidenceById(first.evidence_id)).toEqual(first);
|
||||
expect(session.evidenceById(replay.evidence_id)).toEqual(replay);
|
||||
|
||||
const listed = await mcp.callTool({
|
||||
name: "list_unknowns",
|
||||
arguments: { domain: "native-api" },
|
||||
});
|
||||
expect(listed.isError, JSON.stringify(listed.content)).not.toBe(true);
|
||||
const unknownPage = toolContract("list_unknowns").outputSchema.parse(
|
||||
listed.structuredContent,
|
||||
).result;
|
||||
expect(unknownPage.total).toBe(3);
|
||||
expect(unknownPage.items).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
question: expect.stringContaining(first.evidence_id),
|
||||
supporting_evidence_ids: [first.evidence_id],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
question: expect.stringContaining(replay.evidence_id),
|
||||
supporting_evidence_ids: [replay.evidence_id],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
question: expect.stringContaining(second.evidence_id),
|
||||
supporting_evidence_ids: [second.evidence_id],
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(
|
||||
unknownPage.items.every(({ question }) => question.includes(residual)),
|
||||
).toBe(true);
|
||||
} finally {
|
||||
await harness.close();
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
@@ -56,8 +56,7 @@ it("aggregates comparison Evidence and records an approved runtime gap", async (
|
||||
});
|
||||
expect(session.listUnknowns({ domain: "changed-behavior" })).toMatchObject([
|
||||
{
|
||||
question:
|
||||
"Did both versions behave the same under a complete controlled replay?",
|
||||
question: `Did both versions behave the same under a complete controlled replay? (Evidence ${evidence.evidence_id})`,
|
||||
},
|
||||
]);
|
||||
} finally {
|
||||
@@ -406,8 +405,7 @@ it.each([
|
||||
else
|
||||
expect(session.listUnknowns({ domain: "call-path" })).toMatchObject([
|
||||
{
|
||||
question:
|
||||
"Can the requested call path be established from complete analysis?",
|
||||
question: `Can the requested call path be established from complete analysis? (Evidence ${evidence.evidence_id})`,
|
||||
supporting_evidence_ids: [evidence.evidence_id],
|
||||
},
|
||||
]);
|
||||
@@ -417,32 +415,6 @@ it.each([
|
||||
},
|
||||
);
|
||||
|
||||
it("records a safe unresolved call-path question", async () => {
|
||||
const { session, server, client } = await connected();
|
||||
const dossier = dossierWithCallees(["0x2000"]);
|
||||
expect(session.recordEvidence(dossier).ok).toBe(true);
|
||||
try {
|
||||
const response = await client.callTool({
|
||||
name: "build_call_path",
|
||||
arguments: {
|
||||
...INVESTIGATION_EXAMPLES.build_call_path,
|
||||
functions: [dossier],
|
||||
start: { address: "0x1000" },
|
||||
goal: { address: "0x3000" },
|
||||
},
|
||||
});
|
||||
expect(response.isError, JSON.stringify(response)).not.toBe(true);
|
||||
expect(session.listUnknowns({ domain: "call-path" })).toMatchObject([
|
||||
{
|
||||
question:
|
||||
"Can the requested call path be established from complete analysis?",
|
||||
},
|
||||
]);
|
||||
} finally {
|
||||
await close(session, server, client);
|
||||
}
|
||||
});
|
||||
|
||||
it("records an explicit non-causal static/runtime hypothesis", async () => {
|
||||
const { session, server, client } = await connected();
|
||||
for (const evidence of [
|
||||
@@ -586,7 +558,7 @@ it("cannot omit a session-owned active unknown from reconstruction input", async
|
||||
session.listUnknowns({ domain: "reconstruction-verification" }),
|
||||
).toContainEqual(
|
||||
expect.objectContaining({
|
||||
question: "Does the reconstruction satisfy every declared claim?",
|
||||
question: `Does the reconstruction satisfy every declared claim? (Evidence ${evidence.evidence_id})`,
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
import { expect, it, onTestFinished } from "vitest";
|
||||
|
||||
import {
|
||||
MANAGED_NATIVE_VERIFICATION_EXAMPLE,
|
||||
MANAGED_RECONSTRUCTION_IMPORT_EXAMPLE,
|
||||
} from "../../../src/contracts/managed/managedWorkflowExamples.js";
|
||||
import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js";
|
||||
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
|
||||
import {
|
||||
managedMemberInspectionSchema,
|
||||
managedNativeBoundaryInspectionSchema,
|
||||
} from "../../../src/domain/managed/managedArtifact.js";
|
||||
import { managedMemberComparisonResultSchema } from "../../../src/domain/managed/managedMemberComparison.js";
|
||||
import { managedNativeVerificationResultSchema } from "../../../src/domain/managed/managedNativeVerificationSchemas.js";
|
||||
import { inspectMachoSchema } from "../../../src/domain/native/nativeInspection.js";
|
||||
import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
|
||||
import { createTestBinarySession } from "../../fixtures/binarySession.js";
|
||||
|
||||
const emptyExportEvidence = () => {
|
||||
const original = MANAGED_NATIVE_VERIFICATION_EXAMPLE.native_observations[0];
|
||||
if (original === undefined || original.subject === null)
|
||||
throw new Error("Managed verification example is missing native Evidence");
|
||||
const result = inspectMachoSchema.parse(original.normalized_result);
|
||||
return createEvidence(
|
||||
{
|
||||
path: "/fixtures/empty-exports.dylib",
|
||||
sha256: "8".repeat(64),
|
||||
format: original.subject.format,
|
||||
...(original.subject.architecture === null
|
||||
? {}
|
||||
: { architecture: original.subject.architecture }),
|
||||
},
|
||||
original.provider,
|
||||
{
|
||||
operation: "inspect_macho",
|
||||
parameters: original.parameters,
|
||||
result: jsonValueSchema.parse({
|
||||
...result,
|
||||
exports: {
|
||||
items: [],
|
||||
total: 0,
|
||||
exhaustive: true,
|
||||
limitations: [],
|
||||
},
|
||||
}),
|
||||
rawResult: null,
|
||||
limitations: [],
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
const secondDeclaration = () => {
|
||||
const original = MANAGED_NATIVE_VERIFICATION_EXAMPLE.managed_boundaries;
|
||||
const result = managedNativeBoundaryInspectionSchema.parse(
|
||||
original.normalized_result,
|
||||
);
|
||||
const pinvokes = result.pinvoke_imports;
|
||||
if (pinvokes.length !== 1)
|
||||
throw new Error("Managed verification example must contain one P/Invoke");
|
||||
const pinvoke = pinvokes[0];
|
||||
if (pinvoke === undefined)
|
||||
throw new Error("Managed verification example has invalid P/Invoke data");
|
||||
return createEvidence(undefined, original.provider, {
|
||||
operation: "inspect_managed_native_boundaries",
|
||||
parameters: original.parameters,
|
||||
result: jsonValueSchema.parse({
|
||||
...result,
|
||||
pinvoke_imports: [
|
||||
{
|
||||
...pinvoke,
|
||||
token: "0x1c000002",
|
||||
member_token: "0x06000002",
|
||||
member_name: "OpenNativeAgain",
|
||||
import_name: "open_native_again",
|
||||
},
|
||||
],
|
||||
}),
|
||||
rawResult: null,
|
||||
limitations: original.limitations,
|
||||
});
|
||||
};
|
||||
|
||||
const memberEvidence = (
|
||||
id: string,
|
||||
digest: string,
|
||||
methodName: string | null,
|
||||
partial: boolean,
|
||||
) => {
|
||||
const original = MANAGED_RECONSTRUCTION_IMPORT_EXAMPLE.static_members;
|
||||
const result = managedMemberInspectionSchema.parse(
|
||||
original.normalized_result,
|
||||
);
|
||||
const method = result.methods[0];
|
||||
if (method === undefined)
|
||||
throw new Error("Managed reconstruction example is missing a method");
|
||||
const normalized = managedMemberInspectionSchema.parse({
|
||||
...result,
|
||||
artifact: {
|
||||
...result.artifact,
|
||||
path: `/fixtures/${id}.dll`,
|
||||
sha256: digest,
|
||||
},
|
||||
identity_scope: {
|
||||
...result.identity_scope,
|
||||
requires_artifact_sha256: digest,
|
||||
},
|
||||
methods: methodName === null ? [] : [{ ...method, name: methodName }],
|
||||
metadata: partial
|
||||
? { ...result.metadata, status: "partial" }
|
||||
: result.metadata,
|
||||
coverage: partial
|
||||
? {
|
||||
state: "partial",
|
||||
issues: [
|
||||
{
|
||||
code: "invalid-tables",
|
||||
scope: "MethodDef",
|
||||
offset: null,
|
||||
detail: "The method inventory is incomplete in this fixture.",
|
||||
},
|
||||
],
|
||||
}
|
||||
: result.coverage,
|
||||
});
|
||||
return createEvidence(
|
||||
{ path: `/fixtures/${id}.dll`, sha256: digest, format: "pe" },
|
||||
original.provider,
|
||||
{
|
||||
operation: "inspect_managed_members",
|
||||
parameters: original.parameters,
|
||||
result: jsonValueSchema.parse(normalized),
|
||||
rawResult: null,
|
||||
limitations: normalized.limitations,
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
it("records distinct unresolved managed boundary checks and makes exact replay idempotent", async () => {
|
||||
const harness = await createApplicationMcpHarness({
|
||||
session: createTestBinarySession(() => {
|
||||
throw new Error("Managed/native verification must not launch a provider");
|
||||
}),
|
||||
});
|
||||
onTestFinished(harness.close);
|
||||
const { client, session } = harness;
|
||||
const managed = MANAGED_NATIVE_VERIFICATION_EXAMPLE.managed_boundaries;
|
||||
const native = emptyExportEvidence();
|
||||
|
||||
const verify = async (boundaries: typeof managed) => {
|
||||
const response = await client.callTool({
|
||||
name: "verify_managed_native_boundaries",
|
||||
arguments: {
|
||||
managed_boundaries: boundaries,
|
||||
native_observations: [native],
|
||||
},
|
||||
});
|
||||
expect(response.isError, JSON.stringify(response.content)).not.toBe(true);
|
||||
const evidence = parseEvidence(response.structuredContent);
|
||||
const result = managedNativeVerificationResultSchema.parse(
|
||||
evidence.normalized_result,
|
||||
);
|
||||
expect(result.summary).toMatchObject({ unresolved: 1 });
|
||||
expect(result.pinvoke_imports).toEqual([
|
||||
expect.objectContaining({
|
||||
status: "unresolved",
|
||||
basis: "no-native-candidate",
|
||||
}),
|
||||
]);
|
||||
return evidence;
|
||||
};
|
||||
|
||||
const first = await verify(managed);
|
||||
const firstBundle = session.exportEvidenceBundle();
|
||||
const replay = await verify(managed);
|
||||
expect(replay.evidence_id).toBe(first.evidence_id);
|
||||
expect(session.exportEvidenceBundle()).toEqual(firstBundle);
|
||||
expect(session.listUnknowns()).toHaveLength(1);
|
||||
|
||||
const second = await verify(secondDeclaration());
|
||||
expect(second.evidence_id).not.toBe(first.evidence_id);
|
||||
const unknowns = await client.callTool({
|
||||
name: "list_unknowns",
|
||||
arguments: { domain: "managed-native-verification" },
|
||||
});
|
||||
expect(unknowns.isError, JSON.stringify(unknowns.content)).not.toBe(true);
|
||||
expect(unknowns.structuredContent).toMatchObject({
|
||||
result: {
|
||||
total: 2,
|
||||
items: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
supporting_evidence_ids: [first.evidence_id],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
supporting_evidence_ids: [second.evidence_id],
|
||||
}),
|
||||
]),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("retains distinct unresolved managed member comparisons and exact replay", async () => {
|
||||
const harness = await createApplicationMcpHarness({
|
||||
session: createTestBinarySession(() => {
|
||||
throw new Error("Managed member comparison must not launch a provider");
|
||||
}),
|
||||
});
|
||||
onTestFinished(harness.close);
|
||||
const { client, session } = harness;
|
||||
const right = memberEvidence("members-partial", "a".repeat(64), null, true);
|
||||
|
||||
const compare = async (left: ReturnType<typeof memberEvidence>) => {
|
||||
const response = await client.callTool({
|
||||
name: "compare_managed_members",
|
||||
arguments: { left, right },
|
||||
});
|
||||
expect(response.isError, JSON.stringify(response.content)).not.toBe(true);
|
||||
const evidence = parseEvidence(response.structuredContent);
|
||||
const result = managedMemberComparisonResultSchema.parse(
|
||||
evidence.normalized_result,
|
||||
);
|
||||
expect(result.summary.unknown).toBeGreaterThan(0);
|
||||
return evidence;
|
||||
};
|
||||
|
||||
const first = await compare(
|
||||
memberEvidence("members-first", "b".repeat(64), "Main", false),
|
||||
);
|
||||
const firstBundle = session.exportEvidenceBundle();
|
||||
const replay = await compare(
|
||||
memberEvidence("members-first", "b".repeat(64), "Main", false),
|
||||
);
|
||||
expect(replay.evidence_id).toBe(first.evidence_id);
|
||||
expect(session.exportEvidenceBundle()).toEqual(firstBundle);
|
||||
|
||||
const second = await compare(
|
||||
memberEvidence("members-second", "c".repeat(64), "Open", false),
|
||||
);
|
||||
expect(second.evidence_id).not.toBe(first.evidence_id);
|
||||
const unknowns = await client.callTool({
|
||||
name: "list_unknowns",
|
||||
arguments: { domain: "managed-member-comparison" },
|
||||
});
|
||||
expect(unknowns.isError, JSON.stringify(unknowns.content)).not.toBe(true);
|
||||
expect(unknowns.structuredContent).toMatchObject({
|
||||
result: {
|
||||
total: 2,
|
||||
items: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
supporting_evidence_ids: [first.evidence_id],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
supporting_evidence_ids: [second.evidence_id],
|
||||
}),
|
||||
]),
|
||||
},
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { expect } from "vitest";
|
||||
|
||||
import { parseEvidence } from "../../../src/domain/evidence.js";
|
||||
import { silentLogger } from "../../../src/logger.js";
|
||||
import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
|
||||
import { createTestBinarySession } from "../../fixtures/binarySession.js";
|
||||
import { itWithCaptureCapability } from "../process/processCaptureCapability.js";
|
||||
|
||||
const partialProcessFixture = fileURLToPath(
|
||||
new URL("../../fixtures/processFidelity.mjs", import.meta.url),
|
||||
);
|
||||
|
||||
itWithCaptureCapability(
|
||||
"retains residuals from distinct process captures with the same missing scope",
|
||||
async () => {
|
||||
const harness = await createApplicationMcpHarness({
|
||||
session: createTestBinarySession(() => {
|
||||
throw new Error("Process capture must not launch an analysis provider");
|
||||
}),
|
||||
serverOptions: { logger: silentLogger },
|
||||
});
|
||||
const { client } = harness;
|
||||
try {
|
||||
const captures = [];
|
||||
for (const settleMs of [50, 75]) {
|
||||
const response = await client.callTool({
|
||||
name: "capture_process_scenario",
|
||||
arguments: {
|
||||
executable: process.execPath,
|
||||
arguments: [partialProcessFixture, "partial"],
|
||||
settle_ms: settleMs,
|
||||
},
|
||||
});
|
||||
expect(response.isError, JSON.stringify(response.content)).not.toBe(
|
||||
true,
|
||||
);
|
||||
captures.push(parseEvidence(response.structuredContent));
|
||||
}
|
||||
expect(captures[0]?.evidence_id).not.toBe(captures[1]?.evidence_id);
|
||||
for (const capture of captures)
|
||||
expect(capture.normalized_result).toMatchObject({
|
||||
residual_unknowns: expect.arrayContaining([
|
||||
expect.objectContaining({ scope: "network" }),
|
||||
]),
|
||||
});
|
||||
|
||||
const unknowns = await client.callTool({
|
||||
name: "list_unknowns",
|
||||
arguments: { domain: "process-network" },
|
||||
});
|
||||
expect(unknowns.isError, JSON.stringify(unknowns.content)).not.toBe(true);
|
||||
expect(unknowns.structuredContent).toMatchObject({
|
||||
result: {
|
||||
total: 2,
|
||||
items: expect.arrayContaining(
|
||||
captures.map((capture) =>
|
||||
expect.objectContaining({
|
||||
question: `Was network behavior fully observed during capture? (Evidence ${capture.evidence_id})`,
|
||||
supporting_evidence_ids: [capture.evidence_id],
|
||||
}),
|
||||
),
|
||||
),
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
await harness.close();
|
||||
}
|
||||
},
|
||||
30_000,
|
||||
);
|
||||
@@ -0,0 +1,248 @@
|
||||
import type { Client } from "@modelcontextprotocol/client";
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js";
|
||||
import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js";
|
||||
import { functionDossierSchema } from "../../../src/domain/hopperValues.js";
|
||||
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
|
||||
import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
|
||||
import { createTestBinarySession } from "../../fixtures/binarySession.js";
|
||||
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
|
||||
|
||||
const withClient = async (
|
||||
run: (
|
||||
client: Client,
|
||||
session: ReturnType<typeof createTestBinarySession>,
|
||||
) => Promise<void>,
|
||||
): Promise<void> => {
|
||||
const session = createTestBinarySession(() => {
|
||||
throw new Error("This workflow must not launch an analysis provider");
|
||||
});
|
||||
const harness = await createApplicationMcpHarness({ session });
|
||||
try {
|
||||
await run(harness.client, session);
|
||||
} finally {
|
||||
await harness.close();
|
||||
}
|
||||
};
|
||||
|
||||
const pathSource = () => {
|
||||
const template = FUNCTION_COMPARISON_EXAMPLE.left;
|
||||
if (template.subject === null)
|
||||
throw new Error("Call-path fixtures must be artifact-bound");
|
||||
const dossier = functionDossierSchema.parse(template.normalized_result);
|
||||
return createEvidence(
|
||||
{
|
||||
path: `${template.subject.local_path}/workflow-unknown-path`,
|
||||
sha256: template.subject.digest.sha256,
|
||||
format: template.subject.format,
|
||||
},
|
||||
template.provider,
|
||||
{
|
||||
operation: "analyze_function",
|
||||
parameters: {
|
||||
...template.parameters,
|
||||
observation: "workflow-unknown-path",
|
||||
},
|
||||
result: jsonValueSchema.parse({
|
||||
...dossier,
|
||||
callees: [
|
||||
{
|
||||
address: "0x2000",
|
||||
name: "missing-helper",
|
||||
classification: null,
|
||||
body: { available: false, reason: "No body was observed." },
|
||||
},
|
||||
],
|
||||
}),
|
||||
confidence: template.confidence,
|
||||
authority: template.authority,
|
||||
relationshipCoverage: { status: "complete" },
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
const incompleteRight = (suffix: string) => {
|
||||
const template = FUNCTION_COMPARISON_EXAMPLE.right;
|
||||
if (template.subject === null)
|
||||
throw new Error("Comparison fixtures must be artifact-bound");
|
||||
return createEvidence(
|
||||
{
|
||||
path: `${template.subject.local_path}/${suffix}`,
|
||||
sha256: template.subject.digest.sha256,
|
||||
format: template.subject.format,
|
||||
},
|
||||
template.provider,
|
||||
{
|
||||
operation: "analyze_function",
|
||||
parameters: { ...template.parameters, observation: suffix },
|
||||
result: jsonValueSchema.parse(
|
||||
functionDossierSchema.parse(
|
||||
FUNCTION_COMPARISON_EXAMPLE.left.normalized_result,
|
||||
),
|
||||
),
|
||||
confidence: template.confidence,
|
||||
authority: template.authority,
|
||||
relationshipCoverage: {
|
||||
status: "unknown",
|
||||
reason: "The fixture omits call relationship coverage.",
|
||||
},
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
const compareIncomplete = async (client: Client, suffix: string) => {
|
||||
const response = await client.callTool({
|
||||
name: "compare_functions",
|
||||
arguments: {
|
||||
left: FUNCTION_COMPARISON_EXAMPLE.left,
|
||||
right: incompleteRight(suffix),
|
||||
},
|
||||
});
|
||||
expect(response.isError, JSON.stringify(response)).not.toBe(true);
|
||||
const comparison = parseEvidence(response.structuredContent);
|
||||
expect(comparison.normalized_result).toMatchObject({ status: "unknown" });
|
||||
return comparison;
|
||||
};
|
||||
|
||||
const verifyCallClaim = async (
|
||||
client: Client,
|
||||
comparison: ReturnType<typeof parseEvidence>,
|
||||
) =>
|
||||
client.callTool({
|
||||
name: "verify_reconstruction",
|
||||
arguments: {
|
||||
specification: {
|
||||
name: "Same unresolved call claim",
|
||||
claims: [
|
||||
{
|
||||
kind: "structural-function",
|
||||
claim_id: "call-coverage",
|
||||
title: "Call coverage is complete",
|
||||
comparison_evidence_id: comparison.evidence_id,
|
||||
dimension: "calls",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const assertUnknownEvidence = (
|
||||
response: Awaited<ReturnType<Client["callTool"]>>,
|
||||
) => {
|
||||
expect(response.isError, JSON.stringify(response)).not.toBe(true);
|
||||
expect(
|
||||
parseEvidence(response.structuredContent).normalized_result,
|
||||
).toMatchObject({ status: "unknown" });
|
||||
};
|
||||
|
||||
it("retains separate unresolved call-path workflows in one session", async () => {
|
||||
await withClient(async (client, session) => {
|
||||
const source = pathSource();
|
||||
expect(session.recordEvidence(source).ok).toBe(true);
|
||||
const responses = [];
|
||||
for (const goal of ["0x3000", "0x4000", "0x3000"])
|
||||
responses.push(
|
||||
await client.callTool({
|
||||
name: "build_call_path",
|
||||
arguments: {
|
||||
functions: [source],
|
||||
start: { address: "0x1000" },
|
||||
goal: { address: goal },
|
||||
},
|
||||
}),
|
||||
);
|
||||
responses.forEach(assertUnknownEvidence);
|
||||
const ids = responses.map(
|
||||
(response) => parseEvidence(response.structuredContent).evidence_id,
|
||||
);
|
||||
expect(ids[2]).toBe(ids[0]);
|
||||
expect(ids[1]).not.toBe(ids[0]);
|
||||
expect(
|
||||
session
|
||||
.listUnknowns({ domain: "call-path" })
|
||||
.map(({ supporting_evidence_ids }) => supporting_evidence_ids),
|
||||
).toEqual(expect.arrayContaining([...new Set(ids)].map((id) => [id])));
|
||||
expect(
|
||||
session
|
||||
.listUnknowns({ domain: "call-path" })
|
||||
.map(({ question }) => question)
|
||||
.sort(),
|
||||
).toEqual(
|
||||
[...new Set(ids)]
|
||||
.map(
|
||||
(id) =>
|
||||
`Can the requested call path be established from complete analysis? (Evidence ${id})`,
|
||||
)
|
||||
.sort(),
|
||||
);
|
||||
const unknowns = await client.callTool({
|
||||
name: "list_unknowns",
|
||||
arguments: { domain: "call-path" },
|
||||
});
|
||||
expect(unknowns.structuredContent).toMatchObject({
|
||||
result: { total: 2, items: expect.any(Array) },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("retains separate unresolved reconstruction verifications with identical claim names", async () => {
|
||||
await withClient(async (client, session) => {
|
||||
const firstComparison = await compareIncomplete(
|
||||
client,
|
||||
"verification-unknown-a",
|
||||
);
|
||||
const secondComparison = await compareIncomplete(
|
||||
client,
|
||||
"verification-unknown-b",
|
||||
);
|
||||
const responses = [
|
||||
await verifyCallClaim(client, firstComparison),
|
||||
await verifyCallClaim(client, secondComparison),
|
||||
await verifyCallClaim(client, firstComparison),
|
||||
];
|
||||
responses.forEach(assertUnknownEvidence);
|
||||
const ids = responses.map(
|
||||
(response) => parseEvidence(response.structuredContent).evidence_id,
|
||||
);
|
||||
expect(ids[2]).toBe(ids[0]);
|
||||
expect(ids[1]).not.toBe(ids[0]);
|
||||
expect(
|
||||
session
|
||||
.listUnknowns({ domain: "reconstruction-verification" })
|
||||
.map(({ supporting_evidence_ids }) => supporting_evidence_ids),
|
||||
).toEqual(expect.arrayContaining([...new Set(ids)].map((id) => [id])));
|
||||
const unknowns = await client.callTool({
|
||||
name: "list_unknowns",
|
||||
arguments: { domain: "reconstruction-verification" },
|
||||
});
|
||||
expect(unknowns.structuredContent).toMatchObject({
|
||||
result: { total: 2, items: expect.any(Array) },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("continues rejecting duplicate explicit record_unknown identities", async () => {
|
||||
await withClient(async (client) => {
|
||||
const input = {
|
||||
question: "Does this explicitly submitted question remain unique?",
|
||||
severity: "medium",
|
||||
domain: "manual-duplicate-control",
|
||||
required_authority: null,
|
||||
required_confidence: "derived",
|
||||
required_environment: null,
|
||||
recommended_probes: [],
|
||||
relationships: [],
|
||||
};
|
||||
const first = await client.callTool({
|
||||
name: "record_unknown",
|
||||
arguments: input,
|
||||
});
|
||||
expect(first.isError).not.toBe(true);
|
||||
const duplicate = await client.callTool({
|
||||
name: "record_unknown",
|
||||
arguments: input,
|
||||
});
|
||||
expect(parseMcpToolError(duplicate).error.code).toBe("revision_conflict");
|
||||
});
|
||||
});
|
||||
@@ -199,9 +199,7 @@ itWithCaptureCapability(
|
||||
captured.structuredContent,
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
contract.outputSchema.safeParse(captured.structuredContent).success,
|
||||
).toBe(true);
|
||||
const evidence = contract.outputSchema.parse(captured.structuredContent);
|
||||
const listedUnknowns = await mcp.callTool({
|
||||
name: "list_unknowns",
|
||||
arguments: {},
|
||||
@@ -224,7 +222,7 @@ itWithCaptureCapability(
|
||||
.parse(listedUnknowns.structuredContent).result.items;
|
||||
expect(listed).toContainEqual(
|
||||
expect.objectContaining({
|
||||
question: "Was network behavior fully observed during capture?",
|
||||
question: `Was network behavior fully observed during capture? (Evidence ${evidence.evidence_id})`,
|
||||
domain: "process-network",
|
||||
}),
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user