From 030c47732833c9cedcc31db1309eb3a1ca2c62e8 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:49:17 +0800 Subject: [PATCH] fix(evidence): derive bundle comparison status from classified changes Keep digests as provenance while status follows examined deltas and unresolved coverage. Refs: #3216 --- src/domain/bundleComparison.test.ts | 33 ++++++ src/domain/bundleComparison.ts | 6 +- .../boundary/cli/bundleComparisonCli.test.ts | 105 ++++++++++++++++++ .../boundary/mcp/bundleComparisonMcp.test.ts | 103 +++++++++++++++++ 4 files changed, 242 insertions(+), 5 deletions(-) create mode 100644 tests/boundary/cli/bundleComparisonCli.test.ts diff --git a/src/domain/bundleComparison.test.ts b/src/domain/bundleComparison.test.ts index 9743c3525a..866c09be23 100644 --- a/src/domain/bundleComparison.test.ts +++ b/src/domain/bundleComparison.test.ts @@ -76,6 +76,39 @@ describe("bundle comparison explicit cross-pairs", () => { }); describe("bundle comparison", () => { + it("does not report display-only bundle differences as unexplained changes", () => { + const target = { + path: "/left/artifact.elf", + sha256: "a".repeat(64), + format: "elf" as const, + architecture: "x86_64" as const, + }; + const record = createEvidence(target, PROVIDER, { + operation: "observe", + parameters: {}, + result: { observed: true }, + confidence: "observed", + authority: "shipped-artifact", + }); + const left = createEvidenceBundle([record]); + const rightRecord = { + ...record, + subject: { + ...record.subject!, + name: "renamed.elf", + local_path: "/right/renamed.elf", + }, + }; + const right = { ...left, records: [rightRecord] }; + const result = compareBundles(left, right); + expect(result).toMatchObject({ + status: "unchanged", + summary: { records_unchanged: 1, unresolved: 0 }, + changes: [], + }); + expect(result.left_bundle_sha256).not.toBe(result.right_bundle_sha256); + }); + it("accepts long caller-selected bundle paths", () => { const path = `/${"deep/".repeat(1_000)}bundle.json`; expect( diff --git a/src/domain/bundleComparison.ts b/src/domain/bundleComparison.ts index cb01980a47..3a00c0cb32 100644 --- a/src/domain/bundleComparison.ts +++ b/src/domain/bundleComparison.ts @@ -102,11 +102,7 @@ export const compareBundles = ( ).length; return bundleComparisonResultSchema.parse({ status: - leftDigest === rightDigest && changes.length === 0 - ? "unchanged" - : unresolved > 0 - ? "unknown" - : "changed", + unresolved > 0 ? "unknown" : changes.length > 0 ? "changed" : "unchanged", left_bundle_sha256: leftDigest, right_bundle_sha256: rightDigest, summary: { diff --git a/tests/boundary/cli/bundleComparisonCli.test.ts b/tests/boundary/cli/bundleComparisonCli.test.ts new file mode 100644 index 0000000000..620cd0eaf2 --- /dev/null +++ b/tests/boundary/cli/bundleComparisonCli.test.ts @@ -0,0 +1,105 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect } from "vitest"; + +import { createEvidence } from "../../../src/domain/evidence.js"; +import { bundleComparisonResultSchema } from "../../../src/domain/bundleComparison.js"; +import { + createEvidenceBundle, + serializeEvidenceBundle, +} from "../../../src/domain/evidenceBundle.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "compares Evidence bundles without unexplained display-only changes", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-bundle-comparison-cli-"); + const leftPath = join(root, "left.json"); + const displayPath = join(root, "display.json"); + const changedPath = join(root, "changed.json"); + const target = { + path: join(root, "left.elf"), + sha256: "a".repeat(64), + format: "elf" as const, + architecture: "x86_64" as const, + }; + const record = createEvidence( + target, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "observe", + parameters: {}, + result: { observed: true }, + confidence: "observed", + authority: "shipped-artifact", + }, + ); + const left = createEvidenceBundle([record]); + const displayOnly = { + ...left, + records: [ + { + ...record, + subject: { + ...record.subject!, + name: "renamed.elf", + local_path: join(root, "renamed.elf"), + }, + }, + ], + }; + const different = createEvidenceBundle([ + createEvidence( + target, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "observe", + parameters: {}, + result: { observed: false }, + confidence: "observed", + authority: "shipped-artifact", + }, + ), + ]); + await Promise.all([ + writeFile(leftPath, serializeEvidenceBundle(left)), + writeFile(displayPath, serializeEvidenceBundle(displayOnly)), + writeFile(changedPath, serializeEvidenceBundle(different)), + ]); + const compare = async (path: string) => + cli.run({ arguments: ["compare", leftPath, path, "--json"] }); + const same = await compare(leftPath); + expect(same.exitCode).toBe(0); + expect(bundleComparisonResultSchema.parse(same.json)).toMatchObject({ + status: "unchanged", + changes: [], + }); + const display = await compare(displayPath); + expect(display.exitCode).toBe(0); + const displayResult = bundleComparisonResultSchema.parse(display.json); + expect(displayResult).toMatchObject({ status: "unchanged", changes: [] }); + expect(displayResult.left_bundle_sha256).not.toBe( + displayResult.right_bundle_sha256, + ); + const differentResult = await compare(changedPath); + expect(differentResult.exitCode).toBe(0); + expect( + bundleComparisonResultSchema.parse(differentResult.json), + ).toMatchObject({ + status: "unknown", + summary: { unresolved: 2 }, + changes: [ + expect.objectContaining({ + entity: "evidence", + conclusion_kind: "unresolved_branch", + }), + expect.objectContaining({ + entity: "evidence", + conclusion_kind: "unresolved_branch", + }), + ], + }); + }, +); diff --git a/tests/boundary/mcp/bundleComparisonMcp.test.ts b/tests/boundary/mcp/bundleComparisonMcp.test.ts index ab8838287d..24dd42dbfe 100644 --- a/tests/boundary/mcp/bundleComparisonMcp.test.ts +++ b/tests/boundary/mcp/bundleComparisonMcp.test.ts @@ -9,6 +9,8 @@ import { afterEach, describe, expect, it } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { bundleComparisonResultSchema } from "../../../src/domain/bundleComparison.js"; import { createEvidence } from "../../../src/domain/evidence.js"; import { createEvidenceBundle, @@ -37,6 +39,107 @@ const sourceEvidence = (label: string) => }, ); +describe("bundle comparison MCP semantic status", () => { + it("keeps display-only metadata out of deltas and preserves semantic changes", async () => { + const root = await createTestTempDirectory("rea-bundle-metadata-mcp-"); + roots.push(root); + const leftPath = join(root, "left.json"); + const displayPath = join(root, "display.json"); + const changedPath = join(root, "changed.json"); + const target = { + path: join(root, "left.elf"), + sha256: "a".repeat(64), + format: "elf" as const, + architecture: "x86_64" as const, + }; + const record = createEvidence( + target, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "observe", + parameters: {}, + result: { observed: true }, + confidence: "observed", + authority: "shipped-artifact", + }, + ); + const left = createEvidenceBundle([record]); + const displayRecord = { + ...record, + subject: { + ...record.subject!, + name: "renamed.elf", + local_path: join(root, "renamed.elf"), + }, + }; + const displayOnly = { ...left, records: [displayRecord] }; + const changedRecord = createEvidence( + target, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "observe", + parameters: {}, + result: { observed: false }, + confidence: "observed", + authority: "shipped-artifact", + }, + ); + await Promise.all([ + writeFile(leftPath, serializeEvidenceBundle(left)), + writeFile(displayPath, serializeEvidenceBundle(displayOnly)), + writeFile( + changedPath, + serializeEvidenceBundle(createEvidenceBundle([changedRecord])), + ), + ]); + const connected = await connect(); + try { + const compare = async ( + right: string, + recordPairs: readonly { + left_evidence_id: string; + right_evidence_id: string; + }[] = [], + ) => { + const result = await connected.client.callTool({ + name: "compare_bundles", + arguments: { + left_bundle_path: leftPath, + right_bundle_path: right, + record_pairs: recordPairs, + }, + }); + expect(result.isError).not.toBe(true); + const evidence = toolContract("compare_bundles").outputSchema.parse( + result.structuredContent, + ); + return bundleComparisonResultSchema.parse(evidence.normalized_result); + }; + const same = await compare(leftPath); + expect(same).toMatchObject({ + status: "unchanged", + changes: [], + }); + const display = await compare(displayPath); + expect(display).toMatchObject({ + status: "unchanged", + changes: [], + }); + expect(display.left_bundle_sha256).not.toBe(display.right_bundle_sha256); + const changed = await compare(changedPath, [ + { + left_evidence_id: record.evidence_id, + right_evidence_id: changedRecord.evidence_id, + }, + ]); + expect(changed.status).toBe("changed"); + expect(changed.changes).toHaveLength(1); + } finally { + await connected.close(); + } + }); +}); + describe("bundle comparison MCP integration", () => { it("reads caller-selected bundle files and returns complete Evidence", async () => { const root = await createTestTempDirectory("rea-bundle-mcp-");