fix(evidence): derive bundle comparison status from classified changes

Keep digests as provenance while status follows examined deltas and unresolved coverage.

Refs: #3216
This commit is contained in:
morluto
2026-10-14 17:49:17 +08:00
parent 3a34ad37bd
commit 030c477328
4 changed files with 242 additions and 5 deletions
+33
View File
@@ -76,6 +76,39 @@ describe("bundle comparison explicit cross-pairs", () => {
});
describe("bundle comparison", () => {
it("does not report display-only bundle differences as unexplained changes", () => {
const target = {
path: "/left/artifact.elf",
sha256: "a".repeat(64),
format: "elf" as const,
architecture: "x86_64" as const,
};
const record = createEvidence(target, PROVIDER, {
operation: "observe",
parameters: {},
result: { observed: true },
confidence: "observed",
authority: "shipped-artifact",
});
const left = createEvidenceBundle([record]);
const rightRecord = {
...record,
subject: {
...record.subject!,
name: "renamed.elf",
local_path: "/right/renamed.elf",
},
};
const right = { ...left, records: [rightRecord] };
const result = compareBundles(left, right);
expect(result).toMatchObject({
status: "unchanged",
summary: { records_unchanged: 1, unresolved: 0 },
changes: [],
});
expect(result.left_bundle_sha256).not.toBe(result.right_bundle_sha256);
});
it("accepts long caller-selected bundle paths", () => {
const path = `/${"deep/".repeat(1_000)}bundle.json`;
expect(
+1 -5
View File
@@ -102,11 +102,7 @@ export const compareBundles = (
).length;
return bundleComparisonResultSchema.parse({
status:
leftDigest === rightDigest && changes.length === 0
? "unchanged"
: unresolved > 0
? "unknown"
: "changed",
unresolved > 0 ? "unknown" : changes.length > 0 ? "changed" : "unchanged",
left_bundle_sha256: leftDigest,
right_bundle_sha256: rightDigest,
summary: {
@@ -0,0 +1,105 @@
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect } from "vitest";
import { createEvidence } from "../../../src/domain/evidence.js";
import { bundleComparisonResultSchema } from "../../../src/domain/bundleComparison.js";
import {
createEvidenceBundle,
serializeEvidenceBundle,
} from "../../../src/domain/evidenceBundle.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { cliTest } from "../../support/cli/cliFixture.js";
cliTest(
"compares Evidence bundles without unexplained display-only changes",
async ({ cli }) => {
const root = await createTestTempDirectory("rea-bundle-comparison-cli-");
const leftPath = join(root, "left.json");
const displayPath = join(root, "display.json");
const changedPath = join(root, "changed.json");
const target = {
path: join(root, "left.elf"),
sha256: "a".repeat(64),
format: "elf" as const,
architecture: "x86_64" as const,
};
const record = createEvidence(
target,
{ id: "fixture", name: "Fixture", version: "1" },
{
operation: "observe",
parameters: {},
result: { observed: true },
confidence: "observed",
authority: "shipped-artifact",
},
);
const left = createEvidenceBundle([record]);
const displayOnly = {
...left,
records: [
{
...record,
subject: {
...record.subject!,
name: "renamed.elf",
local_path: join(root, "renamed.elf"),
},
},
],
};
const different = createEvidenceBundle([
createEvidence(
target,
{ id: "fixture", name: "Fixture", version: "1" },
{
operation: "observe",
parameters: {},
result: { observed: false },
confidence: "observed",
authority: "shipped-artifact",
},
),
]);
await Promise.all([
writeFile(leftPath, serializeEvidenceBundle(left)),
writeFile(displayPath, serializeEvidenceBundle(displayOnly)),
writeFile(changedPath, serializeEvidenceBundle(different)),
]);
const compare = async (path: string) =>
cli.run({ arguments: ["compare", leftPath, path, "--json"] });
const same = await compare(leftPath);
expect(same.exitCode).toBe(0);
expect(bundleComparisonResultSchema.parse(same.json)).toMatchObject({
status: "unchanged",
changes: [],
});
const display = await compare(displayPath);
expect(display.exitCode).toBe(0);
const displayResult = bundleComparisonResultSchema.parse(display.json);
expect(displayResult).toMatchObject({ status: "unchanged", changes: [] });
expect(displayResult.left_bundle_sha256).not.toBe(
displayResult.right_bundle_sha256,
);
const differentResult = await compare(changedPath);
expect(differentResult.exitCode).toBe(0);
expect(
bundleComparisonResultSchema.parse(differentResult.json),
).toMatchObject({
status: "unknown",
summary: { unresolved: 2 },
changes: [
expect.objectContaining({
entity: "evidence",
conclusion_kind: "unresolved_branch",
}),
expect.objectContaining({
entity: "evidence",
conclusion_kind: "unresolved_branch",
}),
],
});
},
);
@@ -9,6 +9,8 @@ import { afterEach, describe, expect, it } from "vitest";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { createTestBinarySession } from "../../fixtures/binarySession.js";
import { toolContract } from "../../../src/contracts/toolContracts.js";
import { bundleComparisonResultSchema } from "../../../src/domain/bundleComparison.js";
import { createEvidence } from "../../../src/domain/evidence.js";
import {
createEvidenceBundle,
@@ -37,6 +39,107 @@ const sourceEvidence = (label: string) =>
},
);
describe("bundle comparison MCP semantic status", () => {
it("keeps display-only metadata out of deltas and preserves semantic changes", async () => {
const root = await createTestTempDirectory("rea-bundle-metadata-mcp-");
roots.push(root);
const leftPath = join(root, "left.json");
const displayPath = join(root, "display.json");
const changedPath = join(root, "changed.json");
const target = {
path: join(root, "left.elf"),
sha256: "a".repeat(64),
format: "elf" as const,
architecture: "x86_64" as const,
};
const record = createEvidence(
target,
{ id: "fixture", name: "Fixture", version: "1" },
{
operation: "observe",
parameters: {},
result: { observed: true },
confidence: "observed",
authority: "shipped-artifact",
},
);
const left = createEvidenceBundle([record]);
const displayRecord = {
...record,
subject: {
...record.subject!,
name: "renamed.elf",
local_path: join(root, "renamed.elf"),
},
};
const displayOnly = { ...left, records: [displayRecord] };
const changedRecord = createEvidence(
target,
{ id: "fixture", name: "Fixture", version: "1" },
{
operation: "observe",
parameters: {},
result: { observed: false },
confidence: "observed",
authority: "shipped-artifact",
},
);
await Promise.all([
writeFile(leftPath, serializeEvidenceBundle(left)),
writeFile(displayPath, serializeEvidenceBundle(displayOnly)),
writeFile(
changedPath,
serializeEvidenceBundle(createEvidenceBundle([changedRecord])),
),
]);
const connected = await connect();
try {
const compare = async (
right: string,
recordPairs: readonly {
left_evidence_id: string;
right_evidence_id: string;
}[] = [],
) => {
const result = await connected.client.callTool({
name: "compare_bundles",
arguments: {
left_bundle_path: leftPath,
right_bundle_path: right,
record_pairs: recordPairs,
},
});
expect(result.isError).not.toBe(true);
const evidence = toolContract("compare_bundles").outputSchema.parse(
result.structuredContent,
);
return bundleComparisonResultSchema.parse(evidence.normalized_result);
};
const same = await compare(leftPath);
expect(same).toMatchObject({
status: "unchanged",
changes: [],
});
const display = await compare(displayPath);
expect(display).toMatchObject({
status: "unchanged",
changes: [],
});
expect(display.left_bundle_sha256).not.toBe(display.right_bundle_sha256);
const changed = await compare(changedPath, [
{
left_evidence_id: record.evidence_id,
right_evidence_id: changedRecord.evidence_id,
},
]);
expect(changed.status).toBe("changed");
expect(changed.changes).toHaveLength(1);
} finally {
await connected.close();
}
});
});
describe("bundle comparison MCP integration", () => {
it("reads caller-selected bundle files and returns complete Evidence", async () => {
const root = await createTestTempDirectory("rea-bundle-mcp-");