mirror of
https://github.com/morluto/rea
synced 2026-10-25 22:08:11 -04:00
refactor(verification): separate readiness evidence scopes
Move the synthetic evaluator and its unchanged semantic tests out of the shipped runtime. Retain actual CLI and MCP Evidence separately from the synthetic nine-stage baseline in a versioned verifier artifact, execute its reported conformance checks, and assign the verifier to its CI lane. Validation: evaluator/state/classifier source suites, compiled CLI/MCP views, verify:readiness, static checks, build and docs passed. Readiness observed coverage is static analysis; fixture runtime behavior and reconstruction closure were not executed.
This commit is contained in:
morluto
@@ -28,7 +28,6 @@ const FIXTURE_SOURCES = [
|
||||
"tests/conformance/swift/**",
|
||||
"tests/conformance/versions/**",
|
||||
];
|
||||
const READINESS_SOURCES = ["tests/conformance/readiness/**"];
|
||||
// These reporters affect CI output only; they do not select a host or provider
|
||||
// verification lane when changed on their own.
|
||||
const PORTABLE_CI_REPORTERS = new Set([
|
||||
@@ -281,7 +280,6 @@ if (tier === "reused") {
|
||||
selected.fixtures = true;
|
||||
selected.readiness = true;
|
||||
}
|
||||
if (matchesAny(file, READINESS_SOURCES)) selected.readiness = true;
|
||||
if (file.startsWith("tests/")) continue;
|
||||
if (
|
||||
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test(
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
// Shared build inputs are classified once in plan.mjs rather than repeated in
|
||||
// every workflow.
|
||||
export const SCOPES = {
|
||||
readiness: [
|
||||
"tests/conformance/readiness/**",
|
||||
"scripts/verify-reconstruction-readiness.mjs",
|
||||
],
|
||||
reverse_tools: [
|
||||
".github/workflows/mutation.yml",
|
||||
"src/application/reverse/**",
|
||||
|
||||
@@ -6,14 +6,32 @@ import { promisify } from "node:util";
|
||||
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { tsImport } from "tsx/esm/api";
|
||||
|
||||
import { toolContract } from "../dist/contracts/toolContracts.js";
|
||||
import { RECONSTRUCTION_READINESS_EXAMPLE } from "../dist/contracts/reconstructionReadinessExample.js";
|
||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||
import { createEvidenceBundle } from "../dist/domain/evidenceBundle.js";
|
||||
import { createReconstructionReadinessReport } from "../dist/domain/reconstructionReadiness.js";
|
||||
import { reconstructionReadinessReportSchema } from "../dist/domain/reconstructionReadinessSchemas.js";
|
||||
import { PACKAGE_METADATA } from "../dist/generatedPackageMetadata.js";
|
||||
import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
|
||||
|
||||
const [
|
||||
{ RECONSTRUCTION_READINESS_EXAMPLE },
|
||||
{ createReconstructionReadinessReport },
|
||||
{ reconstructionReadinessReportSchema },
|
||||
] = await Promise.all(
|
||||
[
|
||||
"reconstructionReadinessExample",
|
||||
"reconstructionReadiness",
|
||||
"reconstructionReadinessSchemas",
|
||||
].map((name) =>
|
||||
tsImport(
|
||||
new URL(
|
||||
`../tests/conformance/readiness/report/${name}.ts`,
|
||||
import.meta.url,
|
||||
).href,
|
||||
import.meta.url,
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const root = process.cwd();
|
||||
@@ -50,6 +68,8 @@ const client = new Client({
|
||||
capabilities: { elicitation: {} },
|
||||
});
|
||||
|
||||
let verificationReport;
|
||||
let primaryError;
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const catalog = await client.listTools();
|
||||
@@ -61,6 +81,7 @@ try {
|
||||
{ name: "binary_session", arguments: {} },
|
||||
{ toolDefinition: sessionTool },
|
||||
);
|
||||
assert.notEqual(sessionReply.isError, true);
|
||||
const sessionStatus = toolContract("binary_session").outputSchema.parse(
|
||||
sessionReply.structuredContent,
|
||||
).result;
|
||||
@@ -77,76 +98,93 @@ try {
|
||||
"CLI/MCP Electron analysis Evidence identity drifted",
|
||||
);
|
||||
|
||||
const input = structuredClone(RECONSTRUCTION_READINESS_EXAMPLE);
|
||||
input.identity.cli_version = PACKAGE_METADATA.version;
|
||||
input.identity.server_version = PACKAGE_METADATA.version;
|
||||
input.identity.catalog_digest =
|
||||
sessionStatus.server_identity.catalog.digests.combined_sha256;
|
||||
const records = [
|
||||
...input.evidence_bundle.records,
|
||||
nativeEvidence,
|
||||
javascriptCliEvidence,
|
||||
electronCliEvidence,
|
||||
].map(parseEvidence);
|
||||
input.evidence_bundle = createEvidenceBundle(records);
|
||||
setFixtureDigest(input, "native", nativeEvidence);
|
||||
setFixtureDigest(input, "javascript-cli", javascriptCliEvidence);
|
||||
setFixtureDigest(input, "electron", electronCliEvidence);
|
||||
linkCheck(input, "discover-classify", "artifact-inventory", nativeEvidence);
|
||||
linkCheck(input, "static-analysis", "native-routed", nativeEvidence);
|
||||
linkCheck(
|
||||
input,
|
||||
"static-analysis",
|
||||
"javascript-routed",
|
||||
javascriptCliEvidence,
|
||||
);
|
||||
linkCheck(
|
||||
input,
|
||||
"reactive-scenarios",
|
||||
"electron-correlated",
|
||||
electronCliEvidence,
|
||||
);
|
||||
|
||||
const readinessReport = reconstructionReadinessReportSchema.parse(
|
||||
createReconstructionReadinessReport(input),
|
||||
);
|
||||
assert.equal(readinessReport.status, "pass");
|
||||
if (process.env.REA_READINESS_REPORT_PATH !== undefined)
|
||||
await writeFile(
|
||||
resolve(root, process.env.REA_READINESS_REPORT_PATH),
|
||||
`${JSON.stringify(readinessReport, null, 2)}\n`,
|
||||
);
|
||||
|
||||
const tampered = structuredClone(input);
|
||||
tampered.replay.expected_source_digest = readinessReport.source_digest;
|
||||
tampered.identity.server_version = "tampered";
|
||||
const tamperedReport = reconstructionReadinessReportSchema.parse(
|
||||
createReconstructionReadinessReport(tampered),
|
||||
);
|
||||
assert.equal(tamperedReport.status, "fail");
|
||||
assert.ok(
|
||||
tamperedReport.findings.some(
|
||||
({ code }) => code === "replay-source-digest-mismatch",
|
||||
),
|
||||
);
|
||||
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
status: readinessReport.status,
|
||||
report_id: readinessReport.report_id,
|
||||
report_digest: readinessReport.report_digest,
|
||||
stages: readinessReport.summary.stages,
|
||||
findings: readinessReport.summary.findings,
|
||||
native_evidence_id: nativeEvidence.evidence_id,
|
||||
javascript_evidence_id: javascriptCliEvidence.evidence_id,
|
||||
electron_evidence_id: electronCliEvidence.evidence_id,
|
||||
provider: "local-stdio",
|
||||
})}\n`,
|
||||
);
|
||||
verificationReport = {
|
||||
schema: "ReconstructionReadinessVerificationReport",
|
||||
schema_version: 1,
|
||||
status: "pass",
|
||||
observed_journeys: {
|
||||
evidence_kind: "observed",
|
||||
scope: "static-cli-mcp",
|
||||
status: "pass",
|
||||
server_identity: sessionStatus.server_identity,
|
||||
evidence_bundles: {
|
||||
cli: createEvidenceBundle([
|
||||
nativeEvidence,
|
||||
javascriptCliEvidence,
|
||||
electronCliEvidence,
|
||||
]),
|
||||
mcp: createEvidenceBundle([javascriptMcp, electronMcp]),
|
||||
},
|
||||
journeys: [
|
||||
{
|
||||
journey_id: "native-artifact-cli",
|
||||
operation: "inspect-artifact",
|
||||
surfaces: ["cli"],
|
||||
status: "pass",
|
||||
evidence_ids: [nativeEvidence.evidence_id],
|
||||
checks: ["cli-completed", "evidence-authenticated"],
|
||||
},
|
||||
...[
|
||||
["javascript-static-cli-mcp", javascriptCliEvidence],
|
||||
["electron-static-cli-mcp", electronCliEvidence],
|
||||
].map(([journeyId, evidence]) => ({
|
||||
journey_id: journeyId,
|
||||
operation: "analyze-javascript-application",
|
||||
surfaces: ["cli", "mcp"],
|
||||
status: "pass",
|
||||
evidence_ids: [evidence.evidence_id],
|
||||
checks: [
|
||||
"cli-completed",
|
||||
"mcp-completed",
|
||||
"evidence-authenticated",
|
||||
"evidence-identity-parity",
|
||||
],
|
||||
})),
|
||||
],
|
||||
limitations: [
|
||||
"Native MCP parity was not exercised.",
|
||||
"Fixture application runtime behavior was not executed.",
|
||||
"Authority recovery, cancellation cleanup, behavioral comparison, and reconstruction closure were not exercised.",
|
||||
],
|
||||
},
|
||||
evaluator_conformance: verifySyntheticEvaluator(),
|
||||
};
|
||||
} catch (cause) {
|
||||
primaryError = cause;
|
||||
throw cause;
|
||||
} finally {
|
||||
await client.close();
|
||||
await closeVerifierResources(
|
||||
[() => client.close(), () => transport.close()],
|
||||
primaryError,
|
||||
);
|
||||
}
|
||||
|
||||
if (process.env.REA_READINESS_REPORT_PATH !== undefined)
|
||||
await writeFile(
|
||||
resolve(root, process.env.REA_READINESS_REPORT_PATH),
|
||||
`${JSON.stringify(verificationReport, null, 2)}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
schema: verificationReport.schema,
|
||||
schema_version: verificationReport.schema_version,
|
||||
status: verificationReport.status,
|
||||
observed_journeys: {
|
||||
status: verificationReport.observed_journeys.status,
|
||||
scope: verificationReport.observed_journeys.scope,
|
||||
journeys: verificationReport.observed_journeys.journeys,
|
||||
limitations: verificationReport.observed_journeys.limitations,
|
||||
},
|
||||
evaluator_conformance: {
|
||||
status: verificationReport.evaluator_conformance.status,
|
||||
evidence_kind: verificationReport.evaluator_conformance.evidence_kind,
|
||||
checks: verificationReport.evaluator_conformance.checks,
|
||||
report_digest:
|
||||
verificationReport.evaluator_conformance.baseline_report.report_digest,
|
||||
},
|
||||
})}\n`,
|
||||
);
|
||||
|
||||
async function runCli(arguments_) {
|
||||
const { stdout } = await execute(
|
||||
process.execPath,
|
||||
@@ -166,32 +204,50 @@ async function analyzeMcp(client_, inputPath) {
|
||||
arguments: { input_path: inputPath },
|
||||
});
|
||||
assert.notEqual(response.isError, true);
|
||||
const projected = requireObject(response.structuredContent);
|
||||
return { evidence_id: projected.evidence_id };
|
||||
}
|
||||
|
||||
function requireObject(value) {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value))
|
||||
throw new TypeError("Expected an object");
|
||||
return value;
|
||||
}
|
||||
|
||||
function setFixtureDigest(input, kind, evidence) {
|
||||
const fixture = input.fixtures.find(
|
||||
({ kind: fixtureKind }) => fixtureKind === kind,
|
||||
toolContract("analyze_javascript_application").outputSchema.parse(
|
||||
response.structuredContent,
|
||||
);
|
||||
if (fixture === undefined || evidence.subject === null)
|
||||
throw new TypeError(`Missing ${kind} fixture identity`);
|
||||
fixture.artifact_sha256 = evidence.subject.digest.sha256;
|
||||
return parseEvidence(response.structuredContent);
|
||||
}
|
||||
|
||||
function linkCheck(input, stageId, checkId, evidence) {
|
||||
const stage = input.stages.find(({ stage_id: id }) => id === stageId);
|
||||
const check = stage?.checks.find(({ check_id: id }) => id === checkId);
|
||||
if (stage === undefined || check === undefined)
|
||||
throw new TypeError(`Missing readiness check ${stageId}/${checkId}`);
|
||||
stage.evidence_ids = [
|
||||
...new Set([...stage.evidence_ids, evidence.evidence_id]),
|
||||
].sort();
|
||||
check.evidence_ids = [evidence.evidence_id];
|
||||
function verifySyntheticEvaluator() {
|
||||
const input = structuredClone(RECONSTRUCTION_READINESS_EXAMPLE);
|
||||
const evaluate = (value) =>
|
||||
reconstructionReadinessReportSchema.parse(
|
||||
createReconstructionReadinessReport(value),
|
||||
);
|
||||
const baseline = evaluate(input);
|
||||
assert.equal(baseline.status, "pass");
|
||||
assert.deepEqual(evaluate(structuredClone(input)), baseline);
|
||||
|
||||
const tampered = structuredClone(input);
|
||||
const record = tampered.evidence_bundle.records[0];
|
||||
assert.ok(record, "Synthetic conformance requires Evidence");
|
||||
record.operation += "_tampered";
|
||||
assert.throws(
|
||||
() => evaluate(tampered),
|
||||
/Evidence semantic identifier does not match its record/u,
|
||||
);
|
||||
|
||||
const stale = structuredClone(input);
|
||||
stale.replay.expected_source_digest = baseline.source_digest;
|
||||
stale.identity.server_version += "-changed";
|
||||
const staleReport = evaluate(stale);
|
||||
assert.equal(staleReport.status, "fail");
|
||||
assert.ok(
|
||||
staleReport.findings.some(
|
||||
({ code }) => code === "replay-source-digest-mismatch",
|
||||
),
|
||||
);
|
||||
return {
|
||||
evidence_kind: "synthetic",
|
||||
status: "pass",
|
||||
checks: [
|
||||
"baseline-accepted",
|
||||
"deterministic-evaluation",
|
||||
"tampered-evidence-rejected",
|
||||
"stale-source-rejected",
|
||||
].map((checkId) => ({ check_id: checkId, status: "pass" })),
|
||||
baseline_report: baseline,
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user