mirror of
https://github.com/morluto/rea
synced 2026-10-26 06:16:17 -04:00
feat(conformance): add portable conformance export package format
Define a committed scenario/replay/shim/evidence package format for portable process-fidelity conformance fixtures (#336). The package format is versioned, path-independent, and consumable from a clean checkout. - conformancePackage.ts: top-level package schema with scenario manifests, replay plans, shim plans, expected evidence, and verifier contracts. Deterministic package ID computed from canonical JSON. Cross-reference validation ensures internal consistency. - conformanceTrustGate.ts: trust gates that compare captured runs against committed conformance packages (#337). Classifies semantic diffs from timing/normalization noise. Unknown/truncated evidence is never treated as equivalence. - conformanceReplay.ts: CI replay engine that runs each scenario, evaluates trust gates, and detects drift between captured and committed evidence (#338). Closes #336, #337, #338
This commit is contained in:
morluto
@@ -0,0 +1,218 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import canonicalize from "canonicalize";
|
||||
import { z } from "zod";
|
||||
|
||||
import { evidenceEnvelopeSchema } from "./evidence.js";
|
||||
import { evidenceBundleSchema } from "./evidenceBundle.js";
|
||||
|
||||
/** Schema version for the conformance package format. */
|
||||
export const CONFORMANCE_PACKAGE_VERSION = 1;
|
||||
|
||||
/** A stable, path-independent identifier for a conformance package. */
|
||||
export const conformancePackageIdSchema = z
|
||||
.string()
|
||||
.regex(/^cp_[a-f0-9]{64}$/u);
|
||||
|
||||
/** Identifier for a single scenario within a package. */
|
||||
export const scenarioIdSchema = z
|
||||
.string()
|
||||
.regex(/^[A-Za-z][A-Za-z0-9._-]{0,99}$/u);
|
||||
|
||||
/** Deterministic manifest for a scenario fixture. */
|
||||
export const scenarioManifestSchema = z.strictObject({
|
||||
scenario_id: scenarioIdSchema,
|
||||
name: z.string().min(1),
|
||||
description: z.string().min(1),
|
||||
/** Source-owned fixture path relative to the repository root. */
|
||||
fixture_path: z.string().min(1),
|
||||
/** Expected exit code or null if unspecified. */
|
||||
expected_exit_code: z.number().int().nullable(),
|
||||
/** Expected output patterns that must appear in the capture. */
|
||||
expected_patterns: z.array(z.string()).default([]),
|
||||
});
|
||||
|
||||
/** Replay plan describing how a scenario is replayed deterministically. */
|
||||
export const replayPlanSchema = z.strictObject({
|
||||
scenario_id: scenarioIdSchema,
|
||||
/** Ordered steps to execute during replay. */
|
||||
steps: z
|
||||
.array(
|
||||
z.strictObject({
|
||||
step_id: z.string().min(1),
|
||||
action: z.string().min(1),
|
||||
arguments: z.array(z.string()).default([]),
|
||||
timeout_ms: z.number().int().positive(),
|
||||
}),
|
||||
)
|
||||
.min(1)
|
||||
.max(100),
|
||||
/** Environment variables to set (never inherit host paths). */
|
||||
environment: z.record(z.string(), z.string()).default({}),
|
||||
});
|
||||
|
||||
/** Shim plan for intercepting observable effects. */
|
||||
export const shimPlanSchema = z.strictObject({
|
||||
scenario_id: scenarioIdSchema,
|
||||
/** Shims to install, each intercepting a named effect. */
|
||||
shims: z
|
||||
.array(
|
||||
z.strictObject({
|
||||
shim_id: z.string().min(1),
|
||||
kind: z.enum(["filesystem", "network", "process", "signal"]),
|
||||
target: z.string().min(1),
|
||||
policy: z.enum(["observe", "allow", "block", "emulate"]),
|
||||
}),
|
||||
)
|
||||
.min(0)
|
||||
.max(50),
|
||||
});
|
||||
|
||||
/** Expected evidence for a scenario, checked against the captured run. */
|
||||
export const expectedEvidenceSchema = z.strictObject({
|
||||
scenario_id: scenarioIdSchema,
|
||||
/** Expected evidence envelopes. */
|
||||
envelopes: z.array(evidenceEnvelopeSchema).min(0).max(100),
|
||||
/** Expected evidence bundle. */
|
||||
bundle: evidenceBundleSchema.nullable(),
|
||||
/** Required dimensions that must be present in the evidence. */
|
||||
required_dimensions: z.array(z.string().min(1)).default([]),
|
||||
});
|
||||
|
||||
/** Verifier contract describing how to validate conformance. */
|
||||
export const verifierContractSchema = z.strictObject({
|
||||
scenario_id: scenarioIdSchema,
|
||||
/** Dimensions to verify. */
|
||||
dimensions: z
|
||||
.array(
|
||||
z.strictObject({
|
||||
name: z.string().min(1),
|
||||
required: z.boolean().default(true),
|
||||
comparison: z.enum(["exact", "semantic", "fuzzy"]),
|
||||
}),
|
||||
)
|
||||
.min(1)
|
||||
.max(50),
|
||||
/** Tolerance for timing differences in milliseconds. */
|
||||
timing_tolerance_ms: z.number().int().nonnegative().default(0),
|
||||
});
|
||||
|
||||
/** Top-level conformance package manifest. */
|
||||
export const conformancePackageSchema = z.strictObject({
|
||||
schema_version: z.literal(CONFORMANCE_PACKAGE_VERSION),
|
||||
package_id: conformancePackageIdSchema,
|
||||
name: z.string().min(1),
|
||||
description: z.string().min(1),
|
||||
created_at: z.string().datetime(),
|
||||
/** Scenario manifests. */
|
||||
scenarios: z.array(scenarioManifestSchema).min(1).max(100),
|
||||
/** Replay plans keyed by scenario_id. */
|
||||
replay_plans: z.array(replayPlanSchema).min(1).max(100),
|
||||
/** Shim plans keyed by scenario_id. */
|
||||
shim_plans: z.array(shimPlanSchema).default([]),
|
||||
/** Expected evidence for each scenario. */
|
||||
expected_evidence: z.array(expectedEvidenceSchema).min(1).max(100),
|
||||
/** Verifier contracts for each scenario. */
|
||||
verifier_contracts: z.array(verifierContractSchema).min(1).max(100),
|
||||
});
|
||||
|
||||
export type ConformancePackage = z.infer<typeof conformancePackageSchema>;
|
||||
export type ScenarioManifest = z.infer<typeof scenarioManifestSchema>;
|
||||
export type ReplayPlan = z.infer<typeof replayPlanSchema>;
|
||||
export type ShimPlan = z.infer<typeof shimPlanSchema>;
|
||||
export type ExpectedEvidence = z.infer<typeof expectedEvidenceSchema>;
|
||||
export type VerifierContract = z.infer<typeof verifierContractSchema>;
|
||||
|
||||
/** Error from validation or package operations. */
|
||||
export type ConformancePackageError =
|
||||
| { kind: "invalid_package"; message: string }
|
||||
| { kind: "scenario_not_found"; scenario_id: string }
|
||||
| { kind: "duplicate_scenario"; scenario_id: string };
|
||||
|
||||
/** Result type for package operations. */
|
||||
export type ConformancePackageResult<T> =
|
||||
| { ok: true; value: T }
|
||||
| { ok: false; error: ConformancePackageError };
|
||||
|
||||
/** Validate a conformance package and cross-check internal references. */
|
||||
export function validateConformancePackage(
|
||||
input: unknown,
|
||||
): ConformancePackageResult<ConformancePackage> {
|
||||
const parsed = conformancePackageSchema.safeParse(input);
|
||||
if (!parsed.success)
|
||||
return {
|
||||
ok: false,
|
||||
error: {
|
||||
kind: "invalid_package",
|
||||
message: parsed.error.issues[0]?.message ?? "invalid package",
|
||||
},
|
||||
};
|
||||
|
||||
const pkg = parsed.data;
|
||||
|
||||
// Check for duplicate scenario IDs
|
||||
const seen = new Set<string>();
|
||||
for (const s of pkg.scenarios) {
|
||||
if (seen.has(s.scenario_id))
|
||||
return {
|
||||
ok: false,
|
||||
error: { kind: "duplicate_scenario", scenario_id: s.scenario_id },
|
||||
};
|
||||
seen.add(s.scenario_id);
|
||||
}
|
||||
|
||||
// Check that replay plans reference existing scenarios
|
||||
for (const rp of pkg.replay_plans) {
|
||||
if (!seen.has(rp.scenario_id))
|
||||
return {
|
||||
ok: false,
|
||||
error: {
|
||||
kind: "scenario_not_found",
|
||||
scenario_id: rp.scenario_id,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Check that expected evidence references existing scenarios
|
||||
for (const ee of pkg.expected_evidence) {
|
||||
if (!seen.has(ee.scenario_id))
|
||||
return {
|
||||
ok: false,
|
||||
error: {
|
||||
kind: "scenario_not_found",
|
||||
scenario_id: ee.scenario_id,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Check that verifier contracts reference existing scenarios
|
||||
for (const vc of pkg.verifier_contracts) {
|
||||
if (!seen.has(vc.scenario_id))
|
||||
return {
|
||||
ok: false,
|
||||
error: {
|
||||
kind: "scenario_not_found",
|
||||
scenario_id: vc.scenario_id,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return { ok: true, value: pkg };
|
||||
}
|
||||
|
||||
/** Compute a deterministic package ID from the canonical JSON. */
|
||||
export function computePackageId(
|
||||
pkg: Omit<ConformancePackage, "package_id">,
|
||||
): string {
|
||||
const json = canonicalize(pkg);
|
||||
if (!json) throw new Error("failed to canonicalize package");
|
||||
return `cp_${createHash("sha256").update(json).digest("hex")}`;
|
||||
}
|
||||
|
||||
/** Create a conformance package with an auto-computed package_id. */
|
||||
export function createConformancePackage(
|
||||
pkg: Omit<ConformancePackage, "package_id">,
|
||||
): ConformancePackage {
|
||||
const package_id = computePackageId(pkg);
|
||||
return { package_id, ...pkg };
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
conformancePackageSchema,
|
||||
type ConformancePackage,
|
||||
} from "./conformancePackage.js";
|
||||
import { trustGateResultSchema } from "./conformanceTrustGate.js";
|
||||
|
||||
/** Result of a single scenario replay. */
|
||||
export const scenarioReplayResultSchema = z.strictObject({
|
||||
scenario_id: z.string().min(1),
|
||||
status: z.enum(["pass", "fail", "error", "skipped"]),
|
||||
exit_code: z.number().int().nullable(),
|
||||
duration_ms: z.number().int().nonnegative(),
|
||||
output: z.string().default(""),
|
||||
error: z.string().nullable(),
|
||||
});
|
||||
|
||||
/** Result of replaying an entire conformance package. */
|
||||
export const packageReplayResultSchema = z.strictObject({
|
||||
package_id: z.string().min(1),
|
||||
total_scenarios: z.number().int().positive(),
|
||||
passed: z.number().int().nonnegative(),
|
||||
failed: z.number().int().nonnegative(),
|
||||
errored: z.number().int().nonnegative(),
|
||||
skipped: z.number().int().nonnegative(),
|
||||
scenario_results: z.array(scenarioReplayResultSchema),
|
||||
trust_gate_results: z.array(trustGateResultSchema),
|
||||
drift_detected: z.boolean(),
|
||||
first_drift: z
|
||||
.strictObject({
|
||||
scenario_id: z.string().min(1),
|
||||
dimension: z.string().min(1),
|
||||
message: z.string(),
|
||||
})
|
||||
.nullable(),
|
||||
});
|
||||
|
||||
export type ScenarioReplayResult = z.infer<typeof scenarioReplayResultSchema>;
|
||||
export type PackageReplayResult = z.infer<typeof packageReplayResultSchema>;
|
||||
|
||||
/** Callback interface for replaying a scenario. */
|
||||
export interface ScenarioRunner {
|
||||
(scenarioId: string, fixturePath: string): Promise<ScenarioReplayResult>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default no-op scenario runner that skips all scenarios.
|
||||
* Real CI replays inject an actual runner.
|
||||
*/
|
||||
async function defaultRunner(): Promise<ScenarioReplayResult> {
|
||||
return {
|
||||
scenario_id: "unknown",
|
||||
status: "skipped",
|
||||
exit_code: null,
|
||||
duration_ms: 0,
|
||||
output: "",
|
||||
error: "no runner provided",
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Replay a conformance package from a clean checkout.
|
||||
* Runs each scenario, evaluates trust gates, and detects drift
|
||||
* between the captured run and the committed package.
|
||||
*/
|
||||
export async function replayConformancePackage(
|
||||
packageInput: unknown,
|
||||
actualEvidence: Record<string, Record<string, unknown>>,
|
||||
runner: ScenarioRunner = defaultRunner,
|
||||
options: { truncated?: boolean } = {},
|
||||
): Promise<PackageReplayResult> {
|
||||
const parsed = conformancePackageSchema.safeParse(packageInput);
|
||||
if (!parsed.success)
|
||||
throw new TypeError(
|
||||
`invalid conformance package: ${parsed.error.issues[0]?.message ?? "unknown"}`,
|
||||
);
|
||||
const pkg: ConformancePackage = parsed.data;
|
||||
|
||||
const scenarioResults: ScenarioReplayResult[] = [];
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
let errored = 0;
|
||||
let skipped = 0;
|
||||
|
||||
for (const scenario of pkg.scenarios) {
|
||||
const result = await runner(scenario.scenario_id, scenario.fixture_path);
|
||||
scenarioResults.push(result);
|
||||
|
||||
switch (result.status) {
|
||||
case "pass":
|
||||
passed++;
|
||||
break;
|
||||
case "fail":
|
||||
failed++;
|
||||
break;
|
||||
case "error":
|
||||
errored++;
|
||||
break;
|
||||
case "skipped":
|
||||
skipped++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const trustGateResults = evaluatePackageTrustGates(
|
||||
pkg,
|
||||
actualEvidence,
|
||||
options,
|
||||
);
|
||||
|
||||
const driftGates = trustGateResults.filter((r) => r.verdict === "fail");
|
||||
const drift_detected = driftGates.length > 0;
|
||||
const first_drift_gate = driftGates[0];
|
||||
const first_drift = first_drift_gate?.first_divergence
|
||||
? {
|
||||
scenario_id: first_drift_gate.scenario_id,
|
||||
dimension: first_drift_gate.first_divergence.dimension,
|
||||
message: first_drift_gate.first_divergence.message,
|
||||
}
|
||||
: null;
|
||||
|
||||
return {
|
||||
package_id: pkg.package_id,
|
||||
total_scenarios: pkg.scenarios.length,
|
||||
passed,
|
||||
failed,
|
||||
errored,
|
||||
skipped,
|
||||
scenario_results: scenarioResults,
|
||||
trust_gate_results: trustGateResults,
|
||||
drift_detected,
|
||||
first_drift,
|
||||
};
|
||||
}
|
||||
|
||||
/** Import trust gate evaluator for replay. */
|
||||
import { evaluatePackageTrustGates } from "./conformanceTrustGate.js";
|
||||
@@ -0,0 +1,246 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import type {
|
||||
ConformancePackage,
|
||||
VerifierContract,
|
||||
} from "./conformancePackage.js";
|
||||
|
||||
/** Result of comparing a single dimension. */
|
||||
export const dimensionResultSchema = z.strictObject({
|
||||
name: z.string().min(1),
|
||||
status: z.enum(["match", "mismatch", "unknown", "truncated"]),
|
||||
message: z.string().default(""),
|
||||
evidence_ids: z.array(z.string()).default([]),
|
||||
});
|
||||
|
||||
/** Result of evaluating a trust gate for one scenario. */
|
||||
export const trustGateResultSchema = z.strictObject({
|
||||
scenario_id: z.string().min(1),
|
||||
verdict: z.enum(["pass", "fail", "unknown"]),
|
||||
dimension_results: z.array(dimensionResultSchema),
|
||||
first_divergence: z
|
||||
.strictObject({
|
||||
dimension: z.string().min(1),
|
||||
evidence_id: z.string().min(1),
|
||||
message: z.string(),
|
||||
})
|
||||
.nullable(),
|
||||
});
|
||||
|
||||
export type DimensionResult = z.infer<typeof dimensionResultSchema>;
|
||||
export type TrustGateResult = z.infer<typeof trustGateResultSchema>;
|
||||
|
||||
/** Dimension that is volatile due to timing or normalization. */
|
||||
const VOLATILE_DIMENSIONS = new Set([
|
||||
"timing",
|
||||
"timestamp",
|
||||
"pid",
|
||||
"ppid",
|
||||
"duration_ms",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
]);
|
||||
|
||||
/** Dimension that carries semantic content. */
|
||||
const SEMANTIC_DIMENSIONS = new Set([
|
||||
"exit_code",
|
||||
"stdout",
|
||||
"stderr",
|
||||
"filesystem",
|
||||
"process_tree",
|
||||
"shim_events",
|
||||
"protocol_events",
|
||||
"event_journal",
|
||||
]);
|
||||
|
||||
/** Check if a dimension name is volatile (timing/normalization noise). */
|
||||
export function isVolatileDimension(name: string): boolean {
|
||||
return VOLATILE_DIMENSIONS.has(name);
|
||||
}
|
||||
|
||||
/** Check if a dimension name carries semantic content. */
|
||||
export function isSemanticDimension(name: string): boolean {
|
||||
return SEMANTIC_DIMENSIONS.has(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two values for a dimension, classifying semantic diffs
|
||||
* from timing/normalization noise.
|
||||
*
|
||||
* Volatile dimensions (timing, timestamps, PIDs) are always treated
|
||||
* as matches because their values are expected to differ across
|
||||
* runs and do not carry semantic meaning.
|
||||
*/
|
||||
export function compareDimension(
|
||||
dimensionName: string,
|
||||
expected: unknown,
|
||||
actual: unknown,
|
||||
options: { truncated?: boolean } = {},
|
||||
): DimensionResult {
|
||||
// Volatile dimensions: always match regardless of values
|
||||
if (isVolatileDimension(dimensionName)) {
|
||||
return {
|
||||
name: dimensionName,
|
||||
status: "match",
|
||||
message: "volatile dimension ignored",
|
||||
evidence_ids: [],
|
||||
};
|
||||
}
|
||||
|
||||
if (options.truncated) {
|
||||
return {
|
||||
name: dimensionName,
|
||||
status: "truncated",
|
||||
message: "evidence was truncated",
|
||||
evidence_ids: [],
|
||||
};
|
||||
}
|
||||
|
||||
if (expected === undefined && actual === undefined) {
|
||||
return {
|
||||
name: dimensionName,
|
||||
status: "unknown",
|
||||
message: "both expected and actual are undefined",
|
||||
evidence_ids: [],
|
||||
};
|
||||
}
|
||||
|
||||
if (expected === undefined || actual === undefined) {
|
||||
return {
|
||||
name: dimensionName,
|
||||
status: "mismatch",
|
||||
message: `expected ${expected === undefined ? "present" : "absent"}, actual ${actual === undefined ? "absent" : "present"}`,
|
||||
evidence_ids: [],
|
||||
};
|
||||
}
|
||||
|
||||
// Semantic comparison for structured objects
|
||||
if (typeof expected === "object" && typeof actual === "object") {
|
||||
const expectedJson = JSON.stringify(sortKeys(expected));
|
||||
const actualJson = JSON.stringify(sortKeys(actual));
|
||||
if (expectedJson === actualJson) {
|
||||
return {
|
||||
name: dimensionName,
|
||||
status: "match",
|
||||
message: "",
|
||||
evidence_ids: [],
|
||||
};
|
||||
}
|
||||
return {
|
||||
name: dimensionName,
|
||||
status: "mismatch",
|
||||
message: "semantic content differs",
|
||||
evidence_ids: [],
|
||||
};
|
||||
}
|
||||
|
||||
// Primitive comparison
|
||||
if (expected === actual) {
|
||||
return {
|
||||
name: dimensionName,
|
||||
status: "match",
|
||||
message: "",
|
||||
evidence_ids: [],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name: dimensionName,
|
||||
status: "mismatch",
|
||||
message: `expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`,
|
||||
evidence_ids: [],
|
||||
};
|
||||
}
|
||||
|
||||
/** Recursively sort object keys for deterministic comparison. */
|
||||
function sortKeys(value: unknown): unknown {
|
||||
if (value === null || typeof value !== "object") return value;
|
||||
if (Array.isArray(value)) return value.map(sortKeys);
|
||||
const sorted: Record<string, unknown> = {};
|
||||
for (const key of Object.keys(value as Record<string, unknown>).sort()) {
|
||||
sorted[key] = sortKeys((value as Record<string, unknown>)[key]);
|
||||
}
|
||||
return sorted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate a trust gate for one scenario by comparing expected
|
||||
* evidence against the actual captured run.
|
||||
*/
|
||||
export function evaluateTrustGate(
|
||||
contract: VerifierContract,
|
||||
expectedEvidence: Record<string, unknown>,
|
||||
actualEvidence: Record<string, unknown>,
|
||||
options: { truncated?: boolean } = {},
|
||||
): TrustGateResult {
|
||||
const dimensionResults: DimensionResult[] = [];
|
||||
let firstDivergence: TrustGateResult["first_divergence"] = null;
|
||||
|
||||
for (const dim of contract.dimensions) {
|
||||
if (!dim.required) continue;
|
||||
const expectedValue = expectedEvidence[dim.name];
|
||||
const actualValue = actualEvidence[dim.name];
|
||||
const result = compareDimension(
|
||||
dim.name,
|
||||
expectedValue,
|
||||
actualValue,
|
||||
options,
|
||||
);
|
||||
dimensionResults.push(result);
|
||||
|
||||
if (result.status === "mismatch" && firstDivergence === null) {
|
||||
firstDivergence = {
|
||||
dimension: dim.name,
|
||||
evidence_id: result.evidence_ids[0] ?? "unknown",
|
||||
message: result.message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const hasFail = dimensionResults.some(
|
||||
(r) => r.status === "mismatch" || r.status === "truncated",
|
||||
);
|
||||
const hasUnknown = dimensionResults.some((r) => r.status === "unknown");
|
||||
|
||||
const verdict: TrustGateResult["verdict"] = hasFail
|
||||
? "fail"
|
||||
: hasUnknown
|
||||
? "unknown"
|
||||
: "pass";
|
||||
|
||||
return {
|
||||
scenario_id: contract.scenario_id,
|
||||
verdict,
|
||||
dimension_results: dimensionResults,
|
||||
first_divergence: firstDivergence,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate trust gates for all scenarios in a conformance package.
|
||||
* Rejects runs that differ in required dimensions. Unknown/truncated
|
||||
* evidence is never treated as equivalence.
|
||||
*/
|
||||
export function evaluatePackageTrustGates(
|
||||
pkg: ConformancePackage,
|
||||
actualEvidence: Record<string, Record<string, unknown>>,
|
||||
options: { truncated?: boolean } = {},
|
||||
): TrustGateResult[] {
|
||||
const results: TrustGateResult[] = [];
|
||||
for (const contract of pkg.verifier_contracts) {
|
||||
const expected = pkg.expected_evidence.find(
|
||||
(e) => e.scenario_id === contract.scenario_id,
|
||||
);
|
||||
if (!expected) continue;
|
||||
const expectedEvidence = expected.envelopes[0] ?? {};
|
||||
const actual = actualEvidence[contract.scenario_id] ?? {};
|
||||
const result = evaluateTrustGate(
|
||||
contract,
|
||||
expectedEvidence as Record<string, unknown>,
|
||||
actual,
|
||||
options,
|
||||
);
|
||||
results.push(result);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
CONFORMANCE_PACKAGE_VERSION,
|
||||
conformancePackageSchema,
|
||||
computePackageId,
|
||||
createConformancePackage,
|
||||
validateConformancePackage,
|
||||
type ScenarioManifest,
|
||||
type ReplayPlan,
|
||||
type ShimPlan,
|
||||
type ExpectedEvidence,
|
||||
} from "../src/domain/conformancePackage.js";
|
||||
|
||||
const validPackage = createConformancePackage({
|
||||
schema_version: CONFORMANCE_PACKAGE_VERSION,
|
||||
name: "test-fixture",
|
||||
description: "A test conformance fixture",
|
||||
created_at: "2026-07-28T00:00:00Z",
|
||||
scenarios: [
|
||||
{
|
||||
scenario_id: "s1",
|
||||
name: "Simple spawn",
|
||||
description: "A simple process spawn scenario",
|
||||
fixture_path: "tests/conformance/c/fixture.c",
|
||||
expected_exit_code: 0,
|
||||
expected_patterns: ["Hello", "World"],
|
||||
},
|
||||
],
|
||||
replay_plans: [
|
||||
{
|
||||
scenario_id: "s1",
|
||||
steps: [
|
||||
{
|
||||
step_id: "step1",
|
||||
action: "compile",
|
||||
arguments: ["gcc", "fixture.c"],
|
||||
timeout_ms: 5000,
|
||||
},
|
||||
],
|
||||
environment: {},
|
||||
},
|
||||
],
|
||||
shim_plans: [],
|
||||
expected_evidence: [
|
||||
{
|
||||
scenario_id: "s1",
|
||||
envelopes: [],
|
||||
bundle: null,
|
||||
required_dimensions: ["exit_code", "stdout"],
|
||||
},
|
||||
],
|
||||
verifier_contracts: [
|
||||
{
|
||||
scenario_id: "s1",
|
||||
dimensions: [
|
||||
{
|
||||
name: "exit_code",
|
||||
required: true,
|
||||
comparison: "exact",
|
||||
},
|
||||
],
|
||||
timing_tolerance_ms: 100,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
describe("conformance package format", () => {
|
||||
it("validates a well-formed package", () => {
|
||||
const result = validateConformancePackage(validPackage);
|
||||
expect(result.ok).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects invalid schema version", () => {
|
||||
const result = validateConformancePackage({
|
||||
...validPackage,
|
||||
schema_version: 999,
|
||||
});
|
||||
expect(result.ok).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects duplicate scenario IDs", () => {
|
||||
const result = validateConformancePackage({
|
||||
...validPackage,
|
||||
scenarios: [validPackage.scenarios[0], validPackage.scenarios[0]],
|
||||
});
|
||||
expect(result.ok).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects replay plan referencing non-existent scenario", () => {
|
||||
const result = validateConformancePackage({
|
||||
...validPackage,
|
||||
replay_plans: [
|
||||
{
|
||||
scenario_id: "nonexistent",
|
||||
steps: [
|
||||
{
|
||||
step_id: "step1",
|
||||
action: "run",
|
||||
arguments: [],
|
||||
timeout_ms: 1000,
|
||||
},
|
||||
],
|
||||
environment: {},
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.ok).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects verifier contract referencing non-existent scenario", () => {
|
||||
const result = validateConformancePackage({
|
||||
...validPackage,
|
||||
verifier_contracts: [
|
||||
{
|
||||
scenario_id: "nonexistent",
|
||||
dimensions: [
|
||||
{ name: "exit_code", required: true, comparison: "exact" },
|
||||
],
|
||||
timing_tolerance_ms: 0,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.ok).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects expected evidence referencing non-existent scenario", () => {
|
||||
const result = validateConformancePackage({
|
||||
...validPackage,
|
||||
expected_evidence: [
|
||||
{
|
||||
scenario_id: "nonexistent",
|
||||
envelopes: [],
|
||||
bundle: null,
|
||||
required_dimensions: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.ok).toBe(false);
|
||||
});
|
||||
|
||||
it("computes a deterministic package ID", () => {
|
||||
const id = computePackageId({
|
||||
...validPackage,
|
||||
});
|
||||
expect(id).toMatch(/^cp_[a-f0-9]{64}$/u);
|
||||
// Same input yields same ID
|
||||
const id2 = computePackageId({
|
||||
...validPackage,
|
||||
});
|
||||
expect(id).toBe(id2);
|
||||
});
|
||||
|
||||
it("produces different IDs for different packages", () => {
|
||||
const id1 = computePackageId({ ...validPackage, name: "packageA" });
|
||||
const id2 = computePackageId({ ...validPackage, name: "packageB" });
|
||||
expect(id1).not.toBe(id2);
|
||||
});
|
||||
|
||||
it("uses all exported types", () => {
|
||||
const scenario: ScenarioManifest = validPackage.scenarios[0]!;
|
||||
expect(scenario.scenario_id).toBe("s1");
|
||||
|
||||
const replay: ReplayPlan = validPackage.replay_plans[0]!;
|
||||
expect(replay.scenario_id).toBe("s1");
|
||||
|
||||
const shim: ShimPlan = validPackage.shim_plans[0] ?? {
|
||||
scenario_id: "s1",
|
||||
shims: [],
|
||||
};
|
||||
expect(shim.scenario_id).toBe("s1");
|
||||
|
||||
const evidence: ExpectedEvidence = validPackage.expected_evidence[0]!;
|
||||
expect(evidence.scenario_id).toBe("s1");
|
||||
});
|
||||
|
||||
it("parses via zod schema", () => {
|
||||
const result = conformancePackageSchema.safeParse(validPackage);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,190 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
CONFORMANCE_PACKAGE_VERSION,
|
||||
createConformancePackage,
|
||||
type ConformancePackage,
|
||||
} from "../src/domain/conformancePackage.js";
|
||||
import {
|
||||
replayConformancePackage,
|
||||
type ScenarioReplayResult,
|
||||
} from "../src/domain/conformanceReplay.js";
|
||||
|
||||
const validPackage = createConformancePackage({
|
||||
schema_version: CONFORMANCE_PACKAGE_VERSION,
|
||||
name: "test-fixture",
|
||||
description: "A test conformance fixture",
|
||||
created_at: "2026-07-28T00:00:00Z",
|
||||
scenarios: [
|
||||
{
|
||||
scenario_id: "s1",
|
||||
name: "Simple spawn",
|
||||
description: "A simple process spawn scenario",
|
||||
fixture_path: "tests/conformance/c/fixture.c",
|
||||
expected_exit_code: 0,
|
||||
expected_patterns: [],
|
||||
},
|
||||
],
|
||||
replay_plans: [
|
||||
{
|
||||
scenario_id: "s1",
|
||||
steps: [
|
||||
{
|
||||
step_id: "step1",
|
||||
action: "run",
|
||||
arguments: [],
|
||||
timeout_ms: 1000,
|
||||
},
|
||||
],
|
||||
environment: {},
|
||||
},
|
||||
],
|
||||
shim_plans: [],
|
||||
expected_evidence: [
|
||||
{
|
||||
scenario_id: "s1",
|
||||
envelopes: [],
|
||||
bundle: null,
|
||||
required_dimensions: [],
|
||||
},
|
||||
],
|
||||
verifier_contracts: [
|
||||
{
|
||||
scenario_id: "s1",
|
||||
dimensions: [{ name: "exit_code", required: true, comparison: "exact" }],
|
||||
timing_tolerance_ms: 0,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
async function passingRunner(
|
||||
scenarioId: string,
|
||||
): Promise<ScenarioReplayResult> {
|
||||
return {
|
||||
scenario_id: scenarioId,
|
||||
status: "pass",
|
||||
exit_code: 0,
|
||||
duration_ms: 10,
|
||||
output: "ok",
|
||||
error: null,
|
||||
};
|
||||
}
|
||||
|
||||
async function failingRunner(
|
||||
scenarioId: string,
|
||||
): Promise<ScenarioReplayResult> {
|
||||
return {
|
||||
scenario_id: scenarioId,
|
||||
status: "fail",
|
||||
exit_code: 1,
|
||||
duration_ms: 10,
|
||||
output: "fail",
|
||||
error: null,
|
||||
};
|
||||
}
|
||||
|
||||
describe("conformance CI replay", () => {
|
||||
it("replays a package with passing scenarios", async () => {
|
||||
const result = await replayConformancePackage(
|
||||
validPackage,
|
||||
{},
|
||||
passingRunner,
|
||||
);
|
||||
expect(result.total_scenarios).toBe(1);
|
||||
expect(result.passed).toBe(1);
|
||||
expect(result.failed).toBe(0);
|
||||
expect(result.errored).toBe(0);
|
||||
});
|
||||
|
||||
it("replays a package with failing scenarios", async () => {
|
||||
const result = await replayConformancePackage(
|
||||
validPackage,
|
||||
{},
|
||||
failingRunner,
|
||||
);
|
||||
expect(result.failed).toBe(1);
|
||||
expect(result.passed).toBe(0);
|
||||
});
|
||||
|
||||
it("does not detect drift when expected and actual match", async () => {
|
||||
const result = await replayConformancePackage(
|
||||
validPackage,
|
||||
{},
|
||||
passingRunner,
|
||||
);
|
||||
expect(result.drift_detected).toBe(false);
|
||||
});
|
||||
|
||||
it("throws on invalid package", async () => {
|
||||
await expect(replayConformancePackage({}, {})).rejects.toThrow(
|
||||
/invalid conformance package/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("counts all result types", async () => {
|
||||
const base: ConformancePackage = validPackage;
|
||||
const multiPackage = createConformancePackage({
|
||||
...base,
|
||||
scenarios: [
|
||||
base.scenarios[0]!,
|
||||
{
|
||||
scenario_id: "s2",
|
||||
name: "Second scenario",
|
||||
description: "Second",
|
||||
fixture_path: "tests/conformance/c/fixture2.c",
|
||||
expected_exit_code: 1,
|
||||
expected_patterns: [],
|
||||
},
|
||||
],
|
||||
replay_plans: [
|
||||
base.replay_plans[0]!,
|
||||
{
|
||||
scenario_id: "s2",
|
||||
steps: [
|
||||
{
|
||||
step_id: "step1",
|
||||
action: "run",
|
||||
arguments: [],
|
||||
timeout_ms: 1000,
|
||||
},
|
||||
],
|
||||
environment: {},
|
||||
},
|
||||
],
|
||||
expected_evidence: [
|
||||
base.expected_evidence[0]!,
|
||||
{
|
||||
scenario_id: "s2",
|
||||
envelopes: [],
|
||||
bundle: null,
|
||||
required_dimensions: [],
|
||||
},
|
||||
],
|
||||
verifier_contracts: [
|
||||
base.verifier_contracts[0]!,
|
||||
{
|
||||
scenario_id: "s2",
|
||||
dimensions: [
|
||||
{ name: "exit_code", required: true, comparison: "exact" },
|
||||
],
|
||||
timing_tolerance_ms: 0,
|
||||
},
|
||||
],
|
||||
});
|
||||
const result = await replayConformancePackage(
|
||||
multiPackage,
|
||||
{},
|
||||
async (scenarioId) => ({
|
||||
scenario_id: scenarioId,
|
||||
status: scenarioId === "s1" ? "pass" : "fail",
|
||||
exit_code: scenarioId === "s1" ? 0 : 1,
|
||||
duration_ms: 10,
|
||||
output: "",
|
||||
error: null,
|
||||
}),
|
||||
);
|
||||
expect(result.total_scenarios).toBe(2);
|
||||
expect(result.passed).toBe(1);
|
||||
expect(result.failed).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,113 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { VerifierContract } from "../src/domain/conformancePackage.js";
|
||||
import {
|
||||
compareDimension,
|
||||
evaluateTrustGate,
|
||||
isSemanticDimension,
|
||||
isVolatileDimension,
|
||||
} from "../src/domain/conformanceTrustGate.js";
|
||||
|
||||
const contract: VerifierContract = {
|
||||
scenario_id: "s1",
|
||||
dimensions: [
|
||||
{ name: "exit_code", required: true, comparison: "exact" },
|
||||
{ name: "stdout", required: true, comparison: "semantic" },
|
||||
{ name: "timing", required: true, comparison: "fuzzy" },
|
||||
],
|
||||
timing_tolerance_ms: 100,
|
||||
};
|
||||
|
||||
describe("conformance trust gates", () => {
|
||||
it("classifies volatile dimensions correctly", () => {
|
||||
expect(isVolatileDimension("timing")).toBe(true);
|
||||
expect(isVolatileDimension("timestamp")).toBe(true);
|
||||
expect(isVolatileDimension("pid")).toBe(true);
|
||||
expect(isVolatileDimension("exit_code")).toBe(false);
|
||||
});
|
||||
|
||||
it("classifies semantic dimensions correctly", () => {
|
||||
expect(isSemanticDimension("exit_code")).toBe(true);
|
||||
expect(isSemanticDimension("stdout")).toBe(true);
|
||||
expect(isSemanticDimension("timing")).toBe(false);
|
||||
});
|
||||
|
||||
it("matches identical values", () => {
|
||||
const result = compareDimension("exit_code", 0, 0);
|
||||
expect(result.status).toBe("match");
|
||||
});
|
||||
|
||||
it("detects mismatches", () => {
|
||||
const result = compareDimension("exit_code", 0, 1);
|
||||
expect(result.status).toBe("mismatch");
|
||||
});
|
||||
|
||||
it("ignores volatile dimensions", () => {
|
||||
const result = compareDimension("timing", 100, 200);
|
||||
expect(result.status).toBe("match");
|
||||
expect(result.message).toContain("volatile");
|
||||
});
|
||||
|
||||
it("handles truncated evidence", () => {
|
||||
const result = compareDimension("exit_code", 0, null, {
|
||||
truncated: true,
|
||||
});
|
||||
expect(result.status).toBe("truncated");
|
||||
});
|
||||
|
||||
it("handles undefined values", () => {
|
||||
const result = compareDimension("exit_code", undefined, undefined);
|
||||
expect(result.status).toBe("unknown");
|
||||
});
|
||||
|
||||
it("handles missing actual", () => {
|
||||
const result = compareDimension("exit_code", 0, undefined);
|
||||
expect(result.status).toBe("mismatch");
|
||||
});
|
||||
|
||||
it("passes when all dimensions match", () => {
|
||||
const result = evaluateTrustGate(
|
||||
contract,
|
||||
{ exit_code: 0, stdout: "hello" },
|
||||
{ exit_code: 0, stdout: "hello" },
|
||||
);
|
||||
expect(result.verdict).toBe("pass");
|
||||
});
|
||||
|
||||
it("fails when a required dimension mismatches", () => {
|
||||
const result = evaluateTrustGate(
|
||||
contract,
|
||||
{ exit_code: 0, stdout: "hello" },
|
||||
{ exit_code: 1, stdout: "hello" },
|
||||
);
|
||||
expect(result.verdict).toBe("fail");
|
||||
expect(result.first_divergence).not.toBeNull();
|
||||
expect(result.first_divergence!.dimension).toBe("exit_code");
|
||||
});
|
||||
|
||||
it("reports unknown when both dimensions are undefined", () => {
|
||||
const result = evaluateTrustGate(contract, {}, {});
|
||||
// exit_code and stdout are both undefined -> unknown
|
||||
// timing is volatile -> match
|
||||
expect(result.verdict).toBe("unknown");
|
||||
});
|
||||
|
||||
it("ignores volatile dimension differences", () => {
|
||||
const result = evaluateTrustGate(
|
||||
contract,
|
||||
{ exit_code: 0, stdout: "hello", timing: 100 },
|
||||
{ exit_code: 0, stdout: "hello", timing: 200 },
|
||||
);
|
||||
expect(result.verdict).toBe("pass");
|
||||
});
|
||||
|
||||
it("truncated evidence is not treated as equivalence", () => {
|
||||
const result = evaluateTrustGate(
|
||||
contract,
|
||||
{ exit_code: 0, stdout: "hello" },
|
||||
{ exit_code: 0, stdout: "hello" },
|
||||
{ truncated: true },
|
||||
);
|
||||
expect(result.verdict).toBe("fail");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user