diff --git a/src/domain/conformancePackage.ts b/src/domain/conformancePackage.ts new file mode 100644 index 0000000000..2c347d484e --- /dev/null +++ b/src/domain/conformancePackage.ts @@ -0,0 +1,218 @@ +import { createHash } from "node:crypto"; + +import canonicalize from "canonicalize"; +import { z } from "zod"; + +import { evidenceEnvelopeSchema } from "./evidence.js"; +import { evidenceBundleSchema } from "./evidenceBundle.js"; + +/** Schema version for the conformance package format. */ +export const CONFORMANCE_PACKAGE_VERSION = 1; + +/** A stable, path-independent identifier for a conformance package. */ +export const conformancePackageIdSchema = z + .string() + .regex(/^cp_[a-f0-9]{64}$/u); + +/** Identifier for a single scenario within a package. */ +export const scenarioIdSchema = z + .string() + .regex(/^[A-Za-z][A-Za-z0-9._-]{0,99}$/u); + +/** Deterministic manifest for a scenario fixture. */ +export const scenarioManifestSchema = z.strictObject({ + scenario_id: scenarioIdSchema, + name: z.string().min(1), + description: z.string().min(1), + /** Source-owned fixture path relative to the repository root. */ + fixture_path: z.string().min(1), + /** Expected exit code or null if unspecified. */ + expected_exit_code: z.number().int().nullable(), + /** Expected output patterns that must appear in the capture. */ + expected_patterns: z.array(z.string()).default([]), +}); + +/** Replay plan describing how a scenario is replayed deterministically. */ +export const replayPlanSchema = z.strictObject({ + scenario_id: scenarioIdSchema, + /** Ordered steps to execute during replay. */ + steps: z + .array( + z.strictObject({ + step_id: z.string().min(1), + action: z.string().min(1), + arguments: z.array(z.string()).default([]), + timeout_ms: z.number().int().positive(), + }), + ) + .min(1) + .max(100), + /** Environment variables to set (never inherit host paths). */ + environment: z.record(z.string(), z.string()).default({}), +}); + +/** Shim plan for intercepting observable effects. */ +export const shimPlanSchema = z.strictObject({ + scenario_id: scenarioIdSchema, + /** Shims to install, each intercepting a named effect. */ + shims: z + .array( + z.strictObject({ + shim_id: z.string().min(1), + kind: z.enum(["filesystem", "network", "process", "signal"]), + target: z.string().min(1), + policy: z.enum(["observe", "allow", "block", "emulate"]), + }), + ) + .min(0) + .max(50), +}); + +/** Expected evidence for a scenario, checked against the captured run. */ +export const expectedEvidenceSchema = z.strictObject({ + scenario_id: scenarioIdSchema, + /** Expected evidence envelopes. */ + envelopes: z.array(evidenceEnvelopeSchema).min(0).max(100), + /** Expected evidence bundle. */ + bundle: evidenceBundleSchema.nullable(), + /** Required dimensions that must be present in the evidence. */ + required_dimensions: z.array(z.string().min(1)).default([]), +}); + +/** Verifier contract describing how to validate conformance. */ +export const verifierContractSchema = z.strictObject({ + scenario_id: scenarioIdSchema, + /** Dimensions to verify. */ + dimensions: z + .array( + z.strictObject({ + name: z.string().min(1), + required: z.boolean().default(true), + comparison: z.enum(["exact", "semantic", "fuzzy"]), + }), + ) + .min(1) + .max(50), + /** Tolerance for timing differences in milliseconds. */ + timing_tolerance_ms: z.number().int().nonnegative().default(0), +}); + +/** Top-level conformance package manifest. */ +export const conformancePackageSchema = z.strictObject({ + schema_version: z.literal(CONFORMANCE_PACKAGE_VERSION), + package_id: conformancePackageIdSchema, + name: z.string().min(1), + description: z.string().min(1), + created_at: z.string().datetime(), + /** Scenario manifests. */ + scenarios: z.array(scenarioManifestSchema).min(1).max(100), + /** Replay plans keyed by scenario_id. */ + replay_plans: z.array(replayPlanSchema).min(1).max(100), + /** Shim plans keyed by scenario_id. */ + shim_plans: z.array(shimPlanSchema).default([]), + /** Expected evidence for each scenario. */ + expected_evidence: z.array(expectedEvidenceSchema).min(1).max(100), + /** Verifier contracts for each scenario. */ + verifier_contracts: z.array(verifierContractSchema).min(1).max(100), +}); + +export type ConformancePackage = z.infer; +export type ScenarioManifest = z.infer; +export type ReplayPlan = z.infer; +export type ShimPlan = z.infer; +export type ExpectedEvidence = z.infer; +export type VerifierContract = z.infer; + +/** Error from validation or package operations. */ +export type ConformancePackageError = + | { kind: "invalid_package"; message: string } + | { kind: "scenario_not_found"; scenario_id: string } + | { kind: "duplicate_scenario"; scenario_id: string }; + +/** Result type for package operations. */ +export type ConformancePackageResult = + | { ok: true; value: T } + | { ok: false; error: ConformancePackageError }; + +/** Validate a conformance package and cross-check internal references. */ +export function validateConformancePackage( + input: unknown, +): ConformancePackageResult { + const parsed = conformancePackageSchema.safeParse(input); + if (!parsed.success) + return { + ok: false, + error: { + kind: "invalid_package", + message: parsed.error.issues[0]?.message ?? "invalid package", + }, + }; + + const pkg = parsed.data; + + // Check for duplicate scenario IDs + const seen = new Set(); + for (const s of pkg.scenarios) { + if (seen.has(s.scenario_id)) + return { + ok: false, + error: { kind: "duplicate_scenario", scenario_id: s.scenario_id }, + }; + seen.add(s.scenario_id); + } + + // Check that replay plans reference existing scenarios + for (const rp of pkg.replay_plans) { + if (!seen.has(rp.scenario_id)) + return { + ok: false, + error: { + kind: "scenario_not_found", + scenario_id: rp.scenario_id, + }, + }; + } + + // Check that expected evidence references existing scenarios + for (const ee of pkg.expected_evidence) { + if (!seen.has(ee.scenario_id)) + return { + ok: false, + error: { + kind: "scenario_not_found", + scenario_id: ee.scenario_id, + }, + }; + } + + // Check that verifier contracts reference existing scenarios + for (const vc of pkg.verifier_contracts) { + if (!seen.has(vc.scenario_id)) + return { + ok: false, + error: { + kind: "scenario_not_found", + scenario_id: vc.scenario_id, + }, + }; + } + + return { ok: true, value: pkg }; +} + +/** Compute a deterministic package ID from the canonical JSON. */ +export function computePackageId( + pkg: Omit, +): string { + const json = canonicalize(pkg); + if (!json) throw new Error("failed to canonicalize package"); + return `cp_${createHash("sha256").update(json).digest("hex")}`; +} + +/** Create a conformance package with an auto-computed package_id. */ +export function createConformancePackage( + pkg: Omit, +): ConformancePackage { + const package_id = computePackageId(pkg); + return { package_id, ...pkg }; +} diff --git a/src/domain/conformanceReplay.ts b/src/domain/conformanceReplay.ts new file mode 100644 index 0000000000..2e0640e172 --- /dev/null +++ b/src/domain/conformanceReplay.ts @@ -0,0 +1,138 @@ +import { z } from "zod"; + +import { + conformancePackageSchema, + type ConformancePackage, +} from "./conformancePackage.js"; +import { trustGateResultSchema } from "./conformanceTrustGate.js"; + +/** Result of a single scenario replay. */ +export const scenarioReplayResultSchema = z.strictObject({ + scenario_id: z.string().min(1), + status: z.enum(["pass", "fail", "error", "skipped"]), + exit_code: z.number().int().nullable(), + duration_ms: z.number().int().nonnegative(), + output: z.string().default(""), + error: z.string().nullable(), +}); + +/** Result of replaying an entire conformance package. */ +export const packageReplayResultSchema = z.strictObject({ + package_id: z.string().min(1), + total_scenarios: z.number().int().positive(), + passed: z.number().int().nonnegative(), + failed: z.number().int().nonnegative(), + errored: z.number().int().nonnegative(), + skipped: z.number().int().nonnegative(), + scenario_results: z.array(scenarioReplayResultSchema), + trust_gate_results: z.array(trustGateResultSchema), + drift_detected: z.boolean(), + first_drift: z + .strictObject({ + scenario_id: z.string().min(1), + dimension: z.string().min(1), + message: z.string(), + }) + .nullable(), +}); + +export type ScenarioReplayResult = z.infer; +export type PackageReplayResult = z.infer; + +/** Callback interface for replaying a scenario. */ +export interface ScenarioRunner { + (scenarioId: string, fixturePath: string): Promise; +} + +/** + * Default no-op scenario runner that skips all scenarios. + * Real CI replays inject an actual runner. + */ +async function defaultRunner(): Promise { + return { + scenario_id: "unknown", + status: "skipped", + exit_code: null, + duration_ms: 0, + output: "", + error: "no runner provided", + }; +} + +/** + * Replay a conformance package from a clean checkout. + * Runs each scenario, evaluates trust gates, and detects drift + * between the captured run and the committed package. + */ +export async function replayConformancePackage( + packageInput: unknown, + actualEvidence: Record>, + runner: ScenarioRunner = defaultRunner, + options: { truncated?: boolean } = {}, +): Promise { + const parsed = conformancePackageSchema.safeParse(packageInput); + if (!parsed.success) + throw new TypeError( + `invalid conformance package: ${parsed.error.issues[0]?.message ?? "unknown"}`, + ); + const pkg: ConformancePackage = parsed.data; + + const scenarioResults: ScenarioReplayResult[] = []; + let passed = 0; + let failed = 0; + let errored = 0; + let skipped = 0; + + for (const scenario of pkg.scenarios) { + const result = await runner(scenario.scenario_id, scenario.fixture_path); + scenarioResults.push(result); + + switch (result.status) { + case "pass": + passed++; + break; + case "fail": + failed++; + break; + case "error": + errored++; + break; + case "skipped": + skipped++; + break; + } + } + + const trustGateResults = evaluatePackageTrustGates( + pkg, + actualEvidence, + options, + ); + + const driftGates = trustGateResults.filter((r) => r.verdict === "fail"); + const drift_detected = driftGates.length > 0; + const first_drift_gate = driftGates[0]; + const first_drift = first_drift_gate?.first_divergence + ? { + scenario_id: first_drift_gate.scenario_id, + dimension: first_drift_gate.first_divergence.dimension, + message: first_drift_gate.first_divergence.message, + } + : null; + + return { + package_id: pkg.package_id, + total_scenarios: pkg.scenarios.length, + passed, + failed, + errored, + skipped, + scenario_results: scenarioResults, + trust_gate_results: trustGateResults, + drift_detected, + first_drift, + }; +} + +/** Import trust gate evaluator for replay. */ +import { evaluatePackageTrustGates } from "./conformanceTrustGate.js"; diff --git a/src/domain/conformanceTrustGate.ts b/src/domain/conformanceTrustGate.ts new file mode 100644 index 0000000000..4857684008 --- /dev/null +++ b/src/domain/conformanceTrustGate.ts @@ -0,0 +1,246 @@ +import { z } from "zod"; + +import type { + ConformancePackage, + VerifierContract, +} from "./conformancePackage.js"; + +/** Result of comparing a single dimension. */ +export const dimensionResultSchema = z.strictObject({ + name: z.string().min(1), + status: z.enum(["match", "mismatch", "unknown", "truncated"]), + message: z.string().default(""), + evidence_ids: z.array(z.string()).default([]), +}); + +/** Result of evaluating a trust gate for one scenario. */ +export const trustGateResultSchema = z.strictObject({ + scenario_id: z.string().min(1), + verdict: z.enum(["pass", "fail", "unknown"]), + dimension_results: z.array(dimensionResultSchema), + first_divergence: z + .strictObject({ + dimension: z.string().min(1), + evidence_id: z.string().min(1), + message: z.string(), + }) + .nullable(), +}); + +export type DimensionResult = z.infer; +export type TrustGateResult = z.infer; + +/** Dimension that is volatile due to timing or normalization. */ +const VOLATILE_DIMENSIONS = new Set([ + "timing", + "timestamp", + "pid", + "ppid", + "duration_ms", + "created_at", + "updated_at", +]); + +/** Dimension that carries semantic content. */ +const SEMANTIC_DIMENSIONS = new Set([ + "exit_code", + "stdout", + "stderr", + "filesystem", + "process_tree", + "shim_events", + "protocol_events", + "event_journal", +]); + +/** Check if a dimension name is volatile (timing/normalization noise). */ +export function isVolatileDimension(name: string): boolean { + return VOLATILE_DIMENSIONS.has(name); +} + +/** Check if a dimension name carries semantic content. */ +export function isSemanticDimension(name: string): boolean { + return SEMANTIC_DIMENSIONS.has(name); +} + +/** + * Compare two values for a dimension, classifying semantic diffs + * from timing/normalization noise. + * + * Volatile dimensions (timing, timestamps, PIDs) are always treated + * as matches because their values are expected to differ across + * runs and do not carry semantic meaning. + */ +export function compareDimension( + dimensionName: string, + expected: unknown, + actual: unknown, + options: { truncated?: boolean } = {}, +): DimensionResult { + // Volatile dimensions: always match regardless of values + if (isVolatileDimension(dimensionName)) { + return { + name: dimensionName, + status: "match", + message: "volatile dimension ignored", + evidence_ids: [], + }; + } + + if (options.truncated) { + return { + name: dimensionName, + status: "truncated", + message: "evidence was truncated", + evidence_ids: [], + }; + } + + if (expected === undefined && actual === undefined) { + return { + name: dimensionName, + status: "unknown", + message: "both expected and actual are undefined", + evidence_ids: [], + }; + } + + if (expected === undefined || actual === undefined) { + return { + name: dimensionName, + status: "mismatch", + message: `expected ${expected === undefined ? "present" : "absent"}, actual ${actual === undefined ? "absent" : "present"}`, + evidence_ids: [], + }; + } + + // Semantic comparison for structured objects + if (typeof expected === "object" && typeof actual === "object") { + const expectedJson = JSON.stringify(sortKeys(expected)); + const actualJson = JSON.stringify(sortKeys(actual)); + if (expectedJson === actualJson) { + return { + name: dimensionName, + status: "match", + message: "", + evidence_ids: [], + }; + } + return { + name: dimensionName, + status: "mismatch", + message: "semantic content differs", + evidence_ids: [], + }; + } + + // Primitive comparison + if (expected === actual) { + return { + name: dimensionName, + status: "match", + message: "", + evidence_ids: [], + }; + } + + return { + name: dimensionName, + status: "mismatch", + message: `expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`, + evidence_ids: [], + }; +} + +/** Recursively sort object keys for deterministic comparison. */ +function sortKeys(value: unknown): unknown { + if (value === null || typeof value !== "object") return value; + if (Array.isArray(value)) return value.map(sortKeys); + const sorted: Record = {}; + for (const key of Object.keys(value as Record).sort()) { + sorted[key] = sortKeys((value as Record)[key]); + } + return sorted; +} + +/** + * Evaluate a trust gate for one scenario by comparing expected + * evidence against the actual captured run. + */ +export function evaluateTrustGate( + contract: VerifierContract, + expectedEvidence: Record, + actualEvidence: Record, + options: { truncated?: boolean } = {}, +): TrustGateResult { + const dimensionResults: DimensionResult[] = []; + let firstDivergence: TrustGateResult["first_divergence"] = null; + + for (const dim of contract.dimensions) { + if (!dim.required) continue; + const expectedValue = expectedEvidence[dim.name]; + const actualValue = actualEvidence[dim.name]; + const result = compareDimension( + dim.name, + expectedValue, + actualValue, + options, + ); + dimensionResults.push(result); + + if (result.status === "mismatch" && firstDivergence === null) { + firstDivergence = { + dimension: dim.name, + evidence_id: result.evidence_ids[0] ?? "unknown", + message: result.message, + }; + } + } + + const hasFail = dimensionResults.some( + (r) => r.status === "mismatch" || r.status === "truncated", + ); + const hasUnknown = dimensionResults.some((r) => r.status === "unknown"); + + const verdict: TrustGateResult["verdict"] = hasFail + ? "fail" + : hasUnknown + ? "unknown" + : "pass"; + + return { + scenario_id: contract.scenario_id, + verdict, + dimension_results: dimensionResults, + first_divergence: firstDivergence, + }; +} + +/** + * Evaluate trust gates for all scenarios in a conformance package. + * Rejects runs that differ in required dimensions. Unknown/truncated + * evidence is never treated as equivalence. + */ +export function evaluatePackageTrustGates( + pkg: ConformancePackage, + actualEvidence: Record>, + options: { truncated?: boolean } = {}, +): TrustGateResult[] { + const results: TrustGateResult[] = []; + for (const contract of pkg.verifier_contracts) { + const expected = pkg.expected_evidence.find( + (e) => e.scenario_id === contract.scenario_id, + ); + if (!expected) continue; + const expectedEvidence = expected.envelopes[0] ?? {}; + const actual = actualEvidence[contract.scenario_id] ?? {}; + const result = evaluateTrustGate( + contract, + expectedEvidence as Record, + actual, + options, + ); + results.push(result); + } + return results; +} diff --git a/tests/conformancePackage.test.ts b/tests/conformancePackage.test.ts new file mode 100644 index 0000000000..283f6e806a --- /dev/null +++ b/tests/conformancePackage.test.ts @@ -0,0 +1,181 @@ +import { describe, expect, it } from "vitest"; + +import { + CONFORMANCE_PACKAGE_VERSION, + conformancePackageSchema, + computePackageId, + createConformancePackage, + validateConformancePackage, + type ScenarioManifest, + type ReplayPlan, + type ShimPlan, + type ExpectedEvidence, +} from "../src/domain/conformancePackage.js"; + +const validPackage = createConformancePackage({ + schema_version: CONFORMANCE_PACKAGE_VERSION, + name: "test-fixture", + description: "A test conformance fixture", + created_at: "2026-07-28T00:00:00Z", + scenarios: [ + { + scenario_id: "s1", + name: "Simple spawn", + description: "A simple process spawn scenario", + fixture_path: "tests/conformance/c/fixture.c", + expected_exit_code: 0, + expected_patterns: ["Hello", "World"], + }, + ], + replay_plans: [ + { + scenario_id: "s1", + steps: [ + { + step_id: "step1", + action: "compile", + arguments: ["gcc", "fixture.c"], + timeout_ms: 5000, + }, + ], + environment: {}, + }, + ], + shim_plans: [], + expected_evidence: [ + { + scenario_id: "s1", + envelopes: [], + bundle: null, + required_dimensions: ["exit_code", "stdout"], + }, + ], + verifier_contracts: [ + { + scenario_id: "s1", + dimensions: [ + { + name: "exit_code", + required: true, + comparison: "exact", + }, + ], + timing_tolerance_ms: 100, + }, + ], +}); + +describe("conformance package format", () => { + it("validates a well-formed package", () => { + const result = validateConformancePackage(validPackage); + expect(result.ok).toBe(true); + }); + + it("rejects invalid schema version", () => { + const result = validateConformancePackage({ + ...validPackage, + schema_version: 999, + }); + expect(result.ok).toBe(false); + }); + + it("rejects duplicate scenario IDs", () => { + const result = validateConformancePackage({ + ...validPackage, + scenarios: [validPackage.scenarios[0], validPackage.scenarios[0]], + }); + expect(result.ok).toBe(false); + }); + + it("rejects replay plan referencing non-existent scenario", () => { + const result = validateConformancePackage({ + ...validPackage, + replay_plans: [ + { + scenario_id: "nonexistent", + steps: [ + { + step_id: "step1", + action: "run", + arguments: [], + timeout_ms: 1000, + }, + ], + environment: {}, + }, + ], + }); + expect(result.ok).toBe(false); + }); + + it("rejects verifier contract referencing non-existent scenario", () => { + const result = validateConformancePackage({ + ...validPackage, + verifier_contracts: [ + { + scenario_id: "nonexistent", + dimensions: [ + { name: "exit_code", required: true, comparison: "exact" }, + ], + timing_tolerance_ms: 0, + }, + ], + }); + expect(result.ok).toBe(false); + }); + + it("rejects expected evidence referencing non-existent scenario", () => { + const result = validateConformancePackage({ + ...validPackage, + expected_evidence: [ + { + scenario_id: "nonexistent", + envelopes: [], + bundle: null, + required_dimensions: [], + }, + ], + }); + expect(result.ok).toBe(false); + }); + + it("computes a deterministic package ID", () => { + const id = computePackageId({ + ...validPackage, + }); + expect(id).toMatch(/^cp_[a-f0-9]{64}$/u); + // Same input yields same ID + const id2 = computePackageId({ + ...validPackage, + }); + expect(id).toBe(id2); + }); + + it("produces different IDs for different packages", () => { + const id1 = computePackageId({ ...validPackage, name: "packageA" }); + const id2 = computePackageId({ ...validPackage, name: "packageB" }); + expect(id1).not.toBe(id2); + }); + + it("uses all exported types", () => { + const scenario: ScenarioManifest = validPackage.scenarios[0]!; + expect(scenario.scenario_id).toBe("s1"); + + const replay: ReplayPlan = validPackage.replay_plans[0]!; + expect(replay.scenario_id).toBe("s1"); + + const shim: ShimPlan = validPackage.shim_plans[0] ?? { + scenario_id: "s1", + shims: [], + }; + expect(shim.scenario_id).toBe("s1"); + + const evidence: ExpectedEvidence = validPackage.expected_evidence[0]!; + expect(evidence.scenario_id).toBe("s1"); + }); + + it("parses via zod schema", () => { + const result = conformancePackageSchema.safeParse(validPackage); + expect(result.success).toBe(true); + }); +}); diff --git a/tests/conformanceReplay.test.ts b/tests/conformanceReplay.test.ts new file mode 100644 index 0000000000..078997b849 --- /dev/null +++ b/tests/conformanceReplay.test.ts @@ -0,0 +1,190 @@ +import { describe, expect, it } from "vitest"; + +import { + CONFORMANCE_PACKAGE_VERSION, + createConformancePackage, + type ConformancePackage, +} from "../src/domain/conformancePackage.js"; +import { + replayConformancePackage, + type ScenarioReplayResult, +} from "../src/domain/conformanceReplay.js"; + +const validPackage = createConformancePackage({ + schema_version: CONFORMANCE_PACKAGE_VERSION, + name: "test-fixture", + description: "A test conformance fixture", + created_at: "2026-07-28T00:00:00Z", + scenarios: [ + { + scenario_id: "s1", + name: "Simple spawn", + description: "A simple process spawn scenario", + fixture_path: "tests/conformance/c/fixture.c", + expected_exit_code: 0, + expected_patterns: [], + }, + ], + replay_plans: [ + { + scenario_id: "s1", + steps: [ + { + step_id: "step1", + action: "run", + arguments: [], + timeout_ms: 1000, + }, + ], + environment: {}, + }, + ], + shim_plans: [], + expected_evidence: [ + { + scenario_id: "s1", + envelopes: [], + bundle: null, + required_dimensions: [], + }, + ], + verifier_contracts: [ + { + scenario_id: "s1", + dimensions: [{ name: "exit_code", required: true, comparison: "exact" }], + timing_tolerance_ms: 0, + }, + ], +}); + +async function passingRunner( + scenarioId: string, +): Promise { + return { + scenario_id: scenarioId, + status: "pass", + exit_code: 0, + duration_ms: 10, + output: "ok", + error: null, + }; +} + +async function failingRunner( + scenarioId: string, +): Promise { + return { + scenario_id: scenarioId, + status: "fail", + exit_code: 1, + duration_ms: 10, + output: "fail", + error: null, + }; +} + +describe("conformance CI replay", () => { + it("replays a package with passing scenarios", async () => { + const result = await replayConformancePackage( + validPackage, + {}, + passingRunner, + ); + expect(result.total_scenarios).toBe(1); + expect(result.passed).toBe(1); + expect(result.failed).toBe(0); + expect(result.errored).toBe(0); + }); + + it("replays a package with failing scenarios", async () => { + const result = await replayConformancePackage( + validPackage, + {}, + failingRunner, + ); + expect(result.failed).toBe(1); + expect(result.passed).toBe(0); + }); + + it("does not detect drift when expected and actual match", async () => { + const result = await replayConformancePackage( + validPackage, + {}, + passingRunner, + ); + expect(result.drift_detected).toBe(false); + }); + + it("throws on invalid package", async () => { + await expect(replayConformancePackage({}, {})).rejects.toThrow( + /invalid conformance package/u, + ); + }); + + it("counts all result types", async () => { + const base: ConformancePackage = validPackage; + const multiPackage = createConformancePackage({ + ...base, + scenarios: [ + base.scenarios[0]!, + { + scenario_id: "s2", + name: "Second scenario", + description: "Second", + fixture_path: "tests/conformance/c/fixture2.c", + expected_exit_code: 1, + expected_patterns: [], + }, + ], + replay_plans: [ + base.replay_plans[0]!, + { + scenario_id: "s2", + steps: [ + { + step_id: "step1", + action: "run", + arguments: [], + timeout_ms: 1000, + }, + ], + environment: {}, + }, + ], + expected_evidence: [ + base.expected_evidence[0]!, + { + scenario_id: "s2", + envelopes: [], + bundle: null, + required_dimensions: [], + }, + ], + verifier_contracts: [ + base.verifier_contracts[0]!, + { + scenario_id: "s2", + dimensions: [ + { name: "exit_code", required: true, comparison: "exact" }, + ], + timing_tolerance_ms: 0, + }, + ], + }); + const result = await replayConformancePackage( + multiPackage, + {}, + async (scenarioId) => ({ + scenario_id: scenarioId, + status: scenarioId === "s1" ? "pass" : "fail", + exit_code: scenarioId === "s1" ? 0 : 1, + duration_ms: 10, + output: "", + error: null, + }), + ); + expect(result.total_scenarios).toBe(2); + expect(result.passed).toBe(1); + expect(result.failed).toBe(1); + }); +}); diff --git a/tests/conformanceTrustGate.test.ts b/tests/conformanceTrustGate.test.ts new file mode 100644 index 0000000000..0e03a9f076 --- /dev/null +++ b/tests/conformanceTrustGate.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it } from "vitest"; + +import type { VerifierContract } from "../src/domain/conformancePackage.js"; +import { + compareDimension, + evaluateTrustGate, + isSemanticDimension, + isVolatileDimension, +} from "../src/domain/conformanceTrustGate.js"; + +const contract: VerifierContract = { + scenario_id: "s1", + dimensions: [ + { name: "exit_code", required: true, comparison: "exact" }, + { name: "stdout", required: true, comparison: "semantic" }, + { name: "timing", required: true, comparison: "fuzzy" }, + ], + timing_tolerance_ms: 100, +}; + +describe("conformance trust gates", () => { + it("classifies volatile dimensions correctly", () => { + expect(isVolatileDimension("timing")).toBe(true); + expect(isVolatileDimension("timestamp")).toBe(true); + expect(isVolatileDimension("pid")).toBe(true); + expect(isVolatileDimension("exit_code")).toBe(false); + }); + + it("classifies semantic dimensions correctly", () => { + expect(isSemanticDimension("exit_code")).toBe(true); + expect(isSemanticDimension("stdout")).toBe(true); + expect(isSemanticDimension("timing")).toBe(false); + }); + + it("matches identical values", () => { + const result = compareDimension("exit_code", 0, 0); + expect(result.status).toBe("match"); + }); + + it("detects mismatches", () => { + const result = compareDimension("exit_code", 0, 1); + expect(result.status).toBe("mismatch"); + }); + + it("ignores volatile dimensions", () => { + const result = compareDimension("timing", 100, 200); + expect(result.status).toBe("match"); + expect(result.message).toContain("volatile"); + }); + + it("handles truncated evidence", () => { + const result = compareDimension("exit_code", 0, null, { + truncated: true, + }); + expect(result.status).toBe("truncated"); + }); + + it("handles undefined values", () => { + const result = compareDimension("exit_code", undefined, undefined); + expect(result.status).toBe("unknown"); + }); + + it("handles missing actual", () => { + const result = compareDimension("exit_code", 0, undefined); + expect(result.status).toBe("mismatch"); + }); + + it("passes when all dimensions match", () => { + const result = evaluateTrustGate( + contract, + { exit_code: 0, stdout: "hello" }, + { exit_code: 0, stdout: "hello" }, + ); + expect(result.verdict).toBe("pass"); + }); + + it("fails when a required dimension mismatches", () => { + const result = evaluateTrustGate( + contract, + { exit_code: 0, stdout: "hello" }, + { exit_code: 1, stdout: "hello" }, + ); + expect(result.verdict).toBe("fail"); + expect(result.first_divergence).not.toBeNull(); + expect(result.first_divergence!.dimension).toBe("exit_code"); + }); + + it("reports unknown when both dimensions are undefined", () => { + const result = evaluateTrustGate(contract, {}, {}); + // exit_code and stdout are both undefined -> unknown + // timing is volatile -> match + expect(result.verdict).toBe("unknown"); + }); + + it("ignores volatile dimension differences", () => { + const result = evaluateTrustGate( + contract, + { exit_code: 0, stdout: "hello", timing: 100 }, + { exit_code: 0, stdout: "hello", timing: 200 }, + ); + expect(result.verdict).toBe("pass"); + }); + + it("truncated evidence is not treated as equivalence", () => { + const result = evaluateTrustGate( + contract, + { exit_code: 0, stdout: "hello" }, + { exit_code: 0, stdout: "hello" }, + { truncated: true }, + ); + expect(result.verdict).toBe("fail"); + }); +});