diff --git a/docs/testing.md b/docs/testing.md index 553cf71d27..c6f74193f4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2087,6 +2087,13 @@ Codex CLI. The lane runs sequentially; select scenarios with [agent evaluation reference](client-verification.md#agent-evaluation-and-conformance-records) contains isolation, authentication cleanup, grading, and conformance boundaries. +The Notes evaluation packages `website/public/examples/notes-electron/`. When +changing those example bytes, run `tests/boundary/mcp/agentKnownAnswerMcp.test.ts` +and check its rubric in `tests/evaluation/model-evals/AgentEvaluationFixtures.ts`. +Preserve exact source facts and the coverage reported by the analysis. If a +source change crosses an unresolved static-analysis boundary, keep that +uncertainty and a negative control rejecting claims of complete coverage. + ## Frida instrumentation `npm run verify:frida` builds REA, spawns a short-lived local Node.js fixture, diff --git a/tests/boundary/mcp/agentKnownAnswerMcp.test.ts b/tests/boundary/mcp/agentKnownAnswerMcp.test.ts index a2f5f67971..6a4d050d08 100644 --- a/tests/boundary/mcp/agentKnownAnswerMcp.test.ts +++ b/tests/boundary/mcp/agentKnownAnswerMcp.test.ts @@ -347,7 +347,16 @@ const assertNotesAnswerFailures = ( const assess = (answer: unknown, events = workflow.events) => grade(events, answer, "analyze_javascript_application", claims) .factualAssessment; - expect(assess(manifest).status).toBe("passed"); + const baseline = assess(manifest); + expect( + baseline.status, + JSON.stringify({ + manifestFailures: baseline.manifestFailures, + failedClaims: baseline.claims + .filter((claim) => claim.status !== "passed") + .map(({ id, failures }) => ({ id, failures })), + }), + ).toBe("passed"); const cases = [ { id: "renderer_ipc", @@ -367,6 +376,16 @@ const assertNotesAnswerFailures = ( }, failure: "value_mismatch", }, + { + id: "feature_modules", + patch: { + value: { + paths: ["csv.js", "main.js", "preload.js", "renderer.js"], + coverage: "complete-within-source", + }, + }, + failure: "value_mismatch", + }, { id: "renderer_api", patch: { diff --git a/tests/evaluation/model-evals/AgentEvaluationFixtures.ts b/tests/evaluation/model-evals/AgentEvaluationFixtures.ts index 3fb8d88c04..60a843dd90 100644 --- a/tests/evaluation/model-evals/AgentEvaluationFixtures.ts +++ b/tests/evaluation/model-evals/AgentEvaluationFixtures.ts @@ -265,7 +265,9 @@ const notesClaims = ( id: "feature_modules", expectedValue: { paths: ["csv.js", "main.js", "preload.js", "renderer.js"], - coverage: "complete-within-source", + // The renderer's exportNotes().catch(...) leaves a Promise member + // unresolved in static analysis, so the graph reports partial coverage. + coverage: "partial", }, source: { ...derived,