test(evaluation): preserve partial coverage in the Notes oracle

This commit is contained in:
morluto
2026-10-25 09:07:57 +08:00
parent bb17e90f27
commit ab720b4216
3 changed files with 30 additions and 2 deletions
+7
View File
@@ -2087,6 +2087,13 @@ Codex CLI. The lane runs sequentially; select scenarios with
[agent evaluation reference](client-verification.md#agent-evaluation-and-conformance-records)
contains isolation, authentication cleanup, grading, and conformance boundaries.
The Notes evaluation packages `website/public/examples/notes-electron/`. When
changing those example bytes, run `tests/boundary/mcp/agentKnownAnswerMcp.test.ts`
and check its rubric in `tests/evaluation/model-evals/AgentEvaluationFixtures.ts`.
Preserve exact source facts and the coverage reported by the analysis. If a
source change crosses an unresolved static-analysis boundary, keep that
uncertainty and a negative control rejecting claims of complete coverage.
## Frida instrumentation
`npm run verify:frida` builds REA, spawns a short-lived local Node.js fixture,
+20 -1
View File
@@ -347,7 +347,16 @@ const assertNotesAnswerFailures = (
const assess = (answer: unknown, events = workflow.events) =>
grade(events, answer, "analyze_javascript_application", claims)
.factualAssessment;
expect(assess(manifest).status).toBe("passed");
const baseline = assess(manifest);
expect(
baseline.status,
JSON.stringify({
manifestFailures: baseline.manifestFailures,
failedClaims: baseline.claims
.filter((claim) => claim.status !== "passed")
.map(({ id, failures }) => ({ id, failures })),
}),
).toBe("passed");
const cases = [
{
id: "renderer_ipc",
@@ -367,6 +376,16 @@ const assertNotesAnswerFailures = (
},
failure: "value_mismatch",
},
{
id: "feature_modules",
patch: {
value: {
paths: ["csv.js", "main.js", "preload.js", "renderer.js"],
coverage: "complete-within-source",
},
},
failure: "value_mismatch",
},
{
id: "renderer_api",
patch: {
@@ -265,7 +265,9 @@ const notesClaims = (
id: "feature_modules",
expectedValue: {
paths: ["csv.js", "main.js", "preload.js", "renderer.js"],
coverage: "complete-within-source",
// The renderer's exportNotes().catch(...) leaves a Promise member
// unresolved in static analysis, so the graph reports partial coverage.
coverage: "partial",
},
source: {
...derived,