mirror of
https://github.com/morluto/rea
synced 2026-10-25 22:08:11 -04:00
test(evaluation): preserve partial coverage in the Notes oracle
This commit is contained in:
@@ -2087,6 +2087,13 @@ Codex CLI. The lane runs sequentially; select scenarios with
|
||||
[agent evaluation reference](client-verification.md#agent-evaluation-and-conformance-records)
|
||||
contains isolation, authentication cleanup, grading, and conformance boundaries.
|
||||
|
||||
The Notes evaluation packages `website/public/examples/notes-electron/`. When
|
||||
changing those example bytes, run `tests/boundary/mcp/agentKnownAnswerMcp.test.ts`
|
||||
and check its rubric in `tests/evaluation/model-evals/AgentEvaluationFixtures.ts`.
|
||||
Preserve exact source facts and the coverage reported by the analysis. If a
|
||||
source change crosses an unresolved static-analysis boundary, keep that
|
||||
uncertainty and a negative control rejecting claims of complete coverage.
|
||||
|
||||
## Frida instrumentation
|
||||
|
||||
`npm run verify:frida` builds REA, spawns a short-lived local Node.js fixture,
|
||||
|
||||
@@ -347,7 +347,16 @@ const assertNotesAnswerFailures = (
|
||||
const assess = (answer: unknown, events = workflow.events) =>
|
||||
grade(events, answer, "analyze_javascript_application", claims)
|
||||
.factualAssessment;
|
||||
expect(assess(manifest).status).toBe("passed");
|
||||
const baseline = assess(manifest);
|
||||
expect(
|
||||
baseline.status,
|
||||
JSON.stringify({
|
||||
manifestFailures: baseline.manifestFailures,
|
||||
failedClaims: baseline.claims
|
||||
.filter((claim) => claim.status !== "passed")
|
||||
.map(({ id, failures }) => ({ id, failures })),
|
||||
}),
|
||||
).toBe("passed");
|
||||
const cases = [
|
||||
{
|
||||
id: "renderer_ipc",
|
||||
@@ -367,6 +376,16 @@ const assertNotesAnswerFailures = (
|
||||
},
|
||||
failure: "value_mismatch",
|
||||
},
|
||||
{
|
||||
id: "feature_modules",
|
||||
patch: {
|
||||
value: {
|
||||
paths: ["csv.js", "main.js", "preload.js", "renderer.js"],
|
||||
coverage: "complete-within-source",
|
||||
},
|
||||
},
|
||||
failure: "value_mismatch",
|
||||
},
|
||||
{
|
||||
id: "renderer_api",
|
||||
patch: {
|
||||
|
||||
@@ -265,7 +265,9 @@ const notesClaims = (
|
||||
id: "feature_modules",
|
||||
expectedValue: {
|
||||
paths: ["csv.js", "main.js", "preload.js", "renderer.js"],
|
||||
coverage: "complete-within-source",
|
||||
// The renderer's exportNotes().catch(...) leaves a Promise member
|
||||
// unresolved in static analysis, so the graph reports partial coverage.
|
||||
coverage: "partial",
|
||||
},
|
||||
source: {
|
||||
...derived,
|
||||
|
||||
Reference in New Issue
Block a user