From 0667f7cb7c85f134311c7470ff4ee99ef9cc45f4 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:49:34 +0000 Subject: [PATCH] fix(experiments): bind native claims to observed source evidence Check derived view identities against all earlier producer references. Grade DOS load-image claims only from completed, referenced producer responses, including their actual target and entry identity. Focused synthetic negative transcripts and original real transcript regrades verify the boundary without model reruns. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .../analysis-efficiency/native-entry-views.md | 5 + .../analysis-efficiency/native-entry.mjs | 15 +- .../analysis-efficiency/review-model.py | 20 +- .../cli/nativeEfficiencyComparison.test.ts | 176 +++++++++++++++++- 4 files changed, 205 insertions(+), 11 deletions(-) diff --git a/experiments/analysis-efficiency/native-entry-views.md b/experiments/analysis-efficiency/native-entry-views.md index 05388707b8..1c02fbfd14 100644 --- a/experiments/analysis-efficiency/native-entry-views.md +++ b/experiments/analysis-efficiency/native-entry-views.md @@ -154,6 +154,11 @@ parent ID, while `retained_views` holds its item, coverage, limitations and unkn DOS load-image facts retain their source ID and must match the selected carrier digest before their entry address can be used; a stale result closes the session without querying a procedure. +Derived view IDs are checked against every prior producer reference as well as +other derived views. DOS model grades require the actual completed load-image +response and its reported source ID; status, digest, entry and supplied metadata +are checked against that observation. Missing, failed or unreferenced responses +cannot establish load-image verification. The comparator preserves both runs' reference maps and compares the facts. Existing `entry`/`entry-and-main` answers keep their ID arrays. Historical SDK receipts above retain their original pre-review representation; no replies or measurements were diff --git a/experiments/analysis-efficiency/native-entry.mjs b/experiments/analysis-efficiency/native-entry.mjs index 3f61e5ea21..84bc934ff6 100644 --- a/experiments/analysis-efficiency/native-entry.mjs +++ b/experiments/analysis-efficiency/native-entry.mjs @@ -4,11 +4,17 @@ import { isDeepStrictEqual } from "node:util"; export const NATIVE_ARTIFACTS = ["jq", "sharex.exe", "freedos.edlin.exe"]; const normalized = (value) => value.normalized_result; -const retainedEntryViews = async (call, info, inputSha, ToolFailure) => { +const retainedEntryViews = async ( + call, + info, + inputSha, + ToolFailure, + sourceEvidenceIds, +) => { const n = normalized(info); const retainedViews = []; const retainedViewEvidence = []; - const observedEvidenceIds = new Set([info.evidence_id]); + const observedEvidenceIds = new Set([info.evidence_id, ...sourceEvidenceIds]); let nativeSummaryOutcome, summaryEvidenceId; for (const facet of ["procedure", "pseudocode", "callers", "callees"]) { const selected = await call("inspect_analysis_view", { @@ -175,7 +181,10 @@ export const measureNativeEntry = async ({ const n = normalized(info); const selected = values["native-question"] === "entry-views" - ? await retainedEntryViews(call, info, inputSha, ToolFailure) + ? await retainedEntryViews(call, info, inputSha, ToolFailure, [ + containing.evidence_id, + ...(loadImageEvidenceId === undefined ? [] : [loadImageEvidenceId]), + ]) : {}; const { main, mainEvidenceIds = [] } = values["native-question"] === "entry-and-main" diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index 3f716b40d3..c31a012902 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -216,15 +216,27 @@ def main(): check('procedure.body_ranges', procedure.get('body_ranges', body.get('ranges', body.get('ranges_inclusive'))), expected['body']['ranges']) check('procedure.total_bytes', procedure.get('total_bytes', body.get('total_bytes')), expected['body']['total_bytes']) if 'load_image' in reference: - load = reference['load_image'] + sources = [source for source in producer_results.get('inspect_native_load_image', []) + if isinstance(source, dict) + and source.get('operation') == 'inspect_native_load_image' + and isinstance(source.get('evidence_id'), str) + and source.get('evidence_id') in reported_ids + and isinstance(source.get('normalized_result'), dict)] + checks['load_image_source_observed_and_referenced'] = bool(sources) + load = sources[-1]['normalized_result'] if sources else {} + entry = load.get('entry') + entry = entry if isinstance(entry, dict) else {} + check('load_image.target_sha256', load.get('target_sha256'), reference['artifact_sha256']) + check('load_image.source_status', load.get('status'), reference['load_image']['status']) + check('load_image.source_entry', entry.get('linear_address'), reference['entry_address']) status = answer.get('load_image_status') - check('load_image_status', status.get('status') if isinstance(status, dict) else status, load['status']) + check('load_image_status', status.get('status') if isinstance(status, dict) else status, load.get('status')) if isinstance(status, dict): for key in ['format', 'load_segment', 'header_bytes', 'module_bytes', 'overlay_bytes']: if key in status: - check('load_image.' + key, status[key], load[key]) + check('load_image.' + key, status[key], load.get(key)) if 'entry_linear_address' in status: - check('load_image.entry_linear_address', status['entry_linear_address'], load['entry']['linear_address']) + check('load_image.entry_linear_address', status['entry_linear_address'], entry.get('linear_address')) calls = summary['completed_mcp_calls'] checks['default_analysis_not_disabled'] = all(not call['arguments'].get('skip_auto_analysis', False) for call in calls if call['tool'] == 'open_binary') checks['session_close_completed'] = any(call['tool'] == 'close_binary' and call['status'] == 'completed' for call in calls) diff --git a/tests/boundary/cli/nativeEfficiencyComparison.test.ts b/tests/boundary/cli/nativeEfficiencyComparison.test.ts index 1f9b1bf63c..65542f0ea2 100644 --- a/tests/boundary/cli/nativeEfficiencyComparison.test.ts +++ b/tests/boundary/cli/nativeEfficiencyComparison.test.ts @@ -1,5 +1,5 @@ import { execFile } from "node:child_process"; -import { mkdir, writeFile } from "node:fs/promises"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { promisify } from "node:util"; @@ -16,7 +16,8 @@ import {measureNativeEntry} from './experiments/analysis-efficiency/native-entry const sha='0'.repeat(64); const duplicate=process.argv[1]==='duplicate'; const stale=process.argv[1]==='stale-dos'; -const dos=process.argv[1]==='dos'||stale; +const collision=process.argv[1]; +const dos=process.argv[1]==='dos'||stale||collision==='load-collision'; const dossier={procedure:{address:'0x1000'},pseudocode:'return;',callers:[],callees:[]}; class ToolFailure extends Error {} const build=async digit=>{ @@ -26,9 +27,9 @@ const build=async digit=>{ if(stale)process.stdout.write(name+'\\n'); if(name==='open_binary'||name==='close_binary')return {}; if(name==='inspect_native_load_image')return {evidence_id:'ev_'+digit.repeat(63)+'6',normalized_result:{status:'verified',target_sha256:stale?'f'.repeat(64):sha,entry:{linear_address:'0x1000'}}}; - if(name==='resolve_containing_procedure')return {evidence_id:parent,normalized_result:{found:true}}; + if(name==='resolve_containing_procedure')return {evidence_id:'ev_'+digit.repeat(63)+'7',normalized_result:{found:true}}; if(name==='analyze_function')return {evidence_id:parent,subject:{digest:{sha256:sha}},normalized_result:dossier}; - const id='ev_'+digit.repeat(63)+(duplicate?1:++ordinal); + const id='ev_'+digit.repeat(63)+(collision==='containing-collision'?7:collision==='load-collision'?6:duplicate?1:++ordinal); if(args.view.kind==='summary')return {evidence_id:id}; const facet=args.view.facet; const item=facet==='pseudocode'?{text:dossier.pseudocode,unit:'utf16-code-units'}:dossier[facet]; @@ -39,6 +40,155 @@ const build=async digit=>{ process.stdout.write(JSON.stringify([await build('1'),await build('2')])); `; +it.each([ + "observed", + "missing", + "errored", + "wrong-target", + "wrong-entry", + "unreferenced", +])( + "binds the DOS model grade to its observed load image: %s", + async (scenario) => { + // Synthetic transcripts exercise grading; they are not real-provider or + // resource measurements. Real DOS transcripts are replayed separately. + const root = await createTestTempDirectory("rea-dos-grader-"); + const trial = join(root, "trial"); + const referenceDirectory = join(root, "reference"); + await mkdir(trial); + await mkdir(referenceDirectory); + const sha = "0".repeat(64); + const imageId = "ev_" + "1".repeat(64); + const procedureId = "ev_" + "2".repeat(64); + const procedure = { + address: "0x1000", + name: "entry", + classification: "observed", + signature: null, + locals: [], + body: { ranges: [], total_bytes: 0 }, + }; + const loadImage = { + status: "verified", + target_sha256: sha, + entry: { linear_address: "0x1000" }, + }; + const reference = { + artifact_sha256: sha, + entry_address: "0x1000", + caller_count: 0, + callee_count: 0, + pseudocode_prefix: "return;", + containing_procedure: { procedure }, + procedure, + load_image: loadImage, + }; + const answer = { + ...reference, + load_image_status: "verified", + evidence_ids: [ + procedureId, + ...(["missing", "errored", "unreferenced"].includes(scenario) + ? [] + : [imageId]), + ], + }; + const event = (tool: string, status: string, value: unknown) => ({ + type: "item.completed", + item: { + type: "mcp_tool_call", + tool, + status, + arguments: {}, + result: { structured_content: value }, + }, + }); + const events = [ + event("analyze_function", "completed", { evidence_id: procedureId }), + ]; + if (scenario !== "missing") + events.push( + event( + "inspect_native_load_image", + scenario === "errored" ? "failed" : "completed", + scenario === "errored" + ? { error: { code: "unsupported_target" } } + : { + operation: "inspect_native_load_image", + evidence_id: imageId, + normalized_result: { + ...loadImage, + target_sha256: + scenario === "wrong-target" ? "f".repeat(64) : sha, + entry: { + linear_address: + scenario === "wrong-entry" ? "0x2000" : "0x1000", + }, + }, + }, + ), + ); + const resourceLog = join(root, "resource.log"); + const output = join(root, "review.json"); + for (const [path, value] of [ + [ + join(trial, "summary.json"), + { + completed_mcp_calls: [ + { tool: "close_binary", status: "completed", arguments: {} }, + ], + }, + ], + [join(trial, "trial-settings.json"), {}], + [join(trial, "cleanup.json"), { profile_removed: true }], + [join(trial, "answer.txt"), answer], + [join(referenceDirectory, "answer.json"), reference], + [ + resourceLog, + { + resource_scope: "synthetic-grader-fixture", + abort: null, + resource_counters_verified: true, + resource_events_final: { + "memory.events": { high: 0, max: 0, oom: 0, oom_kill: 0 }, + "pids.events": { max: 0 }, + }, + }, + ], + ] as const) + await writeFile(path, JSON.stringify(value)); + await writeFile( + join(trial, "events.jsonl"), + events.map((value) => JSON.stringify(value)).join("\n"), + ); + const command = execute("python3", [ + resolve("experiments/analysis-efficiency/review-model.py"), + "--domain", + "native", + "--trial", + trial, + "--reference", + referenceDirectory, + "--resource-log", + resourceLog, + "--output", + output, + ]); + if (scenario === "observed") await command; + else await expect(command).rejects.toMatchObject({ code: 1 }); + const report = JSON.parse(await readFile(output, "utf8")); + expect(report.all_selected_checks_pass).toBe(scenario === "observed"); + if (["missing", "errored", "unreferenced"].includes(scenario)) + expect(report.checks.load_image_source_observed_and_referenced).toBe( + false, + ); + if (scenario === "wrong-target") + expect(report.checks["load_image.target_sha256"]).toBe(false); + if (scenario === "wrong-entry") + expect(report.checks["load_image.source_entry"]).toBe(false); + }, +); + it("retains the DOS load-image source for its facts and derived entry seed", async () => { const { stdout } = await execute(process.execPath, [ "--input-type=module", @@ -70,6 +220,24 @@ it("rejects a stale DOS load image before using its entry and closes the session }); }); +it.each(["containing-collision", "load-collision"])( + "rejects a retained view that reuses an earlier producer identity: %s", + async (collision) => { + await expect( + execute(process.execPath, [ + "--input-type=module", + "-e", + generate, + collision, + ]), + ).rejects.toMatchObject({ + stderr: expect.stringContaining( + "Retained native view reused an Evidence identity", + ), + }); + }, +); + it("rejects a producer that reuses a view identity across facets", async () => { await expect( execute(process.execPath, [