diff --git a/experiments/atomic-codemode/README.md b/experiments/atomic-codemode/README.md index 74f733b615..6898e4df42 100644 --- a/experiments/atomic-codemode/README.md +++ b/experiments/atomic-codemode/README.md @@ -19,6 +19,12 @@ The [capability map](capability-map.md) distinguishes verified task coverage, experimental exporter requirements and remaining unknowns before interpreting adapter comparisons. +The [string-span study](string-span-study.md) adds paged incoming occurrence +queries for encoded object spans and independently verifies pooled suffix +references on real SQLite. The same native sparse range index serves Kuna's +existing string aggregates. Equivalent cold occurrence retrieval falls from +48.1 to 1.8 seconds; this is shared analysis reuse, not a Code Mode result. + The [AArch64 reference study](aarch64-xref-study.md) finds and fixes missing ADRP-based string and memory references before the next model comparison. Real SDK and traditional MCP preflights now agree with independent machine diff --git a/experiments/atomic-codemode/api.mjs b/experiments/atomic-codemode/api.mjs index 9bd918e9e3..7fb14a6cf9 100644 --- a/experiments/atomic-codemode/api.mjs +++ b/experiments/atomic-codemode/api.mjs @@ -5,6 +5,7 @@ import { MAX_GRAPH_BLOCKS } from "./graph-limits.mjs"; import { createReference } from "./reference.mjs"; import { definition, hex, page, valueLocation } from "./api-contracts.mjs"; import { storageMethods } from "./storage-contracts.mjs"; +import { referencesMethod } from "./reference-contracts.mjs"; const snapshot = z.string().startsWith("snap_"); const byteWindow = z.strictObject({ @@ -116,29 +117,7 @@ export const methods = { "strings", 'await rea.strings({query:"incorrect header check"})', ), - references: definition( - { - summary: - "Read incoming exact-address or outgoing address/function xrefs with occurrence identity and ownership coverage.", - description: - "Batch incoming or outgoing lifted-code references without decompilation. Address scope selects exact target (incoming) or source instruction (outgoing); function scope selects outgoing native owning-function references and requires an entry address. Unknown function applicability stays unknown. Kind filtering is pushed to the native query; source rows, endpoints and Evidence identity remain. Recovered IR calls are a separate representation and may omit machine calls. Interior string references require a separate extent query.", - }, - { - addresses: z.array(hex).min(1).max(16), - direction: z.enum(["incoming", "outgoing"]).default("incoming"), - scope: z.enum(["address", "function"]).default("address"), - kinds: z - .array(z.enum(["call", "jump", "data", "read", "write"])) - .min(1) - .optional(), - fields: z - .array(z.enum(["fromFunction", "toFunction", "instruction"])) - .optional(), - ...page, - }, - "references", - 'await rea.references({addresses:["0x2000"]})', - ), + references: referencesMethod, snapshot: definition( { summary: @@ -436,15 +415,4 @@ methods.bytes.input = methods.bytes.input.superRefine((value, context) => { message: "Byte range end exceeds the native u64 address format", }); }); -methods.references.input = methods.references.input.superRefine( - (value, context) => { - if (value.direction === "incoming" && value.scope !== "address") - context.addIssue({ - code: "custom", - path: ["scope"], - message: "Incoming references require exact address scope", - }); - }, -); - export const reference = createReference(methods); diff --git a/experiments/atomic-codemode/capability-map.md b/experiments/atomic-codemode/capability-map.md index abb231c78e..71723355ae 100644 --- a/experiments/atomic-codemode/capability-map.md +++ b/experiments/atomic-codemode/capability-map.md @@ -22,6 +22,13 @@ engine profiles and limitations in [the trial record](trials.md). | What machine bytes support the interpretation? | Batched loaded-image bytes and selected linear instruction windows, each with source/coverage | The case models independently derive signed relative targets. These windows do not establish runtime execution/alignment, a complete machine CFG, live memory or arbitrary file-offset mappings. | | Can the answer be audited without returning the report? | Snapshot/source identity, selected fields, batch reads, paging and retained raw/decoded Evidence | Real CLI/MCP checks verify source access and capacity fallbacks. Scope/partial/unknown survive composition; a guest-derived answer keeps its own authority. | +The [string-span experiment](string-span-study.md) extends exact xrefs with +batched incoming target-coordinate spans. It preserves all eleven selected +SQLite data occurrences, including nine pooled suffix pointers, with individual +endpoints/owners and raw source rows. Kuna already had aggregate string-span +counts; the new occurrence query shares its sparse index. Range membership alone +does not establish whole-string use, access overlap or runtime behavior. + The useful grain so far is an identified object plus a relation with its exact qualifiers: an operand needs its slot and PHI binding; a case/out-edge pair needs both identities; an address needs its artifact/address space; an optimized diff --git a/experiments/atomic-codemode/kuna-native-ir.patch b/experiments/atomic-codemode/kuna-native-ir.patch index e616990871..5f8aa0279e 100644 --- a/experiments/atomic-codemode/kuna-native-ir.patch +++ b/experiments/atomic-codemode/kuna-native-ir.patch @@ -2233,7 +2233,7 @@ index e0f09af5..f69ee298 100644 mod kuna_ppclocalentry; mod kuna_switchtable; diff --git a/decompiler/crates/kuna-analysis/src/listing/xrefs.rs b/decompiler/crates/kuna-analysis/src/listing/xrefs.rs -index b39058b1..9dc7ec0f 100644 +index b39058b1..f75c6537 100644 --- a/decompiler/crates/kuna-analysis/src/listing/xrefs.rs +++ b/decompiler/crates/kuna-analysis/src/listing/xrefs.rs @@ -78,7 +78,7 @@ use object::SectionKind; @@ -2245,7 +2245,25 @@ index b39058b1..9dc7ec0f 100644 use super::kuna_poolref::PoolImage; use super::kuna_switchtable; use super::model::{FlowKind, RawOp}; -@@ -554,10 +554,13 @@ impl PcodeEmit for FullCapture { +@@ -285,6 +285,17 @@ impl XrefIndex { + self.by_target.get(&vma).map_or(&[], Vec::as_slice) + } + ++ /// Target-coordinate membership, without callable/slot alias expansion. ++ /// Inclusive endpoints cover the final u64 address without adding one. ++ /// Invalid order is unknown, not an empty reference set. Work follows ++ /// retained target buckets, never the number of bytes in the span. ++ pub fn refs_to_range(&self, start: u64, end: u64) -> Option> { ++ if start > end { ++ return None; ++ } ++ Some(self.by_target.range(start..=end).flat_map(|(_, rows)| rows.iter())) ++ } ++ + /// Everything that references the *callable* `vma` names, rather than the + /// literal address: [`refs_to`](Self::refs_to) taken over `vma`'s whole + /// [`alias_class`](Self::alias_class), with the forwarding jumps that join +@@ -554,10 +565,13 @@ impl PcodeEmit for FullCapture { #[derive(Default)] struct AsmCapture { text: String, @@ -2259,7 +2277,7 @@ index b39058b1..9dc7ec0f 100644 self.text.clear(); self.text.push_str(mnem); if !body.is_empty() { -@@ -698,6 +701,7 @@ fn descend( +@@ -698,6 +712,7 @@ fn descend( // processor). `None` when the program has no data space: no varnode can // match it, so the direct-access projection simply contributes nothing. let data_space = arch.manage().get_default_data_space().cloned(); @@ -2267,7 +2285,7 @@ index b39058b1..9dc7ec0f 100644 // (kuna) `picbase`: the module's PIC base register, when the program has one // this can prove. Detected before the walk because the base a function -@@ -962,10 +966,10 @@ fn descend( +@@ -962,10 +977,10 @@ fn descend( // body before any of it can be attributed (`kuna_picbase::scope`), and // buffering it costs nothing on the overwhelmingly common `None` path. let mut body: Vec = Vec::new(); @@ -2280,7 +2298,7 @@ index b39058b1..9dc7ec0f 100644 while let Some((vma, mode)) = insn_queue.pop_front() { if st.decoded.contains(&vma) { continue; // already decoded (the VisitStat dedup) -@@ -1031,26 +1035,26 @@ fn descend( +@@ -1031,26 +1046,26 @@ fn descend( let fall_through = vma.wrapping_add(len as u64); data_refs(cap.ops(), data_space.as_ref(), &mapped, fall_through, pool.as_ref()) }; @@ -2324,7 +2342,7 @@ index b39058b1..9dc7ec0f 100644 }; if cap.ops().iter().any(|op| op.opcode == OpCode::CPUI_CALLIND) { -@@ -1150,7 +1154,7 @@ fn descend( +@@ -1150,7 +1165,7 @@ fn descend( // and a row whose instruction column is blank does not say // which jump reached the case body. if !cases.is_empty() && text.is_empty() { @@ -2333,7 +2351,7 @@ index b39058b1..9dc7ec0f 100644 } for target in cases { st.file(vma, target, XrefKind::Jump, &text); -@@ -1162,8 +1166,8 @@ fn descend( +@@ -1162,8 +1177,8 @@ fn descend( for &(to, kind) in &drefs { st.file(vma, to, kind, &text); } @@ -2344,7 +2362,7 @@ index b39058b1..9dc7ec0f 100644 } // Both halves the deferred base-relative pass needs are pure // functions of this instruction's ops, so they are computed here -@@ -1201,7 +1205,7 @@ fn descend( +@@ -1201,7 +1216,7 @@ fn descend( // parse of the whole program on any image with a PIC base // (an i386 `__x86.get_pc_thunk` binary: 154,608 renders to // file 151 references). @@ -2353,7 +2371,7 @@ index b39058b1..9dc7ec0f 100644 for &(to, kind) in &cand.refs { st.file(cand.vma, to, kind, &text); } -@@ -1336,7 +1340,7 @@ fn render_fingerprints( +@@ -1336,7 +1351,7 @@ fn render_fingerprints( let mut vma = entry; for _ in 0..2 { let Some(insn) = listing.insns.get_mut(&vma) else { break }; @@ -2362,7 +2380,7 @@ index b39058b1..9dc7ec0f 100644 insn.mnemonic = text.split_whitespace().next().unwrap_or_default().to_string(); vma = vma.wrapping_add(u64::from(insn.len)); } -@@ -1606,14 +1610,14 @@ pub(super) fn decode( +@@ -1606,14 +1621,14 @@ pub(super) fn decode( /// will actually carry the text. On a 466 KB obfuscated i386 image (154,638 /// instructions) rendering every decode cost 0.22 s of a 1.3 s walk, and the /// large majority of instructions file no reference at all. @@ -2381,6 +2399,31 @@ index b39058b1..9dc7ec0f 100644 } /// The data references one instruction's p-code carries. +@@ -1864,6 +1879,24 @@ pub(super) fn range_union(mut ranges: Vec<(u64, u64)>) -> Vec<(u64, u64)> { + + #[cfg(test)] + mod tests { ++ #[test] ++ fn physical_target_spans_cover_u64_without_aliases_or_byte_enumeration() { ++ let mut index=super::empty(); ++ let edge=|from,to|super::Xref {from,to,kind:super::XrefKind::Data,instruction:String::new()}; ++ index.by_target.insert(0,vec![edge(3,0)]); ++ index.by_target.insert(8,vec![edge(4,8),edge(5,8)]); ++ index.by_target.insert(u64::MAX,vec![edge(6,u64::MAX)]); ++ assert!(index.refs_to_range(9,8).is_none()); ++ assert_eq!(index.refs_to_range(1,7).unwrap().count(),0); ++ assert_eq!(index.refs_to_range(0,u64::MAX).unwrap().map(|r|r.from).collect::>(),[3,4,5,6]); ++ assert_eq!(index.refs_to_range(u64::MAX,u64::MAX).unwrap().next().unwrap().from,6); ++ assert_eq!(index.refs_to_range(8,8).unwrap().map(|r|(r.from,r.to)).collect::>(),index.refs_to(8).iter().map(|r|(r.from,r.to)).collect::>()); ++ let aliases=import_index(); ++ for &at in aliases.by_target.keys() { ++ assert_eq!(aliases.refs_to_range(at,at).unwrap().count(),aliases.refs_to(at).len()); ++ } ++ } ++ + use std::rc::Rc; + + use kuna_base::space::{spacetype, AddrSpace}; diff --git a/decompiler/crates/kuna-analysis/src/loader/elf_plt.rs b/decompiler/crates/kuna-analysis/src/loader/elf_plt.rs index 4ab6cbca..abcf8c5b 100644 --- a/decompiler/crates/kuna-analysis/src/loader/elf_plt.rs @@ -3633,13 +3676,13 @@ index 3eb2a6e4..2b951ba2 100644 }; decompile_callee_first(prog, args, targets, explicit, base) diff --git a/decompiler/crates/kuna-cli/src/jsonfmt.rs b/decompiler/crates/kuna-cli/src/jsonfmt.rs -index 198b85a6..71bd9675 100644 +index 198b85a6..299e4a09 100644 --- a/decompiler/crates/kuna-cli/src/jsonfmt.rs +++ b/decompiler/crates/kuna-cli/src/jsonfmt.rs -@@ -28,6 +28,19 @@ pub enum Json { +@@ -28,6 +28,26 @@ pub enum Json { Object(Vec<(String, Json)>), } - + +impl From for Json { + fn from(value: serde_json::Value) -> Self { + match value { @@ -3647,8 +3690,15 @@ index 198b85a6..71bd9675 100644 + serde_json::Value::Bool(value) => Json::Bool(value), + serde_json::Value::Number(value) => Json::Number(value.to_string()), + serde_json::Value::String(value) => Json::Str(value), -+ serde_json::Value::Array(values) => Json::Array(values.into_iter().map(Json::from).collect()), -+ serde_json::Value::Object(values) => Json::Object(values.into_iter().map(|(key, value)| (key, Json::from(value))).collect()), ++ serde_json::Value::Array(values) => { ++ Json::Array(values.into_iter().map(Json::from).collect()) ++ } ++ serde_json::Value::Object(values) => Json::Object( ++ values ++ .into_iter() ++ .map(|(key, value)| (key, Json::from(value))) ++ .collect(), ++ ), + } + } +} @@ -3656,46 +3706,84 @@ index 198b85a6..71bd9675 100644 /// Render `v` exactly as CPython's `json.dumps(v, indent=2)` would (ASCII-safe /// string escaping, `", "`/`": "` item/key separators collapsed to the indent /// form: items separated by `,\n`, keys by `": "`). -@@ -50,6 +63,12 @@ pub fn dumps_compact(v: &Json) -> String { +@@ -50,6 +70,35 @@ pub fn dumps_compact(v: &Json) -> String { writer::render(v, Format::Compact) } - + +/// Enforce the caller's encoded-byte budget before growing the output buffer. +/// Uses the same writer and escaping as the other modes; no length estimator. +pub fn dumps_compact_bounded(v: &Json, limit: usize) -> Option { + writer::render_bounded(v, Format::Compact, limit) +} ++ ++/// Stream selected records through the same bounded escaping owner. The caller ++/// need not construct a complete result tree before checking encoded growth. ++pub struct BoundedJsonWriter(writer::BoundedString); ++impl BoundedJsonWriter { ++ pub fn new(limit: usize) -> Self { ++ Self(writer::BoundedString::new(limit)) ++ } ++ pub fn remaining(&self) -> usize { ++ self.0.remaining() ++ } ++ pub fn value(&mut self, value: &Json) -> std::fmt::Result { ++ writer::write_value(&mut self.0, value, 0, Format::Compact) ++ } ++ pub fn finish(self) -> String { ++ self.0.finish() ++ } ++} ++impl std::fmt::Write for BoundedJsonWriter { ++ fn write_str(&mut self, text: &str) -> std::fmt::Result { ++ std::fmt::Write::write_str(&mut self.0, text) ++ } ++} + /// Extract the JSON array/object span out of a `decomp_dbg` console transcript. /// /// Port of `kuna/catalog.py::_extract_json`: drop the fixed `[decomp]>` prompt diff --git a/decompiler/crates/kuna-cli/src/jsonfmt/writer.rs b/decompiler/crates/kuna-cli/src/jsonfmt/writer.rs -index c4eca1db..06b9d024 100644 +index c4eca1db..6a76bd3f 100644 --- a/decompiler/crates/kuna-cli/src/jsonfmt/writer.rs +++ b/decompiler/crates/kuna-cli/src/jsonfmt/writer.rs @@ -1,7 +1,7 @@ //! JSON layout and ASCII-safe escaping shared by all CLI output modes. - + use super::Json; -use std::fmt::Write as _; +use std::fmt::{self, Write}; - + #[derive(Clone, Copy, PartialEq, Eq)] pub(super) enum Format { -@@ -12,108 +12,142 @@ pub(super) enum Format { - +@@ -12,108 +12,159 @@ pub(super) enum Format { + pub(super) fn render(value: &Json, format: Format) -> String { let mut out = String::new(); - write_value(&mut out, value, 0, format); + write_value(&mut out, value, 0, format).expect("String writer is infallible"); out } - -+struct BoundedString { + ++pub(super) struct BoundedString { + text: String, + limit: usize, +} + ++impl BoundedString { ++ pub(super) fn new(limit: usize) -> Self { ++ Self { ++ text: String::new(), ++ limit, ++ } ++ } ++ pub(super) fn remaining(&self) -> usize { ++ self.limit - self.text.len() ++ } ++ pub(super) fn finish(self) -> String { ++ self.text ++ } ++} ++ +impl Write for BoundedString { + fn write_str(&mut self, value: &str) -> fmt::Result { + let next = self.text.len().checked_add(value.len()).ok_or(fmt::Error)?; @@ -3716,10 +3804,7 @@ index c4eca1db..06b9d024 100644 +} + +pub(super) fn render_bounded(value: &Json, format: Format, limit: usize) -> Option { -+ let mut out = BoundedString { -+ text: String::new(), -+ limit, -+ }; ++ let mut out = BoundedString::new(limit); + write_value(&mut out, value, 0, format).ok()?; + Some(out.text) +} @@ -3738,9 +3823,14 @@ index c4eca1db..06b9d024 100644 + Ok(()) } } - + -fn write_value(out: &mut String, value: &Json, indent: usize, format: Format) { -+fn write_value(out: &mut impl Write, value: &Json, indent: usize, format: Format) -> fmt::Result { ++pub(super) fn write_value( ++ out: &mut impl Write, ++ value: &Json, ++ indent: usize, ++ format: Format, ++) -> fmt::Result { match value { - Json::Null => out.push_str("null"), - Json::Bool(true) => out.push_str("true"), @@ -3784,7 +3874,7 @@ index c4eca1db..06b9d024 100644 } } } - + fn write_pairs<'a>( - out: &mut String, + out: &mut impl Write, @@ -3821,7 +3911,7 @@ index c4eca1db..06b9d024 100644 - out.push('}'); + out.write_char('}') } - + -fn write_string(out: &mut String, s: &str) { - out.push('"'); +fn write_string(out: &mut impl Write, s: &str) -> fmt::Result { @@ -3869,10 +3959,40 @@ index c4eca1db..06b9d024 100644 + out.write_char('"') } diff --git a/decompiler/crates/kuna-cli/src/jsonfmt/writer_tests.rs b/decompiler/crates/kuna-cli/src/jsonfmt/writer_tests.rs -index 6710e12a..ef90468c 100644 +index 6710e12a..d27b3964 100644 --- a/decompiler/crates/kuna-cli/src/jsonfmt/writer_tests.rs +++ b/decompiler/crates/kuna-cli/src/jsonfmt/writer_tests.rs -@@ -94,3 +94,21 @@ fn every_unicode_scalar_round_trips_through_ascii_output() { +@@ -2,6 +2,29 @@ use super::*; + + const RENDERERS: [fn(&Json) -> String; 3] = [dumps_compact, dumps_indent2, dumps_indent2_sorted]; + ++#[test] ++fn streamed_values_use_the_same_escaping_and_total_budget() { ++ use std::fmt::Write; ++ let value = Json::Str("a\n\"\\δΈ­ζ–‡πŸ˜€".into()); ++ let expected = format!("[{},{}]", dumps_compact(&value), dumps_compact(&value)); ++ for limit in 0..=expected.len() + 1 { ++ let mut stream = BoundedJsonWriter::new(limit); ++ let attempt = (|| -> std::fmt::Result { ++ stream.write_char('[')?; ++ stream.value(&value)?; ++ stream.write_char(',')?; ++ stream.value(&value)?; ++ stream.write_char(']') ++ })(); ++ assert_eq!(attempt.is_ok(), limit >= expected.len()); ++ let actual = stream.finish(); ++ assert!(actual.len() <= limit); ++ if attempt.is_ok() { ++ assert_eq!(actual, expected); ++ } ++ } ++} ++ + #[test] + fn nested_containers_and_numeric_spelling_are_byte_stable() { + let value = Json::Object(vec![ +@@ -94,3 +117,21 @@ fn every_unicode_scalar_round_trips_through_ascii_output() { assert_eq!(serde_json::from_str::(&output).unwrap(), text); } } @@ -4173,6 +4293,748 @@ index 00000000..b2a30345 + assert_eq!(value[0]["offset"], "0xffffffffffffffff"); + } +} +diff --git a/decompiler/crates/kuna-cli/src/strings.rs b/decompiler/crates/kuna-cli/src/strings.rs +index ffe33667..d2412227 100644 +--- a/decompiler/crates/kuna-cli/src/strings.rs ++++ b/decompiler/crates/kuna-cli/src/strings.rs +@@ -52,7 +52,7 @@ mod filter; + + use filter::Regex; + +-use std::collections::BTreeMap; ++use std::collections::{BTreeMap, BTreeSet}; + + use kuna_analysis::strings::kuna_stringinv::{self, FoundString, Termination}; + +@@ -132,12 +132,19 @@ pub(crate) fn query(args: &StringsArgs) -> Result { + section: args.section.clone(), + termination: args.termination, + }; +- let inv = match view.as_ref().filter(|v| v.bytes.as_ptr() == scanned.as_ptr()) { ++ let inv = match view ++ .as_ref() ++ .filter(|v| v.bytes.as_ptr() == scanned.as_ptr()) ++ { + Some(view) => kuna_stringinv::inventory_in_image(&file, &query, &view.ranges), + None => kuna_stringinv::inventory(&file, &query), + }; + if let Some(want) = &args.section { +- if !inv.regions.iter().any(|n| n == want || n.strip_prefix('.') == Some(want.as_str())) { ++ if !inv ++ .regions ++ .iter() ++ .any(|n| n == want || n.strip_prefix('.') == Some(want.as_str())) ++ { + let mut have: Vec<&str> = inv.regions.iter().map(String::as_str).collect(); + have.dedup(); + let have = if have.is_empty() { +@@ -168,7 +175,11 @@ pub(crate) fn query(args: &StringsArgs) -> Result { + let rows = if args.no_xrefs || found.is_empty() { + found + .into_iter() +- .map(|found| Row { found, xrefs_count: 0, functions: Vec::new() }) ++ .map(|found| Row { ++ found, ++ xrefs_count: 0, ++ functions: Vec::new(), ++ }) + .collect() + } else { + attribute(args, options, &file, found)? +@@ -181,7 +192,10 @@ pub(crate) fn query(args: &StringsArgs) -> Result { + ); + } + Ok(if args.json { +- format!("{}\n", dumps_indent2(&result_json(args, inv.from_segments, &rows))) ++ format!( ++ "{}\n", ++ dumps_indent2(&result_json(args, inv.from_segments, &rows)) ++ ) + } else { + render_text(args, inv.from_segments, &rows) + }) +@@ -226,43 +240,53 @@ fn attribute( + .map(|e| (e.addr.get_offset(), e.name)) + .collect(); + let seeds: Vec = inventory.keys().copied().collect(); +- let index = kuna_analysis::listing::xrefs::build( +- file, +- prog.arch(), +- prog.arch().translate(), +- &seeds, +- ); ++ let index = ++ kuna_analysis::listing::xrefs::build(file, prog.arch(), prog.arch().translate(), &seeds); + + // A name is looked up once per referencing function, not once per row: a + // prompt string referenced from twenty sites resolves one entry. + let mut names: BTreeMap = BTreeMap::new(); +- Ok(found ++ found + .into_iter() + .map(|found| { + let mut xrefs_count = 0usize; + let mut functions: Vec<(u64, String)> = Vec::new(); +- // A reference may land anywhere in the literal, not only on its first +- // byte β€” `lea rax,[fmt+4]` is still a use of `fmt`. +- for vma in found.addr..found.addr.saturating_add(u64::from(found.byte_len)) { +- for r in index.refs_to(vma) { +- xrefs_count += 1; +- let Some(entry) = owning_function(&prog, &index, r.from) else { +- continue; +- }; +- if functions.iter().any(|(e, _)| *e == entry) { +- continue; +- } +- let name = names +- .entry(entry) +- .or_insert_with(|| function_name(&prog, &inventory, entry)) +- .clone(); +- functions.push((entry, name)); ++ let mut seen = BTreeSet::new(); ++ let end = found ++ .addr ++ .checked_add( ++ u64::from(found.byte_len) ++ .checked_sub(1) ++ .ok_or("string has an empty encoded extent")?, ++ ) ++ .ok_or("string encoded extent exceeds u64 addresses")?; ++ // Target coordinates inside the backing bytes also include pooled ++ // suffixes; membership does not establish use of the whole text. ++ for r in index ++ .refs_to_range(found.addr, end) ++ .ok_or("invalid string extent")? ++ { ++ xrefs_count += 1; ++ let Some(entry) = owning_function(&prog, &index, r.from) else { ++ continue; ++ }; ++ if !seen.insert(entry) { ++ continue; + } ++ let name = names ++ .entry(entry) ++ .or_insert_with(|| function_name(&prog, &inventory, entry)) ++ .clone(); ++ functions.push((entry, name)); + } + functions.sort(); +- Row { found, xrefs_count, functions } ++ Ok(Row { ++ found, ++ xrefs_count, ++ functions, ++ }) + }) +- .collect()) ++ .collect() + } + + // --- rendering --------------------------------------------------------------- +@@ -294,14 +318,32 @@ fn result_json(args: &StringsArgs, from_segments: bool, rows: &[Row]) -> Json { + .map(|row| { + Json::Object(vec![ + ("address".into(), Json::Number(row.found.addr.to_string())), +- ("address_hex".into(), Json::Str(format!("0x{:x}", row.found.addr))), ++ ( ++ "address_hex".into(), ++ Json::Str(format!("0x{:x}", row.found.addr)), ++ ), + ("text".into(), Json::Str(row.found.text.clone())), +- ("length".into(), Json::Number(row.found.char_len.to_string())), +- ("byte_length".into(), Json::Number(row.found.byte_len.to_string())), +- ("nul_terminated".into(), Json::Bool(row.found.nul_terminated)), +- ("encoding".into(), Json::Str(row.found.encoding.as_str().to_string())), ++ ( ++ "length".into(), ++ Json::Number(row.found.char_len.to_string()), ++ ), ++ ( ++ "byte_length".into(), ++ Json::Number(row.found.byte_len.to_string()), ++ ), ++ ( ++ "nul_terminated".into(), ++ Json::Bool(row.found.nul_terminated), ++ ), ++ ( ++ "encoding".into(), ++ Json::Str(row.found.encoding.as_str().to_string()), ++ ), + ("section".into(), optional_str(row.found.section.as_deref())), +- ("xrefs_count".into(), Json::Number(row.xrefs_count.to_string())), ++ ( ++ "xrefs_count".into(), ++ Json::Number(row.xrefs_count.to_string()), ++ ), + ( + "functions".into(), + Json::Array( +@@ -318,11 +360,23 @@ fn result_json(args: &StringsArgs, from_segments: bool, rows: &[Row]) -> Json { + Json::Object(vec![ + ("binary".into(), Json::Str(args.binary.clone())), + ("encoding".into(), Json::Str(args.encoding_label.clone())), +- ("min_length".into(), Json::Number(args.min_length.to_string())), +- ("filter".into(), optional_str(args.filter.as_ref().map(|(p, _)| p.as_str()))), ++ ( ++ "min_length".into(), ++ Json::Number(args.min_length.to_string()), ++ ), ++ ( ++ "filter".into(), ++ optional_str(args.filter.as_ref().map(|(p, _)| p.as_str())), ++ ), + ("section".into(), optional_str(args.section.as_deref())), +- ("termination".into(), Json::Str(termination_label(args.termination).to_string())), +- ("scanned".into(), Json::Str(scanned_label(from_segments).to_string())), ++ ( ++ "termination".into(), ++ Json::Str(termination_label(args.termination).to_string()), ++ ), ++ ( ++ "scanned".into(), ++ Json::Str(scanned_label(from_segments).to_string()), ++ ), + ("xrefs".into(), Json::Bool(!args.no_xrefs)), + ("count".into(), Json::Number(rows.len().to_string())), + ("strings".into(), strings), +@@ -367,7 +421,11 @@ fn render_text(args: &StringsArgs, from_segments: bool, rows: &[Row]) -> String + let functions = if row.functions.is_empty() { + "-".to_string() + } else { +- row.functions.iter().map(|(_, n)| n.as_str()).collect::>().join(",") ++ row.functions ++ .iter() ++ .map(|(_, n)| n.as_str()) ++ .collect::>() ++ .join(",") + }; + let _ = writeln!( + out, +@@ -409,8 +467,9 @@ pub(crate) fn parse_args(argv: &[String]) -> Result { + "--json" => json = true, + "--min-length" => { + let v = take(argv, &mut i, "--min-length")?; +- let n: usize = +- v.parse().map_err(|_| format!("--min-length wants a number, not {v:?}"))?; ++ let n: usize = v ++ .parse() ++ .map_err(|_| format!("--min-length wants a number, not {v:?}"))?; + if n == 0 { + return Err("--min-length must be at least 1".into()); + } +@@ -470,7 +529,9 @@ pub(crate) fn parse_args(argv: &[String]) -> Result { + "utf16" => (false, false, true), + "all" => (true, true, true), + other => { +- return Err(format!("unknown --encoding {other:?} (ascii, utf8, utf16, all)")) ++ return Err(format!( ++ "unknown --encoding {other:?} (ascii, utf8, utf16, all)" ++ )) + } + }; + Ok(StringsArgs { +diff --git a/decompiler/crates/kuna-cli/src/xrefs.rs b/decompiler/crates/kuna-cli/src/xrefs.rs +index c351a097..ec20198c 100644 +--- a/decompiler/crates/kuna-cli/src/xrefs.rs ++++ b/decompiler/crates/kuna-cli/src/xrefs.rs +@@ -3,6 +3,8 @@ + //! ```text + //! kuna xrefs --to [--json] [--kind k,k..] [--mode MODE] + //! kuna xrefs --from [--json] [--kind k,k..] [--mode MODE] ++//! kuna xrefs --to-range 0xSTART:0xEND_INCLUSIVE.. --json ++//! [--offset N] [--limit 1..128] [--output-bytes N] + //! [--option N V].. [--isa auto|arm|thumb] [--slice ARCH] [--target T] [--sleighpath D] + //! ``` + //! +@@ -27,6 +29,9 @@ + + use std::collections::{BTreeMap, BTreeSet}; + ++mod ranges; ++mod row; ++ + use kuna_analysis::listing::xrefs::{Xref, XrefIndex, XrefKind}; + use kuna_console::engine::{ConsoleProgram, EntryLookupError, EntrySelector}; + +@@ -56,6 +61,7 @@ impl Direction { + /// The parsed command line. + struct XrefArgs { + binary: String, ++ ranges: Option, + /// The `--to` / `--from` operand: a symbol name or an address. + spec: String, + direction: Direction, +@@ -250,15 +256,24 @@ fn query(args: &XrefArgs) -> Result { + // "no references" about the very address the caller named. A name several + // entries carry points the walk at all of them and is settled afterwards, + // because whether they are one callable is the walk's own finding. +- let resolution = target_address(&prog, &args.spec)?; ++ let resolution = if args.ranges.is_some() { None } else { Some(target_address(&prog, &args.spec)?) }; ++ let focus = match (&resolution,&args.ranges) { ++ (Some(resolution),_) => resolution.focus(), ++ (_,Some(query)) => query.spans.iter().map(|span|span.start).collect(), ++ _ => return Err("xref query has no selection".into()), ++ }; + let index = kuna_analysis::listing::xrefs::build_with_focus( + &file, + prog.arch(), + prog.arch().translate(), + &seeds, +- &resolution.focus(), ++ &focus, + ); + ++ if let Some(query)=&args.ranges { ++ return ranges::render(args,query,&prog,&index,&names); ++ } ++ let resolution=resolution.ok_or("xref query has no exact target")?; + let target = resolve_target(&prog, &file, &index, &names, resolution.settle(&index)?); + warn_other_entries(&index, &names, &target); + let rows = match args.direction { +@@ -414,10 +429,14 @@ fn name_at(prog: &ConsoleProgram, index: &XrefIndex, names: &EntryNames, vma: u6 + /// first (it knows which entry's descent reached the instruction), then the + /// engine's inventory for an address the walk never decoded. + fn owning_function(prog: &ConsoleProgram, index: &XrefIndex, names: &EntryNames, vma: u64) -> Option<(u64, String)> { +- let entry = index.function_containing(vma).or_else(|| names.at(vma).map(|(address, _)| address))?; ++ let entry = function_entry(index,names,vma)?; + Some((entry, function_name(prog, index, names, entry))) + } + ++fn function_entry(index:&XrefIndex,names:&EntryNames,vma:u64)->Option { ++ index.function_containing(vma).or_else(||names.at(vma).map(|(address,_)|address)) ++} ++ + /// The display name for a function entry, falling back to the engine's own + /// placeholder (`sub_`) so a row is never nameless. + fn function_name(prog: &ConsoleProgram, index: &XrefIndex, names: &EntryNames, entry: u64) -> String { +@@ -426,13 +445,6 @@ fn function_name(prog: &ConsoleProgram, index: &XrefIndex, names: &EntryNames, e + + // --- rendering --------------------------------------------------------------- + +-fn optional_function_json(f: Option<(u64, String)>) -> Json { +- match f { +- Some((addr, name)) => function_json(&name, addr), +- None => Json::Null, +- } +-} +- + /// Build the `xrefs --json` document. + /// + /// Each row carries BOTH ends of the edge explicitly (`from_address` / +@@ -448,34 +460,7 @@ fn result_json( + target: &Target, + rows: &[&Xref], + ) -> Json { +- let xrefs = Json::Array( +- rows.iter() +- .map(|r| { +- let other = match args.direction { +- Direction::To => r.from, +- Direction::From => r.to, +- }; +- Json::Object(vec![ +- ("address".into(), Json::Number(other.to_string())), +- ("address_hex".into(), Json::Str(format!("0x{other:x}"))), +- ("kind".into(), Json::Str(r.kind.as_str().to_string())), +- ("from_address".into(), Json::Number(r.from.to_string())), +- ("from_address_hex".into(), Json::Str(format!("0x{:x}", r.from))), +- ("to_address".into(), Json::Number(r.to.to_string())), +- ("to_address_hex".into(), Json::Str(format!("0x{:x}", r.to))), +- ( +- "from_function".into(), +- optional_function_json(owning_function(prog, index, names, r.from)), +- ), +- ( +- "to_function".into(), +- optional_function_json(owning_function(prog, index, names, r.to)), +- ), +- ("instruction".into(), Json::Str(r.instruction.clone())), +- ]) +- }) +- .collect(), +- ); ++ let xrefs=Json::Array(rows.iter().map(|r|row::ReferenceRow::new(args.direction,prog,index,names,r).into_json()).collect()); + Json::Object(vec![ + ("binary".into(), Json::Str(args.binary.clone())), + ( +@@ -602,6 +587,11 @@ fn site_label(prog: &ConsoleProgram, index: &XrefIndex, names: &EntryNames, vma: + fn parse_args(argv: &[String]) -> Result { + let mut binary: Option = None; + let mut spec: Option<(Direction, String)> = None; ++ let mut spans=Vec::new(); ++ let mut offset=0; ++ let mut limit=20; ++ let mut output_bytes=ranges::OUTPUT_BYTES; ++ let mut range_options=false; + let mut json = false; + let mut kinds: Vec = Vec::new(); + let mut options: Vec<(String, String)> = Vec::new(); +@@ -616,7 +606,23 @@ fn parse_args(argv: &[String]) -> Result { + let a = argv[i].as_str(); + match a { + "--json" => json = true, ++ "--to-range" => { ++ if spec.is_some() { return Err("--to-range cannot combine with --to/--from".into()); } ++ if spans.len()==ranges::MAX_RANGES {return Err("at most 16 target ranges per query".into());} ++ spans.push(ranges::Span::parse(&take(argv,&mut i,a)?)?); ++ } ++ "--offset" | "--limit" | "--output-bytes" => { ++ let value=take(argv,&mut i,a)?.parse::().map_err(|_|format!("{a} requires an unsigned integer"))?; ++ range_options=true; ++ match a { ++ "--offset" => offset=value, ++ "--limit" if (1..=ranges::MAX_PAGE).contains(&value) => limit=value, ++ "--output-bytes" if value>0 => output_bytes=value, ++ _ => return Err(format!("invalid {a} value")), ++ } ++ } + "--to" | "--from" => { ++ if !spans.is_empty() {return Err("--to/--from cannot combine with --to-range".into());} + let dir = if a == "--to" { Direction::To } else { Direction::From }; + let v = take(argv, &mut i, a)?; + if spec.is_some() { +@@ -653,9 +659,19 @@ fn parse_args(argv: &[String]) -> Result { + } + + let binary = binary.ok_or("xrefs requires ")?; +- let (direction, spec) = spec.ok_or("xrefs requires --to or --from ")?; ++ let ranges=if spans.is_empty() { ++ if range_options {return Err("range paging/output options require --to-range".into());} ++ None ++ } else { ++ if !json {return Err("--to-range requires --json".into());} ++ Some(ranges::RangeQuery {spans,offset,limit,output_bytes}) ++ }; ++ let (direction,spec)=if ranges.is_some() {(Direction::To,String::new())} else { ++ spec.ok_or("xrefs requires --to , --from , or --to-range START:END_INCLUSIVE")? ++ }; + Ok(XrefArgs { + binary, ++ ranges, + spec, + direction, + json, +@@ -691,6 +707,9 @@ fn usage() { + \x20 IAT/GOT slot it jumps through -- and both answer the same; target.aliases\n\ + \x20 names the other one. A name also answers over the export's own PLT stub\n\ + \x20 when the library calls it through one.\n\ ++ --to-range START:END_INCLUSIVE selects physical target coordinates; repeat up to 16 times.\n\ ++ \x20 Requires --json; --offset N and --limit 1..128 page each range separately.\n\ ++ \x20 --output-bytes N bounds JSON before serialization growth (default 8 MiB).\n\ + --from everything the target references (its callees and the data it touches)\n\ + \n\ + --json emits {{binary,target:{{name,address,address_hex,aliases}},direction,count,\n\ +diff --git a/decompiler/crates/kuna-cli/src/xrefs/ranges.rs b/decompiler/crates/kuna-cli/src/xrefs/ranges.rs +new file mode 100644 +index 00000000..f486b3e5 +--- /dev/null ++++ b/decompiler/crates/kuna-cli/src/xrefs/ranges.rs +@@ -0,0 +1,216 @@ ++//! Batched physical target-coordinate predicates over one native index. ++use super::*; ++use crate::jsonfmt::BoundedJsonWriter; ++use std::fmt::{self, Write}; ++ ++pub(super) const MAX_RANGES: usize = 16; ++pub(super) const MAX_PAGE: usize = 128; ++pub(super) const OUTPUT_BYTES: usize = 8 * 1024 * 1024; ++ ++#[derive(Debug, Clone, Copy, PartialEq, Eq)] ++pub(super) struct Span { ++ pub start: u64, ++ pub end: u64, ++} ++impl Span { ++ pub(super) fn parse(text: &str) -> Result { ++ let (start, end) = text ++ .split_once(':') ++ .ok_or("range requires 0xSTART:0xEND_INCLUSIVE")?; ++ let address = |value: &str| -> Result { ++ let digits = value ++ .strip_prefix("0x") ++ .ok_or("range endpoints require 0x hexadecimal addresses")?; ++ if digits.is_empty() ++ || digits.len() > 16 ++ || !digits.bytes().all(|b| b.is_ascii_hexdigit()) ++ { ++ return Err("range endpoint is not a u64 address".into()); ++ } ++ u64::from_str_radix(digits, 16).map_err(|e| e.to_string()) ++ }; ++ let span = Self { ++ start: address(start)?, ++ end: address(end)?, ++ }; ++ if span.start > span.end { ++ return Err("range start exceeds inclusive end".into()); ++ } ++ Ok(span) ++ } ++} ++pub(super) struct RangeQuery { ++ pub spans: Vec, ++ pub offset: usize, ++ pub limit: usize, ++ pub output_bytes: usize, ++} ++struct Page<'a> { ++ span: Span, ++ total: usize, ++ rows: Vec<&'a Xref>, ++} ++ ++fn pages<'a>( ++ query: &RangeQuery, ++ index: &'a XrefIndex, ++ kinds: &[XrefKind], ++) -> Result>, String> { ++ query ++ .spans ++ .iter() ++ .map(|&span| { ++ let mut total = 0; ++ let mut rows = Vec::new(); ++ let iter = index ++ .refs_to_range(span.start, span.end) ++ .ok_or("invalid native target range")?; ++ for row in iter.filter(|r| kinds.is_empty() || kinds.contains(&r.kind)) { ++ if total >= query.offset && rows.len() < query.limit { ++ rows.push(row); ++ } ++ total += 1; ++ } ++ Ok(Page { span, total, rows }) ++ }) ++ .collect() ++} ++ ++pub(super) fn render( ++ args: &XrefArgs, ++ query: &RangeQuery, ++ prog: &ConsoleProgram, ++ index: &XrefIndex, ++ names: &EntryNames, ++) -> Result { ++ let pages = pages(query, index, &args.kinds)?; ++ let count: usize = pages.iter().map(|p| p.rows.len()).sum(); ++ let metadata = Json::Array( ++ pages ++ .iter() ++ .enumerate() ++ .map(|(i, p)| { ++ let end = query.offset.saturating_add(p.rows.len()); ++ Json::Object(vec![ ++ ("index".into(), Json::Number(i.to_string())), ++ ("start".into(), Json::Str(format!("0x{:x}", p.span.start))), ++ ( ++ "end_inclusive".into(), ++ Json::Str(format!("0x{:x}", p.span.end)), ++ ), ++ ("offset".into(), Json::Number(query.offset.to_string())), ++ ("limit".into(), Json::Number(query.limit.to_string())), ++ ("total".into(), Json::Number(p.total.to_string())), ++ ("count".into(), Json::Number(p.rows.len().to_string())), ++ ( ++ "next_offset".into(), ++ if end < p.total { ++ Json::Number(end.to_string()) ++ } else { ++ Json::Null ++ }, ++ ), ++ ]) ++ }) ++ .collect(), ++ ); ++ let mut out = BoundedJsonWriter::new(query.output_bytes); ++ let mut write = || -> fmt::Result { ++ out.write_str("{\"schema_version\":1,\"binary\":")?; ++ out.value(&Json::Str(args.binary.clone()))?; ++ out.write_str(",\"direction\":\"to\",\"ranges\":")?; ++ out.value(&metadata)?; ++ write!(out, ",\"count\":{count},\"xrefs\":[")?; ++ let mut first = true; ++ for (i, page) in pages.iter().enumerate() { ++ for r in &page.rows { ++ let row = row::ReferenceRow::new(Direction::To, prog, index, names, r); ++ if row.clone_bytes().is_none_or(|n| n > out.remaining()) { ++ return Err(fmt::Error); ++ } ++ let Json::Object(mut fields) = row.into_json() else { ++ return Err(fmt::Error); ++ }; ++ fields.insert(0, ("range_index".into(), Json::Number(i.to_string()))); ++ if !first { ++ out.write_char(',')?; ++ } ++ first = false; ++ out.value(&Json::Object(fields))?; ++ } ++ } ++ out.write_str("]}\n") ++ }; ++ write().map_err(|_| { ++ format!( ++ "xref range JSON exceeds {} encoded bytes", ++ query.output_bytes ++ ) ++ })?; ++ Ok(out.finish()) ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ fn arguments(values: &[&str]) -> Result { ++ super::super::parse_args(&values.iter().map(|s| s.to_string()).collect::>()) ++ } ++ #[test] ++ fn range_choices_are_explicit_and_cannot_silently_override_exact_queries() { ++ for flags in [ ++ vec!["input", "--to-range", "0x0:0x1"], ++ vec!["input", "--json", "--to-range", "0x0:0x1", "--to", "0x0"], ++ vec!["input", "--json", "--from", "0x0", "--to-range", "0x0:0x1"], ++ vec!["input", "--json", "--to", "0x0", "--offset", "2"], ++ vec!["input", "--json", "--to-range", "0x0:0x1", "--limit", "0"], ++ vec!["input", "--json", "--to-range", "0x0:0x1", "--limit", "129"], ++ vec!["input", "--json", "--to-range", "0x0:0x1", "--offset", "-1"], ++ ] { ++ assert!(arguments(&flags).is_err(), "{flags:?}"); ++ } ++ let valid = arguments(&[ ++ "input", ++ "--json", ++ "--to-range", ++ "0x1:0x2", ++ "--to-range", ++ "0x1:0x2", ++ "--offset", ++ "3", ++ "--limit", ++ "1", ++ ]) ++ .unwrap(); ++ let query = valid.ranges.unwrap(); ++ assert_eq!(query.spans.len(), 2); ++ assert_eq!(query.spans[0], query.spans[1]); ++ assert_eq!((query.offset, query.limit), (3, 1)); ++ let mut flags = vec!["input", "--json"]; ++ for _ in 0..MAX_RANGES { ++ flags.extend(["--to-range", "0x0:0x0"]); ++ } ++ assert!(arguments(&flags).is_ok()); ++ flags.extend(["--to-range", "0x0:0x0"]); ++ assert!(arguments(&flags).is_err()); ++ } ++ #[test] ++ fn inclusive_u64_spans_are_ordered_without_end_increment() { ++ assert_eq!( ++ Span::parse("0xffffffffffffffff:0xffffffffffffffff").unwrap(), ++ Span { ++ start: u64::MAX, ++ end: u64::MAX ++ } ++ ); ++ for s in [ ++ "0x2:0x1", ++ "0x0:0x10000000000000000", ++ "0x0:0x1:0x2", ++ "0:0x1", ++ "0x:0x1", ++ ] { ++ assert!(Span::parse(s).is_err(), "{s}"); ++ } ++ } ++} +diff --git a/decompiler/crates/kuna-cli/src/xrefs/row.rs b/decompiler/crates/kuna-cli/src/xrefs/row.rs +new file mode 100644 +index 00000000..f680fbad +--- /dev/null ++++ b/decompiler/crates/kuna-cli/src/xrefs/row.rs +@@ -0,0 +1,81 @@ ++//! One endpoint/function interpretation for exact and range wire rows. ++use super::*; ++use std::borrow::Cow; ++ ++struct Function<'a> { ++ address: u64, ++ name: Cow<'a, str>, ++} ++fn owner<'a>( ++ prog: &ConsoleProgram, ++ index: &XrefIndex, ++ names: &'a EntryNames, ++ at: u64, ++) -> Option> { ++ let address = function_entry(index, names, at)?; ++ let name = names ++ .at(address) ++ .map(|(_, name)| Cow::Borrowed(name)) ++ .unwrap_or_else(|| Cow::Owned(function_name(prog, index, names, address))); ++ Some(Function { address, name }) ++} ++fn function(value: Option>) -> Json { ++ value ++ .map(|f| function_json(&f.name, f.address)) ++ .unwrap_or(Json::Null) ++} ++ ++pub(super) struct ReferenceRow<'a> { ++ direction: Direction, ++ reference: &'a Xref, ++ from: Option>, ++ to: Option>, ++} ++impl<'a> ReferenceRow<'a> { ++ pub(super) fn new( ++ direction: Direction, ++ prog: &ConsoleProgram, ++ index: &XrefIndex, ++ names: &'a EntryNames, ++ reference: &'a Xref, ++ ) -> Self { ++ Self { ++ direction, ++ reference, ++ from: owner(prog, index, names, reference.from), ++ to: owner(prog, index, names, reference.to), ++ } ++ } ++ /// Lower bound before cloning variable payload. Encoded escaping is checked ++ /// by the shared writer. Only one admitted row exists beside its buffer. ++ pub(super) fn clone_bytes(&self) -> Option { ++ self.reference ++ .instruction ++ .len() ++ .checked_add(self.from.as_ref().map_or(0, |f| f.name.len()))? ++ .checked_add(self.to.as_ref().map_or(0, |f| f.name.len())) ++ } ++ pub(super) fn into_json(self) -> Json { ++ let r = self.reference; ++ let other = if self.direction == Direction::To { ++ r.from ++ } else { ++ r.to ++ }; ++ Json::Object(vec![ ++ ("address".into(), Json::Number(other.to_string())), ++ ("address_hex".into(), Json::Str(format!("0x{other:x}"))), ++ ("kind".into(), Json::Str(r.kind.as_str().to_string())), ++ ("from_address".into(), Json::Number(r.from.to_string())), ++ ( ++ "from_address_hex".into(), ++ Json::Str(format!("0x{:x}", r.from)), ++ ), ++ ("to_address".into(), Json::Number(r.to.to_string())), ++ ("to_address_hex".into(), Json::Str(format!("0x{:x}", r.to))), ++ ("from_function".into(), function(self.from)), ++ ("to_function".into(), function(self.to)), ++ ("instruction".into(), Json::Str(r.instruction.clone())), ++ ]) ++ } ++} diff --git a/decompiler/crates/kuna-console/src/inspect.rs b/decompiler/crates/kuna-console/src/inspect.rs index 74024b10..9e36d55f 100644 --- a/decompiler/crates/kuna-console/src/inspect.rs diff --git a/experiments/atomic-codemode/provider.mjs b/experiments/atomic-codemode/provider.mjs index 7f30c66d2a..3f17c6b8c4 100644 --- a/experiments/atomic-codemode/provider.mjs +++ b/experiments/atomic-codemode/provider.mjs @@ -5,7 +5,7 @@ import { join } from "node:path"; import { parseBinaryTarget } from "../../dist/application/BinaryTargetResolver.js"; import { inspectNativeInstallation } from "./native-installation.mjs"; import { KunaTargetCommands } from "../../dist/kuna/KunaTargetCommands.js"; -import { referenceReader } from "./reference-reads.mjs"; +import { NativeReferenceReads } from "./reference-reads.mjs"; import { PrivateRuntimeRoot } from "../../dist/process/PrivateRuntimeRoot.js"; import { parseNativeIR } from "./native-ir.mjs"; import { NativeEvidenceReads } from "./native-evidence-reads.mjs"; @@ -44,6 +44,11 @@ export class KunaFactOwner { (id) => this.#sourceText(id), () => this.#requireNativeIR(), ); + #references = new NativeReferenceReads( + (...args) => this.#native(...args), + (id) => this.#sourceText(id), + () => this.#requireNativeIR(), + ); #captureDirectory; #evidence = new NativeEvidenceReads(async (id) => ({ text: await this.#sourceText(id), @@ -203,22 +208,7 @@ export class KunaFactOwner { return this.#catalogs.registers(input, signal); } async references(input, signal) { - return readFactBatch( - input.addresses.map((address) => referenceReader.query(input, address)), - (query) => - this.#native( - referenceReader.key(query), - (path) => referenceReader.arguments(path, query), - signal, - ), - async (capture, query) => - referenceReader.read( - await this.#sourceText(capture.source.evidence_id), - capture.input, - query, - input, - ), - ); + return this.#references.read(input, signal); } async snapshot(input, signal) { const capture = await this.#native( @@ -468,6 +458,7 @@ export class KunaFactOwner { this.#work.clear(); this.#snapshots.clear(); this.#catalogs.clear(); + this.#references.clear(); this.#evidence.clear(); } cancel() { diff --git a/experiments/atomic-codemode/reference-contracts.mjs b/experiments/atomic-codemode/reference-contracts.mjs new file mode 100644 index 0000000000..519189abbb --- /dev/null +++ b/experiments/atomic-codemode/reference-contracts.mjs @@ -0,0 +1,63 @@ +// One named reference operation; exact-address and image-span selectors stay distinct. +import { z } from "zod"; +import { hex, page } from "./api-contracts.mjs"; + +const selection = { + kinds: z + .array(z.enum(["call", "jump", "data", "read", "write"])) + .min(1) + .optional(), + fields: z + .array(z.enum(["fromFunction", "toFunction", "instruction"])) + .optional(), + ...page, +}; +const span = z.strictObject({ start: hex, end_inclusive: hex }); +const input = z + .union([ + z.strictObject({ + addresses: z.array(hex).min(1).max(16), + direction: z.enum(["incoming", "outgoing"]).default("incoming"), + scope: z.literal("address").default("address"), + ...selection, + }), + z.strictObject({ + addresses: z.array(hex).min(1).max(16), + direction: z.literal("outgoing"), + scope: z.literal("function"), + ...selection, + }), + z.strictObject({ + ranges: z.array(span).min(1).max(16), + direction: z.literal("incoming").default("incoming"), + scope: z.literal("range").default("range"), + ...selection, + }), + ]) + .superRefine((value, context) => { + for (const [i, range] of (value.ranges ?? []).entries()) { + if ( + !hex.safeParse(range.start).success || + !hex.safeParse(range.end_inclusive).success + ) + continue; + if (BigInt(range.start) > BigInt(range.end_inclusive)) { + context.addIssue({ + code: "custom", + path: ["ranges", i, "end_inclusive"], + message: "Inclusive end precedes start", + }); + } + } + }); + +export const referencesMethod = { + summary: + "Read exact-address/outgoing xrefs, or page batched incoming target-address spans with one native index.", + description: + "Select addresses for incoming exact-target or outgoing address/function references; function scope requires direction:outgoing and an entry. Alternatively select incoming native image ranges [{start,end_inclusive}] in caller order, including overlaps/duplicates; offset/limit apply independently to each range. Range membership tests the reference target coordinate, without callable-alias expansion. It does not prove memory-access overlap, whole-string use, or runtime binding. Both forms preserve occurrence/source-row identity, endpoints, native owning functions, paging and Evidence. A range batch builds one index; fields are projected from the same retained page. Completeness is limited to examined native references; discovery stays unknown. Recovered IR calls are a separate representation and may omit machine calls.", + input, + method: "references", + example: + 'await rea.references({ranges:[{start:"0x2000",end_inclusive:"0x200d"}],kinds:["data"],limit:20})', +}; diff --git a/experiments/atomic-codemode/reference-ranges.test.mjs b/experiments/atomic-codemode/reference-ranges.test.mjs new file mode 100644 index 0000000000..50bc911bc9 --- /dev/null +++ b/experiments/atomic-codemode/reference-ranges.test.mjs @@ -0,0 +1,143 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { referencesMethod } from "./reference-contracts.mjs"; +import { methods } from "./api.mjs"; +import { NativeReferenceReads } from "./reference-reads.mjs"; + +test("range selectors preserve old inputs and reject ambiguous/directional/malformed choices", () => { + assert.equal(methods.references, referencesMethod); + const schema = methods.references.input; + assert.deepEqual(schema.parse({ addresses: ["0x1"] }).addresses, ["0x1"]); + assert.equal( + schema.parse({ + addresses: ["0x1"], + direction: "outgoing", + scope: "function", + }).scope, + "function", + ); + const ranges = [{ start: "0x0", end_inclusive: "0xffffffffffffffff" }]; + assert.equal(schema.parse({ ranges }).scope, "range"); + for (const input of [ + { ranges, addresses: ["0x0"] }, + { addresses: ["0x1"], scope: "function", direction: "incoming" }, + { addresses: ["0x1"], scope: "function" }, + { ranges, scope: "function" }, + { ranges, direction: "outgoing" }, + { ranges: [] }, + { ranges: [{ start: "0x2", end_inclusive: "0x1" }] }, + { ranges: [{ start: "0xg", end_inclusive: "0x1" }] }, + { ranges: [{ start: "0x0", end_inclusive: "0x10000000000000000" }] }, + { ranges, limit: 129 }, + { ranges, offset: -1 }, + ]) + assert.equal(schema.safeParse(input).success, false, JSON.stringify(input)); +}); + +const range = { start: "0x100", end_inclusive: "0x10d" }; +const wire = JSON.stringify({ + schema_version: 1, + binary: "/input", + direction: "to", + count: 1, + ranges: [ + { + ...range, + index: 0, + offset: 0, + limit: 20, + total: 1, + count: 1, + next_offset: null, + }, + ], + xrefs: [ + { + range_index: 0, + address: 8, + address_hex: "0x8", + kind: "data", + from_address: 8, + from_address_hex: "0x8", + to_address: 264, + to_address_hex: "0x108", + from_function: null, + to_function: null, + instruction: "LEA pointer", + }, + ], +}); +const source = { + evidence_id: "ev_page", + sha256: "owned-sha", + bytes: Buffer.byteLength(wire), + elapsed_ms: 1, +}; +test("range pages reuse one immutable parsed source across fields and expire on clear", async () => { + let reads = 0; + const invocations = []; + const reader = new NativeReferenceReads( + async (key, args) => { + invocations.push({ key, args: args("/input") }); + return { input: "/input", source }; + }, + async () => { + reads++; + return wire; + }, + () => {}, + ); + const input = referencesMethod.input.parse({ ranges: [range] }); + const first = await reader.read(input); + const selected = await reader.read({ ...input, fields: [] }); + assert.equal(reads, 1); + assert.equal(invocations[0].key, invocations[1].key); + assert.deepEqual(Object.keys(selected.results[0].items[0]), [ + "source_row", + "from", + "to", + "kind", + ]); + assert.throws(() => { + first.results[0].items[0].instruction = "corrupt"; + }, TypeError); + first.results[0].items.length = 0; + assert.equal((await reader.read(input)).results[0].items.length, 1); + assert.equal( + selected.results[0].coverage.whole_string_use, + "not-established", + ); + reader.clear(); + await reader.read(input); + assert.equal(reads, 2); +}); +test("unavailable and malformed ranges cannot masquerade as empty observations", async () => { + const input = referencesMethod.input.parse({ ranges: [range] }); + const unavailable = new NativeReferenceReads( + () => { + throw Error("must not launch"); + }, + () => {}, + () => { + throw Object.assign(Error("requires native"), { + code: "capability_unavailable", + }); + }, + ); + await assert.rejects( + unavailable.read(input), + (e) => e.code === "capability_unavailable", + ); + const malformed = new NativeReferenceReads( + async () => ({ input: "/input", source }), + async () => "{}", + () => {}, + ); + await assert.rejects( + malformed.read(input), + (e) => + e.code === "invalid_provider_output" && + e.details.evidence_id === source.evidence_id && + e.details.source_retained === true, + ); +}); diff --git a/experiments/atomic-codemode/reference-reads.mjs b/experiments/atomic-codemode/reference-reads.mjs index 586e53772b..7e15968492 100644 --- a/experiments/atomic-codemode/reference-reads.mjs +++ b/experiments/atomic-codemode/reference-reads.mjs @@ -1,7 +1,122 @@ // Shared native reference selection; adapters never reinterpret wire endpoints. import { parseKunaReferenceFacts } from "../../dist/kuna/KunaReferenceFiles.js"; import { pageFacts } from "./fact-reads.mjs"; -import { requireResult } from "./fact-error.mjs"; +import { FactFailure, requireResult } from "./fact-error.mjs"; +import { parseKunaReferenceRanges } from "../../dist/kuna/KunaReferenceRanges.js"; +import { + freezeFacts, + projectFact, + readFactBatch, + sourceSummary, +} from "./fact-reads.mjs"; +import { NATIVE_OUTPUT_BYTES } from "./limits.mjs"; + +export class NativeReferenceReads { + #lastRange; + constructor(capture, readSource, requireRanges) { + this.capture = capture; + this.readSource = readSource; + this.requireRanges = requireRanges; + } + clear() { + this.#lastRange = undefined; + } + async read(input, signal) { + if (input.ranges) return this.#ranges(input, signal); + return readFactBatch( + input.addresses.map((address) => referenceReader.query(input, address)), + (query) => + this.capture( + referenceReader.key(query), + (path) => referenceReader.arguments(path, query), + signal, + ), + async (capture, query) => + referenceReader.read( + await this.readSource(capture.source.evidence_id), + capture.input, + query, + input, + ), + ); + } + async #ranges(input, signal) { + this.requireRanges(); + const capture = await this.capture( + `xref-ranges:${JSON.stringify([input.ranges, input.kinds, input.offset, input.limit])}`, + (path) => [ + "xrefs", + path, + ...input.ranges.flatMap((r) => [ + "--to-range", + `${r.start}:${r.end_inclusive}`, + ]), + "--offset", + String(input.offset), + "--limit", + String(input.limit), + "--output-bytes", + String(NATIVE_OUTPUT_BYTES), + ...(input.kinds ? ["--kind", input.kinds.join(",")] : []), + "--json", + ], + signal, + ); + if (this.#lastRange?.source.evidence_id !== capture.source.evidence_id) { + try { + const facts = requireResult( + parseKunaReferenceRanges( + await this.readSource(capture.source.evidence_id), + capture.input, + input, + ), + "invalid_provider_output", + ); + this.#lastRange = freezeFacts({ + facts, + source: sourceSummary(capture.source), + }); + } catch (error) { + throw new FactFailure( + error.code ?? "invalid_provider_output", + error.message, + { + ...error.details, + evidence_id: capture.source.evidence_id, + source_retained: true, + }, + ); + } + } + const { facts, source } = this.#lastRange; + return { + results: facts.results.map((page) => ({ + ok: true, + range: { ...page.range }, + range_index: page.range_index, + items: page.facts.map((row) => + projectFact(row, input.fields, ["source_row", "from", "to", "kind"]), + ), + total: page.total, + next_offset: page.next_offset, + source, + coverage: { + direction: "incoming", + scope: "target-address membership in the inclusive static image span", + examined_rows: page.total, + returned_rows: page.facts.length, + alias_expansion: "not-applied", + discovery: "unknown", + indirect_targets: "unknown", + machine_semantic_equivalence: "unknown", + runtime_binding: "unknown", + memory_access_overlap: "not-established", + whole_string_use: "not-established", + }, + })), + }; + } +} export const referenceReader = { query: (input, address) => ({ diff --git a/experiments/atomic-codemode/sqlite-string-span-question.txt b/experiments/atomic-codemode/sqlite-string-span-question.txt new file mode 100644 index 0000000000..134b67ab99 --- /dev/null +++ b/experiments/atomic-codemode/sqlite-string-span-question.txt @@ -0,0 +1,25 @@ +Analyze {{artifact_path}} statically, starting with the exact encoded strings +"no such table" and "sqlite_schema" (not longer messages that contain them). + +Find each retained encoded string's inclusive byte span, including its observed +terminator when the inventory includes one. Find every retained incoming data +reference whose target coordinate lies in either span, including interior +targets. Report the distinct instruction occurrences, target offsets and owning +function entries; group counts by target offset without merging repeated sites +in the same function. State what the reference discovery actually covers. + +Explain whether each interior reference establishes use of the complete text. +For the selected suffix use in sqlite3AlterRenameTable, verify the actual bytes, +passed register and next direct call target. For the base-message use in +sqlite3LocateTable, identify the local conditional message-pointer selection: +the tested mask/storage, zero/nonzero behavior, alternate string, passing +register and selected formatting/error call. Corroborate this with focused +machine instructions; distinguish a message component from the separate format +pointer. Do not infer runtime argument values, executed paths or symbol binding. +If using recovered IR, distinguish its identities/model from machine facts. + +Give a focused answer with exact addresses, real Evidence IDs, target digest, +coverage and unknowns. Prefer selected facts/windows; keep full reports and +inventories out of model context. Static span membership alone proves neither +memory-access overlap nor whole-string use. Do not execute the target. Close +the target lease. diff --git a/experiments/atomic-codemode/string-span-example.txt b/experiments/atomic-codemode/string-span-example.txt new file mode 100644 index 0000000000..2adc98cd63 --- /dev/null +++ b/experiments/atomic-codemode/string-span-example.txt @@ -0,0 +1,18 @@ +// Input for cli.mjs; the same named operations are used by both MCP adapters. +const literals = ["no such table", "sqlite_schema"]; +const spans = []; +for (const literal of literals) { + const inventory = await rea.strings({query: literal, limit: 128}); + if (!inventory.ok) return inventory; + const exact = inventory.data.items.filter(row => row.value === literal); + if (exact.length !== 1 || inventory.data.next_offset !== null) + return {unknown: "Resolve the inventory ambiguity or remaining pages", literal}; + const row = exact[0]; + if (row.string.byte_length < 1) + return {unknown: "No nonempty encoded byte extent", literal}; + spans.push({start: row.address, + end_inclusive: "0x" + (BigInt(row.address) + BigInt(row.string.byte_length) - 1n).toString(16)}); +} +const references = await rea.references({ranges: spans, kinds: ["data"], + fields: ["fromFunction"], limit: 20}); +return references; diff --git a/experiments/atomic-codemode/string-span-oracle.md b/experiments/atomic-codemode/string-span-oracle.md new file mode 100644 index 0000000000..144f000f4b --- /dev/null +++ b/experiments/atomic-codemode/string-span-oracle.md @@ -0,0 +1,72 @@ +# Frozen SQLite pooled-string task + +Freeze this question and oracle before the three model runs. The target is the +Linux x86-64 `/usr/lib/x86_64-linux-gnu/libsqlite3.so.0.8.6`, SHA-256 +`2e6eef9a727f081f0d453b4e5e6cbd8b9ef8b6f86cbf7681cbad444d3b0b55c8`. +It is inspected statically, never executed. The independent fixture oracle is +`verify-string-span-machine.py`; its fixed expectations are checked against ELF +loaded bytes, GNU disassembly and relocation records, without reading Kuna output. +This is a selected corpus check, not a proof of global reference completeness. + +Both exact inventory strings are ASCII with an observed NUL and 14 encoded bytes: + +| String | Inclusive span | Target offsets and occurrence counts | +| ------------- | ----------------- | ------------------------------------ | +| no such table | 0x11b376–0x11b383 | 0: 1; 8: 6 | +| sqlite_schema | 0x11b33f–0x11b34c | 0: 1; 7: 3 | + +The complete selected data occurrence set is: + +| From | Target | Owner entry | Native name | +| ------- | -------- | ----------- | ------------------------- | +| 0x4215c | 0x11b376 | 0x420d0 | sqlite3LocateTable | +| 0x2d709 | 0x11b37e | 0x2d600 | sqlite3AlterRenameTable | +| 0x4565e | 0x11b37e | 0x45590 | sqlite3StartTable | +| 0x4573d | 0x11b37e | 0x45590 | sqlite3StartTable | +| 0x48b2e | 0x11b37e | 0x47ff0 | sqlite3EndTable | +| 0xbad38 | 0x11b37e | 0xb90e0 | sqlite3Pragma | +| 0xbd1f0 | 0x11b37e | 0xbd1b0 | sqlite3InitOne | +| 0x422c5 | 0x11b33f | 0x42280 | sqlite3PreferredTableName | +| 0x41f0b | 0x11b346 | 0x41db0 | sqlite3FindTable | +| 0x41f65 | 0x11b346 | 0x41db0 | sqlite3FindTable | +| 0x41fb2 | 0x11b346 | 0x41db0 | sqlite3FindTable | + +Order is irrelevant; all eleven distinct sites and owner entries are required. +Nine interior-target occurrences are suffix pointers (`table`, `schema`), not +nine uses of the entire original inventory literal. Scope is retained static +data references, not exhaustive machine dataflow, dynamic aliases or executed use. + +At `0x2d709`, bytes `488d156edc0e00` form `0x11b37e` in RDX. There is no RDX +overwrite before the direct call at `0x2d716` to `0x29060`. The PLT jump slot is +`0x159fc8`, whose ELF JUMP_SLOT symbol is `sqlite3CheckObjectName`; this does not +establish the runtime bound implementation. Decoding must start at a known +instruction boundary, e.g. `0x2d706`, rather than a guessed mid-instruction start. + +At `0x42158`, `AND R13D,1` (`4183e501`) sets flags. The LEAs at `0x4215c` +(`488d1513920d00`) and `0x42163` load the base `no such table` pointer into RDX +and `no such view` (`0x11b369`) into RAX. These LEAs preserve flags. +`CMOVNE RDX,RAX` at `0x4216a` (`480f45d0`) keeps the table component when the +masked result is zero and selects view when nonzero. The separate test/branch +at `0x4216e`/`0x42171` must not be substituted for this selection. On the +selected fallthrough path, RDX is still the message component at the call at +`0x42185` to `0x2a1e0`; RSI separately receives the format pointer `0x11b38c` +at `0x4217c`. The PLT slot `0x15a888` has symbol `sqlite3ErrorMsg`. +Runtime values, feasibility, effects and binding remain unknown. + +Grade complete answers against these facts and verify cited Evidence IDs, +artifact/engine identity and selected source rows. An answer that misses interior +targets, merges the three FindTable sites, claims whole-message use from span +membership or reverses CMOVNE behavior fails. Record failures and retries rather +than selecting only successful runs. Full real-model task costs include tool +schemas, reference help, rejected calls, responses, cached/uncached input, output +and reasoning tokens, wall time, local family CPU and kernel family peak memory. + +Use one frozen runtime, engine and target for traditional (the complete supported +16-tool selector), atomic Direct (22 tools) and Code Mode (4 tools), serially in +fresh isolated profiles. All have the common client JavaScript executor. +The canonical provider has exact-address xrefs, so complete spans can be +enumerated; only the atomic API exposes the new native span occurrence query. +Consequently this is a workflow/profile comparison, not a controlled test of +adapter overhead alone. Native Kuna already aggregates string-span counts and +owner functions, but that aggregation lacks individual reference endpoints and +is not exposed as an equivalent traditional MCP operation. diff --git a/experiments/atomic-codemode/string-span-study.md b/experiments/atomic-codemode/string-span-study.md new file mode 100644 index 0000000000..36b51c5729 --- /dev/null +++ b/experiments/atomic-codemode/string-span-study.md @@ -0,0 +1,148 @@ +# Incoming reference spans and pooled strings + +This experiment adds individually traceable incoming reference occurrences for +inclusive target-address spans. It does not establish whole-string use from a +reference into shared storage. No PR or integration is intended. + +Before this work, the branch refreshed main to `0c7b6d834` in `c8d831936`. +The native overlay remains based on Kuna `fe01d6294eb0d1053ce343c8943646df88e143fc`. +The immutable tested executable SHA-256 is +`2e398e7229e30b6d2387a802db8a0050764119a6b457020d334855e36e69d6c5`; +the full overlay is `kuna-native-ir.patch`. Node is 24.18.0 / npm 11.16.0. + +## Existing capability and missing composition + +Native Kuna already aggregates xrefs and owner functions over a string's encoded +extent. Its string output does not include each reference's source/target pair. +REA's cheap inventory intentionally skips that analysis; the existing reference +operation selects exact addresses, so complete occurrence retrieval previously +required enumerating the encoded span. This is an interface/composition gap, +not evidence that Kuna had no string-span aggregation. + +On real Linux SQLite, exact `no such table` occupies `0x11b376–0x11b383` and +`sqlite_schema` occupies `0x11b33f–0x11b34c`, both including NUL. Base-address +queries return one occurrence each; their full spans retain seven and four. +Nine interior occurrences target the `table` / `schema` suffixes. Calling all +seven references uses of the complete error message would be wrong. The frozen +[task oracle](string-span-oracle.md) separates occurrence completeness, symbol +membership, bytes, register passing and conditional message selection. + +## Implementation and bounds + +The existing named `references` operation accepts explicit `ranges`, preserving +the old exact-address and outgoing-function selectors. CLI, SDK, Direct MCP and +Code Mode use the same contract and owner. There remain 19 named methods, 22 +Direct tools and four Code Mode tools. Both advertised catalogs were validated +after actual SDK conversion, including valid/invalid reference selectors. + +The native BTreeMap index owns sparse inclusive range lookup, including u64 max +without end-address increment. The string aggregate also uses this owner, +removing its per-byte lookup loop and quadratic function deduplication. Legacy +and range wire rows share endpoint/function interpretation and one JSON escaping +writer. A batch builds one index, supports 16 spans and 128 selected rows per +span, and streams into an 8 MiB encoded-output budget. Selected row pointers are +bounded; serialization failure returns an error without partial JSON stdout. +These bounds cap native work/serialization and match the experimental fact page. + +The TypeScript provider boundary owns exact u64 parsing, row consistency and +normalization for both formats. Range metadata binds by explicit query index, +not catalog/array order. Overlapping and duplicate caller ranges remain separate; +`source_row` indexes the retained batch's original flat xrefs array. It is not a +global occurrence ID. Malformed or unavailable data remains an error with source +Evidence where captured, never an empty observation. + +Native capture cache keys cover target/engine identity, ordered spans, kinds, +offset and limit. Fields project the same retained source. One immutable parsed +range batch is retained per lease and cleared after successful close. Different +pages still rebuild a native index; this is not a persistent analysis database. +Coverage explicitly leaves discovery, indirect targets, runtime binding, memory +access overlap and whole-string use unknown/not established. Target-coordinate +membership does not expand import aliases or establish all bytes touched by an +access. + +## Executed verification + +- Native release tests: 25 `listing::xrefs`, two range CLI tests and 15 JSON + writer tests passed, including sparse/full-u64 selection, selector conflicts + and escaped-byte streaming bounds. Release binary rebuilt after final changes. + A real one-byte output-budget query exited with the explicit capacity error and + zero stdout bytes, confirming fail-closed serialization at the CLI boundary. +- Focused production boundary: 23 existing reference tests plus four new range + groups passed (27 total), with full runtime compilation. Focused owner graph + typecheck, changed-file lint and formatting passed. Full `check:fast` was + attempted but stopped at the 3.25 GiB threshold after 45.806 s; sampled peak + 3,330.8 MiB, sampled OOM=0, final payload counters unavailable. Its processes + exited after owned-scope cleanup. This is not a passing full root typecheck. + `check:pr` and an exhaustive native corpus were not run. Experiment pushes + bypass the resource-limited pre-push gate rather than repeat it unbounded. +- All 90 experimental JavaScript tests passed. Five resource receipt/policy + tests passed, including rejecting unsafe bounds before process acquisition. +- Real SDK, Direct MCP and Code Mode range verifiers passed all eleven exact + occurrences/owners, field projection/source reuse, overlaps, repeated spans, + empty u64-max page, pagination, selected bytes/instructions, raw source hashes + and source-row traceability. Each verified cleanup. +- The public experimental CLI ran `string-span-example.txt` and returned totals + seven/four. Native string aggregate JSON remained semantically identical to + the previous engine, including counts and function lists. Real x86 SSH complete + function/prototype fidelity and AArch64 xref/return verifiers passed on the new + engine. Targets were never executed. +- `verify-string-span-machine.py` independently passed ELF loaded bytes, all + eleven selected GNU LEA targets, exported function interval membership, RIP + displacement math, mask/CMOV bytes and the two ELF PLT-slot relocations. + This is not a proof of dynamic binding or global xref completeness. +- The canonical native overlay passed forward application against a temporary + upstream index and reverse application against the owned patched worktree. + +The first public SDK attempt exposed a stale registry refinement that rejected +the new selector despite the standalone contract passing. That redundant guard +was removed; the regression now checks the final registry, and the second SDK +attempt passed. Preserve the failed attempt (`range-sdk.log`) in the record. +An initial focused build crossed the default 3 GiB early stop, without OOM. A +validated optional threshold allows the build to use 3.25 GiB; its successful +focused run took 35.964 s / 35.703191 CPU s, peak 3,084.7 MiB. The hard cap +remains 4 GiB, swap zero, one CPU and 192 tasks. Analysis/model runs retain the +default 3 GiB early stop. Payload-exit counters verify cleanup and OOM=0 for all +successful bounded runs here; kernel process-family memory includes file cache, +not just RSS. Exited transient cgroup final counters are unavailable. + +## Measured equivalent reference task + +The exact-address baseline enumerates both 14-byte spans and retains all raw +sources. The new range query obtains the same eleven occurrences and owners +with one native batch. These are reference-phase measurements after inventory +selection, not complete model costs; cold means fresh lease/capture cache, not +cold kernel file pages. The same new engine is used in both rows. + +| SDK reference workflow | Cold wall | Local family CPU | Warm wall | +| ------------------------ | --------: | ---------------: | --------: | +| 28 exact-address queries | 48.123 s | 47.791799 s | 9.899 ms | +| One two-span batch | 1.821 s | 1.806206 s | 0.984 ms | + +The selected cold phase is about 26.4 times faster. This is shared native work, +not a Code Mode speedup. Old-engine exact enumeration also agreed on every fact +and took 46.806 s. The independent Direct and Code Mode batch preflights took +1.755 / 1.798 s cold and 3.066 / 9.542 ms warm. They have different transport +overheads and are single observations, not benchmark distributions. + +Whole verifier resource receipts (which include additional validation queries): + +| Run | Wall | Local family CPU | Kernel family peak | +| ------------------------- | -------: | ---------------: | -----------------: | +| New-engine exact baseline | 50.557 s | 50.055582 s | 475.6 MiB | +| SDK range verifier | 14.616 s | 14.279055 s | 482.7 MiB | +| Direct range verifier | 15.120 s | 14.742037 s | 312.6 MiB | +| Code Mode range verifier | 15.630 s | 15.257697 s | 331.8 MiB | + +These suites have different validation work; their peak values do not establish +an equivalent-task memory reduction. Scripted checks have no model token usage. +Raw receipts/captures live in the ignored `.cache/atomic-codemode/range-*` paths. +Selected receipt SHA-256 values: + +- `range-exact-after.log`: `cb1d19a4f58face2d442868add0289c3ca62f070ef14f5e4bf0cc101b3bd9ed8` +- `range-sdk-v2.log`: `f203ab548d7509b37a15c39aaf8e466e61f7591eddcb1868be97477811a698c8` +- `range-direct.log`: `c4efbcee95e087fa676b560f70bbb8f6617475be122289c3052f19af02efb2b8` +- `range-codemode.log`: `1d967347c31c99f8caf71391502a7e24ef5fcdcaaaddc5c8d1272eb5b8d7eeae` + +Real-model results will be recorded separately after the frozen three-profile +trial. No token, general complex-software or adapter superiority claim follows +from these preflights. diff --git a/experiments/atomic-codemode/verify-discovery.mjs b/experiments/atomic-codemode/verify-discovery.mjs index 229aaac79b..1ca7efb817 100644 --- a/experiments/atomic-codemode/verify-discovery.mjs +++ b/experiments/atomic-codemode/verify-discovery.mjs @@ -51,6 +51,7 @@ for (const direct of [false, true]) { ); const prefix = direct ? "facts_" : "rea."; const ids = [ + "references", "spaces", "registers", "switches", @@ -118,6 +119,30 @@ for (const direct of [false, true]) { assert.ok(catalog.tools.some((tool) => tool.name === method.name)); } for (const [name, good, bad] of [ + [ + "references", + { addresses: ["0x1"], direction: "outgoing", scope: "function" }, + { addresses: ["0x1"], direction: "incoming", scope: "function" }, + ], + [ + "references", + { addresses: ["0x1"] }, + { + addresses: ["0x1"], + ranges: [{ start: "0x0", end_inclusive: "0x1" }], + }, + ], + [ + "references", + { + ranges: [{ start: "0x0", end_inclusive: "0xffffffffffffffff" }], + fields: [], + }, + { + ranges: [{ start: "0x0", end_inclusive: "0x1" }], + direction: "outgoing", + }, + ], ["spaces", { indices: [3], fields: ["kind"] }, { indices: ["3"] }], [ "registers", diff --git a/experiments/atomic-codemode/verify-reference-ranges.mjs b/experiments/atomic-codemode/verify-reference-ranges.mjs new file mode 100644 index 0000000000..1c634c02d7 --- /dev/null +++ b/experiments/atomic-codemode/verify-reference-ranges.mjs @@ -0,0 +1,286 @@ +// Real SDK/MCP range facts, source identity, paging and bounded native work. +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { AtomicSession } from "./session.mjs"; +import { familyCounters } from "./family-counters.mjs"; +import { + createVerifierRun, + completeVerifierRun, + assertVerifierCleanupComplete, +} from "../../scripts/lib/verifier-run.mjs"; + +const [binary, output, adapter = "sdk"] = process.argv.slice(2); +assert(binary && output && ["sdk", "direct", "codemode"].includes(adapter)); +const verifier = createVerifierRun(), + session = adapter === "sdk" ? new AtomicSession() : null; +const client = session + ? null + : new Client({ name: "rea-range-verifier", version: "0.1.0" }); +const transport = session + ? null + : new StdioClientTransport({ + command: process.execPath, + args: [resolve("experiments/atomic-codemode/server.mjs")], + env: { ...process.env, REA_ATOMIC_ADAPTER: adapter }, + stderr: "pipe", + }); +const observations = [], + sources = new Map(), + phases = [], + failures = []; +const invoke = async (name, input) => { + const reply = await client.callTool( + { name, arguments: input }, + { timeout: 180000 }, + ); + return JSON.parse(reply.content[0].text); +}; +const read = async (method, input) => { + const reply = session + ? await session.call(method, input) + : adapter === "direct" + ? await invoke(`facts_${method}`, input) + : ( + await invoke("execute_javascript", { + code: `return await rea.${method}(${JSON.stringify(input)});`, + }) + ).value; + assert.equal(reply.ok, true, JSON.stringify(reply)); + assert.equal( + reply.context.artifact_sha256, + "2e6eef9a727f081f0d453b4e5e6cbd8b9ef8b6f86cbf7681cbad444d3b0b55c8", + ); + for (const value of reply.data.results ?? [reply.data]) { + if (Object.hasOwn(value, "ok")) + assert.equal(value.ok, true, JSON.stringify(value)); + if (value.source) sources.set(value.source.evidence_id, value.source); + } + observations.push({ method, input, data: reply.data }); + return reply.data; +}; +const measure = async (phase, work) => { + const a = await familyCounters(), + start = performance.now(), + value = await work(), + b = await familyCounters(); + phases.push({ + phase, + elapsed_ms: performance.now() - start, + family_cpu_seconds: (b.cpu - a.cpu) / 1e6, + cumulative_family_memory_peak_bytes: b.peak, + }); + return value; +}; +const key = (row) => [ + row.from, + row.to, + row.kind, + row.fromFunction?.address ?? null, + row.fromFunction?.name ?? null, +]; +const expected = [ + ["0x4215c", "0x11b376", "data", "0x420d0", "sqlite3LocateTable"], + ["0x2d709", "0x11b37e", "data", "0x2d600", "sqlite3AlterRenameTable"], + ["0x4565e", "0x11b37e", "data", "0x45590", "sqlite3StartTable"], + ["0x4573d", "0x11b37e", "data", "0x45590", "sqlite3StartTable"], + ["0x48b2e", "0x11b37e", "data", "0x47ff0", "sqlite3EndTable"], + ["0xbad38", "0x11b37e", "data", "0xb90e0", "sqlite3Pragma"], + ["0xbd1f0", "0x11b37e", "data", "0xbd1b0", "sqlite3InitOne"], +]; +const schemaExpected = [ + ["0x422c5", "0x11b33f", "data", "0x42280", "sqlite3PreferredTableName"], + ...["0x41f0b", "0x41f65", "0x41fb2"].map((from) => [ + from, + "0x11b346", + "data", + "0x41db0", + "sqlite3FindTable", + ]), +]; +let context; +try { + if (session) { + const opened = await session.open(binary); + context = opened.context; + } else { + await client.connect(transport); + const opened = await invoke("open_binary", { path: binary }); + assert.equal(opened.ok, true); + context = opened.context; + } + const ranges = []; + for (const value of ["no such table", "sqlite_schema"]) { + const strings = await read("strings", { query: value, limit: 128 }); + const matches = strings.items.filter((s) => s.value === value); + assert.equal(matches.length, 1); + const string = matches[0]; + assert.equal(string.string.encoding, "ascii"); + assert.equal(string.string.byte_length, 14); + ranges.push({ + start: string.address, + end_inclusive: `0x${(BigInt(string.address) + BigInt(string.string.byte_length) - 1n).toString(16)}`, + }); + } + const input = { + ranges, + kinds: ["data"], + fields: ["fromFunction", "toFunction", "instruction"], + limit: 128, + }; + const cold = await measure("cold-two-span-batch", () => + read("references", input), + ); + assert.deepEqual(cold.results[0].items.map(key), expected); + assert.deepEqual(cold.results[1].items.map(key), schemaExpected); + assert.equal( + new Set(cold.results.map((r) => r.source.evidence_id)).size, + 1, + "one source covers both ranges", + ); + const warm = await measure("warm-same-two-span-batch", () => + read("references", input), + ); + assert.deepEqual(warm, cold); + const narrow = await read("references", { ...input, fields: [] }); + assert.deepEqual(narrow.results[0].source, cold.results[0].source); + assert.deepEqual(Object.keys(narrow.results[0].items[0]), [ + "source_row", + "from", + "to", + "kind", + ]); + for (const page of cold.results) { + assert.equal(page.coverage.discovery, "unknown"); + assert.equal(page.coverage.whole_string_use, "not-established"); + assert.equal(page.next_offset, null); + } + const overlap = await read("references", { + ranges: [ + ranges[0], + { start: "0x11b37e", end_inclusive: ranges[0].end_inclusive }, + ranges[0], + { start: "0xffffffffffffffff", end_inclusive: "0xffffffffffffffff" }, + ], + kinds: ["data"], + limit: 128, + }); + assert.deepEqual( + overlap.results.map((r) => r.total), + [7, 6, 7, 0], + ); + assert.deepEqual( + overlap.results[0].items.map(key), + overlap.results[2].items.map(key), + ); + const recovered = []; + for (let offset = 0; offset !== null;) { + const reply = await read("references", { + ranges: [ranges[0]], + kinds: ["data"], + offset, + limit: 2, + }); + assert.equal(reply.results[0].total, 7); + recovered.push(...reply.results[0].items.map(key)); + offset = reply.results[0].next_offset; + } + assert.deepEqual(recovered, expected); + const bytes = await read("bytes", { + windows: ranges.map((r) => ({ address: r.start, length: 14 })), + }); + assert.deepEqual( + bytes.results.map((r) => Buffer.from(r.hex, "hex").toString("ascii")), + ["no such table\0", "sqlite_schema\0"], + ); + const machine = await read("instructions", { + windows: [ + { address: "0x2d706", count: 6 }, + { address: "0x42158", count: 12 }, + ], + fields: ["bytes", "text"], + }); + for (const [at, bytes] of [ + ["0x2d709", "488d156edc0e00"], + ["0x4215c", "488d1513920d00"], + ["0x4216a", "480f45d0"], + ]) + assert( + machine.results.some((r) => + r.items.some((i) => i.address === at && i.bytes === bytes), + ), + at, + ); + // Check each source row against its actual native batch position and page. + for (const source of sources.values()) { + let text = "", + offset = 0; + do { + const page = await read("evidence", { + evidence_id: source.evidence_id, + offset, + limit: 16384, + }); + text += page.text; + offset = page.next_offset; + } while (offset !== null); + assert.equal(Buffer.byteLength(text), source.bytes); + assert.equal( + createHash("sha256").update(text).digest("hex"), + source.sha256, + ); + const raw = JSON.parse(text); + const pages = observations + .filter((o) => o.method === "references") + .flatMap((o) => o.data.results) + .filter((p) => p.source.evidence_id === source.evidence_id); + for (const page of pages) + for (const fact of page.items) { + const row = raw.xrefs[fact.source_row]; + assert.equal(row.range_index, page.range_index); + assert.equal(row.from_address_hex, fact.from); + assert.equal(row.to_address_hex, fact.to); + } + } +} catch (error) { + failures.push(error.stack ?? String(error)); +} finally { + if (session) await session.close(); + else { + try { + await invoke("close_binary", {}); + } finally { + await client.close(); + } + } +} +const lifecycle = await completeVerifierRun(verifier); +assertVerifierCleanupComplete(lifecycle.process_lineage); +await writeFile( + output, + JSON.stringify({ + adapter, + context, + observations, + phases, + failures, + lifecycle, + source_count: sources.size, + scope: + "Selected static SDK/MCP range facts and original-source fidelity. No target execution, complete discovery or whole-string-use proof.", + }), + { flag: "wx" }, +); +console.log( + JSON.stringify({ + output, + adapter, + phases, + failures, + source_count: sources.size, + }), +); +if (failures.length) process.exitCode = 1; diff --git a/experiments/atomic-codemode/verify-string-span-machine.py b/experiments/atomic-codemode/verify-string-span-machine.py new file mode 100644 index 0000000000..cebd48e2f9 --- /dev/null +++ b/experiments/atomic-codemode/verify-string-span-machine.py @@ -0,0 +1,125 @@ +"""Independent ELF/GNU oracle for the pinned SQLite pooled-string task.""" +import hashlib +import json +from pathlib import Path +import re +import struct +import subprocess +import sys + +if sys.flags.optimize: + raise RuntimeError('Run the oracle without Python optimization') +binary, output = map(Path, sys.argv[1:]) +raw = binary.read_bytes() +digest = hashlib.sha256(raw).hexdigest() +assert digest == '2e6eef9a727f081f0d453b4e5e6cbd8b9ef8b6f86cbf7681cbad444d3b0b55c8' +assert raw[:6] == b'\x7fELF\x02\x01' and struct.unpack_from('= 56 and phoff + phsize * phcount <= len(raw) +segments = [struct.unpack_from('> 3) & 7) | ((data[0] & 4) << 1) + names = ['rax', 'rcx', 'rdx', 'rbx', 'rsp', 'rbp', 'rsi', 'rdi', + 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15'] + return at + 7 + struct.unpack_from(' { } } -const xrefFunction = z.object({ - name: z.string().min(1), - address, - address_hex: address, -}); -const xrefRow = z.object({ - address, - address_hex: address, - kind, - from_address: address, - from_address_hex: address, - to_address: address, - to_address_hex: address, - from_function: xrefFunction.nullable(), - to_function: xrefFunction.nullable(), - instruction: z.string(), -}); const xrefsDocument = z.object({ binary: z.string(), direction: z.enum(["to", "from"]), @@ -242,20 +231,7 @@ const parseReferenceRow = ( aliases: ReadonlySet, query: KunaReferenceQuery, ) => { - const row = xrefRow.parse(raw); - if ( - row.address !== row.address_hex || - row.address !== - (query.direction === "to" ? row.from_address : row.to_address) || - row.from_address !== row.from_address_hex || - row.to_address !== row.to_address_hex - ) - throw new Error( - "xref endpoints disagree with numeric or directional fields", - ); - for (const fn of [row.from_function, row.to_function]) - if (fn !== null && fn.address !== fn.address_hex) - throw new Error("xref function address disagrees with its hex field"); + const row = validateKunaReferenceRow(xrefRow.parse(raw), query.direction); if (query.direction === "to" && !aliases.has(row.to_address)) throw new Error( "xref row targets neither the requested address nor an advertised alias", @@ -280,22 +256,6 @@ export interface KunaReferenceAddresses { } /** Experimental fact projection, owned by the native protocol parser. */ -export type KunaReferenceFact = { - readonly source_row: number; - readonly from: string; - readonly to: string; - readonly kind: z.infer; - readonly fromFunction: { - readonly address: string; - readonly name: string; - } | null; - readonly toFunction: { - readonly address: string; - readonly name: string; - } | null; - readonly instruction: string; -}; - export type KunaReferenceQuery = ( | { readonly direction: "to"; @@ -328,31 +288,6 @@ const referenceSelection = ( return row.from_function.address === query.address ? "selected" : "scope"; }; -const referenceFact = ( - row: z.infer, - source_row: number, -): KunaReferenceFact => ({ - source_row, - from: row.from_address, - to: row.to_address, - kind: row.kind, - fromFunction: - row.from_function === null - ? null - : { - address: row.from_function.address, - name: row.from_function.name, - }, - toFunction: - row.to_function === null - ? null - : { - address: row.to_function.address, - name: row.to_function.name, - }, - instruction: row.instruction, -}); - /** One native-format boundary, with caller-selected endpoint/function scope. */ const parseReferenceDocument = ( text: string, diff --git a/src/kuna/KunaReferenceQueries.ts b/src/kuna/KunaReferenceQueries.ts index 6d87bc2295..2f85df4594 100644 --- a/src/kuna/KunaReferenceQueries.ts +++ b/src/kuna/KunaReferenceQueries.ts @@ -10,13 +10,13 @@ import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { err, ok, type Result } from "../domain/result.js"; import type { KunaInstallation } from "./KunaInstallation.js"; import { - KUNA_XREF_OUTPUT_BYTES, KunaCallGraphReader, parseKunaIncomingReferences, type KunaCallGraph, type KunaGraphNode, type KunaReferenceAddresses, } from "./KunaReferenceFiles.js"; +import { KUNA_XREF_OUTPUT_BYTES } from "./KunaLimits.js"; import { KUNA_PROVIDER_ID } from "./KunaRelease.js"; import type { KunaPendingQuery, KunaQueryWork } from "./KunaQueryWork.js"; import { kunaUnavailable } from "./KunaUnavailable.js"; diff --git a/src/kuna/KunaReferenceRanges.ts b/src/kuna/KunaReferenceRanges.ts new file mode 100644 index 0000000000..6e746954ac --- /dev/null +++ b/src/kuna/KunaReferenceRanges.ts @@ -0,0 +1,150 @@ +// Paged native range batches: query identity and coverage stay with their owner. +import { createHash } from "node:crypto"; +import { z } from "zod"; +import { err, ok } from "../domain/result.js"; +import { parseKunaJson } from "./KunaJson.js"; +import { KUNA_XREF_OUTPUT_BYTES } from "./KunaLimits.js"; +import { + kunaReferenceAddressKeys, + kunaReferenceAddressSchema as address, + kunaReferenceCountSchema as count, + kunaReferenceKindSchema, + kunaReferenceRowSchema, + validateKunaReferenceRow, + kunaReferenceFact, + type KunaReferenceFact, +} from "./KunaReferenceRows.js"; + +const span = z.strictObject({ start: address, end_inclusive: address }); +const metadata = span.extend({ + index: count, + offset: count, + limit: count.min(1).max(128), + total: count, + count, + next_offset: count.nullable(), +}); +const wire = z.object({ + schema_version: z.literal(1), + binary: z.string(), + direction: z.literal("to"), + ranges: z.array(metadata).min(1).max(16), + count, + xrefs: z.array(kunaReferenceRowSchema.extend({ range_index: count })), +}); +export interface KunaReferenceRangeQuery { + readonly ranges: readonly z.infer[]; + readonly offset: number; + readonly limit: number; + readonly kinds?: readonly z.infer[]; +} +export interface KunaReferenceRangePage { + readonly range: z.infer; + readonly range_index: number; + readonly total: number; + readonly next_offset: number | null; + readonly facts: readonly KunaReferenceFact[]; +} + +type NativeRangeDocument = z.infer; +type ParsedPage = { + metadata: z.infer; + facts: KunaReferenceFact[]; + keys: Set; +}; +const readPages = ( + entries: NativeRangeDocument["ranges"], + query: KunaReferenceRangeQuery, +) => { + const pages = new Map(); + for (const meta of entries) { + const expected = query.ranges[meta.index]; + if ( + pages.has(meta.index) || + expected === undefined || + meta.start !== expected.start || + meta.end_inclusive !== expected.end_inclusive || + BigInt(meta.start) > BigInt(meta.end_inclusive) || + meta.offset !== query.offset || + meta.limit !== query.limit + ) + throw new Error("xref range metadata is not bound to the selected query"); + const selected = Math.min( + meta.limit, + Math.max(0, meta.total - meta.offset), + ); + if ( + meta.count !== selected || + meta.next_offset !== + (meta.offset + selected < meta.total ? meta.offset + selected : null) + ) + throw new Error("xref range pagination contradicts examined coverage"); + pages.set(meta.index, { metadata: meta, facts: [], keys: new Set() }); + } + return pages; +}; +const readRows = ( + rows: NativeRangeDocument["xrefs"], + pages: Map, + kinds: KunaReferenceRangeQuery["kinds"], +) => { + for (const [source_row, raw] of rows.entries()) { + const row = validateKunaReferenceRow(raw, "to"), + page = pages.get(row.range_index); + if ( + page === undefined || + BigInt(row.to_address) < BigInt(page.metadata.start) || + BigInt(row.to_address) > BigInt(page.metadata.end_inclusive) || + (kinds !== undefined && !kinds.includes(row.kind)) + ) + throw new Error("xref row violates selected range or kind scope"); + const key = JSON.stringify([row.from_address, row.to_address, row.kind]); + if (page.keys.has(key)) + throw new Error("xref range page repeats an occurrence"); + page.keys.add(key); + page.facts.push(kunaReferenceFact(row, source_row)); + } +}; + +export const parseKunaReferenceRanges = ( + text: string, + input: string, + query: KunaReferenceRangeQuery, +) => { + try { + const document = wire.parse( + parseKunaJson(text, KUNA_XREF_OUTPUT_BYTES, kunaReferenceAddressKeys), + ); + if ( + document.binary !== input || + document.count !== document.xrefs.length || + document.ranges.length !== query.ranges.length + ) + throw new Error( + "xref range document has foreign input or inconsistent counts", + ); + const pages = readPages(document.ranges, query); + readRows(document.xrefs, pages, query.kinds); + const results: KunaReferenceRangePage[] = query.ranges.map( + (range, index) => { + const page = pages.get(index); + if (page === undefined || page.facts.length !== page.metadata.count) + throw new Error("xref range page is missing declared observations"); + return { + range: { ...range }, + range_index: index, + total: page.metadata.total, + next_offset: page.metadata.next_offset, + facts: page.facts, + }; + }, + ); + return ok({ + results, + bytes: Buffer.byteLength(text), + sha256: createHash("sha256").update(text).digest("hex"), + }); + } catch (cause: unknown) { + return err(cause instanceof Error ? cause.message : String(cause)); + } +}; diff --git a/src/kuna/KunaReferenceRows.ts b/src/kuna/KunaReferenceRows.ts new file mode 100644 index 0000000000..1a2ceaba7e --- /dev/null +++ b/src/kuna/KunaReferenceRows.ts @@ -0,0 +1,102 @@ +// Provider wire endpoints and normalized facts have one interpretation. +import { z } from "zod"; +export const kunaReferenceAddressKeys = new Set([ + "address", + "from_address", + "to_address", +]); +export const kunaReferenceAddressSchema = z + .string() + .regex(/^0x(?:0|[1-9a-f][0-9a-f]{0,15})$/u); +export const kunaReferenceCountSchema = z + .number() + .int() + .min(0) + .max(Number.MAX_SAFE_INTEGER); +export const kunaReferenceKindSchema = z.enum([ + "call", + "jump", + "data", + "read", + "write", +]); +const address = kunaReferenceAddressSchema, + kind = kunaReferenceKindSchema; +export const kunaReferenceFunctionSchema = z.object({ + name: z.string().min(1), + address, + address_hex: address, +}); +export const kunaReferenceRowSchema = z.object({ + address, + address_hex: address, + kind, + from_address: address, + from_address_hex: address, + to_address: address, + to_address_hex: address, + from_function: kunaReferenceFunctionSchema.nullable(), + to_function: kunaReferenceFunctionSchema.nullable(), + instruction: z.string(), +}); +export const validateKunaReferenceRow = < + T extends z.infer, +>( + row: T, + direction: "to" | "from", +): T => { + if ( + row.address !== row.address_hex || + row.address !== (direction === "to" ? row.from_address : row.to_address) || + row.from_address !== row.from_address_hex || + row.to_address !== row.to_address_hex + ) + throw new Error( + "xref endpoints disagree with numeric or directional fields", + ); + for (const fn of [row.from_function, row.to_function]) + if (fn !== null && fn.address !== fn.address_hex) + throw new Error("xref function address disagrees with its hex field"); + return row; +}; + +export type KunaReferenceFact = { + readonly source_row: number; + readonly from: string; + readonly to: string; + readonly kind: z.infer; + readonly fromFunction: { + readonly address: string; + readonly name: string; + } | null; + readonly toFunction: { + readonly address: string; + readonly name: string; + } | null; + readonly instruction: string; +}; + +export const kunaReferenceFact = ( + row: z.infer, + source_row: number, +): KunaReferenceFact => ({ + source_row, + from: row.from_address, + to: row.to_address, + kind: row.kind, + fromFunction: + row.from_function === null + ? null + : { + address: row.from_function.address, + name: row.from_function.name, + }, + toFunction: + row.to_function === null + ? null + : { + address: row.to_function.address, + name: row.to_function.name, + }, + instruction: row.instruction, +}); diff --git a/tests/boundary/providers/kuna/kunaReferenceRanges.test.ts b/tests/boundary/providers/kuna/kunaReferenceRanges.test.ts new file mode 100644 index 0000000000..6264b6af6c --- /dev/null +++ b/tests/boundary/providers/kuna/kunaReferenceRanges.test.ts @@ -0,0 +1,181 @@ +import { expect, it } from "vitest"; +import { parseKunaReferenceRanges } from "../../../../src/kuna/KunaReferenceRanges.js"; + +const input = "/snapshot/program"; +const maximum = "0xffffffffffffffff"; +const query = { + ranges: [ + { start: "0x20000000000001", end_inclusive: maximum }, + { start: maximum, end_inclusive: maximum }, + ], + offset: 0, + limit: 2, + kinds: ["data"] as const, +}; +const encoded = (value: unknown) => + JSON.stringify(value).replace(/"@(\d+)"/gu, "$1"); +const row = (range_index: number) => ({ + range_index, + address: "@9007199254740995", + address_hex: "0x20000000000003", + kind: "data", + from_address: "@9007199254740995", + from_address_hex: "0x20000000000003", + to_address: "@18446744073709551615", + to_address_hex: maximum, + from_function: { + name: "caller δΈ­ζ–‡", + address: "@9007199254740992", + address_hex: "0x20000000000000", + }, + to_function: null, + instruction: "LEA RDX,[target]", +}); +const document = () => ({ + schema_version: 1, + binary: input, + direction: "to", + count: 2, + ranges: query.ranges + .map((range, index) => ({ + ...range, + index, + offset: 0, + limit: 2, + total: 1, + count: 1, + next_offset: null as number | null, + })) + .reverse(), + xrefs: [row(1), row(0)], +}); + +it("binds reordered metadata by named range index, preserves u64 and original source rows", () => { + const result = parseKunaReferenceRanges(encoded(document()), input, query); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect( + result.value.results.map((p) => [ + p.range_index, + p.range, + p.facts[0]?.source_row, + p.total, + ]), + ).toEqual([ + [0, query.ranges[0], 1, 1], + [1, query.ranges[1], 0, 1], + ]); + expect(result.value.results[0]?.facts[0]).toMatchObject({ + from: "0x20000000000003", + to: maximum, + kind: "data", + fromFunction: { name: "caller δΈ­ζ–‡" }, + toFunction: null, + }); + expect(result.value.results[0]?.range).not.toBe(query.ranges[0]); +}); + +it("rejects foreign scope, dangling or repeated queries, malformed endpoints and inconsistent paging", () => { + for (const mutate of [ + (v: ReturnType) => { + v.binary = "/other"; + }, + (v: ReturnType) => { + v.count = 3; + }, + (v: ReturnType) => { + v.ranges[0]!.index = 0; + }, + (v: ReturnType) => { + v.ranges[0]!.end_inclusive = "0x1"; + }, + (v: ReturnType) => { + v.ranges[0]!.offset = 1; + }, + (v: ReturnType) => { + v.ranges[0]!.next_offset = 1; + }, + (v: ReturnType) => { + v.ranges[0]!.total = 2; + }, + (v: ReturnType) => { + v.xrefs[0]!.range_index = 7; + }, + (v: ReturnType) => { + v.xrefs[0]!.kind = "call"; + }, + (v: ReturnType) => { + v.xrefs[0]!.to_address = "@1"; + v.xrefs[0]!.to_address_hex = "0x1"; + }, + (v: ReturnType) => { + v.xrefs[0]!.address = "@1"; + }, + (v: ReturnType) => { + v.xrefs[0]!.from_function.address = "@1"; + }, + ]) { + const value = document(); + mutate(value); + expect(parseKunaReferenceRanges(encoded(value), input, query).ok).toBe( + false, + ); + } +}); + +it("preserves overlap between queries but rejects a duplicated occurrence within one page", () => { + const value = document(); + value.count = 3; + value.xrefs.push(row(0)); + const first = value.ranges.find((r) => r.index === 0)!; + first.total = 2; + first.count = 2; + const result = parseKunaReferenceRanges(encoded(value), input, query); + expect(result).toMatchObject({ + ok: false, + error: expect.stringContaining("repeats an occurrence"), + }); +}); + +it("distinguishes a complete page from an empty out-of-range page with observed matches", () => { + const one = { + ranges: [query.ranges[1]!], + offset: 1, + limit: 1, + kinds: query.kinds, + }; + const value = { + schema_version: 1, + binary: input, + direction: "to", + count: 1, + ranges: [ + { + ...one.ranges[0], + index: 0, + offset: 1, + limit: 1, + total: 3, + count: 1, + next_offset: 2, + }, + ], + xrefs: [row(0)], + }; + expect(parseKunaReferenceRanges(encoded(value), input, one)).toMatchObject({ + ok: true, + value: { results: [{ total: 3, next_offset: 2 }] }, + }); + const empty = { + ...value, + count: 0, + xrefs: [], + ranges: [{ ...value.ranges[0], offset: 4, count: 0, next_offset: null }], + }; + expect( + parseKunaReferenceRanges(encoded(empty), input, { ...one, offset: 4 }), + ).toMatchObject({ + ok: true, + value: { results: [{ total: 3, next_offset: null, facts: [] }] }, + }); +});