From 1417bf943bf49e49b414e8a508bf28a31304be83 Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:48:52 +0800 Subject: [PATCH] feat(native): match retained function inventories across builds (#5390) Match retained native inventories through shared CLI/MCP workflows with conservative symbol/byte correspondence, unresolved structural candidates, complete Evidence provenance and bounded paged reads. Closes #5343. --- .../references/native-and-artifacts.md | 11 + docs/mcp-contracts.md | 4 + docs/native-function-matching.md | 120 +++ .../NativeFunctionMatchingService.test.ts | 707 ++++++++++++++++++ .../native/NativeFunctionMatchingService.ts | 250 +++++++ .../NativeFunctionMatchingViews.test.ts | 101 +++ .../native/NativeMatchingAdmission.ts | 60 ++ .../native/NativeMatchingBundleSources.ts | 165 ++++ src/cli.ts | 2 + src/cli/analysisEvidenceDelivery.ts | 11 +- src/cli/nativeFunctionMatchingCommands.ts | 103 +++ src/cliCommandNames.ts | 1 + src/contracts/evidenceInputContracts.ts | 23 +- .../native/functionMatchingContracts.ts | 30 + src/contracts/sessionToolContracts.ts | 8 +- src/contracts/toolContractExamples.ts | 4 + src/contracts/toolEffects.ts | 1 + src/contracts/toolOutputSchemaGroups.ts | 19 +- src/contracts/toolOutputSchemaPrimitives.ts | 17 +- src/domain/analysisView/analysisView.ts | 27 + .../nativeFunctionMatchingView.ts | 311 ++++++++ src/domain/evidenceReference.ts | 17 + src/domain/functionComparisonSchemas.ts | 6 +- src/domain/native/functionByteCoverage.ts | 193 +++++ src/domain/native/functionInventory.ts | 290 +++++++ src/domain/native/functionMatching.ts | 204 +++++ .../native/functionStructuralCandidate.ts | 54 ++ src/domain/native/matchFunctionInventories.ts | 197 +++++ src/domain/native/nativeByteRead.ts | 22 + src/domain/native/procedureInfo.ts | 18 + src/ghidra/GhidraFunctionValues.ts | 7 +- src/ghidra/GhidraInventoryValues.ts | 23 +- src/server/registerFunctionComparisonTool.ts | 12 +- .../registerNativeFunctionMatchingTool.ts | 65 ++ src/server/registerSessionTools.ts | 2 + .../tools/nativeFunctionMatching.test.ts | 276 +++++++ tests/fixtures/nativeFunctionMatching.c | 32 + tests/fixtures/nativeFunctionMatching.ts | 101 +++ 38 files changed, 3422 insertions(+), 72 deletions(-) create mode 100644 docs/native-function-matching.md create mode 100644 src/application/native/NativeFunctionMatchingService.test.ts create mode 100644 src/application/native/NativeFunctionMatchingService.ts create mode 100644 src/application/native/NativeFunctionMatchingViews.test.ts create mode 100644 src/application/native/NativeMatchingAdmission.ts create mode 100644 src/application/native/NativeMatchingBundleSources.ts create mode 100644 src/cli/nativeFunctionMatchingCommands.ts create mode 100644 src/contracts/native/functionMatchingContracts.ts create mode 100644 src/domain/analysisView/nativeFunctionMatchingView.ts create mode 100644 src/domain/evidenceReference.ts create mode 100644 src/domain/native/functionByteCoverage.ts create mode 100644 src/domain/native/functionInventory.ts create mode 100644 src/domain/native/functionMatching.ts create mode 100644 src/domain/native/functionStructuralCandidate.ts create mode 100644 src/domain/native/matchFunctionInventories.ts create mode 100644 src/domain/native/nativeByteRead.ts create mode 100644 src/domain/native/procedureInfo.ts create mode 100644 src/server/registerNativeFunctionMatchingTool.ts create mode 100644 tests/acceptance/tools/nativeFunctionMatching.test.ts create mode 100644 tests/fixtures/nativeFunctionMatching.c create mode 100644 tests/fixtures/nativeFunctionMatching.ts diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index 675de228cc..a9c49e748c 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -1,5 +1,16 @@ # Native, managed, and packaged artifacts +For release comparisons, export `list_procedures` Evidence from each target before +closing it. Close clears session records: import both bundles after collecting the +two targets, or supply portable inline Evidence. `match_native_functions` pairs unique exact names and, when supplied, +complete observed body bytes. Reuse optional same-context `list_names`, +`procedure_info`, `read_bytes` and existing `analyze_function` Evidence; avoid +decompiling every function merely to match it. Its summary retains a parent for +`inspect_analysis_view` `function-matches` collections `matches` and `functions`. +Select columns, batch IDs and page individual facts; aliases remain unresolved. +Unpaired observations do not prove additions/removals. Existing matched dossier +pairs include inputs for `compare_functions`; otherwise analyze just chosen pairs. + ## Desktop installation directories When an installation directory is supplied and its runtime is unknown, use diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 4f0dd8e6b5..c6e4d876d4 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -1,5 +1,9 @@ # MCP runtime contracts +For pairing native functions before comparing selected dossiers, see +[native function matching](native-function-matching.md). It operates on retained +inventories, preserves aliases/unknowns and exposes selected fact pages. + Use this reference to call REA tools and interpret their replies on one MCP connection. Start with [tool results](#tool-results), [retained Evidence](#retained-application-evidence-inputs), and [session discovery](#identity-and-discovery). See [transports](#transports) diff --git a/docs/native-function-matching.md b/docs/native-function-matching.md new file mode 100644 index 0000000000..c44f043e7d --- /dev/null +++ b/docs/native-function-matching.md @@ -0,0 +1,120 @@ +# Match native functions across builds + +`match_native_functions` pairs two existing `list_procedures` Evidence inventories. Select `list_procedures` `detail: "summary"` to retain +complete source rows without transferring the entire inventory. Its +`parent_evidence_id` is the reference matching consumes. +Targets may be opened sequentially; matching never opens a target, rereads its +file, executes it or starts a provider. Preserve each inventory before closing its +target, or import its Evidence bundle later. `close_binary` clears session +records. Export each bundle before closing; after collecting both sides and +closing the last target, import both bundles, then use their references. Keeping +only an Evidence ID across close is insufficient. Complete inline Evidence and +CLI `source_bundle` inputs are portable alternatives. Pass Evidence or exact +session references as `left` and `right`: + +```json +{ + "left": { "kind": "retained-evidence", "evidence_id": "ev_" }, + "right": { "kind": "retained-evidence", "evidence_id": "ev_" } +} +``` + +Unique exact reported names are paired first. Optional `list_names` Evidence +identifies dynamic/default names to exclude. If origin is unavailable, the +existing automatic-name exclusion rule applies and each function retains that +limitation. Exact names establish name correspondence, not unchanged code or +original-source identity. + +For unmatched candidates, optionally supply `left_observations` and +`right_observations`. `procedure_info` or existing `analyze_function` Evidence +gives exact inclusive body ranges; `read_bytes` supplies memory bytes. Observations +must belong to the side's exact target/provider/profile and document context. +Adjacent, consistent overlapping and partially returned reads compose by their +observed extent; conflicting intervals remain gaps. A byte match requires every +body byte, equal total size, equal ordered range lengths, a unique SHA-256 +fingerprint and the same known architecture. Extents are never guessed from the +next entry, and relocations are not normalized. A larger mapped read can serve +several observed bodies; only needed ranges need to be captured. + +This first slice reuses existing range observations. Stripped builds can require +many lightweight range queries when those facts are not retained yet. Matching +does not hide that acquisition cost or silently start those queries. Full +decompilation is unnecessary. + +Repeated candidates stay `unresolved` groups without a Cartesian pair expansion. +`only_left`/`only_right` mean no exact counterpart in the supplied observations; +renaming, inlining, stripping, changed bytes or incomplete discovery can explain +them. They do not prove source addition/removal. `source_coverage` explicitly keeps +inventory completeness unknown and separately counts known byte fingerprints and +resolved correspondence. Provider limitations remain accessible. + +Default `detail: "summary"` retains complete matching Evidence in MCP and returns +counts, both identities and coverage. Query its `parent_evidence_id`: + +```json +{ + "source": { + "kind": "retained-evidence", + "evidence_id": "ev_" + }, + "view": { + "kind": "function-matches", + "collection": "functions", + "states": ["matched"], + "sides": ["left"], + "fields": ["name", "body_sha256"], + "offset": 0, + "limit": 20 + } +} +``` + +`collection: "matches"` returns relation identities, states, basis and member +counts. Large member lists are omitted by default; select `left`/`right` columns +explicitly or page individual `functions`. Batch `match_ids`/`function_ids`, exact function `names`, state +and side filters intersect. Mandatory identities and provenance survive column +selection. Page coverage covers selected retained facts. Function identity binds +inventory Evidence and the owning inventory’s `source_index`, preserving same-address alternatives. +One validation, lazy member index and bounded recent filter are retained per +immutable parent; no live target state is cached. + +When both functions already have unambiguous `analyze_function` dossiers, a matched +relation includes `compare_functions_input` with directly usable retained +references. Otherwise analyze just the selected canonical addresses in their +respective targets, then compare those dossiers. Matching never claims behavioral +equivalence. Optional informative CFG/call/string fingerprints from supplied +dossiers connect unresolved candidate groups only. They require observed CFG, +callees and strings, complete observed call coverage, no unresolved calls, and +the same analysis-profile identity. Empty leaf-function buckets are excluded. +`structural_candidate_sha256` identifies that derived candidate signature; its +dossier link preserves the full supporting facts. Group membership does not +assert that every member pair is a candidate or equivalent. + +CLI uses the same service and JSON input: + +```bash +rea match-native-functions request.json \ + --source_bundle left-bundle.json --source_bundle right-bundle.json \ + --evidence_output matching-bundle.json --json +``` + +`source_bundle` resolves exact references from authenticated local bundles. +Default summary requires `evidence_output`, retaining full matching Evidence and +its selected source closure, including transitive Evidence links across supplied +bundles. Missing dependencies fail with an integrity error; unrelated records +are omitted. Publication never overwrites. Set `detail: "complete"` +in the JSON for all groups inline. Import the bundle into MCP for views, or +extract its matching Evidence record to a JSON file for the existing CLI +`inspect-analysis-view` `evidence-file` source. Input bundles are unchanged. + +Representation admission is bounded at 96 MiB before cloning selected observations +and expanding matching indexes and repeated per-function source links. Work yields to cancellation. No worker/provider +concurrency is introduced; this budget is not a bound on total process RSS. + +The self-owned source fixture is [nativeFunctionMatching.c](https://github.com/morluto/rea/blob/main/tests/fixtures/nativeFunctionMatching.c). +On Linux x86-64, compile with `cc -O2 -g0 -fno-pie -no-pie -DVERSION=1` and +`-DVERSION=2`; copy the second binary and run `strip --strip-all` on that copy. +It includes unchanged/changed code, a byte-identical rename, a removed/added +observation, an inlined function, identical-code aliases and a renamed changed +function with an observed call/string signature. Analyze without executing the +fixture. Full-provider verification still needs caller-supplied Ghidra and Java. diff --git a/src/application/native/NativeFunctionMatchingService.test.ts b/src/application/native/NativeFunctionMatchingService.test.ts new file mode 100644 index 0000000000..a8c3ef6987 --- /dev/null +++ b/src/application/native/NativeFunctionMatchingService.test.ts @@ -0,0 +1,707 @@ +import { expect, it } from "vitest"; +import { createHash } from "node:crypto"; +import { matchNativeFunctions } from "./NativeFunctionMatchingService.js"; +import { nativeFunctionMatchingSchema } from "../../domain/native/functionMatching.js"; +import { createEvidence } from "../../domain/evidence.js"; +import { createAnalysisProfile } from "../../domain/analysisProfile.js"; +import { functionDossierSchema } from "../../domain/hopperValues.js"; +import { + nativeMatchingInventory, + nativeMatchingDetails, + nativeMatchingTarget, + nativeMatchingProvider, +} from "../../../tests/fixtures/nativeFunctionMatching.js"; + +const compare = async (input: unknown) => { + const result = await matchNativeFunctions(input); + if (!result.ok) throw result.error; + return { + ...result.value, + report: nativeFunctionMatchingSchema.parse( + result.value.evidence.normalized_result, + ), + }; +}; + +it("retains transitive provenance without treating dependencies as native observations and fails closed for a missing ancestor", async () => { + const ancestor = createEvidence(undefined, nativeMatchingProvider, { + operation: "fixture_provenance", + parameters: {}, + result: { provenance: "Synthetic ancestor" }, + }); + const dependency = createEvidence(undefined, nativeMatchingProvider, { + operation: "fixture_dependency", + parameters: {}, + result: { provenance: "Synthetic dependency" }, + evidenceLinks: [ancestor.evidence_id], + }); + const left = nativeMatchingInventory( + "1", + [{ address: "0x10", value: "main" }], + [dependency.evidence_id], + ); + const right = nativeMatchingInventory( + "2", + [{ address: "0x20", value: "main" }], + [dependency.evidence_id], + ); + const dependencies = new Map( + [ancestor, dependency].map((record) => [record.evidence_id, record]), + ); + const result = await matchNativeFunctions({ left, right }, undefined, (id) => + dependencies.get(id), + ); + if (!result.ok) throw result.error; + expect( + new Set(result.value.sources.map((record) => record.evidence_id)), + ).toEqual( + new Set( + [left, right, dependency, ancestor].map((record) => record.evidence_id), + ), + ); + expect(new Set(result.value.evidence.evidence_links)).toEqual( + new Set([left.evidence_id, right.evidence_id]), + ); + expect( + nativeFunctionMatchingSchema.parse(result.value.evidence.normalized_result) + .summary.matched, + ).toBe(1); + const missing = await matchNativeFunctions( + { left, right }, + undefined, + (id) => (id === dependency.evidence_id ? dependency : undefined), + ); + expect(missing.ok).toBe(false); + if (missing.ok) throw new Error("Missing ancestor accepted"); + expect(missing.error).toMatchObject({ + evidenceId: ancestor.evidence_id, + reason: "missing", + }); +}); + +it("keeps symbol origin unknown when supplied metadata describes a different same-address alias", async () => { + const left = nativeMatchingInventory("1", [ + { address: "0x10", value: "reported_name" }, + ]); + const right = nativeMatchingInventory("2", [ + { address: "0x20", value: "reported_name" }, + ]); + const alias = createEvidence( + nativeMatchingTarget("1"), + nativeMatchingProvider, + { + operation: "list_names", + parameters: {}, + result: [ + { + address: "0x10", + value: "other_alias", + symbol: { + primary: false, + dynamic: false, + external: false, + type: "function", + source: "user_defined", + }, + }, + ], + }, + ); + const { report } = await compare({ left, right, left_observations: [alias] }); + expect(report.summary.matched).toBe(1); + expect(report.matches[0]?.left[0]?.limitations).toContain( + "Symbol origin unavailable; exact-name matching uses the reported name and the existing automatic-name exclusion rule.", + ); + expect(report.matches[0]?.left[0]?.evidence_links).toContain( + alias.evidence_id, + ); +}); + +it("preserves contradictory symbol origins from inventory/name facts or multiple name observations, independently of byte correspondence", async () => { + const observe = (operation: string, generated: boolean) => + createEvidence(nativeMatchingTarget("1"), nativeMatchingProvider, { + operation, + parameters: {}, + result: [ + { + address: "0x10", + value: "same_name", + symbol: { + primary: true, + dynamic: generated, + external: false, + type: "function", + source: generated ? "default" : "user_defined", + }, + }, + ], + }); + const right = nativeMatchingInventory("2", [ + { address: "0x20", value: "same_name" }, + ]); + for (const [left, names] of [ + [observe("list_procedures", false), [observe("list_names", true)]], + [observe("list_procedures", true), [observe("list_names", false)]], + [ + nativeMatchingInventory("1", [{ address: "0x10", value: "same_name" }]), + [observe("list_names", false), observe("list_names", true)], + ], + [ + nativeMatchingInventory("1", [{ address: "0x10", value: "same_name" }]), + [observe("list_names", true), observe("list_names", false)], + ], + ] as const) { + for (const withBytes of [false, true]) { + const { report } = await compare({ + left, + right, + left_observations: [ + ...names, + ...(withBytes ? nativeMatchingDetails("1", "0x10", "c3") : []), + ], + right_observations: withBytes + ? nativeMatchingDetails("2", "0x20", "c3") + : [], + }); + expect(report.summary.matched).toBe(withBytes ? 1 : 0); + expect(report.matches[0]!.basis).toBe( + withBytes ? "identical-bytes" : null, + ); + const member = report.matches[0]!.left[0]!; + expect(member.limitations).toContain( + "Conflicting generated and named symbol-origin observations; exact-name matching is unavailable. Independent complete body bytes remain eligible.", + ); + expect(member.limitations.join(" ")).not.toContain( + "Symbol origin unavailable", + ); + for (const source of names) + expect(member.evidence_links).toContain(source.evidence_id); + } + } +}); + +it("keeps informative CFG/call/string fingerprints as unresolved candidates and refuses incomplete structural observations", async () => { + const profile = createAnalysisProfile(nativeMatchingProvider, { + fixture: "structural-candidates-v1", + }); + const observe = ( + side: "1" | "2", + operation: string, + result: Parameters[2]["result"], + complete = true, + ) => + createEvidence(nativeMatchingTarget(side), nativeMatchingProvider, { + operation, + parameters: {}, + result, + analysisProfile: profile, + relationshipCoverage: complete + ? { status: "complete" } + : { status: "unknown", reason: "Missing direct call coverage" }, + }); + const left = observe("1", "list_procedures", [ + { address: "0x10", value: "old_behavior" }, + ]); + const right = observe("2", "list_procedures", [ + { address: "0x20", value: "new_behavior" }, + ]); + const dossier = (side: "1" | "2", address: string) => { + const value = functionDossierSchema.parse( + nativeMatchingDetails(side, address, "0102", true)[0]!.normalized_result, + ); + value.basic_blocks = [ + { + start: address, + end: `0x${(BigInt(address) + 2n).toString(16)}`, + successors: [], + }, + ]; + value.callees = [ + { + address: "0x500", + name: "puts", + classification: null, + body: { available: false, reason: "Synthetic external target" }, + }, + ]; + value.unresolved_calls = []; + value.referenced_strings = [ + { address: "0x900", value: "marker", source_address: address }, + ]; + return value; + }; + const l = dossier("1", "0x10"), + r = dossier("2", "0x20"); + const { report } = await compare({ + left, + right, + left_observations: [observe("1", "analyze_function", l)], + right_observations: [observe("2", "analyze_function", r)], + }); + expect(report.summary).toEqual({ + matched: 0, + only_left: 0, + only_right: 0, + unresolved: 1, + }); + expect(report.matches[0]?.left[0]?.structural_candidate_sha256).toMatch( + /^[a-f0-9]{64}$/u, + ); + expect(report.matches[0]?.left[0]?.structural_candidate_sha256).toBe( + report.matches[0]?.right[0]?.structural_candidate_sha256, + ); + expect(report.matches[0]?.compare_functions_input).toBeNull(); + const incomplete = await compare({ + left, + right, + left_observations: [observe("1", "analyze_function", l, false)], + right_observations: [observe("2", "analyze_function", r)], + }); + expect(incomplete.report.summary.unresolved).toBe(0); + expect(incomplete.report.source_coverage.left_structural_candidates).toBe(0); + const legacy = { ...l }; + delete legacy.facet_availability; + const missingFacets = await compare({ + left, + right, + left_observations: [observe("1", "analyze_function", legacy)], + right_observations: [observe("2", "analyze_function", r)], + }); + expect(missingFacets.report.source_coverage.left_structural_candidates).toBe( + 0, + ); + expect(missingFacets.report.summary.unresolved).toBe(0); + l.callees = []; + l.referenced_strings = []; + r.callees = []; + r.referenced_strings = []; + const uninformative = await compare({ + left, + right, + left_observations: [observe("1", "analyze_function", l)], + right_observations: [observe("2", "analyze_function", r)], + }); + expect(uninformative.report.source_coverage.left_structural_candidates).toBe( + 0, + ); + expect(uninformative.report.summary.unresolved).toBe(0); +}); + +it("matches symbols before bytes, retains exact renamed bytes, changed pairs, unpaired observations and duplicate candidates", async () => { + const left = nativeMatchingInventory("1", [ + { address: "0x1000", value: "stable" }, + { address: "0x1010", value: "changed" }, + { address: "0x1020", value: "old_name" }, + { address: "0x1030", value: "removed_observation" }, + { address: "0x1040", value: "duplicate" }, + { address: "0x1050", value: "duplicate" }, + ]); + const right = nativeMatchingInventory("2", [ + { address: "0x2000", value: "stable" }, + { address: "0x2010", value: "changed" }, + { address: "0x2020", value: "renamed" }, + { address: "0x2030", value: "added_observation" }, + { address: "0x2040", value: "duplicate" }, + ]); + const result = await compare({ + left, + right, + left_observations: [ + ...nativeMatchingDetails("1", "0x1010", "0102"), + ...nativeMatchingDetails("1", "0x1020", "030405", true), + ], + right_observations: [ + ...nativeMatchingDetails("2", "0x2010", "aabb"), + ...nativeMatchingDetails("2", "0x2020", "030405", true), + ], + }); + expect(result.report.summary).toEqual({ + matched: 3, + only_left: 1, + only_right: 1, + unresolved: 1, + }); + const renamed = result.report.matches.find( + (row) => row.basis === "identical-bytes", + )!; + expect(renamed.left[0]).toMatchObject({ + name: "old_name", + body_bytes: 3, + body_sha256: createHash("sha256") + .update(Buffer.from("030405", "hex")) + .digest("hex"), + }); + expect(renamed.right[0]?.name).toBe("renamed"); + expect(renamed.compare_functions_input?.left.evidence_id).toBe( + renamed.left[0]?.dossier_evidence_id, + ); + expect( + result.report.matches.find((row) => row.left[0]?.name === "changed"), + ).toMatchObject({ state: "matched", basis: "exact-symbol" }); + expect( + result.report.matches.find((row) => row.state === "unresolved"), + ).toMatchObject({ + left: [expect.anything(), expect.anything()], + right: [expect.anything()], + }); + expect(new Set(result.evidence.evidence_links)).toEqual( + new Set(result.sources.map((x) => x.evidence_id)), + ); + expect(result.report.limitations.join(" ")).toContain("never prove absence"); +}); + +it("does not force repeated identical-byte candidates after an exact name consumes an alias", async () => { + const left = nativeMatchingInventory("1", [ + { address: "0x10", value: "known" }, + { address: "0x20", value: "FUN_20" }, + ]); + const right = nativeMatchingInventory("2", [ + { address: "0x30", value: "known" }, + { address: "0x40", value: "FUN_40" }, + ]); + const { report } = await compare({ + left, + right, + left_observations: [ + ...nativeMatchingDetails("1", "0x10", "c3"), + ...nativeMatchingDetails("1", "0x20", "c3"), + ], + right_observations: [ + ...nativeMatchingDetails("2", "0x30", "c3"), + ...nativeMatchingDetails("2", "0x40", "c3"), + ], + }); + expect(report.summary).toEqual({ + matched: 1, + unresolved: 1, + only_left: 0, + only_right: 0, + }); + expect( + report.matches.find((x) => x.state === "unresolved")?.left[0]?.name, + ).toBe("FUN_20"); +}); + +it("reconstructs discontiguous bodies from adjacent byte reads without next-procedure guesses and keeps address-space/64-bit identity", async () => { + const address = "ram:0xfffffffffffffff0"; + const left = nativeMatchingInventory("1", [{ address, value: "FUN_f0" }]); + const right = nativeMatchingInventory("2", [ + { address: "0x20", value: "FUN_20" }, + ]); + const body = { + available: true, + provenance: "synthetic noncontiguous body", + ranges: [ + { start: address, end: "ram:0xfffffffffffffff1" }, + { start: "ram:0xfffffffffffffff8", end: "ram:0xfffffffffffffff8" }, + ], + total_bytes: 3, + span_bytes: 9, + non_contiguous: true, + contains_entry: true, + }; + const info = createEvidence( + nativeMatchingTarget("1"), + nativeMatchingProvider, + { + operation: "procedure_info", + parameters: {}, + result: { + name: "a", + entrypoint: address, + basicblock_count: 1, + length: 3, + signature: null, + locals: [], + classification: null, + body, + }, + }, + ); + const read = (address: string, hex: string) => + createEvidence(nativeMatchingTarget("1"), nativeMatchingProvider, { + operation: "read_bytes", + parameters: {}, + result: { + address, + requested_bytes: hex.length / 2, + returned_bytes: hex.length / 2, + bytes_hex: hex, + complete: true, + }, + }); + const rightDetails = nativeMatchingDetails("2", "0x20", "010203"); + const { report } = await compare({ + left, + right, + left_observations: [ + info, + read(address, "01"), + read("ram:0xfffffffffffffff1", "02"), + read("ram:0xfffffffffffffff8", "03"), + ], + right_observations: rightDetails, + }); + // Same concatenated bytes with a different body layout do not become a pair. + expect(report.summary.matched).toBe(0); + expect(report.matches[0]?.left[0]).toMatchObject({ + address, + body_bytes: 3, + body_sha256: createHash("sha256") + .update(Buffer.from("010203", "hex")) + .digest("hex"), + }); +}); + +it("preserves byte correspondence when consistent dossiers differ only in request identity, without choosing an arbitrary dossier", async () => { + const left = nativeMatchingInventory("1", [ + { address: "0x10", value: "old_name" }, + ]); + const right = nativeMatchingInventory("2", [ + { address: "0x20", value: "new_name" }, + ]); + const leftDetails = nativeMatchingDetails("1", "0x10", "0102", true); + const another = createEvidence( + nativeMatchingTarget("1"), + nativeMatchingProvider, + { + operation: "analyze_function", + parameters: { procedure: "old_name" }, + result: leftDetails[0]!.normalized_result, + relationshipCoverage: { status: "complete" }, + }, + ); + for (const observations of [ + [...leftDetails, another], + [another, ...leftDetails], + ]) { + const { report } = await compare({ + left, + right, + left_observations: observations, + right_observations: nativeMatchingDetails("2", "0x20", "0102", true), + }); + expect(report.summary.matched).toBe(1); + expect(report.matches[0]).toMatchObject({ + basis: "identical-bytes", + compare_functions_input: null, + left: [ + { + dossier_evidence_id: null, + body_bytes: 2, + body_sha256: createHash("sha256") + .update(Buffer.from("0102", "hex")) + .digest("hex"), + }, + ], + }); + expect(report.matches[0]!.left[0]!.evidence_links).toContain( + another.evidence_id, + ); + } + const conflicting = nativeMatchingDetails("1", "0x10", "010203", true)[0]!; + const { report } = await compare({ + left, + right, + left_observations: [...leftDetails, conflicting], + right_observations: nativeMatchingDetails("2", "0x20", "0102", true), + }); + expect(report.summary.matched).toBe(0); + expect(report.matches[0]!.left[0]!.body_sha256).toBeNull(); +}); + +it("rejects repeated per-occurrence source-link expansion within the shared representation budget", async () => { + const base = nativeMatchingDetails("1", "0x10", "c3")[0]!; + const observations = Array.from({ length: 250 }, (_, observation) => + createEvidence(nativeMatchingTarget("1"), nativeMatchingProvider, { + operation: "procedure_info", + parameters: { procedure: "0x10", observation }, + result: base.normalized_result, + }), + ); + const result = await matchNativeFunctions({ + left: nativeMatchingInventory( + "1", + Array.from({ length: 1000 }, (_, i) => ({ + address: "0x10", + value: `alias_${i}`, + })), + ), + right: nativeMatchingInventory("2", []), + left_observations: observations, + }); + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error._tag).toBe("AnalysisResourceConstraintError"); +}); + +it("retains missing bytes and conflicting overlapping observations as unknown instead of exact matches", async () => { + const left = nativeMatchingInventory("1", [ + { address: "0x10", value: "FUN_10" }, + ]); + const right = nativeMatchingInventory("2", [ + { address: "0x20", value: "FUN_20" }, + ]); + const observations = [ + ...nativeMatchingDetails("1", "0x10", "0102"), + nativeMatchingDetails("1", "0x10", "ffff")[1]!, + ]; + const { report } = await compare({ + left, + right, + left_observations: observations, + right_observations: nativeMatchingDetails("2", "0x20", "0102"), + }); + expect(report.summary.matched).toBe(0); + expect(report.matches[0]?.left[0]?.body_sha256).toBeNull(); +}); + +it("composes consistent overlapping and partial byte reads without losing complete body coverage", async () => { + const left = nativeMatchingInventory("1", [ + { address: "0x10", value: "FUN_10" }, + ]); + const right = nativeMatchingInventory("2", [ + { address: "0x20", value: "FUN_20" }, + ]); + const info = nativeMatchingDetails("1", "0x10", "01020304")[0]!; + const read = (address: string, hex: string, requested = hex.length / 2) => + createEvidence(nativeMatchingTarget("1"), nativeMatchingProvider, { + operation: "read_bytes", + parameters: { address, length: requested }, + result: { + address, + requested_bytes: requested, + returned_bytes: hex.length / 2, + bytes_hex: hex, + complete: requested === hex.length / 2, + }, + }); + const { report } = await compare({ + left, + right, + left_observations: [ + info, + read("0x10", "010203", 6), + read("0x11", "020304"), + ], + right_observations: nativeMatchingDetails("2", "0x20", "01020304"), + }); + expect(report.summary.matched).toBe(1); + expect(report.source_coverage).toMatchObject({ + status: "complete-within-inputs", + left_byte_fingerprints: 1, + right_byte_fingerprints: 1, + left_inventory_completeness: "unknown", + }); +}); + +it("does not use generated symbols and preserves literal named FUN_ symbols when producer metadata proves their origin", async () => { + const entries = [ + { address: "0x10", value: "FUN_10" }, + { address: "0x20", value: "apparently_named" }, + ]; + const names = (side: "1" | "2") => + createEvidence(nativeMatchingTarget(side), nativeMatchingProvider, { + operation: "list_names", + parameters: {}, + result: entries.map((entry, i) => ({ + ...entry, + symbol: { + primary: true, + dynamic: i === 1, + external: false, + type: "function", + source: i === 0 ? "user_defined" : "default", + }, + })), + }); + const { report } = await compare({ + left: nativeMatchingInventory("1", entries), + right: nativeMatchingInventory("2", entries), + left_observations: [names("1")], + right_observations: [names("2")], + }); + expect(report.summary.matched).toBe(1); + expect(report.matches[0]?.left[0]?.name).toBe("FUN_10"); + expect(report.matches[0]?.left[0]?.evidence_links).toContain( + names("1").evidence_id, + ); +}); + +it("rejects tampered, non-inventory and mixed-target observations with typed failures", async () => { + const left = nativeMatchingInventory("1", [ + { address: "0x10", value: "main" }, + ]); + for (const input of [ + { left: { ...left, normalized_result: [] }, right: left }, + { left: nativeMatchingDetails("1", "0x10", "c3")[0], right: left }, + { + left, + right: left, + left_observations: nativeMatchingDetails("2", "0x10", "c3"), + }, + { + left, + right: left, + left_observations: [ + { kind: "retained-evidence", evidence_id: `ev_${"a".repeat(64)}` }, + ], + }, + ]) { + const result = await matchNativeFunctions(input); + expect(result.ok).toBe(false); + if (!result.ok) + expect([ + "AnalysisInputError", + "EvidenceIntegrityError", + "EvidenceReferenceError", + ]).toContain(result.error._tag); + } +}); + +it("cooperatively preserves the exact cancellation reason and rejects expansion before cloning oversized input", async () => { + const controller = new AbortController(), + reason = new Error("stop matching"); + const left = nativeMatchingInventory( + "1", + Array.from({ length: 2000 }, (_, i) => ({ + address: `0x${i.toString(16)}`, + value: `item${i}`, + })), + ); + const pending = matchNativeFunctions( + { left, right: left }, + { signal: controller.signal }, + ); + setImmediate(() => controller.abort(reason)); + const cancelled = await pending; + expect(cancelled.ok).toBe(false); + if (!cancelled.ok) { + expect(cancelled.error._tag).toBe("AnalysisCancelledError"); + expect(cancelled.error.cause).toBe(reason); + } + const large = await matchNativeFunctions({ + left: { padding: "x".repeat(26 * 1024 * 1024) }, + right: left, + }); + expect(large.ok).toBe(false); + if (!large.ok) + expect(large.error._tag).toBe("AnalysisResourceConstraintError"); +}); + +it("keeps duplicate candidate storage linear and emits every input occurrence exactly once", async () => { + const entries = Array.from({ length: 1000 }, (_, i) => ({ + address: `0x${i.toString(16)}`, + value: "same", + })); + const { report } = await compare({ + left: nativeMatchingInventory("1", entries), + right: nativeMatchingInventory("2", entries), + }); + expect(report.matches).toHaveLength(1); + expect(report.matches[0]?.left).toHaveLength(1000); + expect(report.matches[0]?.right).toHaveLength(1000); + expect(new Set(report.matches[0]?.left.map((x) => x.function_id)).size).toBe( + 1000, + ); +}); diff --git a/src/application/native/NativeFunctionMatchingService.ts b/src/application/native/NativeFunctionMatchingService.ts new file mode 100644 index 0000000000..deff5d57d5 --- /dev/null +++ b/src/application/native/NativeFunctionMatchingService.ts @@ -0,0 +1,250 @@ +import { NativeMatchingAdmission } from "./NativeMatchingAdmission.js"; +import { loadNativeMatchingBundleSources } from "./NativeMatchingBundleSources.js"; +import { RegularFileCleanupFailure } from "../RegularFileRead.js"; +import { matchFunctionInventories } from "../../domain/native/matchFunctionInventories.js"; +import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; +import { setImmediate } from "node:timers/promises"; +import { matchNativeFunctionsInputSchema } from "../../contracts/native/functionMatchingContracts.js"; +import { retainedEvidenceReferenceSchema } from "../../domain/evidenceReference.js"; +import { + createEvidence, + parseEvidence, + type Evidence, +} from "../../domain/evidence.js"; +import { readFunctionInventory } from "../../domain/native/functionInventory.js"; +import { nativeFunctionMatchingSchema } from "../../domain/native/functionMatching.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisInputError, +} from "../../domain/analysisErrorCore.js"; +import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { + resolveEvidenceInput, + type EvidenceLookup, +} from "../EvidenceInputResolver.js"; +import type { ExecutionOptions } from "../AnalysisProvider.js"; +import { FUNCTION_COMPARISON_PROVIDER } from "../InvestigationProviders.js"; + +const OPERATION = "match_native_functions"; +export interface NativeFunctionMatchingResult { + readonly evidence: Evidence; + readonly sources: readonly Evidence[]; +} + +/** Pair existing observations through one source/analysis admission. */ +export const matchNativeFunctions = ( + rawInput: unknown, + options?: ExecutionOptions, + lookup?: EvidenceLookup, +): Promise> => + matchWithAdmission( + rawInput, + options, + lookup, + new NativeMatchingAdmission(OPERATION, options?.signal), + ); + +const matchWithAdmission = async ( + rawInput: unknown, + options: ExecutionOptions | undefined, + lookup: EvidenceLookup | undefined, + admission: NativeMatchingAdmission, +): Promise> => { + try { + const checkpoint = admission.checkpoint; + await setImmediate(); + await checkpoint(); + const parsed = matchNativeFunctionsInputSchema.safeParse(rawInput); + if (!parsed.success) + return err( + analysisInputErrorFromIssues(OPERATION, parsed.error.issues, rawInput), + ); + const sources = new Map(); + const resolve = async (input: unknown): Promise => { + const reference = retainedEvidenceReferenceSchema.safeParse(input); + let raw = input; + if (reference.success) { + const found = resolveEvidenceInput(reference.data, lookup); + if (!found.ok) throw found.error; + raw = found.value; + } + await admission.admit(raw); + const evidence = parseEvidence(raw); + const cached = sources.get(evidence.evidence_id); + if (cached !== undefined) return cached; + sources.set(evidence.evidence_id, evidence); + return evidence; + }; + const left = await resolve(parsed.data.left), + right = await resolve(parsed.data.right); + const observations = async (inputs: readonly unknown[]) => { + const result: Evidence[] = []; + const seen = new Set(); + for (const input of inputs) { + const record = await resolve(input); + if (!seen.has(record.evidence_id)) { + seen.add(record.evidence_id); + result.push(record); + } + } + return result; + }; + const leftObservations = await observations(parsed.data.left_observations); + const rightObservations = await observations( + parsed.data.right_observations, + ); + const directLinks = [...sources.keys()].sort(); + // Expand provenance separately from role observations: dependencies may be + // any Evidence operation and must not be interpreted as native body facts. + for (const source of sources.values()) + for (const id of source.evidence_links) + if (!sources.has(id)) + await resolve({ kind: "retained-evidence", evidence_id: id }); + const leftFunctions = await readFunctionInventory( + left, + leftObservations, + checkpoint, + ); + const rightFunctions = await readFunctionInventory( + right, + rightObservations, + checkpoint, + ); + const compatible = + left.subject?.architecture != null && + left.subject.architecture === right.subject?.architecture; + const { matches, summary } = await matchFunctionInventories( + leftFunctions, + rightFunctions, + { identicalBytesComparable: compatible, checkpoint }, + ); + const limitations = [ + ...new Set([ + ...left.limitations.map((x) => `Left: ${x}`), + ...right.limitations.map((x) => `Right: ${x}`), + "Coverage is restricted to the supplied provider-observed inventories; missing/truncated inventory facts never prove absence from either binary.", + "Only unique exact symbols and bytes are paired. Compatible observed CFG/call/string fingerprints propose unresolved candidates, never identity or equivalence. Missing structural coverage stays unknown.", + "Name or byte correspondence is not behavioral equivalence; use analyze_function for selected pairs, then compare_functions. Existing dossier pairs include directly usable retained-reference inputs.", + ...(!compatible + ? [ + "Identical-byte matching unavailable: input architectures are missing or differ.", + ] + : []), + ]), + ]; + const report = nativeFunctionMatchingSchema.parse({ + left: { + evidence_id: left.evidence_id, + sha256: left.subject!.digest.sha256, + functions: leftFunctions.length, + }, + right: { + evidence_id: right.evidence_id, + sha256: right.subject!.digest.sha256, + functions: rightFunctions.length, + }, + scope: "provider-observed-inputs", + source_coverage: { + status: + summary.only_left + summary.only_right + summary.unresolved > 0 + ? "partial" + : "complete-within-inputs", + left_inventory_completeness: "unknown", + right_inventory_completeness: "unknown", + left_byte_fingerprints: leftFunctions.filter( + (x) => x.fingerprint !== null, + ).length, + left_structural_candidates: leftFunctions.filter( + (x) => x.member.structural_candidate_sha256 !== null, + ).length, + right_structural_candidates: rightFunctions.filter( + (x) => x.member.structural_candidate_sha256 !== null, + ).length, + right_byte_fingerprints: rightFunctions.filter( + (x) => x.fingerprint !== null, + ).length, + }, + summary, + matches, + limitations, + }); + await checkpoint(); + const evidence = createEvidence(undefined, FUNCTION_COMPARISON_PROVIDER, { + operation: OPERATION, + predicateType: "rea.native-function-matching", + confidence: "derived", + authority: "analyst-inference", + parameters: { + left_evidence_id: left.evidence_id, + right_evidence_id: right.evidence_id, + left_observation_ids: leftObservations.map((x) => x.evidence_id), + right_observation_ids: rightObservations.map((x) => x.evidence_id), + algorithm: "exact-symbol-body-bytes-and-structural-candidates-v1", + }, + result: report, + limitations, + evidenceLinks: directLinks, + }); + await checkpoint(); + return ok({ evidence, sources: [...sources.values()] }); + } catch (cause: unknown) { + if (options?.signal?.aborted) + return err( + new AnalysisCancelledError(OPERATION, { cause: options.signal.reason }), + ); + if (cause instanceof AnalysisError) return err(cause); + return err( + new AnalysisInputError(OPERATION, { cause }, [ + { + path: [], + reason: "invalid_value", + message: + cause instanceof Error + ? cause.message + : "Invalid native matching observations", + }, + ]), + ); + } +}; + +/** Load selected portable records; do not retain unrelated records across bundles. */ +export const matchNativeFunctionsFromBundles = async ( + input: unknown, + paths: readonly string[], + resources: ArtifactResourceScope, + options?: ExecutionOptions, +): Promise> => { + try { + const parsed = matchNativeFunctionsInputSchema.safeParse(input); + if (!parsed.success) + return err( + analysisInputErrorFromIssues(OPERATION, parsed.error.issues, input), + ); + const admission = new NativeMatchingAdmission(OPERATION, options?.signal); + const loaded = await loadNativeMatchingBundleSources({ + input: parsed.data, + paths, + resources, + admission, + }); + return await matchWithAdmission( + loaded.input, + options, + loaded.lookup, + admission, + ); + } catch (cause: unknown) { + // This marker carries the exact descriptor owner; the caller must finish + // that owner before classifying cancellation or unexpected I/O failures. + if (cause instanceof RegularFileCleanupFailure) throw cause; + if (options?.signal?.aborted) + return err( + new AnalysisCancelledError(OPERATION, { cause: options.signal.reason }), + ); + if (cause instanceof AnalysisError) return err(cause); + return err(new AnalysisInputError(OPERATION, { cause })); + } +}; diff --git a/src/application/native/NativeFunctionMatchingViews.test.ts b/src/application/native/NativeFunctionMatchingViews.test.ts new file mode 100644 index 0000000000..72da044d7a --- /dev/null +++ b/src/application/native/NativeFunctionMatchingViews.test.ts @@ -0,0 +1,101 @@ +import { expect, it } from "vitest"; +import { matchNativeFunctions } from "./NativeFunctionMatchingService.js"; +import { inspectAnalysisView } from "../analysisView/AnalysisViewService.js"; +import { nativeMatchingInventory } from "../../../tests/fixtures/nativeFunctionMatching.js"; +import { analysisViewResultSchema } from "../../domain/analysisView/analysisView.js"; + +it("pages individual function facts without transferring an entire ambiguous alias group, preserving mandatory identity and source coverage", async () => { + const entries = Array.from({ length: 500 }, (_, i) => ({ + address: `0x${i.toString(16)}`, + value: "same", + })); + const matched = await matchNativeFunctions({ + left: nativeMatchingInventory("1", entries), + right: nativeMatchingInventory("2", entries), + }); + if (!matched.ok) throw matched.error; + const source = { kind: "inline", evidence: matched.value.evidence }; + const view = (query: unknown) => { + const result = inspectAnalysisView({ source, view: query }); + if (!result.ok) throw result.error; + return analysisViewResultSchema.parse(result.value.normalized_result); + }; + const groups = view({ + kind: "function-matches", + collection: "matches", + fields: ["basis"], + limit: 1, + }); + expect(groups).toMatchObject({ + kind: "function-matches", + coverage: { examined: 1, total: 1, exhausted: true }, + items: [ + { state: "unresolved", basis: null, left_count: 500, right_count: 500 }, + ], + }); + if (groups.kind !== "function-matches") throw new Error("Wrong view"); + expect(groups.items[0]).not.toHaveProperty("left"); + const member = `${matched.value.sources[0]!.evidence_id}/123`; + const functions = view({ + kind: "function-matches", + collection: "functions", + function_ids: [member], + fields: ["name"], + limit: 1, + }); + expect(functions).toMatchObject({ + kind: "function-matches", + coverage: { examined: 1, total: 1, exhausted: true }, + items: [ + { + function_id: member, + source_index: 123, + address: "0x7b", + name: "same", + side: "left", + state: "unresolved", + evidence_links: [matched.value.sources[0]!.evidence_id], + }, + ], + }); + expect(JSON.stringify(functions).length).toBeLessThan(6000); + expect( + view({ + kind: "function-matches", + collection: "functions", + names: [], + limit: 1, + }), + ).toMatchObject({ coverage: { total: 0 } }); + expect( + view({ + kind: "function-matches", + collection: "functions", + names: ["same"], + sides: ["left"], + limit: 1, + }), + ).toMatchObject({ coverage: { total: 500, next_offset: 1 } }); + expect( + view({ + kind: "function-matches", + collection: "functions", + sides: ["right"], + offset: 499, + limit: 2, + }), + ).toMatchObject({ + coverage: { examined: 1, total: 500, exhausted: true, next_offset: null }, + }); + expect( + view({ kind: "function-matches", collection: "functions", states: [] }), + ).toMatchObject({ coverage: { total: 0, exhausted: true } }); + expect(view({ kind: "summary" })).toMatchObject({ + summary: { + scope: "provider-observed-inputs", + summary: { unresolved: 1 }, + left: { functions: 500 }, + right: { functions: 500 }, + }, + }); +}); diff --git a/src/application/native/NativeMatchingAdmission.ts b/src/application/native/NativeMatchingAdmission.ts new file mode 100644 index 0000000000..1acb781bb1 --- /dev/null +++ b/src/application/native/NativeMatchingAdmission.ts @@ -0,0 +1,60 @@ +import { setImmediate } from "node:timers/promises"; +import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js"; + +const NATIVE_MATCHING_REPRESENTATION_BYTES = 96 * 1024 * 1024; + +/** Budget the actual selected observations, before cloning or index allocation. */ +export class NativeMatchingAdmission { + #remaining = NATIVE_MATCHING_REPRESENTATION_BYTES; + #work = 0; + readonly #seen = new WeakSet(); + constructor( + readonly operation: string, + readonly signal?: AbortSignal, + ) {} + readonly checkpoint = async (bytes = 0): Promise => { + this.#remaining -= bytes; + if (this.#remaining < 0) + throw new AnalysisResourceConstraintError( + this.operation, + "capacity", + "Native function matching exceeds representation capacity", + { maximum_bytes: NATIVE_MATCHING_REPRESENTATION_BYTES }, + ); + this.signal?.throwIfAborted(); + if (++this.#work % 256 === 0) + await setImmediate( + undefined, + this.signal === undefined ? {} : { signal: this.signal }, + ); + }; + async admit(root: unknown): Promise { + const pending: Iterator[] = [[root][Symbol.iterator]()]; + while (pending.length) { + const next = pending[pending.length - 1]!.next(); + if (next.done) { + pending.pop(); + continue; + } + const value = next.value; + if (typeof value === "object" && value !== null) { + if (this.#seen.has(value)) continue; + this.#seen.add(value); + await this.checkpoint(128); + // Iterator traversal avoids a second width-sized array of child values. + pending.push( + (function* () { + for (const key in value) + if (Object.hasOwn(value, key)) { + yield key; + yield (value as Record)[key]; + } + })(), + ); + } else + await this.checkpoint( + typeof value === "string" ? 64 + value.length * 4 : 128, + ); + } + } +} diff --git a/src/application/native/NativeMatchingBundleSources.ts b/src/application/native/NativeMatchingBundleSources.ts new file mode 100644 index 0000000000..378a823cff --- /dev/null +++ b/src/application/native/NativeMatchingBundleSources.ts @@ -0,0 +1,165 @@ +import { createHash } from "node:crypto"; +import type { z } from "zod"; +import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; +import type { EvidenceBundle } from "../../domain/evidenceBundle.js"; +import { immutableEvidence, type Evidence } from "../../domain/evidence.js"; +import { + EvidenceIntegrityError, + EvidenceReferenceError, +} from "../../domain/evidenceErrors.js"; +import { digestCanonicalValue } from "../../domain/canonicalDigest.js"; +import { retainedEvidenceReferenceSchema } from "../../domain/evidenceReference.js"; +import type { matchNativeFunctionsInputSchema } from "../../contracts/native/functionMatchingContracts.js"; +import { readImmutableEvidenceBundle } from "../EvidenceBundleFiles.js"; +import type { EvidenceLookup } from "../EvidenceInputResolver.js"; +import type { NativeMatchingAdmission } from "./NativeMatchingAdmission.js"; + +type Request = z.output; +interface SourceEntry { + readonly file: number; + readonly links: readonly string[]; +} +interface Options { + readonly input: Request; + readonly paths: readonly string[]; + readonly resources: ArtifactResourceScope; + readonly admission: NativeMatchingAdmission; +} +const reference = (evidence_id: string) => ({ + kind: "retained-evidence" as const, + evidence_id, +}); +// IDs and links have authenticated ASCII digest syntax. Own these short strings +// independently of the discarded bundle's potentially large JSON backing store. +const ownId = (value: string): string => + Buffer.from(value, "ascii").toString("ascii"); + +/** IDs bind semantic payloads; subjects add the display metadata IDs intentionally omit. */ +const recordSnapshot = async ( + bundle: EvidenceBundle, + admission: NativeMatchingAdmission, +): Promise => { + const hash = createHash("sha256"); + for (const record of bundle.records) { + await admission.checkpoint(); + hash + .update(record.evidence_id) + .update(digestCanonicalValue(record.subject)); + } + return hash.digest("hex"); +}; + +/** Index only graph metadata; keep at most the last whole bundle between passes. */ +export const loadNativeMatchingBundleSources = async ({ + input, + paths, + resources, + admission, +}: Options): Promise<{ input: Request; lookup: EvidenceLookup }> => { + const records = new Map(), + index = new Map(); + const rootIds: string[] = []; + for (const value of [ + input.left, + input.right, + ...input.left_observations, + ...input.right_observations, + ]) { + const parsed = retainedEvidenceReferenceSchema.safeParse(value); + if (parsed.success) rootIds.push(parsed.data.evidence_id); + else { + await admission.admit(value); + const record = immutableEvidence(value); + rootIds.push(record.evidence_id); + records.set(record.evidence_id, record); + index.set(record.evidence_id, { file: -1, links: record.evidence_links }); + } + } + const snapshots: string[] = []; + let last: { file: number; bundle: EvidenceBundle } | undefined; + for (let file = 0; file < paths.length; file++) { + last = undefined; + const loaded = await readImmutableEvidenceBundle( + paths[file]!, + admission.signal, + resources, + ); + if (!loaded.ok) throw loaded.error; + snapshots.push(await recordSnapshot(loaded.value, admission)); + for (const record of loaded.value.records) { + await admission.checkpoint( + 256 + + 4 * + (record.evidence_id.length + + record.evidence_links.reduce((sum, id) => sum + id.length, 0)), + ); + if (!index.has(record.evidence_id)) + index.set(ownId(record.evidence_id), { + file, + links: record.evidence_links.map(ownId), + }); + } + last = { file, bundle: loaded.value }; + } + const needed = new Set(rootIds), + files = new Set(); + // Set iteration includes newly discovered nodes. Shared ancestors and cycles + // are visited once, independent of source-file and record ordering. + for (const id of needed) { + await admission.checkpoint(128); + const entry = index.get(id); + if (entry === undefined) + throw new EvidenceReferenceError( + id, + "missing", + "Complete Evidence closure in the supplied bundles or inline inputs", + null, + ); + if (entry.file >= 0) files.add(entry.file); + for (const linked of entry.links) needed.add(linked); + } + const retain = async ( + bundle: EvidenceBundle, + file: number, + ): Promise => { + for (const record of bundle.records) + if ( + needed.has(record.evidence_id) && + index.get(record.evidence_id)?.file === file + ) { + await admission.admit(record); + records.set(record.evidence_id, record); + } + }; + if (last !== undefined && files.has(last.file)) { + await retain(last.bundle, last.file); + files.delete(last.file); + } + last = undefined; + for (const file of files) { + const loaded = await readImmutableEvidenceBundle( + paths[file]!, + admission.signal, + resources, + ); + if (!loaded.ok) throw loaded.error; + if ((await recordSnapshot(loaded.value, admission)) !== snapshots[file]) + throw new EvidenceIntegrityError( + `Evidence bundle records changed during closure lookup: ${paths[file]}`, + ); + await retain(loaded.value, file); + } + const left = rootIds[0]!, + right = rootIds[1]!; + const split = 2 + input.left_observations.length; + return { + input: { + ...input, + left: reference(left), + right: reference(right), + left_observations: rootIds.slice(2, split).map(reference), + right_observations: rootIds.slice(split).map(reference), + }, + lookup: (id) => records.get(id), + }; +}; diff --git a/src/cli.ts b/src/cli.ts index 8c31905328..d2a1a90d32 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,3 +1,4 @@ +import { registerNativeFunctionMatchingCommands } from "./cli/nativeFunctionMatchingCommands.js"; import { registerCaptureCommands } from "./cli/captureCommands.js"; import { registerSignalCommands } from "./cli/signalCommands.js"; import { registerWasmCommands } from "./cli/wasmCommands.js"; @@ -134,6 +135,7 @@ export const createCli = ( registerFirmwareCommands(cli, logger, environment); registerBinaryDiagnosticsCommands(cli, logger, environment); registerAnalysisViewCommands(cli, logger); + registerNativeFunctionMatchingCommands(cli, logger); registerFridaCommands(cli, logger, fridaInstrumentation); registerEvidenceCommands(cli, logger); registerEnvironmentCommands(cli, logger, environment); diff --git a/src/cli/analysisEvidenceDelivery.ts b/src/cli/analysisEvidenceDelivery.ts index 2c42544a4b..76f8b6be19 100644 --- a/src/cli/analysisEvidenceDelivery.ts +++ b/src/cli/analysisEvidenceDelivery.ts @@ -39,6 +39,8 @@ interface CliEvidenceDelivery { readonly signal: AbortSignal; readonly detail: "complete" | "summary"; readonly evidenceOutput: string | undefined; + /** Derived workflows publish their selected source closure with the result. */ + readonly sourceEvidence?: readonly Evidence[]; } export const cliEvidenceDeliveryInputError = ( @@ -115,7 +117,14 @@ export const publishCliAnalysisEvidence = async ( const delivered = await finishAnalysisCommand( async () => { const written = await writeEvidenceBundle( - createEvidenceBundle([complete]), + createEvidenceBundle([ + ...new Map( + [...(input.sourceEvidence ?? []), complete].map((record) => [ + record.evidence_id, + record, + ]), + ).values(), + ]), absoluteCallerPath(evidenceOutput), false, resources, diff --git a/src/cli/nativeFunctionMatchingCommands.ts b/src/cli/nativeFunctionMatchingCommands.ts new file mode 100644 index 0000000000..cc43069bd6 --- /dev/null +++ b/src/cli/nativeFunctionMatchingCommands.ts @@ -0,0 +1,103 @@ +import { z } from "incur"; +import type { Logger } from "pino"; +import { matchNativeFunctionsFromBundles } from "../application/native/NativeFunctionMatchingService.js"; +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { matchNativeFunctionsInputSchema } from "../contracts/native/functionMatchingContracts.js"; +import { CLI_COMMANDS } from "../cliCommandNames.js"; +import { parseCliJsonInput } from "../cliJsonInput.js"; +import { logCliCommand } from "../cliLogging.js"; +import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import type { Evidence } from "../domain/evidence.js"; +import { withCommandCancellation } from "./commandCancellation.js"; +import { + cliEvidenceDeliveryInputError, + publishCliAnalysisEvidence, +} from "./analysisEvidenceDelivery.js"; +import { finishAnalysisCommand } from "./finishAnalysisCommand.js"; +import type { CliInstance } from "./types.js"; + +/** Resolve portable input bundles, then use the same read-only matching service as MCP. */ +export const registerNativeFunctionMatchingCommands = ( + cli: CliInstance, + logger: Logger, +): void => { + cli.command(CLI_COMMANDS.matchNativeFunctions, { + description: + "Match two observed native function inventories without executing targets", + args: z.object({ + input: z + .string() + .describe( + "JSON or JSON file: left/right inventories, optional observations and detail", + ), + }), + options: z.object({ + source_bundle: z + .array(z.string()) + .default([]) + .describe( + "Canonical source Evidence bundle for retained references; repeat for each input", + ), + evidence_output: z + .string() + .min(1) + .optional() + .describe( + "Save full matching Evidence and selected sources; required for default summary; never overwrites", + ), + }), + run: ({ args, options }) => + withCommandCancellation((signal) => + logCliCommand(logger, CLI_COMMANDS.matchNativeFunctions, async () => { + const input = await parseCliJsonInput( + args.input, + "match_native_functions", + signal, + ); + if (!input.ok) return input.error; + const parsed = matchNativeFunctionsInputSchema.safeParse(input.value); + if (!parsed.success) + return projectAnalysisError( + analysisInputErrorFromIssues( + "match_native_functions", + parsed.error.issues, + input.value, + ), + ); + const delivery = { + providerId: "rea-function-comparison", + operation: "match_native_functions", + signal, + detail: parsed.data.detail, + evidenceOutput: options.evidence_output, + }; + const invalid = cliEvidenceDeliveryInputError(delivery); + if (invalid !== undefined) return projectAnalysisError(invalid); + const resources = new ArtifactResourceScope(); + let sources: readonly Evidence[] = []; + const complete = await finishAnalysisCommand( + async () => { + const result = await matchNativeFunctionsFromBundles( + parsed.data, + options.source_bundle, + resources, + { signal }, + ); + if (!result.ok) return result; + sources = result.value.sources; + return { ok: true as const, value: result.value.evidence }; + }, + resources, + delivery.providerId, + delivery.operation, + signal, + ); + return publishCliAnalysisEvidence(complete, { + ...delivery, + sourceEvidence: sources, + }); + }), + ), + }); +}; diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index 3df7981e3c..a400264d3d 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -4,6 +4,7 @@ export const CLI_COMMANDS = Object.freeze({ inspectPacketCapture: "inspect-packet-capture", inspectSignalDecoders: "inspect-signal-decoders", decodeSignalRecording: "decode-signal-recording", + matchNativeFunctions: "match-native-functions", compareWasmInterfaces: "compare-wasm-interfaces", inspectWasmArtifact: "inspect-wasm-artifact", inspectSqliteDatabase: "inspect-sqlite-database", diff --git a/src/contracts/evidenceInputContracts.ts b/src/contracts/evidenceInputContracts.ts index 02b2bd0e64..0419ba5d58 100644 --- a/src/contracts/evidenceInputContracts.ts +++ b/src/contracts/evidenceInputContracts.ts @@ -1,18 +1,5 @@ -import { z } from "zod"; -import { evidenceSchema } from "../domain/evidence.js"; -import { prefixedDigestSchema } from "../domain/digests.js"; - -/** Exact reference to Evidence already retained by the current session. */ -export const retainedEvidenceReferenceSchema = z.strictObject({ - kind: z.literal("retained-evidence"), - evidence_id: prefixedDigestSchema("ev"), -}); - -/** Portable inline Evidence or an exact, session-scoped reference. */ -export const evidenceInputSchema = z.union([ - evidenceSchema, - retainedEvidenceReferenceSchema, -]); - -/** Evidence input after parsing the advertised boundary contract. */ -export type EvidenceInput = z.output; +/** Canonical Evidence selection contracts are owned by the domain. */ +export { + evidenceInputSchema, + type EvidenceInput, +} from "../domain/evidenceReference.js"; diff --git a/src/contracts/native/functionMatchingContracts.ts b/src/contracts/native/functionMatchingContracts.ts new file mode 100644 index 0000000000..e8bc5c38c8 --- /dev/null +++ b/src/contracts/native/functionMatchingContracts.ts @@ -0,0 +1,30 @@ +import { z } from "zod"; +import { evidenceInputSchema } from "../evidenceInputContracts.js"; + +// Borrow inline payloads until application admission; advertise the exact public +// Evidence/reference union without a pre-admission recursive SDK clone. +const input = z.unknown().nonoptional(); +input._zod.processJSONSchema = evidenceInputSchema._zod.processJSONSchema; + +/** Inventories stay portable; optional observations avoid all-function decompilation. */ +export const matchNativeFunctionsInputSchema = z.strictObject({ + left: input.describe( + "Left list_procedures Evidence or exact retained reference", + ), + right: input.describe( + "Right list_procedures Evidence or exact retained reference", + ), + left_observations: z + .array(input) + .default([]) + .describe( + "Optional same-context list_names, procedure_info, read_bytes or analyze_function Evidence", + ), + right_observations: z + .array(input) + .default([]) + .describe( + "Optional same-context list_names, procedure_info, read_bytes or analyze_function Evidence", + ), + detail: z.enum(["summary", "complete"]).default("summary"), +}); diff --git a/src/contracts/sessionToolContracts.ts b/src/contracts/sessionToolContracts.ts index 215345c792..bbd12c5fe6 100644 --- a/src/contracts/sessionToolContracts.ts +++ b/src/contracts/sessionToolContracts.ts @@ -1,3 +1,4 @@ +import { matchNativeFunctionsInputSchema } from "./native/functionMatchingContracts.js"; import { z } from "zod"; import { isAbsolute } from "node:path"; @@ -107,9 +108,14 @@ export const SESSION_TOOL_CONTRACTS = [ "Compare complete artifact inventories by logical occurrence path, content identity, metadata, and graph relations. Pass the inventory Evidence nested in each inspect_artifact result as left and right, or pass its inspect_analysis_view summary while the complete parent Evidence remains retained in this session. Summary facts are not treated as a complete inventory; comparison resolves the exact retained parent. Every delta cites both complete inputs, and gaps yield truncated or unknown, never equivalence. Returns every change inline.", artifactComparisonInputSchema, ), + session( + "match_native_functions", + "Match two retained list_procedures inventories by unique exact reported symbols, then unique identical observed function bytes. Optional same-context list_names/procedure_info/read_bytes/analyze_function Evidence supplies symbol origin, exact bodies, bytes and dossier links as supplied static facts. Optional compatible CFG/call/string fingerprints from supplied dossiers propose unresolved groups only. Duplicate candidates remain unresolved groups; only_left/right mean unpaired within supplied observations, never proof of addition/removal. Default summary retains complete Evidence for filtered, selected-column, paged function-matches views. Complete detail returns every group. Existing dossier pairs provide direct retained-reference compare_functions inputs.", + matchNativeFunctionsInputSchema, + ), session( "compare_functions", - "Compare two explicit sets of analyze_function Evidence across identity, exact provider text, calls, references, strings, and address-normalized CFG topology. Missing or provider-incompatible facets remain truncated or unknown; every conclusion cites both Evidence sets.", + "Compare two explicit analyze_function Evidence records or exact retained session references across identity, exact provider text, calls, references, strings, and address-normalized CFG topology. Missing or provider-incompatible facets remain truncated or unknown; every conclusion cites both Evidence sets.", functionComparisonInputSchema, ), session( diff --git a/src/contracts/toolContractExamples.ts b/src/contracts/toolContractExamples.ts index dc6567e39f..e2f96d1583 100644 --- a/src/contracts/toolContractExamples.ts +++ b/src/contracts/toolContractExamples.ts @@ -74,6 +74,10 @@ export const TOOL_EXAMPLE_OVERRIDES: Readonly< left: ARTIFACT_COMPARISON_EXAMPLE.left, right: ARTIFACT_COMPARISON_EXAMPLE.right, }, + match_native_functions: { + left: { kind: "retained-evidence", evidence_id: `ev_${"1".repeat(64)}` }, + right: { kind: "retained-evidence", evidence_id: `ev_${"2".repeat(64)}` }, + }, compare_functions: { left: FUNCTION_COMPARISON_EXAMPLE.left, right: FUNCTION_COMPARISON_EXAMPLE.right, diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index d0e87c5c15..7bdb7fdedb 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -705,6 +705,7 @@ export const TOOL_EFFECTS: Readonly> = { compare_artifacts: sessionEvidence, compare_wasm_interfaces: sessionEvidence, compare_functions: sessionEvidence, + match_native_functions: sessionEvidence, compare_bundles: effects({ mutatesSession: true }), find_changed_behavior: sessionEvidence, build_call_path: sessionEvidence, diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index 0073793f45..c1c1ba5413 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -1,3 +1,6 @@ +import { nativeFunctionMatchingSchema } from "../domain/native/functionMatching.js"; +import { nativeFunctionMatchingSummaryViewResultSchema } from "../domain/analysisView/analysisView.js"; +import { nativeByteReadSchema } from "../domain/native/nativeByteRead.js"; import { analysisDetailOutputSchemaOf } from "./analysisDetail.js"; import { nativeInventorySummaryViewResultSchema } from "../domain/analysisView/analysisView.js"; import { nativeInventoryRowsSchemas } from "../domain/native/nativeInventory.js"; @@ -158,15 +161,7 @@ export const officialOutputSchemas: Readonly> = { procedure_callers: evidenceResultOf(addressList), procedure_info: procedureInfoOutput, read_function_instructions: evidenceResultOf(functionInstructionWindowSchema), - read_bytes: evidenceResultOf( - z.object({ - address: z.string(), - requested_bytes: z.number().int().min(1), - returned_bytes: z.number().int().min(0), - bytes_hex: z.string().regex(/^(?:[a-f0-9]{2})*$/u), - complete: z.boolean(), - }), - ), + read_bytes: evidenceResultOf(nativeByteReadSchema), address_to_file_offset: evidenceResultOf( z.object({ address: z.string(), @@ -595,6 +590,12 @@ export const sessionOutputSchemas = { compare_process_captures: evidenceResultOf(processCaptureComparisonSchema), compare_artifacts: evidenceResultOf(artifactComparisonResultSchema), compare_functions: evidenceResultOf(functionComparisonResultSchema), + match_native_functions: evidenceResultOf( + z.union([ + nativeFunctionMatchingSchema, + nativeFunctionMatchingSummaryViewResultSchema, + ]), + ), compare_bundles: evidenceResultOf(bundleComparisonResultSchema), find_changed_behavior: evidenceResultOf(changedBehaviorResultSchema), build_call_path: evidenceResultOf(callPathResultSchema), diff --git a/src/contracts/toolOutputSchemaPrimitives.ts b/src/contracts/toolOutputSchemaPrimitives.ts index 6221d57cb4..058388b201 100644 --- a/src/contracts/toolOutputSchemaPrimitives.ts +++ b/src/contracts/toolOutputSchemaPrimitives.ts @@ -1,11 +1,9 @@ +import { procedureInfoSchema } from "../domain/native/procedureInfo.js"; import { z } from "zod"; import { - procedureClassificationSchema, procedureIdentitySchema, - localVariableSchema, functionDossierSchema, - functionBodySchema, } from "../domain/hopperValues.js"; import { analysisProfileSchema } from "../domain/analysisProfile.js"; import { @@ -598,18 +596,7 @@ export const segmentOutput = evidenceResultOf( z.array(memoryRegionOutput.extend({ sections: z.array(memoryRegionOutput) })), ); -export const procedureInfoOutput = evidenceResultOf( - z.object({ - name: z.string(), - entrypoint: z.string(), - basicblock_count: z.number().int().min(0), - length: z.number().min(0), - signature: nullableText, - locals: z.array(localVariableSchema), - classification: procedureClassificationSchema.nullable(), - body: functionBodySchema, - }), -); +export const procedureInfoOutput = evidenceResultOf(procedureInfoSchema); export const symbolDiscoveryOutput = (property: "classes" | "protocols") => evidenceResultOf( diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 2ed46a7a28..ba9b70da48 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -1,3 +1,9 @@ +import { + projectNativeFunctionMatchingView, + nativeFunctionMatchingViewRequestSchema, + nativeFunctionMatchingViewItemsSchema, + nativeFunctionMatchingViewContextSchema, +} from "./nativeFunctionMatchingView.js"; import { protobufViewRequestSchema, protobufViewItemsSchema, @@ -243,6 +249,7 @@ const nativeViewSchema = z.strictObject({ /** Caller-selected projection of already completed analysis Evidence. */ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ + nativeFunctionMatchingViewRequestSchema, summaryViewSchema, desktopViewRequestSchema, facetViewSchema, @@ -377,6 +384,7 @@ const viewResultBase = { "inspect_dart_aot", "inspect_il2cpp_metadata", ...NATIVE_INVENTORY_OPERATIONS, + "match_native_functions", ]), parent_digest: digestSchema, view_digest: digestSchema, @@ -524,8 +532,25 @@ export const nativeInventorySummaryViewResultSchema = z.strictObject({ summary: nativeInventorySummarySchema, }); +/** Compact native matching discovery retains the complete matching report. */ +export const nativeFunctionMatchingSummaryViewResultSchema = z.strictObject({ + ...summaryViewResultBase, + artifact: z.null(), + parent_operation: z.literal("match_native_functions"), + summary: nativeFunctionMatchingViewContextSchema, +}); + /** Projected facts for one selected view of retained analysis Evidence. */ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + ...viewResultBase, + ...nativeFunctionMatchingViewContextSchema.shape, + kind: z.literal("function-matches"), + artifact: z.null(), + view: nativeFunctionMatchingViewRequestSchema, + items: nativeFunctionMatchingViewItemsSchema, + parent_operation: z.literal("match_native_functions"), + }), z.strictObject({ ...viewResultBase, ...apiSurfaceFields, @@ -574,6 +599,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ dartAotSummaryViewResultSchema, il2cppMetadataSummaryViewResultSchema, nativeInventorySummaryViewResultSchema, + nativeFunctionMatchingSummaryViewResultSchema, ]), z.strictObject({ ...viewResultBase, @@ -844,6 +870,7 @@ const retainedViewProjectors = new Map([ ["inspect_android_package", projectAndroidPackageView], ["inspect_dart_aot", projectDartAotView], ["inspect_il2cpp_metadata", projectIl2cppMetadataView], + ["match_native_functions", projectNativeFunctionMatchingView], ["inspect_artifact", projectArtifactInventoryView], ["inventory_artifact", projectArtifactInventoryView], ]); diff --git a/src/domain/analysisView/nativeFunctionMatchingView.ts b/src/domain/analysisView/nativeFunctionMatchingView.ts new file mode 100644 index 0000000000..fc0b11e03e --- /dev/null +++ b/src/domain/analysisView/nativeFunctionMatchingView.ts @@ -0,0 +1,311 @@ +import { z } from "zod"; +import { + nativeFunctionMatchingSchema, + nativeFunctionMatchingContextSchema, + nativeFunctionMatchSchema, + nativeFunctionMatchMemberSchema, + nativeMatchStateSchema, + type NativeFunctionMatching, +} from "../native/functionMatching.js"; +import { isImmutableJsonSnapshot } from "../immutableJson.js"; +import { parseAnalysisViewResult } from "./analysisViewParent.js"; +import { analysisViewInputError } from "./analysisViewErrors.js"; +import { + boundedViewMessages, + completeWithinViewCoverage, + pageViewCoverage, +} from "./analysisViewCoverage.js"; +import { err, ok, type Result } from "../result.js"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import type { + AnalysisViewParent, + AnalysisViewRequest, + UnsignedAnalysisView, +} from "./analysisView.js"; +import type { JsonValue } from "../jsonValue.js"; + +const matchRow = nativeFunctionMatchSchema.extend({ + left_count: z.number().int().nonnegative(), + right_count: z.number().int().nonnegative(), +}); +const functionRow = nativeFunctionMatchMemberSchema.extend({ + match_id: nativeFunctionMatchSchema.shape.match_id, + state: nativeMatchStateSchema, + side: z.enum(["left", "right"]), +}); +const page = { + kind: z.literal("function-matches"), + states: z.array(nativeMatchStateSchema).optional(), + match_ids: z.array(z.string()).optional(), + function_ids: z.array(z.string()).optional(), + offset: z.number().int().nonnegative().default(0), + limit: z.number().int().positive().default(64), +}; +export const nativeFunctionMatchingViewRequestSchema = z.discriminatedUnion( + "collection", + [ + z.strictObject({ + ...page, + collection: z.literal("matches"), + fields: z.array(matchRow.keyof()).optional(), + }), + z.strictObject({ + ...page, + collection: z.literal("functions"), + sides: z.array(z.enum(["left", "right"])).optional(), + names: z.array(z.string()).optional(), + fields: z.array(functionRow.keyof()).optional(), + }), + ], +); +export const nativeFunctionMatchingViewItemsSchema = z.union([ + z.array( + matchRow.partial().required({ + match_id: true, + state: true, + left_count: true, + right_count: true, + }), + ), + z.array( + functionRow.partial().required({ + function_id: true, + match_id: true, + state: true, + side: true, + inventory_evidence_id: true, + source_index: true, + address: true, + evidence_links: true, + }), + ), +]); +export const nativeFunctionMatchingViewContextSchema = + nativeFunctionMatchingContextSchema; +interface FunctionRow { + readonly member: NativeFunctionMatching["matches"][number]["left"][number]; + readonly match_id: string; + readonly state: NativeFunctionMatching["matches"][number]["state"]; + readonly side: "left" | "right"; +} +interface Checked { + readonly report: NativeFunctionMatching; + functions?: readonly FunctionRow[]; + recent?: { readonly key: string; readonly rows: readonly number[] }; +} +const checked = new WeakMap(); + +type Request = z.output; +const checkedReport = ( + parent: AnalysisViewParent, +): Result => { + const source = parent.normalizedResult; + const reusable = + typeof source === "object" && + source !== null && + parent.sealedResult === true && + isImmutableJsonSnapshot(source); + const cached = reusable ? checked.get(source) : undefined; + if (cached !== undefined) return ok(cached); + const parsed = parseAnalysisViewResult(nativeFunctionMatchingSchema, parent); + if (!parsed.ok) return parsed; + const entry: Checked = { + report: reusable ? (source as NativeFunctionMatching) : parsed.value, + }; + if (reusable) checked.set(source, entry); + return ok(entry); +}; +const functionRows = (entry: Checked): readonly FunctionRow[] => { + if (entry.functions !== undefined) return entry.functions; + const functions: FunctionRow[] = []; + for (const match of entry.report.matches) + for (const side of ["left", "right"] as const) + for (const member of match[side]) + functions.push({ + member, + match_id: match.match_id, + state: match.state, + side, + }); + entry.functions = functions; + return functions; +}; +const sharedContext = ( + parent: AnalysisViewParent, + report: NativeFunctionMatching, +) => ({ + parent_evidence_id: parent.evidenceId, + parent_operation: "match_native_functions" as const, + parent_digest: parent.evidenceId.slice(3), + artifact: null, + limitations: boundedViewMessages( + [...parent.limitations, ...report.limitations], + parent.evidenceId, + "limitations", + ), + unknowns: [ + "Unpaired observations and page exhaustion do not establish binary-wide function absence or behavioral equivalence.", + ], +}); +const selected = (values: readonly string[] | undefined) => + values === undefined ? undefined : new Set(values); +const filterRows = (entry: Checked, view: Request): readonly number[] => { + const key = JSON.stringify([ + view.collection, + view.states, + view.match_ids, + view.function_ids, + view.collection === "functions" ? view.sides : null, + view.collection === "functions" ? view.names : null, + ]); + if (entry.recent?.key === key) return entry.recent.rows; + const states = selected(view.states), + ids = selected(view.match_ids), + functions = selected(view.function_ids), + sides = selected(view.collection === "functions" ? view.sides : undefined), + names = selected(view.collection === "functions" ? view.names : undefined); + const matchesIdentity = (row: { state: string; match_id: string }): boolean => + (states === undefined || states.has(row.state)) && + (ids === undefined || ids.has(row.match_id)); + const rows: number[] = []; + if (view.collection === "matches") + entry.report.matches.forEach((row, index) => { + if ( + matchesIdentity(row) && + (functions === undefined || + [...row.left, ...row.right].some((member) => + functions.has(member.function_id), + )) + ) + rows.push(index); + }); + else + functionRows(entry).forEach((row, index) => { + if ( + matchesIdentity(row) && + (functions === undefined || functions.has(row.member.function_id)) && + (sides === undefined || sides.has(row.side)) && + (names === undefined || names.has(row.member.name)) + ) + rows.push(index); + }); + // One recent selector and at most one row index per source occurrence. + const count = entry.report.left.functions + entry.report.right.functions; + if (key.length <= Math.max(256, count * 8)) entry.recent = { key, rows }; + return rows; +}; +const selectedItems = ( + entry: Checked, + view: Request, + rows: readonly number[], +) => { + const indices = rows.slice(view.offset, view.offset + view.limit); + const defaults = + view.collection === "matches" + ? [ + "match_id", + "state", + "basis", + "reason", + "left_count", + "right_count", + "compare_functions_input", + ] + : [ + "function_id", + "match_id", + "state", + "side", + "inventory_evidence_id", + "source_index", + "address", + "name", + "body_sha256", + "structural_candidate_sha256", + "body_bytes", + "dossier_evidence_id", + "evidence_links", + ]; + const fields = new Set(view.fields ?? defaults); + const columns = (row: Readonly>) => + Object.fromEntries([...fields].map((field) => [field, row[field]!])); + if (view.collection === "matches") + return indices.map((index) => { + const row = entry.report.matches[index]!, + left_count = row.left.length, + right_count = row.right.length; + return { + ...columns({ ...row, left_count, right_count }), + match_id: row.match_id, + state: row.state, + left_count, + right_count, + }; + }); + const functions = functionRows(entry); + return indices.map((index) => { + const row = functions[index]!, + member = row.member; + return { + ...columns({ + ...member, + match_id: row.match_id, + state: row.state, + side: row.side, + }), + function_id: member.function_id, + match_id: row.match_id, + state: row.state, + side: row.side, + inventory_evidence_id: member.inventory_evidence_id, + source_index: member.source_index, + address: member.address, + evidence_links: member.evidence_links, + }; + }); +}; + +/** Query immutable matching facts without copying full alias groups into selected pages. */ +export const projectNativeFunctionMatchingView = ( + parent: AnalysisViewParent, + view: AnalysisViewRequest, +): Result => { + if (view.kind !== "summary" && view.kind !== "function-matches") + return err( + analysisViewInputError([ + { + path: ["view"], + reason: "invalid_value", + message: + "Native matching supports summary and function-matches pages", + }, + ]), + ); + const parsed = checkedReport(parent); + if (!parsed.ok) return parsed; + const entry = parsed.value; + const { + matches: _matches, + limitations: _limitations, + ...context + } = entry.report; + const shared = sharedContext(parent, entry.report); + if (view.kind === "summary") + return ok({ + kind: "summary", + view, + ...shared, + summary: context, + coverage: completeWithinViewCoverage(1, 1), + }); + const rows = filterRows(entry, view), + items = selectedItems(entry, view, rows); + return ok({ + kind: "function-matches", + view, + ...shared, + ...context, + items, + coverage: pageViewCoverage(view.offset, items.length, rows.length), + }); +}; diff --git a/src/domain/evidenceReference.ts b/src/domain/evidenceReference.ts new file mode 100644 index 0000000000..61c24c1fb4 --- /dev/null +++ b/src/domain/evidenceReference.ts @@ -0,0 +1,17 @@ +import { z } from "zod"; +import { prefixedDigestSchema } from "./digests.js"; + +/** Exact reference to Evidence already retained by the current session. */ +export const retainedEvidenceReferenceSchema = z.strictObject({ + kind: z.literal("retained-evidence"), + evidence_id: prefixedDigestSchema("ev"), +}); + +import { evidenceSchema } from "./evidence.js"; + +/** Portable observation or exact retained identity; semantic authentication follows admission. */ +export const evidenceInputSchema = z.union([ + evidenceSchema, + retainedEvidenceReferenceSchema, +]); +export type EvidenceInput = z.output; diff --git a/src/domain/functionComparisonSchemas.ts b/src/domain/functionComparisonSchemas.ts index 844bdc9b9d..9c340028ee 100644 --- a/src/domain/functionComparisonSchemas.ts +++ b/src/domain/functionComparisonSchemas.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import { overallStatus } from "./functionComparisonResults.js"; -import { evidenceSchema } from "./evidence.js"; +import { evidenceInputSchema } from "./evidenceReference.js"; import { digestSchema } from "./../domain/digests.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; @@ -19,8 +19,8 @@ const evidenceIdSchema = prefixedDigestSchema("ev"); /** Inputs for explicit function-to-function comparison. */ export const functionComparisonInputSchema = z.strictObject({ - left: evidenceSchema, - right: evidenceSchema, + left: evidenceInputSchema, + right: evidenceInputSchema, }); const textDeltaSchema = z.object({ diff --git a/src/domain/native/functionByteCoverage.ts b/src/domain/native/functionByteCoverage.ts new file mode 100644 index 0000000000..b3889fc636 --- /dev/null +++ b/src/domain/native/functionByteCoverage.ts @@ -0,0 +1,193 @@ +import { createHash } from "node:crypto"; +import type { z } from "zod"; +import type { functionBodySchema } from "../hopperValues.js"; +import type { nativeByteReadSchema } from "./nativeByteRead.js"; +import { addressCoordinate } from "./addressCoordinate.js"; +import { parseNativeAddress } from "./nativeAddress.js"; + +export type NativeMatchingCheckpoint = (bytes?: number) => Promise; +interface Interval { + start: bigint; + end: bigint; +} +interface Piece extends Interval { + readonly hex: string; + readonly links: readonly string[]; +} +interface Fingerprint { + readonly sha256: string; + readonly key: string; + readonly links: readonly string[]; +} +const slice = (piece: Piece, start: bigint, end: bigint): Piece => ({ + ...piece, + start, + end, + hex: + start === piece.start && end === piece.end + ? piece.hex + : piece.hex.slice( + Number(start - piece.start) * 2, + Number(end - piece.start) * 2, + ), +}); +const firstOverlapping = (pieces: readonly Piece[], start: bigint): number => { + let lower = 0, + upper = pieces.length; + while (lower < upper) { + const middle = (lower + upper) >>> 1; + if (pieces[middle]!.end <= start) lower = middle + 1; + else upper = middle; + } + return lower; +}; +const mergeIntervals = (intervals: Interval[]): Interval[] => { + intervals.sort((a, b) => + a.start < b.start ? -1 : a.start > b.start ? 1 : 0, + ); + const merged: Interval[] = []; + for (const interval of intervals) { + const previous = merged.at(-1); + if (previous !== undefined && interval.start <= previous.end) { + if (interval.end > previous.end) previous.end = interval.end; + } else merged.push({ ...interval }); + } + return merged; +}; +const excludeConflicts = ( + pieces: readonly Piece[], + conflicts: readonly Interval[], +): Piece[] => { + const safe: Piece[] = []; + let first = 0; + for (const piece of pieces) { + while (first < conflicts.length && conflicts[first]!.end <= piece.start) + first++; + let cursor = piece.start; + for ( + let index = first; + index < conflicts.length && conflicts[index]!.start < piece.end; + index++ + ) { + const conflict = conflicts[index]!; + if (conflict.start > cursor) + safe.push(slice(piece, cursor, conflict.start)); + if (conflict.end > cursor) cursor = conflict.end; + if (cursor >= piece.end) break; + } + if (cursor < piece.end) safe.push(slice(piece, cursor, piece.end)); + } + return safe; +}; + +/** Own exact observed byte intervals, their conflicts and whole-body hashing. */ +export class FunctionByteCoverage { + readonly #pieces = new Map(); + #prepared = false; + constructor(readonly checkpoint: NativeMatchingCheckpoint) {} + async add( + read: z.output, + evidenceId: string, + ): Promise { + if (this.#prepared) + throw new TypeError( + "Cannot append byte observations after fingerprinting", + ); + if (read.returned_bytes === 0) return; + const coordinate = addressCoordinate(parseNativeAddress(read.address))!; + let group = this.#pieces.get(coordinate.space); + if (group === undefined) this.#pieces.set(coordinate.space, (group = [])); + await this.checkpoint(192 + read.bytes_hex.length * 2); + group.push({ + start: coordinate.offset, + end: coordinate.offset + BigInt(read.returned_bytes), + hex: read.bytes_hex, + links: [evidenceId], + }); + } + async #conflict(left: Piece, right: Piece): Promise { + const start = left.start > right.start ? left.start : right.start; + const end = left.end < right.end ? left.end : right.end; + for (let cursor = start; cursor < end;) { + const next = cursor + 65536n < end ? cursor + 65536n : end; + await this.checkpoint(); + if (slice(left, cursor, next).hex !== slice(right, cursor, next).hex) + return { start, end }; + cursor = next; + } + return undefined; + } + async #prepareSpace(source: Piece[]): Promise { + source.sort((a, b) => + a.start < b.start + ? -1 + : a.start > b.start + ? 1 + : a.end > b.end + ? -1 + : a.end < b.end + ? 1 + : 0, + ); + const admitted: Piece[] = [], + conflicts: Interval[] = []; + for (const incoming of source) { + await this.checkpoint(); + for ( + let index = firstOverlapping(admitted, incoming.start); + index < admitted.length && admitted[index]!.start < incoming.end; + index++ + ) { + const conflict = await this.#conflict(admitted[index]!, incoming); + if (conflict !== undefined) conflicts.push(conflict); + } + const end = admitted.at(-1)?.end; + if (end === undefined || incoming.start >= end) admitted.push(incoming); + else if (incoming.end > end) + admitted.push(slice(incoming, end, incoming.end)); + } + return excludeConflicts(admitted, mergeIntervals(conflicts)); + } + async fingerprint( + body: z.output | undefined, + ): Promise { + if (!body?.available || !body.contains_entry || body.total_bytes === 0) + return null; + if (!this.#prepared) { + for (const [space, pieces] of this.#pieces) + this.#pieces.set(space, await this.#prepareSpace(pieces)); + this.#prepared = true; + } + const hash = createHash("sha256"), + lengths: string[] = [], + links = new Set(); + for (const range of body.ranges) { + const start = addressCoordinate(parseNativeAddress(range.start))!, + end = addressCoordinate(parseNativeAddress(range.end))!; + const pieces = this.#pieces.get(start.space) ?? []; + lengths.push(String(end.offset - start.offset + 1n)); + let index = firstOverlapping(pieces, start.offset), + cursor = start.offset; + while (cursor <= end.offset) { + const piece = pieces[index++]; + if (piece === undefined || piece.start > cursor || piece.end <= cursor) + return null; + const stop = piece.end < end.offset + 1n ? piece.end : end.offset + 1n; + for (let chunk = cursor; chunk < stop;) { + const next = chunk + 65536n < stop ? chunk + 65536n : stop; + await this.checkpoint(); + hash.update(Buffer.from(slice(piece, chunk, next).hex, "hex")); + chunk = next; + } + for (const link of piece.links) links.add(link); + cursor = stop; + } + } + const sha256 = hash.digest("hex"); + return { + sha256, + key: JSON.stringify([lengths, sha256]), + links: [...links], + }; + } +} diff --git a/src/domain/native/functionInventory.ts b/src/domain/native/functionInventory.ts new file mode 100644 index 0000000000..a89e09802b --- /dev/null +++ b/src/domain/native/functionInventory.ts @@ -0,0 +1,290 @@ +import { functionStructuralCandidate } from "./functionStructuralCandidate.js"; +import type { z } from "zod"; +import type { Evidence } from "../evidence.js"; +import { EvidenceIntegrityError } from "../evidenceErrors.js"; +import { digestCanonicalValue } from "../canonicalDigest.js"; +import { + functionBodySchema, + functionBodyEntryAgrees, + functionDossierSchema, +} from "../hopperValues.js"; +import { isAutoName } from "../functionComparisonNormalization.js"; +import { + addressedValueSchema, + nativeInventoryRowsSchemas, +} from "./nativeInventory.js"; +import { parseNativeAddress } from "./nativeAddress.js"; +import { nativeByteReadSchema } from "./nativeByteRead.js"; +import { procedureInfoSchema } from "./procedureInfo.js"; +import type { NativeFunctionMatchMember } from "./functionMatching.js"; +import { + FunctionByteCoverage, + type NativeMatchingCheckpoint, +} from "./functionByteCoverage.js"; + +type Body = z.infer; +type AddressedValue = z.infer; +export interface MatchingFunction { + readonly member: NativeFunctionMatchMember; + readonly symbol: string | null; + readonly fingerprint: string | null; +} +interface Detail { + body?: Body; + dossier?: string | null; + candidate?: string | null; + conflict: boolean; + readonly links: string[]; +} +interface Symbols { + readonly values: AddressedValue[]; + readonly links: Set; +} +interface Facts { + readonly details: Map; + readonly symbols: Map; + readonly bytes: FunctionByteCoverage; +} + +/** Supplementary observations must belong to this exact provider/target/profile/document. */ +const requireContext = (inventory: Evidence, observation: Evidence): void => { + if ( + observation.predicate_type !== "rea.analysis" || + digestCanonicalValue(observation.subject) !== + digestCanonicalValue(inventory.subject) || + digestCanonicalValue(observation.provider) !== + digestCanonicalValue(inventory.provider) || + digestCanonicalValue(observation.analysis_profile) !== + digestCanonicalValue(inventory.analysis_profile) || + (observation.parameters.document ?? null) !== + (inventory.parameters.document ?? null) + ) + throw new EvidenceIntegrityError( + "Function observation does not belong to its selected inventory context", + ); +}; +const addBody = (observation: Evidence, details: Facts["details"]): void => { + const dossier = + observation.operation === "analyze_function" + ? functionDossierSchema.parse(observation.normalized_result) + : undefined; + const procedure = + dossier?.procedure ?? + (() => { + const info = procedureInfoSchema.parse(observation.normalized_result); + return { address: info.entrypoint, body: info.body }; + })(); + const address = parseNativeAddress(procedure.address); + if (!functionBodyEntryAgrees(procedure.body, address)) + throw new EvidenceIntegrityError( + "Procedure body entry membership contradicts its address", + ); + let detail = details.get(address); + if (detail === undefined) + details.set(address, (detail = { conflict: false, links: [] })); + if ( + detail.body !== undefined && + digestCanonicalValue(detail.body) !== digestCanonicalValue(procedure.body) + ) + detail.conflict = true; + detail.body = procedure.body; + if (observation.operation === "analyze_function") { + const candidate = functionStructuralCandidate(observation, dossier!); + if (detail.dossier === undefined) { + detail.dossier = observation.evidence_id; + detail.candidate = candidate; + } else { + // Request identity and dossier selection are independent of body agreement. + if (detail.dossier !== observation.evidence_id) detail.dossier = null; + if (detail.candidate !== candidate) detail.candidate = null; + } + } + detail.links.push(observation.evidence_id); +}; +const addNames = async ( + observation: Evidence, + symbols: Facts["symbols"], + checkpoint: NativeMatchingCheckpoint, +): Promise => { + for (const symbol of nativeInventoryRowsSchemas.names.parse( + observation.normalized_result, + )) { + await checkpoint(128 + symbol.value.length * 2); + const address = parseNativeAddress(symbol.address); + let group = symbols.get(address); + if (group === undefined) + symbols.set(address, (group = { values: [], links: new Set() })); + group.values.push(symbol); + group.links.add(observation.evidence_id); + } +}; +const addBytes = async ( + observation: Evidence, + bytes: FunctionByteCoverage, +): Promise => { + const read = nativeByteReadSchema.parse(observation.normalized_result); + if ( + (typeof observation.parameters.address === "string" && + parseNativeAddress(observation.parameters.address) !== + parseNativeAddress(read.address)) || + (typeof observation.parameters.length === "number" && + observation.parameters.length !== read.requested_bytes) + ) + throw new EvidenceIntegrityError( + "Read-byte coordinates contradict the selected request", + ); + await bytes.add(read, observation.evidence_id); +}; +const reportedSymbol = ( + row: AddressedValue, + known: Symbols | undefined, +): { + readonly value: string | null; + readonly origin: "known" | "unknown" | "conflicting"; +} => { + const names = [ + ...(row.symbol === undefined ? [] : [row]), + ...(known?.values.filter((symbol) => symbol.value === row.value) ?? []), + ]; + const generated = names.some( + (symbol) => + symbol.symbol?.dynamic === true || symbol.symbol?.source === "default", + ); + const named = names.some( + (symbol) => + symbol.symbol?.dynamic === false && symbol.symbol.source !== "default", + ); + return { + value: + row.value.length === 0 || generated || (!named && isAutoName(row.value)) + ? null + : row.value, + origin: + generated && named + ? "conflicting" + : names.some((symbol) => symbol.symbol !== undefined) + ? "known" + : "unknown", + }; +}; +const functionFact = async ( + { + inventory, + row, + sourceIndex, + }: { inventory: Evidence; row: AddressedValue; sourceIndex: number }, + facts: Facts, + checkpoint: NativeMatchingCheckpoint, +): Promise => { + const address = parseNativeAddress(row.address), + detail = facts.details.get(address), + symbols = facts.symbols.get(address); + // Bound expansion before creating per-occurrence sets, arrays and subsequent + // schema/Evidence serialization copies. Same-address aliases repeat these links. + for (const link of detail?.links ?? []) + await checkpoint(256 + 8 * link.length); + for (const link of symbols?.links ?? []) + await checkpoint(256 + 8 * link.length); + await checkpoint(256 + 8 * inventory.evidence_id.length); + const links = new Set([inventory.evidence_id]); + for (const link of detail?.links ?? []) links.add(link); + for (const link of symbols?.links ?? []) links.add(link); + const body = detail?.conflict ? undefined : detail?.body; + const fingerprint = await facts.bytes.fingerprint(body); + for (const link of fingerprint?.links ?? []) { + await checkpoint(256 + 8 * link.length); + links.add(link); + } + const limitations: string[] = []; + const symbol = reportedSymbol(row, symbols); + if (fingerprint === null) + limitations.push( + detail?.conflict + ? "Conflicting procedure observations; body and dossier remain unknown." + : "No complete unambiguous bytes for the observed function body.", + ); + if (symbol.origin === "unknown") + limitations.push( + "Symbol origin unavailable; exact-name matching uses the reported name and the existing automatic-name exclusion rule.", + ); + if (symbol.origin === "conflicting") + limitations.push( + "Conflicting generated and named symbol-origin observations; exact-name matching is unavailable. Independent complete body bytes remain eligible.", + ); + if (detail?.dossier === null) + limitations.push( + "Multiple supplied dossiers describe this address; no dossier is selected automatically. Agreed body bytes and structural facts remain usable.", + ); + return { + member: { + function_id: `${inventory.evidence_id}/${sourceIndex}`, + inventory_evidence_id: inventory.evidence_id, + source_index: sourceIndex, + address, + name: row.value, + body_sha256: fingerprint?.sha256 ?? null, + structural_candidate_sha256: detail?.conflict + ? null + : (detail?.candidate ?? null), + body_bytes: body?.available ? body.total_bytes : null, + dossier_evidence_id: detail?.conflict ? null : (detail?.dossier ?? null), + evidence_links: [...links].sort(), + limitations, + }, + symbol: symbol.value, + fingerprint: fingerprint?.key ?? null, + }; +}; + +/** Consume normalized producer facts once; never infer body extents from neighbor entries. */ +export const readFunctionInventory = async ( + inventory: Evidence, + observations: readonly Evidence[], + checkpoint: NativeMatchingCheckpoint, +): Promise => { + if ( + inventory.operation !== "list_procedures" || + inventory.predicate_type !== "rea.analysis" || + inventory.subject === null + ) + throw new EvidenceIntegrityError( + "Expected artifact-bound list_procedures Evidence", + ); + const rows = nativeInventoryRowsSchemas.names.parse( + inventory.normalized_result, + ); + const facts: Facts = { + details: new Map(), + symbols: new Map(), + bytes: new FunctionByteCoverage(checkpoint), + }; + for (const observation of observations) { + await checkpoint(); + requireContext(inventory, observation); + switch (observation.operation) { + case "list_names": + await addNames(observation, facts.symbols, checkpoint); + break; + case "read_bytes": + await addBytes(observation, facts.bytes); + break; + case "procedure_info": + case "analyze_function": + addBody(observation, facts.details); + break; + default: + throw new EvidenceIntegrityError( + "Function matching observations must be list_names, procedure_info, read_bytes or analyze_function Evidence", + ); + } + } + const functions: MatchingFunction[] = []; + for (let sourceIndex = 0; sourceIndex < rows.length; sourceIndex++) { + const row = rows[sourceIndex]!; + await checkpoint(1024 + 4 * (row.address.length + row.value.length)); + functions.push( + await functionFact({ inventory, row, sourceIndex }, facts, checkpoint), + ); + } + return functions; +}; diff --git a/src/domain/native/functionMatching.ts b/src/domain/native/functionMatching.ts new file mode 100644 index 0000000000..e2b6039d31 --- /dev/null +++ b/src/domain/native/functionMatching.ts @@ -0,0 +1,204 @@ +import { digestCanonicalValue } from "../canonicalDigest.js"; +import { z } from "zod"; +import { digestSchema, prefixedDigestSchema } from "../digests.js"; +import { retainedEvidenceReferenceSchema } from "../evidenceReference.js"; + +const evidenceId = prefixedDigestSchema("ev"); +export const nativeMatchStateSchema = z.enum([ + "matched", + "only_left", + "only_right", + "unresolved", +]); +export const nativeFunctionMatchMemberSchema = z.strictObject({ + function_id: z.string().min(1), + inventory_evidence_id: evidenceId, + source_index: z.number().int().nonnegative(), + address: z.string(), + name: z.string(), + body_sha256: digestSchema.nullable(), + structural_candidate_sha256: digestSchema.nullable(), + body_bytes: z.number().int().nonnegative().nullable(), + dossier_evidence_id: evidenceId.nullable(), + evidence_links: z.array(evidenceId).min(1), + limitations: z.array(z.string()), +}); +export const nativeFunctionMatchSchema = z.strictObject({ + match_id: prefixedDigestSchema("match"), + state: nativeMatchStateSchema, + basis: z.enum(["exact-symbol", "identical-bytes"]).nullable(), + left: z.array(nativeFunctionMatchMemberSchema), + right: z.array(nativeFunctionMatchMemberSchema), + reason: z.string(), + compare_functions_input: z + .strictObject({ + left: retainedEvidenceReferenceSchema, + right: retainedEvidenceReferenceSchema, + }) + .nullable(), +}); +export const nativeFunctionMatchingSummarySchema = z.strictObject({ + matched: z.number().int().nonnegative(), + only_left: z.number().int().nonnegative(), + only_right: z.number().int().nonnegative(), + unresolved: z.number().int().nonnegative(), +}); +export const nativeFunctionMatchingContextSchema = z.strictObject({ + left: z.strictObject({ + evidence_id: evidenceId, + sha256: digestSchema, + functions: z.number().int().nonnegative(), + }), + right: z.strictObject({ + evidence_id: evidenceId, + sha256: digestSchema, + functions: z.number().int().nonnegative(), + }), + scope: z.literal("provider-observed-inputs"), + source_coverage: z.strictObject({ + status: z.enum(["complete-within-inputs", "partial"]), + left_inventory_completeness: z.literal("unknown"), + right_inventory_completeness: z.literal("unknown"), + left_byte_fingerprints: z.number().int().nonnegative(), + right_byte_fingerprints: z.number().int().nonnegative(), + left_structural_candidates: z.number().int().nonnegative(), + right_structural_candidates: z.number().int().nonnegative(), + }), + summary: nativeFunctionMatchingSummarySchema, +}); +export const nativeFunctionMatchingSchema = nativeFunctionMatchingContextSchema + .extend({ + matches: z.array(nativeFunctionMatchSchema), + limitations: z.array(z.string()), + }) + .superRefine((report, context) => { + const seen = new Set(); + const members = { left: new Set(), right: new Set() }; + const fingerprints = { left: 0, right: 0 }; + const candidates = { left: 0, right: 0 }; + const counts = { matched: 0, only_left: 0, only_right: 0, unresolved: 0 }; + let left = 0, + right = 0; + for (const match of report.matches) { + counts[match.state]++; + left += match.left.length; + right += match.right.length; + if (seen.has(match.match_id)) + context.addIssue({ + code: "custom", + message: "Repeated match identity", + }); + seen.add(match.match_id); + if ( + match.match_id !== + `match_${digestCanonicalValue({ state: match.state, basis: match.basis, left: match.left.map((x) => x.function_id), right: match.right.map((x) => x.function_id) })}` + ) + context.addIssue({ + code: "custom", + message: + "Match identity does not bind its selected occurrences and basis", + }); + for (const side of ["left", "right"] as const) + for (const member of match[side]) { + if ( + member.inventory_evidence_id !== report[side].evidence_id || + member.source_index >= report[side].functions || + member.function_id !== + `${report[side].evidence_id}/${member.source_index}` || + members[side].has(member.function_id) || + !member.evidence_links.includes(member.inventory_evidence_id) || + (member.dossier_evidence_id !== null && + !member.evidence_links.includes(member.dossier_evidence_id)) + ) + context.addIssue({ + code: "custom", + message: + "Function occurrence identity, provenance or uniqueness disagrees with its selected inventory", + }); + members[side].add(member.function_id); + if (member.body_sha256 !== null) fingerprints[side]++; + if (member.structural_candidate_sha256 !== null) candidates[side]++; + } + if ( + match.left.length + match.right.length === 0 || + (match.state !== "matched" && + (match.basis !== null || match.compare_functions_input !== null)) + ) + context.addIssue({ + code: "custom", + message: + "Unresolved/unpaired group cannot contain a definitive basis or comparison input", + }); + if ( + match.basis === "identical-bytes" && + (match.left[0]?.body_sha256 == null || + match.left[0]?.body_sha256 !== match.right[0]?.body_sha256 || + match.left[0]?.body_bytes !== match.right[0]?.body_bytes) + ) + context.addIssue({ + code: "custom", + message: + "Identical-byte match disagrees with its complete body digests and sizes", + }); + if ( + match.compare_functions_input !== null && + (match.compare_functions_input.left.evidence_id !== + match.left[0]?.dossier_evidence_id || + match.compare_functions_input.right.evidence_id !== + match.right[0]?.dossier_evidence_id) + ) + context.addIssue({ + code: "custom", + message: "Comparison input does not identify this pair's dossiers", + }); + + if ( + (match.state === "matched" && + (match.left.length !== 1 || + match.right.length !== 1 || + match.basis === null)) || + (match.state === "only_left" && + (match.left.length !== 1 || match.right.length !== 0)) || + (match.state === "only_right" && + (match.right.length !== 1 || match.left.length !== 0)) + ) + context.addIssue({ + code: "custom", + message: "Match state contradicts its occurrences", + }); + } + if ( + report.source_coverage.status !== + (counts.only_left + counts.only_right + counts.unresolved > 0 + ? "partial" + : "complete-within-inputs") || + report.source_coverage.left_byte_fingerprints !== fingerprints.left || + report.source_coverage.right_byte_fingerprints !== fingerprints.right || + report.source_coverage.left_structural_candidates !== candidates.left || + report.source_coverage.right_structural_candidates !== candidates.right + ) + context.addIssue({ + code: "custom", + message: "Source/matching coverage contradicts examined facts", + }); + if ( + left !== report.left.functions || + right !== report.right.functions || + Object.keys(counts).some( + (key) => + counts[key as keyof typeof counts] !== + report.summary[key as keyof typeof counts], + ) + ) + context.addIssue({ + code: "custom", + message: "Matching summary or examined inventory coverage disagrees", + }); + }); +export type NativeFunctionMatch = z.infer; +export type NativeFunctionMatchMember = z.infer< + typeof nativeFunctionMatchMemberSchema +>; +export type NativeFunctionMatching = z.infer< + typeof nativeFunctionMatchingSchema +>; diff --git a/src/domain/native/functionStructuralCandidate.ts b/src/domain/native/functionStructuralCandidate.ts new file mode 100644 index 0000000000..4897396e00 --- /dev/null +++ b/src/domain/native/functionStructuralCandidate.ts @@ -0,0 +1,54 @@ +import type { Evidence } from "../evidence.js"; +import type { FunctionDossier } from "../hopperValues.js"; +import { digestCanonicalValue } from "../canonicalDigest.js"; +import { + isAutoName, + normalizeCfg, +} from "../functionComparisonNormalization.js"; +import { assessFunctionCallCoverage } from "../functionCallCoverage.js"; +import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; + +/** An observed structural signature proposes candidates; it never proves identity. */ +export const functionStructuralCandidate = ( + evidence: Evidence, + dossier: FunctionDossier, +): string | null => { + if ( + evidence.subject?.architecture == null || + dossier.facet_availability?.cfg?.available !== true || + dossier.facet_availability?.callees?.available !== true || + dossier.facet_availability?.strings_names?.available !== true || + dossier.unresolved_calls === undefined || + assessFunctionCallCoverage(evidence.relationship_coverage, dossier).blockers + .length > 0 || + (dossier.swift_string_literal_omissions?.length ?? 0) > 0 || + evidence.analysis_profile === null + ) + return null; + const cfg = normalizeCfg(dossier.basic_blocks); + if ( + cfg === null || + cfg.length === 0 || + dossier.callees.some( + (callee) => isAutoName(callee.name) || callee.name.length === 0, + ) + ) + return null; + const calls = dossier.callees + .map((callee) => callee.name) + .sort(compareUnicodeCodePoints); + const strings = [ + ...dossier.referenced_strings.map((value) => value.value), + ...(dossier.swift_string_literals ?? []).map((value) => value.value), + ].sort(compareUnicodeCodePoints); + // Empty leaf functions otherwise produce huge uninformative CFG-only buckets. + if (calls.length === 0 && strings.length === 0) return null; + return digestCanonicalValue({ + algorithm: "observed-cfg-calls-strings-v1", + profile: evidence.analysis_profile.digest, + architecture: evidence.subject.architecture, + cfg, + calls, + strings, + }); +}; diff --git a/src/domain/native/matchFunctionInventories.ts b/src/domain/native/matchFunctionInventories.ts new file mode 100644 index 0000000000..8352150815 --- /dev/null +++ b/src/domain/native/matchFunctionInventories.ts @@ -0,0 +1,197 @@ +import { digestCanonicalValue } from "../canonicalDigest.js"; +import type { MatchingFunction } from "./functionInventory.js"; +import type { NativeMatchingCheckpoint } from "./functionByteCoverage.js"; +import type { + NativeFunctionMatch, + NativeFunctionMatching, +} from "./functionMatching.js"; + +type Groups = ReadonlyMap; +type PairingIndexes = readonly [Groups, Groups]; +interface MatchInput { + readonly state: NativeFunctionMatch["state"]; + readonly basis: NativeFunctionMatch["basis"]; + readonly left: MatchingFunction[]; + readonly right: MatchingFunction[]; + readonly reason: string; +} +const index = ( + items: readonly MatchingFunction[], + key: (item: MatchingFunction) => string | null, +): Groups => { + const groups = new Map(); + for (const item of items) { + const value = key(item); + if (value === null) continue; + let group = groups.get(value); + if (group === undefined) groups.set(value, (group = [])); + group.push(item); + } + return groups; +}; + +/** One request owns pairing state; ambiguous components never expand into pair products. */ +class InventoryMatcher { + readonly consumed = new Set(); + readonly matches: NativeFunctionMatch[] = []; + readonly #parent = new Map(); + readonly #ambiguous = new Set(); + constructor(readonly checkpoint: NativeMatchingCheckpoint) {} + add(input: MatchInput): void { + const left = input.left.map((x) => x.member), + right = input.right.map((x) => x.member); + const leftDossier = left[0]?.dossier_evidence_id, + rightDossier = right[0]?.dossier_evidence_id; + this.matches.push({ + match_id: `match_${digestCanonicalValue({ state: input.state, basis: input.basis, left: left.map((x) => x.function_id), right: right.map((x) => x.function_id) })}`, + state: input.state, + basis: input.basis, + left, + right, + reason: input.reason, + compare_functions_input: + input.state === "matched" && leftDossier != null && rightDossier != null + ? { + left: { kind: "retained-evidence", evidence_id: leftDossier }, + right: { kind: "retained-evidence", evidence_id: rightDossier }, + } + : null, + }); + for (const item of [...input.left, ...input.right]) this.consumed.add(item); + } + async pairUnique( + groups: PairingIndexes, + basis: NonNullable, + ): Promise { + for (const [key, left] of groups[0]) { + await this.checkpoint(); + const right = groups[1].get(key); + // Global multiplicity still matters after a different symbol consumes an alias. + if ( + left.length !== 1 || + right?.length !== 1 || + this.consumed.has(left[0]!) || + this.consumed.has(right[0]!) + ) + continue; + this.add({ + state: "matched", + basis, + left, + right, + reason: + basis === "exact-symbol" + ? "Unique exact reported symbol; this does not establish unchanged code or semantic equivalence." + : "Unique identical observed function bytes, total size and ordered range lengths; relocations are not normalized.", + }); + } + } + #find(item: MatchingFunction): MatchingFunction { + let root = item; + while (this.#parent.has(root) && this.#parent.get(root) !== root) + root = this.#parent.get(root)!; + let current = item; + while (this.#parent.has(current) && this.#parent.get(current) !== root) { + const next = this.#parent.get(current)!; + this.#parent.set(current, root); + current = next; + } + return root; + } + async connectCandidates(indexes: readonly PairingIndexes[]): Promise { + for (const groups of indexes) + for (const [key, left] of groups[0]) { + await this.checkpoint(); + const right = groups[1].get(key); + if (right === undefined) continue; + const pending = [...left, ...right].filter( + (item) => !this.consumed.has(item), + ), + first = pending[0]; + if (first === undefined) continue; + for (const item of pending) { + this.#parent.set(this.#find(item), this.#find(first)); + this.#ambiguous.add(item); + } + } + } + async finish( + left: readonly MatchingFunction[], + right: readonly MatchingFunction[], + ): Promise { + const groups = new Map< + MatchingFunction, + { left: MatchingFunction[]; right: MatchingFunction[] } + >(); + for (const [side, items] of [ + ["left", left], + ["right", right], + ] as const) + for (const item of items) { + await this.checkpoint(); + if (this.consumed.has(item)) continue; + if (!this.#ambiguous.has(item)) { + this.add({ + state: side === "left" ? "only_left" : "only_right", + basis: null, + left: side === "left" ? [item] : [], + right: side === "right" ? [item] : [], + reason: + "No exact correspondence in the supplied observed inventories. This is not proof of source addition/removal: renaming, inlining, stripping, byte changes or omitted functions can explain it.", + }); + continue; + } + const root = this.#find(item); + let group = groups.get(root); + if (group === undefined) + groups.set(root, (group = { left: [], right: [] })); + group[side].push(item); + } + for (const group of groups.values()) + this.add({ + state: "unresolved", + basis: null, + ...group, + reason: + "Repeated exact names/bytes, aliases or observed CFG/call/string fingerprints leave correspondence unresolved. Connected group membership does not imply pairwise equivalence. No candidate is forced into a pair.", + }); + } +} + +/** Match only exact unique facts, preserving every occurrence and all candidate ambiguity. */ +export const matchFunctionInventories = async ( + left: readonly MatchingFunction[], + right: readonly MatchingFunction[], + options: { + readonly identicalBytesComparable: boolean; + readonly checkpoint: NativeMatchingCheckpoint; + }, +): Promise<{ + matches: NativeFunctionMatch[]; + summary: NativeFunctionMatching["summary"]; +}> => { + const names: PairingIndexes = [ + index(left, (x) => x.symbol), + index(right, (x) => x.symbol), + ]; + const bytes: PairingIndexes = [ + index(left, (x) => + options.identicalBytesComparable ? x.fingerprint : null, + ), + index(right, (x) => + options.identicalBytesComparable ? x.fingerprint : null, + ), + ]; + const structural: PairingIndexes = [ + index(left, (x) => x.member.structural_candidate_sha256), + index(right, (x) => x.member.structural_candidate_sha256), + ]; + const matcher = new InventoryMatcher(options.checkpoint); + await matcher.pairUnique(names, "exact-symbol"); + await matcher.pairUnique(bytes, "identical-bytes"); + await matcher.connectCandidates([names, bytes, structural]); + await matcher.finish(left, right); + const summary = { matched: 0, only_left: 0, only_right: 0, unresolved: 0 }; + for (const match of matcher.matches) summary[match.state]++; + return { matches: matcher.matches, summary }; +}; diff --git a/src/domain/native/nativeByteRead.ts b/src/domain/native/nativeByteRead.ts new file mode 100644 index 0000000000..031cb21032 --- /dev/null +++ b/src/domain/native/nativeByteRead.ts @@ -0,0 +1,22 @@ +import { z } from "zod"; + +/** Provider-owned bytes and exact coverage of one requested memory interval. */ +export const nativeByteReadSchema = z + .object({ + address: z.string(), + requested_bytes: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), + returned_bytes: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + bytes_hex: z.string().regex(/^(?:[a-f0-9]{2})*$/u), + complete: z.boolean(), + }) + .superRefine((value, context) => { + if ( + value.returned_bytes * 2 !== value.bytes_hex.length || + value.returned_bytes > value.requested_bytes || + value.complete !== (value.returned_bytes === value.requested_bytes) + ) + context.addIssue({ + code: "custom", + message: "Read-byte length and completeness disagree", + }); + }); diff --git a/src/domain/native/procedureInfo.ts b/src/domain/native/procedureInfo.ts new file mode 100644 index 0000000000..f06ea4f09a --- /dev/null +++ b/src/domain/native/procedureInfo.ts @@ -0,0 +1,18 @@ +import { z } from "zod"; +import { + functionBodySchema, + localVariableSchema, + procedureClassificationSchema, +} from "../hopperValues.js"; + +/** Provider-neutral procedure metadata, including exact body ranges. */ +export const procedureInfoSchema = z.object({ + name: z.string(), + entrypoint: z.string(), + basicblock_count: z.number().int().min(0), + length: z.number().min(0), + signature: z.string().nullable(), + locals: z.array(localVariableSchema), + classification: procedureClassificationSchema.nullable(), + body: functionBodySchema, +}); diff --git a/src/ghidra/GhidraFunctionValues.ts b/src/ghidra/GhidraFunctionValues.ts index 8de099c083..2f2f7e8e7b 100644 --- a/src/ghidra/GhidraFunctionValues.ts +++ b/src/ghidra/GhidraFunctionValues.ts @@ -1,3 +1,4 @@ +import { procedureInfoSchema } from "../domain/native/procedureInfo.js"; import { z } from "zod"; import { nativeFunctionAnnotationsInputSchema, @@ -181,13 +182,11 @@ const procedureReferences = z ), }) .strict(); -const procedureInfo = z - .object({ +const procedureInfo = procedureInfoSchema + .safeExtend({ name: z.string().min(1), entrypoint: ghidraCanonicalAddressSchema, - basicblock_count: z.number().int().min(0), length: z.number().int().min(0), - signature: z.string().nullable(), locals: z.array(localVariable), classification, body: ghidraFunctionBody, diff --git a/src/ghidra/GhidraInventoryValues.ts b/src/ghidra/GhidraInventoryValues.ts index 36ddeb5159..938dea74bb 100644 --- a/src/ghidra/GhidraInventoryValues.ts +++ b/src/ghidra/GhidraInventoryValues.ts @@ -1,3 +1,4 @@ +import { nativeByteReadSchema } from "../domain/native/nativeByteRead.js"; import { z } from "zod"; import { bytePatternFields, @@ -320,25 +321,9 @@ const resultSchemas = { }); }), inspect_native_load_image: nativeLoadImageObservationSchema, - read_bytes: z - .strictObject({ - address: canonicalAddress, - requested_bytes: z.number().int().min(1), - returned_bytes: z.number().int().min(0), - bytes_hex: z.string().regex(/^(?:[a-f0-9]{2})*$/u), - complete: z.boolean(), - }) - .superRefine((value, context) => { - if ( - value.returned_bytes * 2 !== value.bytes_hex.length || - value.returned_bytes > value.requested_bytes || - value.complete !== (value.returned_bytes === value.requested_bytes) - ) - context.addIssue({ - code: "custom", - message: "Read-byte length and completeness disagree", - }); - }), + read_bytes: nativeByteReadSchema + .safeExtend({ address: canonicalAddress }) + .strict(), address_to_file_offset: z.strictObject({ address: canonicalAddress, file_offset: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), diff --git a/src/server/registerFunctionComparisonTool.ts b/src/server/registerFunctionComparisonTool.ts index 405efd713d..15e14d5cdd 100644 --- a/src/server/registerFunctionComparisonTool.ts +++ b/src/server/registerFunctionComparisonTool.ts @@ -1,3 +1,4 @@ +import { resolvePairedEvidenceRequest } from "../application/EvidenceInputResolver.js"; import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import type { BinarySessionPort } from "../application/binary/BinarySessionPort.js"; @@ -38,11 +39,18 @@ export const registerFunctionComparisonTool = ( contract.name, context.mcpReq.signal, async () => { + const resolved = resolvePairedEvidenceRequest( + input, + (id) => + session.evidenceForAnalysis?.(id) ?? session.evidenceById(id), + ); + if (!resolved.ok) + return server.delivery.toCallToolResult(resolved, contract); let leftEvidence: Evidence; let rightEvidence: Evidence; try { - leftEvidence = parseEvidence(input.left); - rightEvidence = parseEvidence(input.right); + leftEvidence = parseEvidence(resolved.value.left); + rightEvidence = parseEvidence(resolved.value.right); } catch (cause: unknown) { return server.delivery.toCallToolResult( err( diff --git a/src/server/registerNativeFunctionMatchingTool.ts b/src/server/registerNativeFunctionMatchingTool.ts new file mode 100644 index 0000000000..9d27ed723b --- /dev/null +++ b/src/server/registerNativeFunctionMatchingTool.ts @@ -0,0 +1,65 @@ +import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; +import type { BinarySessionPort } from "../application/binary/BinarySessionPort.js"; +import { toolContract } from "../contracts/toolContracts.js"; +import { matchNativeFunctions } from "../application/native/NativeFunctionMatchingService.js"; +import { summarizeRetainedAnalysis } from "../application/analysisView/AnalysisViewService.js"; +import { withAdmittedAnalysis } from "./analysisAdmission.js"; +import { createEvidenceToolHandler } from "./contractToolHandler.js"; +import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; +import { recordSessionEvidenceSources } from "./sessionEvidence.js"; +import { matchNativeFunctionsInputSchema } from "../contracts/native/functionMatchingContracts.js"; +import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; +import { err, ok } from "../domain/result.js"; +import type { Logger } from "pino"; + +/** Bind one named contract to the same retained-Evidence workflow used by CLI. */ +export const registerNativeFunctionMatchingTool = ( + server: EvidenceMcpServer, + session: BinarySessionPort, + logger: Logger, +): void => { + const contract = toolContract("match_native_functions"); + const evidenceById = (id: string) => + session.evidenceForAnalysis?.(id) ?? session.evidenceById(id); + const recordEvidence: BinarySessionPort["recordEvidence"] = (evidence) => + session.recordEvidence(evidence); + const handler = createEvidenceToolHandler( + server, + async (input, options) => { + const parsed = matchNativeFunctionsInputSchema.safeParse(input); + if (!parsed.success) + return err( + analysisInputErrorFromIssues( + contract.name, + parsed.error.issues, + input, + ), + ); + const result = await matchNativeFunctions( + parsed.data, + options, + evidenceById, + ); + if (!result.ok) return result; + const recorded = recordSessionEvidenceSources( + recordEvidence, + result.value.sources, + ); + if (!recorded.ok) return recorded; + return parsed.data.detail === "summary" + ? summarizeRetainedAnalysis(result.value.evidence, { + evidenceById, + recordEvidence, + }) + : ok(result.value.evidence); + }, + logger, + recordEvidence, + withAdmittedAnalysis({ kind: "session", session }), + ); + server.registerTool( + contract.name, + toolRegistrationOptions(contract), + handler(contract), + ); +}; diff --git a/src/server/registerSessionTools.ts b/src/server/registerSessionTools.ts index dedc3ff590..68c6ddbaa6 100644 --- a/src/server/registerSessionTools.ts +++ b/src/server/registerSessionTools.ts @@ -1,3 +1,4 @@ +import { registerNativeFunctionMatchingTool } from "./registerNativeFunctionMatchingTool.js"; import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { workflowUnknownInput } from "../application/investigation/WorkflowUnknown.js"; @@ -375,6 +376,7 @@ export const registerSessionTools = ( registerProcessComparisonTool(server, session, compareContract); registerArtifactComparisonTool(server, session, compareArtifactsContract); registerFunctionComparisonTool(server, session, compareFunctionsContract); + registerNativeFunctionMatchingTool(server, session, logger); registerBundleComparisonTool( server, session, diff --git a/tests/acceptance/tools/nativeFunctionMatching.test.ts b/tests/acceptance/tools/nativeFunctionMatching.test.ts new file mode 100644 index 0000000000..dbab41cbd8 --- /dev/null +++ b/tests/acceptance/tools/nativeFunctionMatching.test.ts @@ -0,0 +1,276 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; +import { cliTest, type TestCli } from "../../support/cli/cliFixture.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { + nativeMatchingInventory, + nativeMatchingDetails, +} from "../../fixtures/nativeFunctionMatching.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { nativeMatchingProvider } from "../../fixtures/nativeFunctionMatching.js"; +import { + createEvidenceBundle, + parseEvidenceBundle, +} from "../../../src/domain/evidenceBundle.js"; +import { nativeFunctionMatchingSchema } from "../../../src/domain/native/functionMatching.js"; + +const createMatchingFixture = async () => { + const root = await createTestTempDirectory("rea-native-matching-"); + const ancestor = createEvidence(undefined, nativeMatchingProvider, { + operation: "fixture_provenance", + parameters: {}, + result: { source: "Synthetic ancestor" }, + }); + const unrelated = createEvidence(undefined, nativeMatchingProvider, { + operation: "fixture_unrelated", + parameters: {}, + result: { source: "Unselected synthetic record" }, + }); + const dependency = createEvidence(undefined, nativeMatchingProvider, { + operation: "fixture_dependency", + parameters: {}, + result: { source: "Synthetic dependency" }, + evidenceLinks: [ancestor.evidence_id], + }); + const left = nativeMatchingInventory( + "1", + [ + { address: "0x10", value: "main" }, + { address: "0x20", value: "old_name" }, + ], + [dependency.evidence_id], + ); + const right = nativeMatchingInventory("2", [ + { address: "0x30", value: "main" }, + { address: "0x40", value: "new_name" }, + ]); + const leftDetails = [ + ...nativeMatchingDetails("1", "0x10", "9000", true), + ...nativeMatchingDetails("1", "0x20", "c3", true), + ]; + const rightDetails = [ + ...nativeMatchingDetails("2", "0x30", "9001", true), + ...nativeMatchingDetails("2", "0x40", "c3", true), + ]; + const ancestorsPath = join(root, "ancestors.json"); + await writeFile( + ancestorsPath, + JSON.stringify(createEvidenceBundle([ancestor, unrelated])), + ); + const importedPath = join(root, "dependencies.json"); + await writeFile( + importedPath, + JSON.stringify(createEvidenceBundle([ancestor, dependency])), + ); + return { + root, + ancestor, + dependency, + unrelated, + left, + right, + leftDetails, + rightDetails, + ancestorsPath, + importedPath, + }; +}; +const verifyPortableCli = async ( + cli: TestCli, + fixture: Awaited>, + input: Record, + evidenceId: string, +) => { + const { + root, + ancestor, + dependency, + unrelated, + left, + right, + leftDetails, + rightDetails, + ancestorsPath, + } = fixture; + const bundlePath = join(root, "sources.json"), + inputPath = join(root, "request.json"), + outputPath = join(root, "result.json"); + await writeFile( + bundlePath, + JSON.stringify( + createEvidenceBundle([ + left, + right, + ...leftDetails, + ...rightDetails, + dependency, + ]), + ), + ); + const original = await readFile(bundlePath); + await writeFile(inputPath, JSON.stringify({ ...input, detail: "complete" })); + const cliFull = await cli.run({ + arguments: [ + "match-native-functions", + inputPath, + "--source_bundle", + ancestorsPath, + "--source_bundle", + bundlePath, + "--json", + ], + }); + expect(cliFull.exitCode, cliFull.stderr).toBe(0); + expect(parseEvidence(cliFull.json).evidence_id).toBe(evidenceId); + await writeFile(inputPath, JSON.stringify(input)); + const cliSummary = await cli.run({ + arguments: [ + "match-native-functions", + inputPath, + "--source_bundle", + ancestorsPath, + "--source_bundle", + bundlePath, + "--evidence_output", + outputPath, + "--json", + ], + }); + expect(cliSummary.exitCode, cliSummary.stderr).toBe(0); + expect(cliSummary.json).toMatchObject({ + summary: { + normalized_result: { parent_evidence_id: evidenceId }, + }, + evidence_bundle: { path: outputPath }, + }); + const bundle = parseEvidenceBundle( + JSON.parse(await readFile(outputPath, "utf8")), + ); + const ids = new Set(bundle.records.map((x) => x.evidence_id)); + for (const record of bundle.records) + for (const id of record.evidence_links) + expect( + ids.has(id), + `Missing dependency ${id} of ${record.evidence_id}`, + ).toBe(true); + expect(ids.has(ancestor.evidence_id)).toBe(true); + expect(ids.has(dependency.evidence_id)).toBe(true); + expect(ids.has(unrelated.evidence_id)).toBe(false); + expect(ids.has(evidenceId)).toBe(true); + expect(await readFile(bundlePath)).toEqual(original); + const incomplete = await cli.run({ + arguments: [ + "match-native-functions", + inputPath, + "--source_bundle", + bundlePath, + "--evidence_output", + join(root, "incomplete.json"), + "--json", + ], + }); + expect(incomplete.json).toMatchObject({ + code: "evidence_integrity_mismatch", + }); +}; + +cliTest( + "shared public CLI/MCP matching retains sources, pages exact facts, and feeds selected dossier pairs directly to compare_functions", + async ({ cli }) => { + const fixture = await createMatchingFixture(); + const { left, right, leftDetails, rightDetails, importedPath } = fixture; + const { call } = await connectLocalToolsMcp(); + const imported = await call("import_evidence_bundle", { + path: importedPath, + }); + expect(imported.isError, JSON.stringify(imported)).not.toBe(true); + const full = await call("match_native_functions", { + left, + right, + left_observations: leftDetails, + right_observations: rightDetails, + detail: "complete", + }); + expect(full.isError, JSON.stringify(full)).not.toBe(true); + const evidence = parseEvidence(full.structuredContent), + report = nativeFunctionMatchingSchema.parse(evidence.normalized_result); + expect(report.summary).toEqual({ + matched: 2, + only_left: 0, + only_right: 0, + unresolved: 0, + }); + const reference = (evidence_id: string) => ({ + kind: "retained-evidence", + evidence_id, + }); + const input = { + left: reference(left.evidence_id), + right: reference(right.evidence_id), + left_observations: leftDetails.map((x) => reference(x.evidence_id)), + right_observations: rightDetails.map((x) => reference(x.evidence_id)), + }; + const summary = await call("match_native_functions", input); + expect(summary.isError, JSON.stringify(summary)).not.toBe(true); + expect( + parseEvidence(summary.structuredContent).normalized_result, + ).toMatchObject({ + kind: "summary", + artifact: null, + parent_evidence_id: evidence.evidence_id, + summary: { summary: report.summary }, + }); + expect(parseEvidence(summary.structuredContent).locations).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: "artifact-path" }), + ]), + ); + const page = await call("inspect_analysis_view", { + source: reference(evidence.evidence_id), + view: { + kind: "function-matches", + collection: "functions", + sides: ["left"], + fields: ["name", "body_sha256"], + offset: 1, + limit: 1, + }, + }); + expect(page.isError, JSON.stringify(page)).not.toBe(true); + expect( + parseEvidence(page.structuredContent).normalized_result, + ).toMatchObject({ + artifact: null, + coverage: { total: 2, next_offset: null }, + items: [ + { + name: "old_name", + function_id: `${left.evidence_id}/1`, + address: "0x20", + }, + ], + }); + expect(parseEvidence(page.structuredContent).locations).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: "artifact-path" }), + ]), + ); + const compared = await call( + "compare_functions", + report.matches[0]!.compare_functions_input!, + ); + expect(compared.isError, JSON.stringify(compared)).not.toBe(true); + expect(parseEvidence(compared.structuredContent).operation).toBe( + "compare_functions", + ); + await verifyPortableCli(cli, fixture, input, evidence.evidence_id); + const missing = await call("match_native_functions", { + left: reference(`ev_${"a".repeat(64)}`), + right: reference(right.evidence_id), + }); + expect(missing.isError).toBe(true); + }, + 60000, +); diff --git a/tests/fixtures/nativeFunctionMatching.c b/tests/fixtures/nativeFunctionMatching.c new file mode 100644 index 0000000000..d7a062a958 --- /dev/null +++ b/tests/fixtures/nativeFunctionMatching.c @@ -0,0 +1,32 @@ +#include +#include +#ifndef VERSION +#define VERSION 1 +#endif +#define FN __attribute__((noinline, used)) +FN int stable(int x) { return x * 3 + 7; } +FN int changed(int x) { return x + VERSION * 5; } +#if VERSION == 1 +FN int old_behavior(int x) { puts("behavior marker"); return x + 7; } +FN int old_name(int x) { return x ^ 0x13579; } +FN int removed_observation(int x) { return x * 13 + 99; } +FN int formerly_separate(int x) { return x + 12345; } +#else +FN int new_behavior(int x) { puts("behavior marker"); return x + 8; } +FN int renamed(int x) { return x ^ 0x13579; } +FN int added_observation(int x) { return x * 17 + 77; } +static inline int formerly_separate(int x) { return x + 12345; } +#endif +FN int alias_one(int x) { return x + 42; } +FN int alias_two(int x) { return x + 42; } +int main(int argc, char **argv) { + int x = argc > 1 ? atoi(argv[1]) : 2; + int result = stable(x) + changed(x) + formerly_separate(x) + alias_one(x) + alias_two(x); +#if VERSION == 1 + result += old_name(x) + removed_observation(x) + old_behavior(x); +#else + result += renamed(x) + added_observation(x) + new_behavior(x); +#endif + printf("%d\n", result); + return 0; +} diff --git a/tests/fixtures/nativeFunctionMatching.ts b/tests/fixtures/nativeFunctionMatching.ts new file mode 100644 index 0000000000..f20dd77c00 --- /dev/null +++ b/tests/fixtures/nativeFunctionMatching.ts @@ -0,0 +1,101 @@ +import { createEvidence, type Evidence } from "../../src/domain/evidence.js"; +import { jsonValueSchema } from "../../src/domain/jsonValue.js"; +import { FUNCTION_COMPARISON_EXAMPLE } from "../../src/contracts/functionComparisonExample.js"; +import { functionDossierSchema } from "../../src/domain/hopperValues.js"; + +/** Explicit synthetic producer facts; they do not qualify any real provider. */ +export const nativeMatchingProvider = { + id: "matching-fixture", + name: "Synthetic native matching facts", + version: "1", +}; +export const nativeMatchingTarget = (side: "1" | "2") => ({ + path: `/tmp/matching-${side}.elf`, + sha256: side.repeat(64), + format: "elf" as const, + architecture: "x86_64" as const, +}); +export const nativeMatchingInventory = ( + side: "1" | "2", + entries: readonly { address: string; value: string }[], + evidenceLinks: readonly string[] = [], +) => + createEvidence(nativeMatchingTarget(side), nativeMatchingProvider, { + operation: "list_procedures", + parameters: {}, + result: jsonValueSchema.parse(entries), + evidenceLinks: [...evidenceLinks], + }); +export const nativeMatchingBody = (address: string, bytes: number) => ({ + available: true as const, + provenance: "explicit synthetic body", + ranges: [ + { + start: address, + end: `0x${(BigInt(address) + BigInt(bytes) - 1n).toString(16)}`, + }, + ], + total_bytes: bytes, + span_bytes: bytes, + non_contiguous: false, + contains_entry: true, +}); +export const nativeMatchingDetails = ( + side: "1" | "2", + address: string, + hex: string, + dossier = false, +): Evidence[] => { + const target = nativeMatchingTarget(side), + body = nativeMatchingBody(address, hex.length / 2); + const procedure = { + name: "fixture", + entrypoint: address, + basicblock_count: 1, + length: hex.length / 2, + signature: null, + locals: [], + classification: null, + body, + }; + let observation: Evidence; + if (dossier) { + const value = functionDossierSchema.parse( + FUNCTION_COMPARISON_EXAMPLE.left.normalized_result, + ); + value.procedure = { + address, + name: "fixture", + classification: null, + body, + signature: null, + locals: [], + }; + value.basic_blocks = []; + observation = createEvidence(target, nativeMatchingProvider, { + operation: "analyze_function", + parameters: { procedure: address }, + result: jsonValueSchema.parse(value), + relationshipCoverage: { status: "complete" }, + }); + } else + observation = createEvidence(target, nativeMatchingProvider, { + operation: "procedure_info", + parameters: { procedure: address }, + result: procedure, + }); + return [ + observation, + createEvidence(target, nativeMatchingProvider, { + operation: "read_bytes", + parameters: { address, length: hex.length / 2 }, + result: { + address, + requested_bytes: hex.length / 2, + returned_bytes: hex.length / 2, + bytes_hex: hex, + complete: true, + }, + }), + ]; +};