diff --git a/bridge/hopper_bridge.py b/bridge/hopper_bridge.py index cb0e9c6dc6..0c1da8a57e 100644 --- a/bridge/hopper_bridge.py +++ b/bridge/hopper_bridge.py @@ -454,11 +454,51 @@ def _procedure_map(document): def _strings(document): - result = {} - for segment in document.getSegmentsList(): - for value, address in segment.getStringsList(): - result[_hex(address)] = value - return result + return {address: record["value"] for address, record in _string_inventory(document).items()} + + +def _string_record(segment, address, display): + """Read one native typed object, retaining its shortened display separately.""" + length = segment.getObjectLength(address) + end = segment.getStartingAddress() + segment.getLength() + if isinstance(length, bool) or not isinstance(length, int) or length <= 0 or length > end - address: + raise CapabilityUnavailableError("Hopper reported an invalid string object extent at %s" % _hex(address)) + raw = segment.readBytes(address, length) + if not isinstance(raw, (bytes, bytearray)) or len(raw) != length: + raise CapabilityUnavailableError("Hopper could not read the complete string object at %s" % _hex(address)) + kind = segment.getTypeAtAddress(address) + encodings = ("utf-8", "latin-1") if kind == segment.TYPE_ASCII else ("utf-16-le", "utf-16-be") if kind == segment.TYPE_UNICODE else () + # Native display escaping is not invertible (a literal backslash-n and a + # newline can render alike). Decode byte strings directly; for UTF-16, + # compare rendered candidates to establish an unambiguous byte order. + candidates = [] + for encoding in encodings: + terminator = b"\x00\x00" if encoding.startswith("utf-16") else b"\x00" + terminated = raw.endswith(terminator) + payload = raw[:-len(terminator)] if terminated else raw + try: + value = payload.decode(encoding) + except UnicodeDecodeError: + continue + record = {"value": value, "provider_value": display, "string": {"encoding": encoding, "encoding_status": "inferred", "termination": "present_or_not_required" if terminated else "missing", "byte_length": length}} + if kind == segment.TYPE_ASCII: + return record + rendered = value.replace('"', '\\"').replace("\n", "\\n").replace("\r", "\\r").replace("\t", "\\t") + if rendered == display or (display.endswith("\u2026") and display[:-1] and rendered.startswith(display[:-1])): + candidates.append(record) + if candidates and len({candidate["value"] for candidate in candidates}) == 1: + return candidates[0] + raise CapabilityUnavailableError("Hopper's string display cannot be reconciled with its typed bytes at %s; inspect the bytes and encoding explicitly" % _hex(address)) + + +def _string_inventory(document): + key = (id(document), "string_records") + inventory = _search_inventory_cache.get(key) + if inventory is None: + records = {_hex(address): _string_record(segment, address, display) for segment in document.getSegmentsList() for display, address in segment.getStringsList()} + inventory = dict(sorted(records.items(), key=lambda item: int(item[0], 16))) + _search_inventory_cache[key] = inventory + return inventory def _invalidate_search_inventory(document): @@ -529,7 +569,7 @@ def _search_results(document, kind, params): if not matches(item[1]): continue selected.append(item) - return [ + return [{"address": address, **_string_inventory(document)[address]} for address, _ in selected] if kind == "string" else [ { "address": address, "value": value, @@ -894,12 +934,12 @@ def _dispatch(method, params): if method == "list_procedures": return [{"address": address, "value": value} for address, value in _search_inventory(document, "procedure")] if method == "list_strings": - values = dict(_search_inventory(document, "string")) + values = _string_inventory(document) requested = params.get("address") if requested is not None: key = _hex(_address(document, requested)) values = {key: values[key]} if key in values else {} - return [{"address": address, "value": value} for address, value in values.items()] + return [{"address": address, **record} for address, record in values.items()] if method == "list_names": result = dict(_search_inventory(document, "name")) requested = params.get("address") diff --git a/docs/installation.md b/docs/installation.md index 62e07ee768..459d2d1ce4 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -341,6 +341,11 @@ code-unit limit. Oversized names fail before any batch edits; bookmarks and literal string results are not subject to that symbol-name limit. Rename success requires exact final readback. New bookmarks must point into mapped memory; existing legacy bookmarks outside it can still be removed. +String results read each native typed object's complete bytes, retain the original +provider display in `provider_value`, and report its encoding, byte length, and +termination. `encoding_status: inferred` distinguishes REA's decoding from an +observed source encoding. Hopper can split long literals into adjacent +unterminated objects; search matches each object's decoded bytes independently. Closing or switching a target closes its bound Hopper document, shuts down REA's bridge and removes its temporary socket directory while preserving the Hopper diff --git a/docs/testing.md b/docs/testing.md index a58381590a..2c2cbab541 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -203,6 +203,14 @@ and whitespace. Disposable binary copies prove that switching and closing actually removes the native document, and that CLI byte, function-dossier, literal-trace results and invalid-address diagnostics agree with MCP. No provider is mocked in this lane. +Real search probes cover Unicode names, literal metacharacters, case and regex +modes, annotation cache invalidation, complete native fragments of long literals +checked against byte reads, escaped UTF-8/control text, byte-preserving Latin-1 +decoding, and Hopper's UTF-16 symbol-name truncation boundary. When the macOS +Objective-C fixture is present, the lane also verifies native UTF-16 string objects +and their inferred encodings against their actual bytes. +Unmapped annotation destinations and +oversized later batch names fail before any earlier edit is applied. The Linux demo lane remains a separate `verify:hopper:linux` command. `verify:hopper:fat` is a separate macOS lane requiring installed Hopper and the diff --git a/scripts/lib/real-hopper-boundaries.mjs b/scripts/lib/real-hopper-boundaries.mjs index f98d25e8ae..c53044bd47 100644 --- a/scripts/lib/real-hopper-boundaries.mjs +++ b/scripts/lib/real-hopper-boundaries.mjs @@ -9,6 +9,7 @@ import addFormats from "ajv-formats"; import { mcpTextValue, requireMcpResult } from "./mcp-verifier-results.mjs"; import { verifyHopperNavigationAndText } from "./real-hopper-navigation.mjs"; import { verifyHopperWorkflows } from "./real-hopper-workflows.mjs"; +import { verifyHopperSearch } from "./real-hopper-search.mjs"; /** Exercise real Hopper annotation and malformed-input boundaries over MCP. */ export async function verifyHopperBoundaryContracts( @@ -74,6 +75,7 @@ export async function verifyHopperBoundaryContracts( address, ); const workflows = await verifyHopperWorkflows(call, invalid, procedure); + const search = await verifyHopperSearch(call, invalid, procedure); try { if (foreignDocument !== undefined) { await invalid( @@ -278,6 +280,7 @@ export async function verifyHopperBoundaryContracts( foreignDocumentVerified: foreignDocument !== undefined, navigationAndText, workflows, + search, }; } diff --git a/scripts/lib/real-hopper-fixture.mjs b/scripts/lib/real-hopper-fixture.mjs index f6b9376990..8d8ad67a54 100644 --- a/scripts/lib/real-hopper-fixture.mjs +++ b/scripts/lib/real-hopper-fixture.mjs @@ -21,12 +21,26 @@ export async function loadRealHopperFixtureTargets(manifestPath) { bindFixtureArtifact(root, large), ], ); - requireDistinctArtifacts([primaryArtifact, secondaryArtifact, largeArtifact]); + const objc = manifest.fixtures.find((fixture) => fixture?.name === "objc"); + const unicodeArtifact = + objc === undefined + ? undefined + : await bindFixtureArtifact( + root, + fixtureEntry(manifest.fixtures, "objc"), + ); + requireDistinctArtifacts([ + primaryArtifact, + secondaryArtifact, + largeArtifact, + ...(unicodeArtifact === undefined ? [] : [unicodeArtifact]), + ]); return { manifestPath: canonicalManifest, primary: primaryArtifact, secondary: secondaryArtifact, large: largeArtifact, + ...(unicodeArtifact === undefined ? {} : { unicode: unicodeArtifact }), oracle, largeOracle, }; diff --git a/scripts/lib/real-hopper-search.mjs b/scripts/lib/real-hopper-search.mjs new file mode 100644 index 0000000000..349958814a --- /dev/null +++ b/scripts/lib/real-hopper-search.mjs @@ -0,0 +1,205 @@ +import assert from "node:assert/strict"; + +/** Exercise literal/regex search, Unicode identities and cache invalidation in Hopper. */ +export async function verifyHopperSearch(call, invalid, procedure) { + const address = procedure.address; + const original = await call("address_name", { address }); + try { + await verifySymbolNames(call, invalid, address); + await verifySearchModes(call, address); + } finally { + await call("set_address_name", { address, name: original }); + } + assert.deepEqual( + await call("search_procedures", { + pattern: original, + mode: "literal", + case_sensitive: true, + }), + [{ address, value: original }], + ); + return { + unicodeIdentity: true, + literalMetacharacters: true, + longValuePreserved: true, + symbolUtf16LimitValidatedBeforeMutation: true, + regexAndCaseModes: true, + annotationCacheInvalidation: true, + escapedUtf8AndByteStrings: true, + }; +} + +async function verifySymbolNames(call, invalid, address) { + const unicode = "REA_é_e\u0301_😀_Straße_ſ_İ"; + const extentNames = ["x".repeat(1024), "é".repeat(1024), "😀".repeat(512)]; + for (const name of ["REA_a.b[0]", unicode, ...extentNames]) { + assert.equal(await call("set_address_name", { address, name }), true); + assert.equal(await call("address_name", { address }), name); + assert.deepEqual(await call("list_names", { address }), [ + { address, value: name }, + ]); + assert.deepEqual( + await call("search_procedures", { + pattern: name, + mode: "literal", + case_sensitive: true, + }), + [{ address, value: name }], + ); + assert.equal(await call("procedure_address", { procedure: name }), address); + } + assert.deepEqual( + await call("search_procedures", { pattern: unicode, mode: "literal" }), + [], + ); + const other = (await call("list_procedures")).find( + (item) => item.address !== address, + ); + assert.ok(other); + for (const name of ["x".repeat(1025), "é".repeat(1025), "😀".repeat(513)]) { + await invalid( + "set_address_name", + { address, name }, + /UTF-16 code-unit symbol-name limit/u, + ); + await invalid( + "set_addresses_names", + { names: { [other.address]: "must_not_apply", [address]: name } }, + /UTF-16 code-unit symbol-name limit/u, + ); + assert.equal(await call("address_name", { address }), extentNames.at(-1)); + assert.equal( + await call("address_name", { address: other.address }), + other.value, + ); + } +} + +async function verifySearchModes(call, address) { + await call("set_address_name", { address, name: "REA_Search_Upper" }); + assert.deepEqual( + await call("search_procedures", { + pattern: "rea_search_upper", + mode: "literal", + case_sensitive: false, + }), + [{ address, value: "REA_Search_Upper" }], + ); + assert.deepEqual( + await call("search_procedures", { + pattern: "rea_search_upper", + mode: "literal", + case_sensitive: true, + }), + [], + ); + assert.deepEqual( + await call("search_procedures", { + pattern: "^REA_Search_(?:Upper|Lower)$", + mode: "regex", + case_sensitive: true, + }), + [{ address, value: "REA_Search_Upper" }], + ); + const strings = await call("list_strings"); + const long = "REA_LONG_LITERAL_" + "x".repeat(4096) + "needle"; + const longItem = strings.find((item) => + item.value.startsWith("REA_LONG_LITERAL_"), + ); + assert.ok(longItem, "fixture omitted its real long literal"); + assert.ok(long.startsWith(longItem.value)); + assert.ok(longItem.provider_value.endsWith("…")); + assert.equal(longItem.string.termination, "missing"); + const fragments = strings.filter((item) => { + const offset = BigInt(item.address) - BigInt(longItem.address); + return offset >= 0n && offset < BigInt(Buffer.byteLength(long)); + }); + assert.equal(fragments.map((item) => item.value).join(""), long); + for (const fragment of fragments) { + assert.equal( + Buffer.byteLength(fragment.value), + fragment.string.byte_length - + (fragment.string.termination === "missing" ? 0 : 1), + ); + const bytes = await call("read_bytes", { + address: fragment.address, + length: fragment.string.byte_length, + }); + const expected = Buffer.from( + fragment.value + (fragment.string.termination === "missing" ? "" : "\0"), + ); + assert.equal(bytes.bytes_hex, expected.toString("hex")); + } + assert.deepEqual( + await call("search_strings", { + pattern: "needle", + mode: "literal", + case_sensitive: true, + }), + [fragments.at(-1)], + ); + const expected = strings.filter((item) => + /^REA_C_(?:ENTRY|LEAF)$/u.test(item.value), + ); + assert.equal(expected.length, 2); + assert.deepEqual( + await call("search_strings", { + pattern: "^REA_C_(?:ENTRY|LEAF)$", + mode: "regex", + case_sensitive: true, + }), + expected, + ); + await verifyHopperStringObjects(call, [ + { value: "REA_UTF8_é_😀", encoding: "utf-8" }, + { value: 'REA_ESCAPED_"\\line\nend\t\r', encoding: "utf-8" }, + { value: "REA_LITERAL_BACKSLASH_\\n", encoding: "utf-8" }, + { value: "REA_LITERAL_…", encoding: "utf-8" }, + { value: "REA_LATIN1_ÿ", encoding: "latin-1" }, + ]); +} + +/** Compare native string identities and values with the source fixture's bytes. */ +export async function verifyHopperStringObjects(call, expectations) { + const strings = await call("list_strings"); + for (const { value, encoding } of expectations) { + const item = strings.find((candidate) => candidate.value === value); + assert.ok( + item, + `Hopper omitted the decoded string ${JSON.stringify(value)}`, + ); + assert.equal(item.string.encoding, encoding); + assert.equal(item.string.encoding_status, "inferred"); + assert.equal(item.string.termination, "present_or_not_required"); + const expectedBytes = Buffer.from( + value + "\0", + encoding.replaceAll("-", ""), + ); + assert.equal(item.string.byte_length, expectedBytes.length); + const bytes = await call("read_bytes", { + address: item.address, + length: expectedBytes.length, + }); + assert.equal(bytes.bytes_hex, expectedBytes.toString("hex")); + assert.deepEqual(await call("list_strings", { address: item.address }), [ + item, + ]); + assert.deepEqual( + await call("search_strings", { + pattern: value, + mode: "literal", + case_sensitive: true, + }), + [item], + ); + } + assert.deepEqual( + await call("search_strings", { pattern: "\\xC3\\xA9", mode: "literal" }), + [], + ); + return { + sourceBytesVerified: true, + inferredEncodingReported: true, + displayEscapesExcludedFromMatching: true, + }; +} diff --git a/scripts/verify-real-hopper.mjs b/scripts/verify-real-hopper.mjs index 6920f2657f..4277ef1fa6 100644 --- a/scripts/verify-real-hopper.mjs +++ b/scripts/verify-real-hopper.mjs @@ -15,6 +15,7 @@ import { HOPPER_PROVIDER_IDENTITY } from "../dist/hopper/HopperProvider.js"; import { HOPPER_TARGET_LEASE_DIRECTORY } from "../dist/hopper/HopperTargetLease.js"; import { REA_WORKFLOW_PROVIDER } from "../dist/application/InvestigationProviders.js"; import { loadRealHopperFixtureTargets } from "./lib/real-hopper-fixture.mjs"; +import { verifyHopperStringObjects } from "./lib/real-hopper-search.mjs"; import { requireBridgeDiagnostic, requireBridgeProgress, @@ -463,6 +464,21 @@ try { options, ); if (closed.isError === true) throw new Error(textValue(closed)); + let unicodeStrings = null; + if (fixtureTargets.unicode !== undefined) { + const call = async (name, args = {}) => + requireSuccessfulTool( + await client.callTool({ name, arguments: args }, options), + name, + ); + await call("open_binary", { path: fixtureTargets.unicode.path }); + unicodeStrings = await verifyHopperStringObjects(call, [ + { value: "REA_UTF16_é_😀", encoding: "utf-16-le" }, + { value: 'REA_UTF16_ESCAPED_é_"\\line\nend\t\r', encoding: "utf-16-le" }, + { value: "\nREA_UTF16_LEADING_é", encoding: "utf-16-le" }, + ]); + await call("close_binary"); + } const lifecycleAndCli = await verifyHopperLifecycleAndCli(client, options, { primary: targetA, secondary: targetB, @@ -492,6 +508,7 @@ try { segmentCount: firstOverview.segment_count, analyses: [firstAnalysis, secondAnalysis], fixtureAnalysis, + unicodeStrings, boundaryContracts, lifecycleAndCli, inventoryCounts, diff --git a/src/contracts/officialToolContracts.ts b/src/contracts/officialToolContracts.ts index 0377f4b61c..b45221311f 100644 --- a/src/contracts/officialToolContracts.ts +++ b/src/contracts/officialToolContracts.ts @@ -126,7 +126,7 @@ export const OFFICIAL_TOOL_CONTRACTS = [ ), official( "list_strings", - "List every provider-defined string, or filter to one address, as address/value pairs. Ghidra also reports encoding, terminator status, and byte length.", + "List every provider-defined string object, or filter to one address, as address/value pairs with encoding, terminator status, and byte length when available. Hopper reads complete typed objects and retains its original display text as provider_value; long literals may span adjacent unterminated objects.", z.object({ document, address: optionalAddress }), ), official( @@ -198,7 +198,7 @@ export const OFFICIAL_TOOL_CONTRACTS = [ ), official( "search_strings", - "Search every analyzed string using literal matching by default or regex when requested. Results are deterministic and complete.", + "Search every analyzed string object using literal matching by default or regex when requested. Results are deterministic and complete over the native inventory. Hopper searches complete typed bytes rather than shortened displays; adjacent fragments of a long literal are matched independently, with their termination and original display retained.", z.object(analysisSearchInput), ), official( diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index b60608ae16..202c6b0978 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -97,9 +97,21 @@ const bookmarkFacetSchema = z.discriminatedUnion("state", [ const addressedString = z.object({ address: z.string(), value: z.string(), + provider_value: z + .string() + .exactOptional() + .describe( + "Original provider display text, which may shorten the typed object.", + ), string: z .object({ encoding: z.string().min(1), + encoding_status: z + .enum(["observed", "inferred"]) + .exactOptional() + .describe( + "Whether the provider declares the encoding or REA infers it from the typed bytes and native display. This does not establish the original source's intended encoding.", + ), termination: z.enum(["missing", "present_or_not_required"]), byte_length: z.number().int().min(0), }) @@ -197,9 +209,7 @@ export const officialOutputSchemas: Readonly> = { search_procedures: resultOf( z.array(z.object({ address: z.string(), value: z.string() })), ), - search_strings: resultOf( - z.array(z.object({ address: z.string(), value: z.string() })), - ), + search_strings: resultOf(z.array(addressedString)), set_address_name: resultOf(z.boolean()), set_addresses_names: resultOf(z.record(z.string(), z.boolean())), set_bookmark: resultOf(z.boolean()), diff --git a/src/hopper/HopperProviderCapabilities.ts b/src/hopper/HopperProviderCapabilities.ts index 105bec8703..c803962948 100644 --- a/src/hopper/HopperProviderCapabilities.ts +++ b/src/hopper/HopperProviderCapabilities.ts @@ -79,6 +79,12 @@ export const CAPABILITIES: readonly CapabilityDescriptor[] = Object.freeze( }), limitations: Object.freeze([ "Results depend on Hopper's completed static analysis.", + ...(operation === "list_strings" || operation === "search_strings" + ? [ + "String values cover native typed objects. Hopper may split long literals into adjacent unterminated fragments; string searches match each object independently.", + "Reported string encodings are inferred from typed bytes and native display; the original source's intended encoding is unknown.", + ] + : []), ]), }), ), diff --git a/tests/boundary/providers/hopper/bridgeSearch.test.ts b/tests/boundary/providers/hopper/bridgeSearch.test.ts deleted file mode 100644 index 2f0d0ce986..0000000000 --- a/tests/boundary/providers/hopper/bridgeSearch.test.ts +++ /dev/null @@ -1,285 +0,0 @@ -import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; -import { fileURLToPath } from "node:url"; - -import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; -import { z } from "zod"; - -const bridgePath = fileURLToPath( - new URL("../../../../bridge/hopper_bridge.py", import.meta.url), -); -const probePath = fileURLToPath( - new URL("../../../fixtures/bridgeSearchProbe.py", import.meta.url), -); - -const searchResultSchema = z.object({ - action: z.literal("search"), - ok: z.literal(true), - result: z.array(z.object({ address: z.string(), value: z.string() })), -}); -const probeResultSchema = searchResultSchema; -type ProbeResult = z.infer; - -type ProbeInput = { - readonly action: "search"; - readonly items: readonly (readonly [string, string])[]; - readonly params: Readonly>; -}; - -interface PendingProbe { - readonly resolve: (result: ProbeResult) => void; - readonly reject: (error: Error) => void; - timeout: NodeJS.Timeout | undefined; -} - -const PROBE_TIMEOUT_MS = 3_000; -const PROBE_BUFFER_LIMIT = 1_024 * 1_024; -const pendingProbes: PendingProbe[] = []; -let probeProcess: ChildProcessWithoutNullStreams | undefined; -let probeStdout = ""; -let probeStderr = ""; -let probeFailure: Error | undefined; -let stoppingProbe = false; - -const failureWithStderr = (message: string): Error => { - const stderr = probeStderr.trim(); - return new Error(stderr.length === 0 ? message : `${message}: ${stderr}`); -}; - -const rejectPendingProbes = (error: Error): void => { - for (const pending of pendingProbes.splice(0)) { - if (pending.timeout !== undefined) clearTimeout(pending.timeout); - pending.reject(error); - } -}; - -const failProbeProcess = (error: Error): void => { - probeFailure ??= error; - rejectPendingProbes(probeFailure); - if ( - !stoppingProbe && - probeProcess?.exitCode === null && - probeProcess.signalCode === null - ) - probeProcess.kill("SIGKILL"); -}; - -const acceptProbeLine = (line: string): void => { - const pending = pendingProbes.shift(); - if (pending === undefined) { - failProbeProcess( - failureWithStderr("Python probe returned an extra result"), - ); - return; - } - if (pending.timeout !== undefined) clearTimeout(pending.timeout); - try { - const decoded: unknown = JSON.parse(line); - pending.resolve(probeResultSchema.parse(decoded)); - } catch (cause: unknown) { - const error = - cause instanceof Error - ? cause - : failureWithStderr("Python probe returned an invalid result"); - pending.reject(error); - failProbeProcess(error); - } -}; - -const acceptProbeOutput = (chunk: string): void => { - probeStdout += chunk; - if (Buffer.byteLength(probeStdout) > PROBE_BUFFER_LIMIT) { - failProbeProcess(failureWithStderr("Python probe output exceeded 1 MiB")); - return; - } - let newline = probeStdout.indexOf("\n"); - while (newline >= 0) { - const line = probeStdout.slice(0, newline).replace(/\r$/u, ""); - probeStdout = probeStdout.slice(newline + 1); - acceptProbeLine(line); - newline = probeStdout.indexOf("\n"); - } -}; - -const startProbeProcess = async (): Promise => { - stoppingProbe = false; - probeFailure = undefined; - probeStdout = ""; - probeStderr = ""; - const child = spawn("python3", [probePath, bridgePath], { - stdio: ["pipe", "pipe", "pipe"], - }); - probeProcess = child; - child.stdout.setEncoding("utf8"); - child.stderr.setEncoding("utf8"); - child.stdout.on("data", acceptProbeOutput); - child.stderr.on("data", (chunk: string) => { - probeStderr += chunk; - if (Buffer.byteLength(probeStderr) > PROBE_BUFFER_LIMIT) - failProbeProcess(new Error("Python probe stderr exceeded 1 MiB")); - }); - child.stdin.on("error", (error) => { - if (!stoppingProbe) failProbeProcess(error); - }); - child.on("error", (error) => { - if (!stoppingProbe) failProbeProcess(error); - }); - child.on("exit", (code, signal) => { - if (!stoppingProbe) - failProbeProcess( - failureWithStderr( - `Python probe exited before responding (code ${String(code)}, signal ${String(signal)})`, - ), - ); - }); - await new Promise((resolve, reject) => { - const onSpawn = (): void => { - child.off("error", onError); - resolve(); - }; - const onError = (error: Error): void => { - child.off("spawn", onSpawn); - reject(error); - }; - child.once("spawn", onSpawn); - child.once("error", onError); - }); -}; - -const hasExited = (child: ChildProcessWithoutNullStreams): boolean => - child.exitCode !== null || child.signalCode !== null; - -const waitForExit = ( - child: ChildProcessWithoutNullStreams, - timeoutMs: number, -): Promise => { - if (hasExited(child)) return Promise.resolve(true); - return new Promise((resolve) => { - const onExit = (): void => { - clearTimeout(timeout); - resolve(true); - }; - const timeout = setTimeout(() => { - child.off("exit", onExit); - resolve(false); - }, timeoutMs); - child.once("exit", onExit); - }); -}; - -const stopProbeProcess = async (): Promise => { - const child = probeProcess; - if (child === undefined) return; - stoppingProbe = true; - rejectPendingProbes(new Error("Python probe stopped before responding")); - if (!hasExited(child)) child.stdin.end(); - if (!(await waitForExit(child, 500))) child.kill("SIGTERM"); - if (!(await waitForExit(child, 500))) child.kill("SIGKILL"); - if (!(await waitForExit(child, 500))) - throw new Error("Python probe did not stop after SIGKILL"); - probeProcess = undefined; -}; - -const probe = (input: ProbeInput): Promise => { - const child = probeProcess; - if (probeFailure !== undefined) return Promise.reject(probeFailure); - if (child === undefined || hasExited(child)) - return Promise.reject(new Error("Python probe is not running")); - return new Promise((resolve, reject) => { - const pending: PendingProbe = { resolve, reject, timeout: undefined }; - pending.timeout = setTimeout(() => { - failProbeProcess( - new Error( - `Python probe request timed out after ${String(PROBE_TIMEOUT_MS)} ms`, - ), - ); - }, PROBE_TIMEOUT_MS); - pendingProbes.push(pending); - child.stdin.write(`${JSON.stringify(input)}\n`, (error) => { - if (error !== null && error !== undefined) failProbeProcess(error); - }); - }); -}; - -beforeAll(startProbeProcess); -beforeEach(async () => { - if ( - probeFailure === undefined && - probeProcess !== undefined && - !hasExited(probeProcess) - ) - return; - await stopProbeProcess(); - await startProbeProcess(); -}); -afterAll(stopProbeProcess); - -describe("Hopper bridge search", () => { - it("returns matching literal values in deterministic order", async () => { - await expect( - probe({ - action: "search", - items: [ - ["0x1000", "REA_main_1"], - ["0x2000", "unrelated"], - ["0x3000", "rea_helper_42"], - ], - params: { - pattern: "REA", - mode: "literal", - case_sensitive: false, - }, - }), - ).resolves.toEqual({ - action: "search", - ok: true, - result: [ - { address: "0x1000", value: "REA_main_1" }, - { address: "0x3000", value: "rea_helper_42" }, - ], - }); - }); - - it("returns matching entries for regex searches", async () => { - await expect( - probe({ - action: "search", - items: [ - ["0x1000", "REA_GHIDRA_INVENTORY_ENTRY"], - ["0x2000", "unrelated"], - ["0x3000", "REA_GHIDRA_LEAF_VALUE"], - ], - params: { - pattern: "^REA_GHIDRA_(?:INVENTORY_ENTRY|LEAF_VALUE)$", - mode: "regex", - case_sensitive: true, - }, - }), - ).resolves.toEqual({ - action: "search", - ok: true, - result: [ - { address: "0x1000", value: "REA_GHIDRA_INVENTORY_ENTRY" }, - { address: "0x3000", value: "REA_GHIDRA_LEAF_VALUE" }, - ], - }); - }); - - it("returns the complete value for a long literal match", async () => { - const value = `${"x".repeat(4_096)}needle`; - await expect( - probe({ - action: "search", - items: [["0x1000", value]], - params: { - pattern: "needle", - mode: "literal", - case_sensitive: true, - }, - }), - ).resolves.toMatchObject({ - action: "search", - ok: true, - result: [{ address: "0x1000", value }], - }); - }); -}); diff --git a/tests/boundary/providers/hopper/hopperBridgeFacade.test.ts b/tests/boundary/providers/hopper/hopperBridgeFacade.test.ts index 5544ea53f9..62975437df 100644 --- a/tests/boundary/providers/hopper/hopperBridgeFacade.test.ts +++ b/tests/boundary/providers/hopper/hopperBridgeFacade.test.ts @@ -48,7 +48,6 @@ const probeResultSchema = z.strictObject({ reason: z.string(), }), }), - strings: z.strictObject({ "0x401234": z.literal("fixture string") }), procedure_references: z.strictObject({ procedure: z.strictObject({ address: z.literal("0x401000"), @@ -64,14 +63,6 @@ const probeResultSchema = z.strictObject({ unresolved_calls: z.array(z.unknown()), references: z.array(z.unknown()), }), - inventory_replies: z.array( - z.strictObject({ - id: z.number().int(), - result: z.array( - z.strictObject({ address: z.string(), value: z.string() }), - ), - }), - ), provider_faults: z.array( z.strictObject({ id: z.literal(1), @@ -198,28 +189,6 @@ describe("Hopper API facade", () => { }, { id: 9, result: "0x401000" }, ]); - expect(result.inventory_replies).toEqual([ - { - id: 1, - result: [ - { address: "0x2", value: "string-2" }, - { address: "0x10", value: "string-16" }, - { address: "0x100", value: "string-256" }, - ], - }, - { - id: 2, - result: [ - { address: "0x2", value: "name-2" }, - { address: "0x10", value: "name-16" }, - { address: "0x100", value: "name-256" }, - ], - }, - { id: 3, result: [{ address: "0x10", value: "string-16" }] }, - { id: 4, result: [{ address: "0x10", value: "name-16" }] }, - { id: 5, result: [] }, - { id: 6, result: [] }, - ]); expect(stdout).not.toContain("supersecret"); expect(result.analysis_guard.message).toContain( "requires completed Hopper background analysis", diff --git a/tests/conformance/c/fixture.c b/tests/conformance/c/fixture.c index 07dd3a57b7..66d4854366 100644 --- a/tests/conformance/c/fixture.c +++ b/tests/conformance/c/fixture.c @@ -2,6 +2,18 @@ volatile int rea_c_global = 7; +#define REA_LONG_CHUNK "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +#define REA_LONG_256 REA_LONG_CHUNK REA_LONG_CHUNK REA_LONG_CHUNK REA_LONG_CHUNK +#define REA_LONG_1024 REA_LONG_256 REA_LONG_256 REA_LONG_256 REA_LONG_256 +__attribute__((noinline, used)) void rea_long_literal_procedure(void) { + puts("REA_LONG_LITERAL_" REA_LONG_1024 REA_LONG_1024 REA_LONG_1024 REA_LONG_1024 "needle"); + puts("REA_UTF8_é_😀"); + puts("REA_ESCAPED_\"\\line\nend\t\r"); + puts("REA_LITERAL_BACKSLASH_\\n"); + puts("REA_LITERAL_…"); + puts("REA_LATIN1_\xff"); +} + __attribute__((noinline, used)) int rea_leaf(int value) { puts("REA_C_LEAF"); return value + rea_c_global; diff --git a/tests/conformance/objc/fixture.m b/tests/conformance/objc/fixture.m index bc9f5ea2d6..f3cc07b979 100644 --- a/tests/conformance/objc/fixture.m +++ b/tests/conformance/objc/fixture.m @@ -16,6 +16,9 @@ int main(void) { @autoreleasepool { Protocol *protocol = @protocol(REAWidgetDelegate); NSLog(@"%@", NSStringFromProtocol(protocol)); + NSLog(@"REA_UTF16_é_😀"); + NSLog(@"REA_UTF16_ESCAPED_é_\"\\line\nend\t\r"); + NSLog(@"\nREA_UTF16_LEADING_é"); return [[[REAWidget alloc] init] performAction:@"run" error:0] ? 0 : 1; } } diff --git a/tests/fixtures/bridgeSearchProbe.py b/tests/fixtures/bridgeSearchProbe.py deleted file mode 100644 index e9f7c89e91..0000000000 --- a/tests/fixtures/bridgeSearchProbe.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Exercise REA's bridge inventory search without starting a Hopper socket.""" - -import json -from pathlib import Path -import sys - - -def _load_bridge(path): - source = Path(path).read_text(encoding="utf-8") - namespace = {"__file__": path, "__name__": "rea_hopper_bridge"} - exec(compile(source, path, "exec"), namespace) - return namespace - - -def _probe(namespace, payload): - action = payload["action"] - try: - if action == "search": - inventory = tuple(tuple(item) for item in payload["items"]) - namespace["_search_inventory"] = lambda _document, _kind: inventory - return { - "action": action, - "ok": True, - "result": namespace["_search_results"]( - object(), "string", payload["params"] - ), - } - raise ValueError("unknown probe action") - except Exception as error: - return { - "action": action, - "ok": False, - "type": type(error).__name__, - "diagnostic_type": namespace["_diagnostic_type"](error), - "message": str(error), - } - - -if __name__ == "__main__": - bridge = _load_bridge(sys.argv[1]) - if len(sys.argv) >= 3: - print(json.dumps(_probe(bridge, json.loads(sys.argv[2])), sort_keys=True)) - else: - for line in sys.stdin: - if not line.strip(): - continue - request = json.loads(line) - print(json.dumps(_probe(bridge, request), sort_keys=True), flush=True) diff --git a/tests/fixtures/hopperBridgeFacadeProbe.py b/tests/fixtures/hopperBridgeFacadeProbe.py index 6981c20733..097be9f6b5 100644 --- a/tests/fixtures/hopperBridgeFacadeProbe.py +++ b/tests/fixtures/hopperBridgeFacadeProbe.py @@ -30,7 +30,7 @@ class FakeDocument: return 0x401000 def getSegmentsList(self): - return [FakeInventorySegment()] + return [] class FakeDocumentProvider: @@ -47,39 +47,6 @@ class FakeDocumentProvider: return cls.current -class FakeStringSegment: - def getStringsList(self): - return [("fixture string", FakeStringAddress())] - - -class FakeStringAddress: - def __init__(self, value="0x401234"): - self.value = value - - def __str__(self): - return self.value - - -class FakeInventorySegment: - addresses = [FakeStringAddress("0x10"), 0x100, 0x2] - - def getStringsList(self): - return [ - ("string-" + str(self.number(address)), address) - for address in self.addresses - ] - - def getNamedAddresses(self): - return self.addresses - - def getNameAtAddress(self, address): - return "name-" + str(self.number(address)) - - @staticmethod - def number(address): - return address if isinstance(address, int) else int(str(address), 16) - - class FakeReferenceSegment: def getNameAtAddress(self, address): return "fixture-procedure" @@ -120,13 +87,6 @@ class FakeBlock: def getEndingAddress(self): return 0x401004 - - -class FakeStringsDocument: - def getSegmentsList(self): - return [FakeStringSegment()] - - def load_bridge(path): namespace = { "__file__": path, @@ -181,7 +141,6 @@ def main(): bridge["REA_OWNS_PROCESS_LIFETIME"] = False bridge["_configure_hopper_api"](FakeDocumentProvider) bridge["_bind_session_document"]() - strings = bridge["_strings"](FakeStringsDocument()) current = bridge["_dispatch"]("current_document", {}) current_address = bridge["_dispatch"]("current_address", {}) selected = bridge["_session_document"]() is FakeDocumentProvider.current @@ -219,17 +178,6 @@ def main(): procedure_info = bridge["_dispatch"]( "procedure_info", {"procedure": "0x401000"} ) - inventories = bridge_replies(bridge, [ - request(method, params, index) - for index, (method, params) in enumerate([ - ("list_strings", {}), - ("list_names", {}), - ("list_strings", {"address": "0x10"}), - ("list_names", {"address": "0x10"}), - ("list_strings", {"address": "0xff"}), - ("list_names", {"address": "0xff"}), - ], 1) - ]) provider_faults = [] for error_type in (TypeError, ValueError, KeyError): FakeDocumentProvider.current.address_error = error_type("credential=supersecret") @@ -279,8 +227,6 @@ def main(): "imported_without_hopper": unavailable, "current_document": current, "current_address": current_address, - "strings": strings, - "inventory_replies": inventories, "containing_procedure": containing_procedure, "procedure_references": procedure_references, "procedure_info": procedure_info,