mirror of
https://github.com/morluto/rea
synced 2026-10-25 22:08:11 -04:00
fix(hopper): preserve typed string values and verify real searches
This commit is contained in:
morluto
+48
-8
@@ -454,11 +454,51 @@ def _procedure_map(document):
|
||||
|
||||
|
||||
def _strings(document):
|
||||
result = {}
|
||||
for segment in document.getSegmentsList():
|
||||
for value, address in segment.getStringsList():
|
||||
result[_hex(address)] = value
|
||||
return result
|
||||
return {address: record["value"] for address, record in _string_inventory(document).items()}
|
||||
|
||||
|
||||
def _string_record(segment, address, display):
|
||||
"""Read one native typed object, retaining its shortened display separately."""
|
||||
length = segment.getObjectLength(address)
|
||||
end = segment.getStartingAddress() + segment.getLength()
|
||||
if isinstance(length, bool) or not isinstance(length, int) or length <= 0 or length > end - address:
|
||||
raise CapabilityUnavailableError("Hopper reported an invalid string object extent at %s" % _hex(address))
|
||||
raw = segment.readBytes(address, length)
|
||||
if not isinstance(raw, (bytes, bytearray)) or len(raw) != length:
|
||||
raise CapabilityUnavailableError("Hopper could not read the complete string object at %s" % _hex(address))
|
||||
kind = segment.getTypeAtAddress(address)
|
||||
encodings = ("utf-8", "latin-1") if kind == segment.TYPE_ASCII else ("utf-16-le", "utf-16-be") if kind == segment.TYPE_UNICODE else ()
|
||||
# Native display escaping is not invertible (a literal backslash-n and a
|
||||
# newline can render alike). Decode byte strings directly; for UTF-16,
|
||||
# compare rendered candidates to establish an unambiguous byte order.
|
||||
candidates = []
|
||||
for encoding in encodings:
|
||||
terminator = b"\x00\x00" if encoding.startswith("utf-16") else b"\x00"
|
||||
terminated = raw.endswith(terminator)
|
||||
payload = raw[:-len(terminator)] if terminated else raw
|
||||
try:
|
||||
value = payload.decode(encoding)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
record = {"value": value, "provider_value": display, "string": {"encoding": encoding, "encoding_status": "inferred", "termination": "present_or_not_required" if terminated else "missing", "byte_length": length}}
|
||||
if kind == segment.TYPE_ASCII:
|
||||
return record
|
||||
rendered = value.replace('"', '\\"').replace("\n", "\\n").replace("\r", "\\r").replace("\t", "\\t")
|
||||
if rendered == display or (display.endswith("\u2026") and display[:-1] and rendered.startswith(display[:-1])):
|
||||
candidates.append(record)
|
||||
if candidates and len({candidate["value"] for candidate in candidates}) == 1:
|
||||
return candidates[0]
|
||||
raise CapabilityUnavailableError("Hopper's string display cannot be reconciled with its typed bytes at %s; inspect the bytes and encoding explicitly" % _hex(address))
|
||||
|
||||
|
||||
def _string_inventory(document):
|
||||
key = (id(document), "string_records")
|
||||
inventory = _search_inventory_cache.get(key)
|
||||
if inventory is None:
|
||||
records = {_hex(address): _string_record(segment, address, display) for segment in document.getSegmentsList() for display, address in segment.getStringsList()}
|
||||
inventory = dict(sorted(records.items(), key=lambda item: int(item[0], 16)))
|
||||
_search_inventory_cache[key] = inventory
|
||||
return inventory
|
||||
|
||||
|
||||
def _invalidate_search_inventory(document):
|
||||
@@ -529,7 +569,7 @@ def _search_results(document, kind, params):
|
||||
if not matches(item[1]):
|
||||
continue
|
||||
selected.append(item)
|
||||
return [
|
||||
return [{"address": address, **_string_inventory(document)[address]} for address, _ in selected] if kind == "string" else [
|
||||
{
|
||||
"address": address,
|
||||
"value": value,
|
||||
@@ -894,12 +934,12 @@ def _dispatch(method, params):
|
||||
if method == "list_procedures":
|
||||
return [{"address": address, "value": value} for address, value in _search_inventory(document, "procedure")]
|
||||
if method == "list_strings":
|
||||
values = dict(_search_inventory(document, "string"))
|
||||
values = _string_inventory(document)
|
||||
requested = params.get("address")
|
||||
if requested is not None:
|
||||
key = _hex(_address(document, requested))
|
||||
values = {key: values[key]} if key in values else {}
|
||||
return [{"address": address, "value": value} for address, value in values.items()]
|
||||
return [{"address": address, **record} for address, record in values.items()]
|
||||
if method == "list_names":
|
||||
result = dict(_search_inventory(document, "name"))
|
||||
requested = params.get("address")
|
||||
|
||||
@@ -341,6 +341,11 @@ code-unit limit. Oversized names fail before any batch edits; bookmarks and
|
||||
literal string results are not subject to that symbol-name limit. Rename success
|
||||
requires exact final readback. New bookmarks must point into mapped memory;
|
||||
existing legacy bookmarks outside it can still be removed.
|
||||
String results read each native typed object's complete bytes, retain the original
|
||||
provider display in `provider_value`, and report its encoding, byte length, and
|
||||
termination. `encoding_status: inferred` distinguishes REA's decoding from an
|
||||
observed source encoding. Hopper can split long literals into adjacent
|
||||
unterminated objects; search matches each object's decoded bytes independently.
|
||||
|
||||
Closing or switching a target closes its bound Hopper document, shuts down REA's
|
||||
bridge and removes its temporary socket directory while preserving the Hopper
|
||||
|
||||
@@ -203,6 +203,14 @@ and whitespace. Disposable binary copies prove that switching and closing
|
||||
actually removes the native document, and that CLI byte, function-dossier,
|
||||
literal-trace results and invalid-address diagnostics agree with MCP.
|
||||
No provider is mocked in this lane.
|
||||
Real search probes cover Unicode names, literal metacharacters, case and regex
|
||||
modes, annotation cache invalidation, complete native fragments of long literals
|
||||
checked against byte reads, escaped UTF-8/control text, byte-preserving Latin-1
|
||||
decoding, and Hopper's UTF-16 symbol-name truncation boundary. When the macOS
|
||||
Objective-C fixture is present, the lane also verifies native UTF-16 string objects
|
||||
and their inferred encodings against their actual bytes.
|
||||
Unmapped annotation destinations and
|
||||
oversized later batch names fail before any earlier edit is applied.
|
||||
The Linux demo lane remains a separate `verify:hopper:linux` command.
|
||||
|
||||
`verify:hopper:fat` is a separate macOS lane requiring installed Hopper and the
|
||||
|
||||
@@ -9,6 +9,7 @@ import addFormats from "ajv-formats";
|
||||
import { mcpTextValue, requireMcpResult } from "./mcp-verifier-results.mjs";
|
||||
import { verifyHopperNavigationAndText } from "./real-hopper-navigation.mjs";
|
||||
import { verifyHopperWorkflows } from "./real-hopper-workflows.mjs";
|
||||
import { verifyHopperSearch } from "./real-hopper-search.mjs";
|
||||
|
||||
/** Exercise real Hopper annotation and malformed-input boundaries over MCP. */
|
||||
export async function verifyHopperBoundaryContracts(
|
||||
@@ -74,6 +75,7 @@ export async function verifyHopperBoundaryContracts(
|
||||
address,
|
||||
);
|
||||
const workflows = await verifyHopperWorkflows(call, invalid, procedure);
|
||||
const search = await verifyHopperSearch(call, invalid, procedure);
|
||||
try {
|
||||
if (foreignDocument !== undefined) {
|
||||
await invalid(
|
||||
@@ -278,6 +280,7 @@ export async function verifyHopperBoundaryContracts(
|
||||
foreignDocumentVerified: foreignDocument !== undefined,
|
||||
navigationAndText,
|
||||
workflows,
|
||||
search,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -21,12 +21,26 @@ export async function loadRealHopperFixtureTargets(manifestPath) {
|
||||
bindFixtureArtifact(root, large),
|
||||
],
|
||||
);
|
||||
requireDistinctArtifacts([primaryArtifact, secondaryArtifact, largeArtifact]);
|
||||
const objc = manifest.fixtures.find((fixture) => fixture?.name === "objc");
|
||||
const unicodeArtifact =
|
||||
objc === undefined
|
||||
? undefined
|
||||
: await bindFixtureArtifact(
|
||||
root,
|
||||
fixtureEntry(manifest.fixtures, "objc"),
|
||||
);
|
||||
requireDistinctArtifacts([
|
||||
primaryArtifact,
|
||||
secondaryArtifact,
|
||||
largeArtifact,
|
||||
...(unicodeArtifact === undefined ? [] : [unicodeArtifact]),
|
||||
]);
|
||||
return {
|
||||
manifestPath: canonicalManifest,
|
||||
primary: primaryArtifact,
|
||||
secondary: secondaryArtifact,
|
||||
large: largeArtifact,
|
||||
...(unicodeArtifact === undefined ? {} : { unicode: unicodeArtifact }),
|
||||
oracle,
|
||||
largeOracle,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
/** Exercise literal/regex search, Unicode identities and cache invalidation in Hopper. */
|
||||
export async function verifyHopperSearch(call, invalid, procedure) {
|
||||
const address = procedure.address;
|
||||
const original = await call("address_name", { address });
|
||||
try {
|
||||
await verifySymbolNames(call, invalid, address);
|
||||
await verifySearchModes(call, address);
|
||||
} finally {
|
||||
await call("set_address_name", { address, name: original });
|
||||
}
|
||||
assert.deepEqual(
|
||||
await call("search_procedures", {
|
||||
pattern: original,
|
||||
mode: "literal",
|
||||
case_sensitive: true,
|
||||
}),
|
||||
[{ address, value: original }],
|
||||
);
|
||||
return {
|
||||
unicodeIdentity: true,
|
||||
literalMetacharacters: true,
|
||||
longValuePreserved: true,
|
||||
symbolUtf16LimitValidatedBeforeMutation: true,
|
||||
regexAndCaseModes: true,
|
||||
annotationCacheInvalidation: true,
|
||||
escapedUtf8AndByteStrings: true,
|
||||
};
|
||||
}
|
||||
|
||||
async function verifySymbolNames(call, invalid, address) {
|
||||
const unicode = "REA_é_e\u0301_😀_Straße_ſ_İ";
|
||||
const extentNames = ["x".repeat(1024), "é".repeat(1024), "😀".repeat(512)];
|
||||
for (const name of ["REA_a.b[0]", unicode, ...extentNames]) {
|
||||
assert.equal(await call("set_address_name", { address, name }), true);
|
||||
assert.equal(await call("address_name", { address }), name);
|
||||
assert.deepEqual(await call("list_names", { address }), [
|
||||
{ address, value: name },
|
||||
]);
|
||||
assert.deepEqual(
|
||||
await call("search_procedures", {
|
||||
pattern: name,
|
||||
mode: "literal",
|
||||
case_sensitive: true,
|
||||
}),
|
||||
[{ address, value: name }],
|
||||
);
|
||||
assert.equal(await call("procedure_address", { procedure: name }), address);
|
||||
}
|
||||
assert.deepEqual(
|
||||
await call("search_procedures", { pattern: unicode, mode: "literal" }),
|
||||
[],
|
||||
);
|
||||
const other = (await call("list_procedures")).find(
|
||||
(item) => item.address !== address,
|
||||
);
|
||||
assert.ok(other);
|
||||
for (const name of ["x".repeat(1025), "é".repeat(1025), "😀".repeat(513)]) {
|
||||
await invalid(
|
||||
"set_address_name",
|
||||
{ address, name },
|
||||
/UTF-16 code-unit symbol-name limit/u,
|
||||
);
|
||||
await invalid(
|
||||
"set_addresses_names",
|
||||
{ names: { [other.address]: "must_not_apply", [address]: name } },
|
||||
/UTF-16 code-unit symbol-name limit/u,
|
||||
);
|
||||
assert.equal(await call("address_name", { address }), extentNames.at(-1));
|
||||
assert.equal(
|
||||
await call("address_name", { address: other.address }),
|
||||
other.value,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function verifySearchModes(call, address) {
|
||||
await call("set_address_name", { address, name: "REA_Search_Upper" });
|
||||
assert.deepEqual(
|
||||
await call("search_procedures", {
|
||||
pattern: "rea_search_upper",
|
||||
mode: "literal",
|
||||
case_sensitive: false,
|
||||
}),
|
||||
[{ address, value: "REA_Search_Upper" }],
|
||||
);
|
||||
assert.deepEqual(
|
||||
await call("search_procedures", {
|
||||
pattern: "rea_search_upper",
|
||||
mode: "literal",
|
||||
case_sensitive: true,
|
||||
}),
|
||||
[],
|
||||
);
|
||||
assert.deepEqual(
|
||||
await call("search_procedures", {
|
||||
pattern: "^REA_Search_(?:Upper|Lower)$",
|
||||
mode: "regex",
|
||||
case_sensitive: true,
|
||||
}),
|
||||
[{ address, value: "REA_Search_Upper" }],
|
||||
);
|
||||
const strings = await call("list_strings");
|
||||
const long = "REA_LONG_LITERAL_" + "x".repeat(4096) + "needle";
|
||||
const longItem = strings.find((item) =>
|
||||
item.value.startsWith("REA_LONG_LITERAL_"),
|
||||
);
|
||||
assert.ok(longItem, "fixture omitted its real long literal");
|
||||
assert.ok(long.startsWith(longItem.value));
|
||||
assert.ok(longItem.provider_value.endsWith("…"));
|
||||
assert.equal(longItem.string.termination, "missing");
|
||||
const fragments = strings.filter((item) => {
|
||||
const offset = BigInt(item.address) - BigInt(longItem.address);
|
||||
return offset >= 0n && offset < BigInt(Buffer.byteLength(long));
|
||||
});
|
||||
assert.equal(fragments.map((item) => item.value).join(""), long);
|
||||
for (const fragment of fragments) {
|
||||
assert.equal(
|
||||
Buffer.byteLength(fragment.value),
|
||||
fragment.string.byte_length -
|
||||
(fragment.string.termination === "missing" ? 0 : 1),
|
||||
);
|
||||
const bytes = await call("read_bytes", {
|
||||
address: fragment.address,
|
||||
length: fragment.string.byte_length,
|
||||
});
|
||||
const expected = Buffer.from(
|
||||
fragment.value + (fragment.string.termination === "missing" ? "" : "\0"),
|
||||
);
|
||||
assert.equal(bytes.bytes_hex, expected.toString("hex"));
|
||||
}
|
||||
assert.deepEqual(
|
||||
await call("search_strings", {
|
||||
pattern: "needle",
|
||||
mode: "literal",
|
||||
case_sensitive: true,
|
||||
}),
|
||||
[fragments.at(-1)],
|
||||
);
|
||||
const expected = strings.filter((item) =>
|
||||
/^REA_C_(?:ENTRY|LEAF)$/u.test(item.value),
|
||||
);
|
||||
assert.equal(expected.length, 2);
|
||||
assert.deepEqual(
|
||||
await call("search_strings", {
|
||||
pattern: "^REA_C_(?:ENTRY|LEAF)$",
|
||||
mode: "regex",
|
||||
case_sensitive: true,
|
||||
}),
|
||||
expected,
|
||||
);
|
||||
await verifyHopperStringObjects(call, [
|
||||
{ value: "REA_UTF8_é_😀", encoding: "utf-8" },
|
||||
{ value: 'REA_ESCAPED_"\\line\nend\t\r', encoding: "utf-8" },
|
||||
{ value: "REA_LITERAL_BACKSLASH_\\n", encoding: "utf-8" },
|
||||
{ value: "REA_LITERAL_…", encoding: "utf-8" },
|
||||
{ value: "REA_LATIN1_ÿ", encoding: "latin-1" },
|
||||
]);
|
||||
}
|
||||
|
||||
/** Compare native string identities and values with the source fixture's bytes. */
|
||||
export async function verifyHopperStringObjects(call, expectations) {
|
||||
const strings = await call("list_strings");
|
||||
for (const { value, encoding } of expectations) {
|
||||
const item = strings.find((candidate) => candidate.value === value);
|
||||
assert.ok(
|
||||
item,
|
||||
`Hopper omitted the decoded string ${JSON.stringify(value)}`,
|
||||
);
|
||||
assert.equal(item.string.encoding, encoding);
|
||||
assert.equal(item.string.encoding_status, "inferred");
|
||||
assert.equal(item.string.termination, "present_or_not_required");
|
||||
const expectedBytes = Buffer.from(
|
||||
value + "\0",
|
||||
encoding.replaceAll("-", ""),
|
||||
);
|
||||
assert.equal(item.string.byte_length, expectedBytes.length);
|
||||
const bytes = await call("read_bytes", {
|
||||
address: item.address,
|
||||
length: expectedBytes.length,
|
||||
});
|
||||
assert.equal(bytes.bytes_hex, expectedBytes.toString("hex"));
|
||||
assert.deepEqual(await call("list_strings", { address: item.address }), [
|
||||
item,
|
||||
]);
|
||||
assert.deepEqual(
|
||||
await call("search_strings", {
|
||||
pattern: value,
|
||||
mode: "literal",
|
||||
case_sensitive: true,
|
||||
}),
|
||||
[item],
|
||||
);
|
||||
}
|
||||
assert.deepEqual(
|
||||
await call("search_strings", { pattern: "\\xC3\\xA9", mode: "literal" }),
|
||||
[],
|
||||
);
|
||||
return {
|
||||
sourceBytesVerified: true,
|
||||
inferredEncodingReported: true,
|
||||
displayEscapesExcludedFromMatching: true,
|
||||
};
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import { HOPPER_PROVIDER_IDENTITY } from "../dist/hopper/HopperProvider.js";
|
||||
import { HOPPER_TARGET_LEASE_DIRECTORY } from "../dist/hopper/HopperTargetLease.js";
|
||||
import { REA_WORKFLOW_PROVIDER } from "../dist/application/InvestigationProviders.js";
|
||||
import { loadRealHopperFixtureTargets } from "./lib/real-hopper-fixture.mjs";
|
||||
import { verifyHopperStringObjects } from "./lib/real-hopper-search.mjs";
|
||||
import {
|
||||
requireBridgeDiagnostic,
|
||||
requireBridgeProgress,
|
||||
@@ -463,6 +464,21 @@ try {
|
||||
options,
|
||||
);
|
||||
if (closed.isError === true) throw new Error(textValue(closed));
|
||||
let unicodeStrings = null;
|
||||
if (fixtureTargets.unicode !== undefined) {
|
||||
const call = async (name, args = {}) =>
|
||||
requireSuccessfulTool(
|
||||
await client.callTool({ name, arguments: args }, options),
|
||||
name,
|
||||
);
|
||||
await call("open_binary", { path: fixtureTargets.unicode.path });
|
||||
unicodeStrings = await verifyHopperStringObjects(call, [
|
||||
{ value: "REA_UTF16_é_😀", encoding: "utf-16-le" },
|
||||
{ value: 'REA_UTF16_ESCAPED_é_"\\line\nend\t\r', encoding: "utf-16-le" },
|
||||
{ value: "\nREA_UTF16_LEADING_é", encoding: "utf-16-le" },
|
||||
]);
|
||||
await call("close_binary");
|
||||
}
|
||||
const lifecycleAndCli = await verifyHopperLifecycleAndCli(client, options, {
|
||||
primary: targetA,
|
||||
secondary: targetB,
|
||||
@@ -492,6 +508,7 @@ try {
|
||||
segmentCount: firstOverview.segment_count,
|
||||
analyses: [firstAnalysis, secondAnalysis],
|
||||
fixtureAnalysis,
|
||||
unicodeStrings,
|
||||
boundaryContracts,
|
||||
lifecycleAndCli,
|
||||
inventoryCounts,
|
||||
|
||||
@@ -126,7 +126,7 @@ export const OFFICIAL_TOOL_CONTRACTS = [
|
||||
),
|
||||
official(
|
||||
"list_strings",
|
||||
"List every provider-defined string, or filter to one address, as address/value pairs. Ghidra also reports encoding, terminator status, and byte length.",
|
||||
"List every provider-defined string object, or filter to one address, as address/value pairs with encoding, terminator status, and byte length when available. Hopper reads complete typed objects and retains its original display text as provider_value; long literals may span adjacent unterminated objects.",
|
||||
z.object({ document, address: optionalAddress }),
|
||||
),
|
||||
official(
|
||||
@@ -198,7 +198,7 @@ export const OFFICIAL_TOOL_CONTRACTS = [
|
||||
),
|
||||
official(
|
||||
"search_strings",
|
||||
"Search every analyzed string using literal matching by default or regex when requested. Results are deterministic and complete.",
|
||||
"Search every analyzed string object using literal matching by default or regex when requested. Results are deterministic and complete over the native inventory. Hopper searches complete typed bytes rather than shortened displays; adjacent fragments of a long literal are matched independently, with their termination and original display retained.",
|
||||
z.object(analysisSearchInput),
|
||||
),
|
||||
official(
|
||||
|
||||
@@ -97,9 +97,21 @@ const bookmarkFacetSchema = z.discriminatedUnion("state", [
|
||||
const addressedString = z.object({
|
||||
address: z.string(),
|
||||
value: z.string(),
|
||||
provider_value: z
|
||||
.string()
|
||||
.exactOptional()
|
||||
.describe(
|
||||
"Original provider display text, which may shorten the typed object.",
|
||||
),
|
||||
string: z
|
||||
.object({
|
||||
encoding: z.string().min(1),
|
||||
encoding_status: z
|
||||
.enum(["observed", "inferred"])
|
||||
.exactOptional()
|
||||
.describe(
|
||||
"Whether the provider declares the encoding or REA infers it from the typed bytes and native display. This does not establish the original source's intended encoding.",
|
||||
),
|
||||
termination: z.enum(["missing", "present_or_not_required"]),
|
||||
byte_length: z.number().int().min(0),
|
||||
})
|
||||
@@ -197,9 +209,7 @@ export const officialOutputSchemas: Readonly<Record<string, z.ZodObject>> = {
|
||||
search_procedures: resultOf(
|
||||
z.array(z.object({ address: z.string(), value: z.string() })),
|
||||
),
|
||||
search_strings: resultOf(
|
||||
z.array(z.object({ address: z.string(), value: z.string() })),
|
||||
),
|
||||
search_strings: resultOf(z.array(addressedString)),
|
||||
set_address_name: resultOf(z.boolean()),
|
||||
set_addresses_names: resultOf(z.record(z.string(), z.boolean())),
|
||||
set_bookmark: resultOf(z.boolean()),
|
||||
|
||||
@@ -79,6 +79,12 @@ export const CAPABILITIES: readonly CapabilityDescriptor[] = Object.freeze(
|
||||
}),
|
||||
limitations: Object.freeze([
|
||||
"Results depend on Hopper's completed static analysis.",
|
||||
...(operation === "list_strings" || operation === "search_strings"
|
||||
? [
|
||||
"String values cover native typed objects. Hopper may split long literals into adjacent unterminated fragments; string searches match each object independently.",
|
||||
"Reported string encodings are inferred from typed bytes and native display; the original source's intended encoding is unknown.",
|
||||
]
|
||||
: []),
|
||||
]),
|
||||
}),
|
||||
),
|
||||
|
||||
@@ -1,285 +0,0 @@
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
|
||||
const bridgePath = fileURLToPath(
|
||||
new URL("../../../../bridge/hopper_bridge.py", import.meta.url),
|
||||
);
|
||||
const probePath = fileURLToPath(
|
||||
new URL("../../../fixtures/bridgeSearchProbe.py", import.meta.url),
|
||||
);
|
||||
|
||||
const searchResultSchema = z.object({
|
||||
action: z.literal("search"),
|
||||
ok: z.literal(true),
|
||||
result: z.array(z.object({ address: z.string(), value: z.string() })),
|
||||
});
|
||||
const probeResultSchema = searchResultSchema;
|
||||
type ProbeResult = z.infer<typeof probeResultSchema>;
|
||||
|
||||
type ProbeInput = {
|
||||
readonly action: "search";
|
||||
readonly items: readonly (readonly [string, string])[];
|
||||
readonly params: Readonly<Record<string, string | number | boolean>>;
|
||||
};
|
||||
|
||||
interface PendingProbe {
|
||||
readonly resolve: (result: ProbeResult) => void;
|
||||
readonly reject: (error: Error) => void;
|
||||
timeout: NodeJS.Timeout | undefined;
|
||||
}
|
||||
|
||||
const PROBE_TIMEOUT_MS = 3_000;
|
||||
const PROBE_BUFFER_LIMIT = 1_024 * 1_024;
|
||||
const pendingProbes: PendingProbe[] = [];
|
||||
let probeProcess: ChildProcessWithoutNullStreams | undefined;
|
||||
let probeStdout = "";
|
||||
let probeStderr = "";
|
||||
let probeFailure: Error | undefined;
|
||||
let stoppingProbe = false;
|
||||
|
||||
const failureWithStderr = (message: string): Error => {
|
||||
const stderr = probeStderr.trim();
|
||||
return new Error(stderr.length === 0 ? message : `${message}: ${stderr}`);
|
||||
};
|
||||
|
||||
const rejectPendingProbes = (error: Error): void => {
|
||||
for (const pending of pendingProbes.splice(0)) {
|
||||
if (pending.timeout !== undefined) clearTimeout(pending.timeout);
|
||||
pending.reject(error);
|
||||
}
|
||||
};
|
||||
|
||||
const failProbeProcess = (error: Error): void => {
|
||||
probeFailure ??= error;
|
||||
rejectPendingProbes(probeFailure);
|
||||
if (
|
||||
!stoppingProbe &&
|
||||
probeProcess?.exitCode === null &&
|
||||
probeProcess.signalCode === null
|
||||
)
|
||||
probeProcess.kill("SIGKILL");
|
||||
};
|
||||
|
||||
const acceptProbeLine = (line: string): void => {
|
||||
const pending = pendingProbes.shift();
|
||||
if (pending === undefined) {
|
||||
failProbeProcess(
|
||||
failureWithStderr("Python probe returned an extra result"),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (pending.timeout !== undefined) clearTimeout(pending.timeout);
|
||||
try {
|
||||
const decoded: unknown = JSON.parse(line);
|
||||
pending.resolve(probeResultSchema.parse(decoded));
|
||||
} catch (cause: unknown) {
|
||||
const error =
|
||||
cause instanceof Error
|
||||
? cause
|
||||
: failureWithStderr("Python probe returned an invalid result");
|
||||
pending.reject(error);
|
||||
failProbeProcess(error);
|
||||
}
|
||||
};
|
||||
|
||||
const acceptProbeOutput = (chunk: string): void => {
|
||||
probeStdout += chunk;
|
||||
if (Buffer.byteLength(probeStdout) > PROBE_BUFFER_LIMIT) {
|
||||
failProbeProcess(failureWithStderr("Python probe output exceeded 1 MiB"));
|
||||
return;
|
||||
}
|
||||
let newline = probeStdout.indexOf("\n");
|
||||
while (newline >= 0) {
|
||||
const line = probeStdout.slice(0, newline).replace(/\r$/u, "");
|
||||
probeStdout = probeStdout.slice(newline + 1);
|
||||
acceptProbeLine(line);
|
||||
newline = probeStdout.indexOf("\n");
|
||||
}
|
||||
};
|
||||
|
||||
const startProbeProcess = async (): Promise<void> => {
|
||||
stoppingProbe = false;
|
||||
probeFailure = undefined;
|
||||
probeStdout = "";
|
||||
probeStderr = "";
|
||||
const child = spawn("python3", [probePath, bridgePath], {
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
probeProcess = child;
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", acceptProbeOutput);
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
probeStderr += chunk;
|
||||
if (Buffer.byteLength(probeStderr) > PROBE_BUFFER_LIMIT)
|
||||
failProbeProcess(new Error("Python probe stderr exceeded 1 MiB"));
|
||||
});
|
||||
child.stdin.on("error", (error) => {
|
||||
if (!stoppingProbe) failProbeProcess(error);
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
if (!stoppingProbe) failProbeProcess(error);
|
||||
});
|
||||
child.on("exit", (code, signal) => {
|
||||
if (!stoppingProbe)
|
||||
failProbeProcess(
|
||||
failureWithStderr(
|
||||
`Python probe exited before responding (code ${String(code)}, signal ${String(signal)})`,
|
||||
),
|
||||
);
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onSpawn = (): void => {
|
||||
child.off("error", onError);
|
||||
resolve();
|
||||
};
|
||||
const onError = (error: Error): void => {
|
||||
child.off("spawn", onSpawn);
|
||||
reject(error);
|
||||
};
|
||||
child.once("spawn", onSpawn);
|
||||
child.once("error", onError);
|
||||
});
|
||||
};
|
||||
|
||||
const hasExited = (child: ChildProcessWithoutNullStreams): boolean =>
|
||||
child.exitCode !== null || child.signalCode !== null;
|
||||
|
||||
const waitForExit = (
|
||||
child: ChildProcessWithoutNullStreams,
|
||||
timeoutMs: number,
|
||||
): Promise<boolean> => {
|
||||
if (hasExited(child)) return Promise.resolve(true);
|
||||
return new Promise((resolve) => {
|
||||
const onExit = (): void => {
|
||||
clearTimeout(timeout);
|
||||
resolve(true);
|
||||
};
|
||||
const timeout = setTimeout(() => {
|
||||
child.off("exit", onExit);
|
||||
resolve(false);
|
||||
}, timeoutMs);
|
||||
child.once("exit", onExit);
|
||||
});
|
||||
};
|
||||
|
||||
const stopProbeProcess = async (): Promise<void> => {
|
||||
const child = probeProcess;
|
||||
if (child === undefined) return;
|
||||
stoppingProbe = true;
|
||||
rejectPendingProbes(new Error("Python probe stopped before responding"));
|
||||
if (!hasExited(child)) child.stdin.end();
|
||||
if (!(await waitForExit(child, 500))) child.kill("SIGTERM");
|
||||
if (!(await waitForExit(child, 500))) child.kill("SIGKILL");
|
||||
if (!(await waitForExit(child, 500)))
|
||||
throw new Error("Python probe did not stop after SIGKILL");
|
||||
probeProcess = undefined;
|
||||
};
|
||||
|
||||
const probe = (input: ProbeInput): Promise<ProbeResult> => {
|
||||
const child = probeProcess;
|
||||
if (probeFailure !== undefined) return Promise.reject(probeFailure);
|
||||
if (child === undefined || hasExited(child))
|
||||
return Promise.reject(new Error("Python probe is not running"));
|
||||
return new Promise((resolve, reject) => {
|
||||
const pending: PendingProbe = { resolve, reject, timeout: undefined };
|
||||
pending.timeout = setTimeout(() => {
|
||||
failProbeProcess(
|
||||
new Error(
|
||||
`Python probe request timed out after ${String(PROBE_TIMEOUT_MS)} ms`,
|
||||
),
|
||||
);
|
||||
}, PROBE_TIMEOUT_MS);
|
||||
pendingProbes.push(pending);
|
||||
child.stdin.write(`${JSON.stringify(input)}\n`, (error) => {
|
||||
if (error !== null && error !== undefined) failProbeProcess(error);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
beforeAll(startProbeProcess);
|
||||
beforeEach(async () => {
|
||||
if (
|
||||
probeFailure === undefined &&
|
||||
probeProcess !== undefined &&
|
||||
!hasExited(probeProcess)
|
||||
)
|
||||
return;
|
||||
await stopProbeProcess();
|
||||
await startProbeProcess();
|
||||
});
|
||||
afterAll(stopProbeProcess);
|
||||
|
||||
describe("Hopper bridge search", () => {
|
||||
it("returns matching literal values in deterministic order", async () => {
|
||||
await expect(
|
||||
probe({
|
||||
action: "search",
|
||||
items: [
|
||||
["0x1000", "REA_main_1"],
|
||||
["0x2000", "unrelated"],
|
||||
["0x3000", "rea_helper_42"],
|
||||
],
|
||||
params: {
|
||||
pattern: "REA",
|
||||
mode: "literal",
|
||||
case_sensitive: false,
|
||||
},
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
action: "search",
|
||||
ok: true,
|
||||
result: [
|
||||
{ address: "0x1000", value: "REA_main_1" },
|
||||
{ address: "0x3000", value: "rea_helper_42" },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("returns matching entries for regex searches", async () => {
|
||||
await expect(
|
||||
probe({
|
||||
action: "search",
|
||||
items: [
|
||||
["0x1000", "REA_GHIDRA_INVENTORY_ENTRY"],
|
||||
["0x2000", "unrelated"],
|
||||
["0x3000", "REA_GHIDRA_LEAF_VALUE"],
|
||||
],
|
||||
params: {
|
||||
pattern: "^REA_GHIDRA_(?:INVENTORY_ENTRY|LEAF_VALUE)$",
|
||||
mode: "regex",
|
||||
case_sensitive: true,
|
||||
},
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
action: "search",
|
||||
ok: true,
|
||||
result: [
|
||||
{ address: "0x1000", value: "REA_GHIDRA_INVENTORY_ENTRY" },
|
||||
{ address: "0x3000", value: "REA_GHIDRA_LEAF_VALUE" },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("returns the complete value for a long literal match", async () => {
|
||||
const value = `${"x".repeat(4_096)}needle`;
|
||||
await expect(
|
||||
probe({
|
||||
action: "search",
|
||||
items: [["0x1000", value]],
|
||||
params: {
|
||||
pattern: "needle",
|
||||
mode: "literal",
|
||||
case_sensitive: true,
|
||||
},
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
action: "search",
|
||||
ok: true,
|
||||
result: [{ address: "0x1000", value }],
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -48,7 +48,6 @@ const probeResultSchema = z.strictObject({
|
||||
reason: z.string(),
|
||||
}),
|
||||
}),
|
||||
strings: z.strictObject({ "0x401234": z.literal("fixture string") }),
|
||||
procedure_references: z.strictObject({
|
||||
procedure: z.strictObject({
|
||||
address: z.literal("0x401000"),
|
||||
@@ -64,14 +63,6 @@ const probeResultSchema = z.strictObject({
|
||||
unresolved_calls: z.array(z.unknown()),
|
||||
references: z.array(z.unknown()),
|
||||
}),
|
||||
inventory_replies: z.array(
|
||||
z.strictObject({
|
||||
id: z.number().int(),
|
||||
result: z.array(
|
||||
z.strictObject({ address: z.string(), value: z.string() }),
|
||||
),
|
||||
}),
|
||||
),
|
||||
provider_faults: z.array(
|
||||
z.strictObject({
|
||||
id: z.literal(1),
|
||||
@@ -198,28 +189,6 @@ describe("Hopper API facade", () => {
|
||||
},
|
||||
{ id: 9, result: "0x401000" },
|
||||
]);
|
||||
expect(result.inventory_replies).toEqual([
|
||||
{
|
||||
id: 1,
|
||||
result: [
|
||||
{ address: "0x2", value: "string-2" },
|
||||
{ address: "0x10", value: "string-16" },
|
||||
{ address: "0x100", value: "string-256" },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
result: [
|
||||
{ address: "0x2", value: "name-2" },
|
||||
{ address: "0x10", value: "name-16" },
|
||||
{ address: "0x100", value: "name-256" },
|
||||
],
|
||||
},
|
||||
{ id: 3, result: [{ address: "0x10", value: "string-16" }] },
|
||||
{ id: 4, result: [{ address: "0x10", value: "name-16" }] },
|
||||
{ id: 5, result: [] },
|
||||
{ id: 6, result: [] },
|
||||
]);
|
||||
expect(stdout).not.toContain("supersecret");
|
||||
expect(result.analysis_guard.message).toContain(
|
||||
"requires completed Hopper background analysis",
|
||||
|
||||
@@ -2,6 +2,18 @@
|
||||
|
||||
volatile int rea_c_global = 7;
|
||||
|
||||
#define REA_LONG_CHUNK "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
|
||||
#define REA_LONG_256 REA_LONG_CHUNK REA_LONG_CHUNK REA_LONG_CHUNK REA_LONG_CHUNK
|
||||
#define REA_LONG_1024 REA_LONG_256 REA_LONG_256 REA_LONG_256 REA_LONG_256
|
||||
__attribute__((noinline, used)) void rea_long_literal_procedure(void) {
|
||||
puts("REA_LONG_LITERAL_" REA_LONG_1024 REA_LONG_1024 REA_LONG_1024 REA_LONG_1024 "needle");
|
||||
puts("REA_UTF8_é_😀");
|
||||
puts("REA_ESCAPED_\"\\line\nend\t\r");
|
||||
puts("REA_LITERAL_BACKSLASH_\\n");
|
||||
puts("REA_LITERAL_…");
|
||||
puts("REA_LATIN1_\xff");
|
||||
}
|
||||
|
||||
__attribute__((noinline, used)) int rea_leaf(int value) {
|
||||
puts("REA_C_LEAF");
|
||||
return value + rea_c_global;
|
||||
|
||||
@@ -16,6 +16,9 @@ int main(void) {
|
||||
@autoreleasepool {
|
||||
Protocol *protocol = @protocol(REAWidgetDelegate);
|
||||
NSLog(@"%@", NSStringFromProtocol(protocol));
|
||||
NSLog(@"REA_UTF16_é_😀");
|
||||
NSLog(@"REA_UTF16_ESCAPED_é_\"\\line\nend\t\r");
|
||||
NSLog(@"\nREA_UTF16_LEADING_é");
|
||||
return [[[REAWidget alloc] init] performAction:@"run" error:0] ? 0 : 1;
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
-48
@@ -1,48 +0,0 @@
|
||||
"""Exercise REA's bridge inventory search without starting a Hopper socket."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
|
||||
def _load_bridge(path):
|
||||
source = Path(path).read_text(encoding="utf-8")
|
||||
namespace = {"__file__": path, "__name__": "rea_hopper_bridge"}
|
||||
exec(compile(source, path, "exec"), namespace)
|
||||
return namespace
|
||||
|
||||
|
||||
def _probe(namespace, payload):
|
||||
action = payload["action"]
|
||||
try:
|
||||
if action == "search":
|
||||
inventory = tuple(tuple(item) for item in payload["items"])
|
||||
namespace["_search_inventory"] = lambda _document, _kind: inventory
|
||||
return {
|
||||
"action": action,
|
||||
"ok": True,
|
||||
"result": namespace["_search_results"](
|
||||
object(), "string", payload["params"]
|
||||
),
|
||||
}
|
||||
raise ValueError("unknown probe action")
|
||||
except Exception as error:
|
||||
return {
|
||||
"action": action,
|
||||
"ok": False,
|
||||
"type": type(error).__name__,
|
||||
"diagnostic_type": namespace["_diagnostic_type"](error),
|
||||
"message": str(error),
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
bridge = _load_bridge(sys.argv[1])
|
||||
if len(sys.argv) >= 3:
|
||||
print(json.dumps(_probe(bridge, json.loads(sys.argv[2])), sort_keys=True))
|
||||
else:
|
||||
for line in sys.stdin:
|
||||
if not line.strip():
|
||||
continue
|
||||
request = json.loads(line)
|
||||
print(json.dumps(_probe(bridge, request), sort_keys=True), flush=True)
|
||||
+1
-55
@@ -30,7 +30,7 @@ class FakeDocument:
|
||||
return 0x401000
|
||||
|
||||
def getSegmentsList(self):
|
||||
return [FakeInventorySegment()]
|
||||
return []
|
||||
|
||||
|
||||
class FakeDocumentProvider:
|
||||
@@ -47,39 +47,6 @@ class FakeDocumentProvider:
|
||||
return cls.current
|
||||
|
||||
|
||||
class FakeStringSegment:
|
||||
def getStringsList(self):
|
||||
return [("fixture string", FakeStringAddress())]
|
||||
|
||||
|
||||
class FakeStringAddress:
|
||||
def __init__(self, value="0x401234"):
|
||||
self.value = value
|
||||
|
||||
def __str__(self):
|
||||
return self.value
|
||||
|
||||
|
||||
class FakeInventorySegment:
|
||||
addresses = [FakeStringAddress("0x10"), 0x100, 0x2]
|
||||
|
||||
def getStringsList(self):
|
||||
return [
|
||||
("string-" + str(self.number(address)), address)
|
||||
for address in self.addresses
|
||||
]
|
||||
|
||||
def getNamedAddresses(self):
|
||||
return self.addresses
|
||||
|
||||
def getNameAtAddress(self, address):
|
||||
return "name-" + str(self.number(address))
|
||||
|
||||
@staticmethod
|
||||
def number(address):
|
||||
return address if isinstance(address, int) else int(str(address), 16)
|
||||
|
||||
|
||||
class FakeReferenceSegment:
|
||||
def getNameAtAddress(self, address):
|
||||
return "fixture-procedure"
|
||||
@@ -120,13 +87,6 @@ class FakeBlock:
|
||||
|
||||
def getEndingAddress(self):
|
||||
return 0x401004
|
||||
|
||||
|
||||
class FakeStringsDocument:
|
||||
def getSegmentsList(self):
|
||||
return [FakeStringSegment()]
|
||||
|
||||
|
||||
def load_bridge(path):
|
||||
namespace = {
|
||||
"__file__": path,
|
||||
@@ -181,7 +141,6 @@ def main():
|
||||
bridge["REA_OWNS_PROCESS_LIFETIME"] = False
|
||||
bridge["_configure_hopper_api"](FakeDocumentProvider)
|
||||
bridge["_bind_session_document"]()
|
||||
strings = bridge["_strings"](FakeStringsDocument())
|
||||
current = bridge["_dispatch"]("current_document", {})
|
||||
current_address = bridge["_dispatch"]("current_address", {})
|
||||
selected = bridge["_session_document"]() is FakeDocumentProvider.current
|
||||
@@ -219,17 +178,6 @@ def main():
|
||||
procedure_info = bridge["_dispatch"](
|
||||
"procedure_info", {"procedure": "0x401000"}
|
||||
)
|
||||
inventories = bridge_replies(bridge, [
|
||||
request(method, params, index)
|
||||
for index, (method, params) in enumerate([
|
||||
("list_strings", {}),
|
||||
("list_names", {}),
|
||||
("list_strings", {"address": "0x10"}),
|
||||
("list_names", {"address": "0x10"}),
|
||||
("list_strings", {"address": "0xff"}),
|
||||
("list_names", {"address": "0xff"}),
|
||||
], 1)
|
||||
])
|
||||
provider_faults = []
|
||||
for error_type in (TypeError, ValueError, KeyError):
|
||||
FakeDocumentProvider.current.address_error = error_type("credential=supersecret")
|
||||
@@ -279,8 +227,6 @@ def main():
|
||||
"imported_without_hopper": unavailable,
|
||||
"current_document": current,
|
||||
"current_address": current_address,
|
||||
"strings": strings,
|
||||
"inventory_replies": inventories,
|
||||
"containing_procedure": containing_procedure,
|
||||
"procedure_references": procedure_references,
|
||||
"procedure_info": procedure_info,
|
||||
|
||||
Reference in New Issue
Block a user