diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 7661827134..b9b010ed8e 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -331,8 +331,9 @@ complete observation. 64 MiB of input, 200,000 objects/references and 128 nesting levels. XML dictionary entries decoded as `__proto__` are omitted with an explicit count because REA results cannot represent that key. XML tag whitespace and - comments do not change that handling. Other values and the original archive - digest are preserved; key interpretation follows the pinned plist decoder. + comments do not change that handling. XML key names combine their text and + CDATA segments without trimming or expanding literal CDATA entities. Other + values and the original archive digest are preserved. - `decode_interface_builder` / `rea decode-interface-builder ` reads compiled storyboard and nib resources into objects, outlets, actions, connections and controller/class names. A full app graph is often several diff --git a/src/domain/propertyListKeys.test.ts b/src/domain/propertyListKeys.test.ts index ff3ca7bf32..259204a227 100644 --- a/src/domain/propertyListKeys.test.ts +++ b/src/domain/propertyListKeys.test.ts @@ -53,9 +53,9 @@ it("keeps __proto__ text in strings, comments, and similar keys", () => { }); }); -it("keeps an entity-encoded key that decodes to the placeholder", () => { +it("keeps a segmented entity-encoded key that decodes to the placeholder", () => { const text = plist( - "__rea_prototype_key__kept__proto__x", + "__reakept__proto__x", ); expect(parseXmlPropertyList(text)).toEqual({ value: { __rea_prototype_key__: "kept" }, @@ -83,16 +83,75 @@ it("omits own __proto__ entries that JSON decoding creates", () => { }); }); -it("preserves the pinned decoder's first-child key semantics", () => { +it("omits complete prototype keys split across character-data nodes", () => { const text = plist( "__prokept__proto__omitted", ); expect(parseXmlPropertyList(text)).toEqual({ - value: { __pro: "kept" }, + value: {}, omittedPrototypeKeys: 1, }); }); +it("preserves literal key text, XML characters and encoded line endings across segments", () => { + const text = plist( + 'pre]]> …
]]>posta b', + ); + expect(parseXmlPropertyList(text)).toEqual({ + value: { "pre&\r\u0085\u2028]]>post": "a\rb" }, + omittedPrototypeKeys: 0, + }); +}); + +it("does not infer key identity from nested elements", () => { + expect( + parseXmlPropertyList( + plist( + "prenestedkept__proto__nestedomitted", + ), + ), + ).toEqual({ value: { pre: "kept" }, omittedPrototypeKeys: 1 }); +}); + +it.each([ + ["ab", "a", "ab"], + ["a", "ab", "a"], + ["ab", "ab", "ab"], +])("retains last-value-wins for mixed spellings %j", (...keys) => { + const entries = keys + .map( + (key, index) => + `${key}${String(index + 1)}`, + ) + .join(""); + expect(parseXmlPropertyList(plist(`${entries}`))).toEqual({ + value: { ab: 3 }, + omittedPrototypeKeys: 0, + }); +}); + +it("keeps empty keys and alias-looking source keys distinct", () => { + const entries = + "12" + + "3__rea_plist_key_0__literal" + + "ma42"; + expect(parseXmlPropertyList(plist(`${entries}`))).toEqual({ + value: { "": 3, __rea_plist_key_0__: "literal", main: 42 }, + omittedPrototypeKeys: 0, + }); +}); + +it.each([ + "ma", + "ma__rea_plist_key_0__", +])("does not alias key elements used as scalar values in %s", (body) => { + const text = plist(body); + expect(parseXmlPropertyList(text)).toEqual({ + value: parse(text), + omittedPrototypeKeys: 0, + }); +}); + it("preserves OpenStep dictionary and array autodetection", () => { expect( parseXmlPropertyList( diff --git a/src/domain/propertyListKeys.ts b/src/domain/propertyListKeys.ts index 7ed9157f83..12e3e7dba1 100644 --- a/src/domain/propertyListKeys.ts +++ b/src/domain/propertyListKeys.ts @@ -19,9 +19,9 @@ export const omittedPrototypeKeysLimitation = (count: number | null): string => ? "Dictionary entries keyed __proto__ were omitted because REA results cannot represent that key." : `${String(count)} dictionary ${count === 1 ? "entry" : "entries"} keyed __proto__ ${count === 1 ? "was" : "were"} omitted because REA results cannot represent that key.`; -const withoutKey = ( +const projectDictionaryKeys = ( value: unknown, - key: string, + projectName: (name: string) => string | undefined, ): PropertyListWithoutPrototypeKeys => { let omitted = 0; const strip = (item: unknown): unknown => { @@ -35,7 +35,8 @@ const withoutKey = ( return item; return Object.fromEntries( Object.entries(item).flatMap(([name, entry]) => { - if (name !== key) return [[name, strip(entry)]]; + const projected = projectName(name); + if (projected !== undefined) return [[projected, strip(entry)]]; omitted += 1; return []; }), @@ -48,7 +49,10 @@ const withoutKey = ( /** Remove own `__proto__` entries, such as JSON.parse creates, and count them. */ export const omitPrototypeKeys = ( value: unknown, -): PropertyListWithoutPrototypeKeys => withoutKey(value, PROTOTYPE_KEY); +): PropertyListWithoutPrototypeKeys => + projectDictionaryKeys(value, (name) => + name === PROTOTYPE_KEY ? undefined : name, + ); const XML_ENTITIES: Readonly> = { amp: "&", @@ -82,15 +86,14 @@ export const decodeXmlText = (text: string): string => ); /** - * Decode an XML plist whose dictionaries may use the legal key `__proto__`, - * which the `plist` decoder rejects outright. Those entries are omitted and - * counted so callers can report them instead of failing the whole document. + * Decode complete XML character-data keys before the pinned value decoder. + * Entries keyed `__proto__` are omitted and counted at the same boundary. */ export const parseXmlPropertyList = ( text: string, ): PropertyListWithoutPrototypeKeys => { // Preserve the pinned decoder's existing OpenStep autodetection, also used - // by app Info.plist resolution. Only XML needs the key-element substitution. + // by app Info.plist resolution. Only XML needs key normalization. const first = text.trimStart()[0]; if (first === "{" || first === "(") { // The pinned decoder assigns a `__proto__` key to the object prototype, so @@ -115,11 +118,11 @@ export const parseXmlPropertyList = ( if (!parsed.ok) throw parsed.error; const document = parsed.value; assertXmlPlistDepthWithinLimit(document); - const { substituted, placeholder } = substitutePrototypeKeys( - normalized, - document, - ); - return withoutKey(parse(substituted), placeholder); + const { substituted, names } = normalizeXmlKeys(normalized, document); + return projectDictionaryKeys(parse(substituted), (name) => { + const decoded = names.get(name) ?? name; + return decoded === PROTOTYPE_KEY ? undefined : decoded; + }); }; interface OpenStepContainer { @@ -323,17 +326,48 @@ const assertXmlPlistDepthWithinLimit = (document: Document): void => { } }; -/** Locate keys with the same XML parser and first-child semantics as plist. */ -const substitutePrototypeKeys = (text: string, document: Document) => { - const keys = Array.from(document.getElementsByTagName("key")); - const decodedKeys = new Set(keys.map((key) => key.firstChild?.nodeValue)); - let placeholder = "__rea_prototype_key__"; - while (decodedKeys.has(placeholder)) placeholder = `_${placeholder}`; - - const omitted = keys - .map((key) => key.firstChild) - .filter((node) => node?.nodeValue === PROTOTYPE_KEY); - if (omitted.length === 0) return { substituted: text, placeholder }; +/** The validated DOM owns key identities; aliases bypass first-child-only decoding. */ +const normalizeXmlKeys = (text: string, document: Document) => { + const keys = Array.from(document.getElementsByTagName("key")) + // The decoder also accepts as a scalar outside dictionaries. Such + // values have no dictionary-name restoration boundary and must not alias. + .filter((key) => key.parentNode?.nodeName === "dict") + .map((key) => { + const first = key.firstChild; + // Nested elements are unsupported key markup. Retain the pinned decoder's + // interpretation instead of treating descendant element text as key data. + const characterDataOnly = !Array.from(key.childNodes).some( + (child) => child.nodeType === child.ELEMENT_NODE, + ); + return { + node: first ?? key, + empty: first === null, + name: characterDataOnly + ? (key.textContent ?? "") + : first === null + ? "" + : String(first.nodeValue), + firstName: first?.nodeValue ?? "", + }; + }); + const occupied = new Set( + keys.flatMap(({ name, firstName }) => [name, firstName]), + ); + const aliases = new Map(); + const names = new Map(); + for (const { name, firstName } of keys) { + if ((name !== PROTOTYPE_KEY && name === firstName) || aliases.has(name)) + continue; + let alias = `__rea_plist_key_${String(aliases.size)}__`; + while (occupied.has(alias)) alias = `_${alias}`; + occupied.add(alias); + aliases.set(name, alias); + names.set(alias, name); + } + // Alias every occurrence of a corrected name, including plain spellings, + // so the decoder retains its last-value-wins behavior for duplicate keys. + const replacements = keys.filter(({ name }) => aliases.has(name)); + if (replacements.length === 0) return { substituted: text, names }; // xmldom uses one-based locators in document order. Scan forward only to // needed lines instead of retaining an offset for every source newline. @@ -341,7 +375,7 @@ const substitutePrototypeKeys = (text: string, document: Document) => { let lineStart = 0; const parts: string[] = []; let cursor = 0; - for (const node of omitted) { + for (const { node, name, empty } of replacements) { const nodeLine = node?.lineNumber; const column = node?.columnNumber; if (nodeLine === undefined || column === undefined) @@ -354,9 +388,27 @@ const substitutePrototypeKeys = (text: string, document: Document) => { line += 1; } const offset = lineStart + column - 1; - parts.push(text.slice(cursor, offset), ``); - cursor = offset; + const marker = ``; + parts.push(text.slice(cursor, offset)); + if (empty) { + // Empty keys have no child locator. Replace only their opening tag, + // consuming quoted attributes whole, and expand self-closing spellings. + const opening = /^<(?:[^>"']|"[^"]*"|'[^']*')*>/u.exec( + text.slice(offset), + )?.[0]; + if (opening === undefined) + throw new TypeError("XML plist key has an invalid opening tag"); + parts.push( + /\/\s*>$/u.test(opening) + ? `${opening.replace(/\/\s*>$/u, ">")}${marker}` + : `${opening}${marker}`, + ); + cursor = offset + opening.length; + } else { + parts.push(marker); + cursor = offset; + } } parts.push(text.slice(cursor)); - return { substituted: parts.join(""), placeholder }; + return { substituted: parts.join(""), names }; }; diff --git a/src/native/parsers/plist.test.ts b/src/native/parsers/plist.test.ts index db1cbf1352..8d6be612de 100644 --- a/src/native/parsers/plist.test.ts +++ b/src/native/parsers/plist.test.ts @@ -6,6 +6,20 @@ import { plistJsonNeedsNumberTypes, } from "./plist.js"; +it("retains complete segmented XML keys in native plist fallback evidence", () => { + const parsed = parsePlistXml( + "user42__proomittedBlobAA==", + ); + if (!parsed.ok) throw parsed.error; + expect(parsed.value.value).toEqual({ + userID: 42, + Blob: { $plist_type: "data", base64: "AA==" }, + }); + expect(parsed.value.limitations).toContain( + "1 dictionary entry keyed __proto__ was omitted because REA results cannot represent that key.", + ); +}); + it("keeps exact decimal text for JSON integers that XML shows are integers", () => { const parsed = parsePlistJson( '{"Max":18446744073709551615,"Min":-9223372036854775808,"Near":9007199254740993,"Edge":9007199254740992,"Safe":9007199254740991,"Real":1e21,"Ratio":0.5}', diff --git a/tests/acceptance/applications/interfaceBuilderPlistNumbers.test.ts b/tests/acceptance/applications/interfaceBuilderPlistNumbers.test.ts index a70f3e1aa2..3c7049e3b9 100644 --- a/tests/acceptance/applications/interfaceBuilderPlistNumbers.test.ts +++ b/tests/acceptance/applications/interfaceBuilderPlistNumbers.test.ts @@ -18,6 +18,7 @@ const scenarios = [ "integer-real collision", "real", "mixed XML", + "segmented keys", ] as const; cliTest.for(scenarios)( @@ -35,7 +36,9 @@ cliTest.for(scenarios)( ); await writeFile( join(contents, "Info.plist"), - build({ CFBundleExecutable: "Fixture" }), + scenario === "segmented keys" + ? "CFBundleFixture" + : build({ CFBundleExecutable: "Fixture" }), ); const source = { $archiver: "NSKeyedArchiver", @@ -54,7 +57,7 @@ cliTest.for(scenarios)( { $classname: "UIView", $classes: ["UIView", "NSObject"] }, ], }; - const bytes = + let bytes = scenario === "binary" ? Buffer.from(buildBinary(source)) : Buffer.from( @@ -86,6 +89,14 @@ cliTest.for(scenarios)( bytes[offset] = 0x13; bytes.writeBigInt64BE(9007199254740993n, offset + 1); } + if (scenario === "segmented keys") + bytes = Buffer.from( + bytes + .toString("utf8") + .replaceAll("$class", "$cl") + .replaceAll("children", "chil") + .replaceAll("safe", "sa"), + ); await writeFile(join(resources, "Main.nib"), bytes); const output = await cli.run({ arguments: ["decode-interface-builder", app, "--json"], @@ -109,7 +120,9 @@ cliTest.for(scenarios)( id: 'ib:"Contents/Resources/Main.nib":object:["1"]', attributes: { large: - scenario === "XML" || scenario === "binary" + scenario === "XML" || + scenario === "binary" || + scenario === "segmented keys" ? { $plist_type: "integer", decimal: "9007199254740993" } : 9007199254740992, safe: 42, @@ -126,7 +139,11 @@ cliTest.for(scenarios)( relation: "contains", }), ); - if (scenario === "XML" || scenario === "binary") + if ( + scenario === "XML" || + scenario === "binary" || + scenario === "segmented keys" + ) expect(result.limitations).toContainEqual( expect.stringContaining("1 unambiguous integer value"), ); diff --git a/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts b/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts index 79716b6cd1..a686d012f6 100644 --- a/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts +++ b/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts @@ -1,3 +1,7 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { buildBinary } from "plist"; import { expect } from "vitest"; import { parseEvidence } from "../../../src/domain/evidence.js"; @@ -6,6 +10,9 @@ import { keyedArchivePrototypeKeyFixture, } from "../../fixtures/keyedArchivePrototypeKeys.js"; import { cliTest } from "../../support/cli/cliFixture.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; cliTest( "compiled CLI decodes formatted XML prototype keys", @@ -22,3 +29,92 @@ cliTest( ); }, ); + +cliTest( + "preserves segmented XML root and field identities through inspection and root selection", + async ({ cli }) => { + const directory = await createTestTempDirectory( + "rea-keyed-character-data-", + ); + const { call } = await connectLocalToolsMcp({ + HOME: directory, + USERPROFILE: directory, + }); + const mixed = Buffer.from(` + +$archiverNSKeyedArchiver +$version100000 +$objects$null +user42 + +$topma +CF$UID1 +`); + const plain = Buffer.from( + mixed + .toString("utf8") + .replace("user", "userID") + .replace("ma", "main"), + ); + // Binary encoding is an independent control, not a decode of the XML fixture. + const binary = Buffer.from( + buildBinary({ + $archiver: "NSKeyedArchiver", + $version: 100000, + $objects: ["$null", { userID: 42 }], + $top: { main: { CF$UID: 1 } }, + }), + ); + for (const [name, bytes] of [ + ["mixed", mixed], + ["plain", plain], + ["binary", binary], + ] as const) { + const path = join(directory, `${name}.plist`); + await writeFile(path, bytes); + for (const rootArguments of [[], ["--root", "main"]]) { + const output = await cli.run({ + arguments: [ + "inspect-keyed-archive", + path, + ...rootArguments, + "--json", + ], + environment: { + REA_LOG_LEVEL: "silent", + HOME: directory, + USERPROFILE: directory, + }, + }); + expect(output.exitCode, output.stdout).toBe(0); + const result = keyedArchiveResultSchema.parse( + parseEvidence(output.json).normalized_result, + ); + expect(result.archive_sha256).toBe( + createHash("sha256").update(bytes).digest("hex"), + ); + expect(Object.keys(result.roots)).toEqual(["main"]); + expect(result.objects[1]).toMatchObject({ + id: 1, + kind: "dictionary", + status: "observed", + value: { userID: 42 }, + }); + expect(result.references).toMatchObject([ + { source: null, path: ["main"], target: 1, status: "resolved" }, + ]); + expect(result.truncated).toBe(false); + const opened = await call("open_binary", { path }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const mcp = await call("inspect_keyed_archive", (rootArguments.length === 0 ? {} : { root: "main" })); + expect(mcp.isError, JSON.stringify(mcp)).not.toBe(true); + expect( + keyedArchiveResultSchema.parse( + parseEvidence(mcp.structuredContent).normalized_result, + ), + ).toEqual(result); + } + expect(await readFile(path)).toEqual(bytes); + } + }, +);