From 78cf6100f33ad2412c0a3d60b83f8a44a43fbbb5 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 17 Oct 2026 20:28:01 +0800 Subject: [PATCH] fix(plist): preserve complete XML dictionary key identities Use the validated DOM's character-data key names for both identity and prototype omission. Replace first-child reinterpretation with canonical, collision-free aliases inside the pinned decoder boundary; restore names through the existing dictionary projection. Alias every spelling of the same name to preserve duplicate assignment order. Handle empty key spellings and keep scalar key elements outside aliasing. Preserve value text and bytes. Adapt @thangldw's tests and proposal from PR #3426. Avoid the proposal's closing-content scanner and re-escaping source names. Add duplicate, empty, alias-collision, scalar-key, native fallback and Interface Builder controls. Validation: 180 focused checks across 20 files pass, including built CLI and actual stdio MCP, build, check:fast and docs:check. Baseline public inspection truncates main to ma. 24 tiny completed CLI cost controls preserve ordinary XML/binary full replies and Evidence IDs byte for byte. No statistical speed claim, heavy profiling, investigated-target or real deep-provider execution. Fixes #3371 --- docs/native-investigation.md | 5 +- src/domain/propertyListKeys.test.ts | 67 ++++++++++- src/domain/propertyListKeys.ts | 108 +++++++++++++----- src/native/parsers/plist.test.ts | 14 +++ .../interfaceBuilderPlistNumbers.test.ts | 25 +++- .../cli/keyedArchivePrototypeKeyCli.test.ts | 96 ++++++++++++++++ 6 files changed, 277 insertions(+), 38 deletions(-) diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 7661827134..b9b010ed8e 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -331,8 +331,9 @@ complete observation. 64 MiB of input, 200,000 objects/references and 128 nesting levels. XML dictionary entries decoded as `__proto__` are omitted with an explicit count because REA results cannot represent that key. XML tag whitespace and - comments do not change that handling. Other values and the original archive - digest are preserved; key interpretation follows the pinned plist decoder. + comments do not change that handling. XML key names combine their text and + CDATA segments without trimming or expanding literal CDATA entities. Other + values and the original archive digest are preserved. - `decode_interface_builder` / `rea decode-interface-builder ` reads compiled storyboard and nib resources into objects, outlets, actions, connections and controller/class names. A full app graph is often several diff --git a/src/domain/propertyListKeys.test.ts b/src/domain/propertyListKeys.test.ts index ff3ca7bf32..259204a227 100644 --- a/src/domain/propertyListKeys.test.ts +++ b/src/domain/propertyListKeys.test.ts @@ -53,9 +53,9 @@ it("keeps __proto__ text in strings, comments, and similar keys", () => { }); }); -it("keeps an entity-encoded key that decodes to the placeholder", () => { +it("keeps a segmented entity-encoded key that decodes to the placeholder", () => { const text = plist( - "__rea_prototype_key__kept__proto__x", + "__reakept__proto__x", ); expect(parseXmlPropertyList(text)).toEqual({ value: { __rea_prototype_key__: "kept" }, @@ -83,16 +83,75 @@ it("omits own __proto__ entries that JSON decoding creates", () => { }); }); -it("preserves the pinned decoder's first-child key semantics", () => { +it("omits complete prototype keys split across character-data nodes", () => { const text = plist( "__prokept__proto__omitted", ); expect(parseXmlPropertyList(text)).toEqual({ - value: { __pro: "kept" }, + value: {}, omittedPrototypeKeys: 1, }); }); +it("preserves literal key text, XML characters and encoded line endings across segments", () => { + const text = plist( + 'pre]]> …
]]>posta b', + ); + expect(parseXmlPropertyList(text)).toEqual({ + value: { "pre&\r\u0085\u2028]]>post": "a\rb" }, + omittedPrototypeKeys: 0, + }); +}); + +it("does not infer key identity from nested elements", () => { + expect( + parseXmlPropertyList( + plist( + "prenestedkept__proto__nestedomitted", + ), + ), + ).toEqual({ value: { pre: "kept" }, omittedPrototypeKeys: 1 }); +}); + +it.each([ + ["ab", "a", "ab"], + ["a", "ab", "a"], + ["ab", "ab", "ab"], +])("retains last-value-wins for mixed spellings %j", (...keys) => { + const entries = keys + .map( + (key, index) => + `${key}${String(index + 1)}`, + ) + .join(""); + expect(parseXmlPropertyList(plist(`${entries}`))).toEqual({ + value: { ab: 3 }, + omittedPrototypeKeys: 0, + }); +}); + +it("keeps empty keys and alias-looking source keys distinct", () => { + const entries = + "12" + + "3__rea_plist_key_0__literal" + + "ma42"; + expect(parseXmlPropertyList(plist(`${entries}`))).toEqual({ + value: { "": 3, __rea_plist_key_0__: "literal", main: 42 }, + omittedPrototypeKeys: 0, + }); +}); + +it.each([ + "ma", + "ma__rea_plist_key_0__", +])("does not alias key elements used as scalar values in %s", (body) => { + const text = plist(body); + expect(parseXmlPropertyList(text)).toEqual({ + value: parse(text), + omittedPrototypeKeys: 0, + }); +}); + it("preserves OpenStep dictionary and array autodetection", () => { expect( parseXmlPropertyList( diff --git a/src/domain/propertyListKeys.ts b/src/domain/propertyListKeys.ts index 7ed9157f83..12e3e7dba1 100644 --- a/src/domain/propertyListKeys.ts +++ b/src/domain/propertyListKeys.ts @@ -19,9 +19,9 @@ export const omittedPrototypeKeysLimitation = (count: number | null): string => ? "Dictionary entries keyed __proto__ were omitted because REA results cannot represent that key." : `${String(count)} dictionary ${count === 1 ? "entry" : "entries"} keyed __proto__ ${count === 1 ? "was" : "were"} omitted because REA results cannot represent that key.`; -const withoutKey = ( +const projectDictionaryKeys = ( value: unknown, - key: string, + projectName: (name: string) => string | undefined, ): PropertyListWithoutPrototypeKeys => { let omitted = 0; const strip = (item: unknown): unknown => { @@ -35,7 +35,8 @@ const withoutKey = ( return item; return Object.fromEntries( Object.entries(item).flatMap(([name, entry]) => { - if (name !== key) return [[name, strip(entry)]]; + const projected = projectName(name); + if (projected !== undefined) return [[projected, strip(entry)]]; omitted += 1; return []; }), @@ -48,7 +49,10 @@ const withoutKey = ( /** Remove own `__proto__` entries, such as JSON.parse creates, and count them. */ export const omitPrototypeKeys = ( value: unknown, -): PropertyListWithoutPrototypeKeys => withoutKey(value, PROTOTYPE_KEY); +): PropertyListWithoutPrototypeKeys => + projectDictionaryKeys(value, (name) => + name === PROTOTYPE_KEY ? undefined : name, + ); const XML_ENTITIES: Readonly> = { amp: "&", @@ -82,15 +86,14 @@ export const decodeXmlText = (text: string): string => ); /** - * Decode an XML plist whose dictionaries may use the legal key `__proto__`, - * which the `plist` decoder rejects outright. Those entries are omitted and - * counted so callers can report them instead of failing the whole document. + * Decode complete XML character-data keys before the pinned value decoder. + * Entries keyed `__proto__` are omitted and counted at the same boundary. */ export const parseXmlPropertyList = ( text: string, ): PropertyListWithoutPrototypeKeys => { // Preserve the pinned decoder's existing OpenStep autodetection, also used - // by app Info.plist resolution. Only XML needs the key-element substitution. + // by app Info.plist resolution. Only XML needs key normalization. const first = text.trimStart()[0]; if (first === "{" || first === "(") { // The pinned decoder assigns a `__proto__` key to the object prototype, so @@ -115,11 +118,11 @@ export const parseXmlPropertyList = ( if (!parsed.ok) throw parsed.error; const document = parsed.value; assertXmlPlistDepthWithinLimit(document); - const { substituted, placeholder } = substitutePrototypeKeys( - normalized, - document, - ); - return withoutKey(parse(substituted), placeholder); + const { substituted, names } = normalizeXmlKeys(normalized, document); + return projectDictionaryKeys(parse(substituted), (name) => { + const decoded = names.get(name) ?? name; + return decoded === PROTOTYPE_KEY ? undefined : decoded; + }); }; interface OpenStepContainer { @@ -323,17 +326,48 @@ const assertXmlPlistDepthWithinLimit = (document: Document): void => { } }; -/** Locate keys with the same XML parser and first-child semantics as plist. */ -const substitutePrototypeKeys = (text: string, document: Document) => { - const keys = Array.from(document.getElementsByTagName("key")); - const decodedKeys = new Set(keys.map((key) => key.firstChild?.nodeValue)); - let placeholder = "__rea_prototype_key__"; - while (decodedKeys.has(placeholder)) placeholder = `_${placeholder}`; - - const omitted = keys - .map((key) => key.firstChild) - .filter((node) => node?.nodeValue === PROTOTYPE_KEY); - if (omitted.length === 0) return { substituted: text, placeholder }; +/** The validated DOM owns key identities; aliases bypass first-child-only decoding. */ +const normalizeXmlKeys = (text: string, document: Document) => { + const keys = Array.from(document.getElementsByTagName("key")) + // The decoder also accepts as a scalar outside dictionaries. Such + // values have no dictionary-name restoration boundary and must not alias. + .filter((key) => key.parentNode?.nodeName === "dict") + .map((key) => { + const first = key.firstChild; + // Nested elements are unsupported key markup. Retain the pinned decoder's + // interpretation instead of treating descendant element text as key data. + const characterDataOnly = !Array.from(key.childNodes).some( + (child) => child.nodeType === child.ELEMENT_NODE, + ); + return { + node: first ?? key, + empty: first === null, + name: characterDataOnly + ? (key.textContent ?? "") + : first === null + ? "" + : String(first.nodeValue), + firstName: first?.nodeValue ?? "", + }; + }); + const occupied = new Set( + keys.flatMap(({ name, firstName }) => [name, firstName]), + ); + const aliases = new Map(); + const names = new Map(); + for (const { name, firstName } of keys) { + if ((name !== PROTOTYPE_KEY && name === firstName) || aliases.has(name)) + continue; + let alias = `__rea_plist_key_${String(aliases.size)}__`; + while (occupied.has(alias)) alias = `_${alias}`; + occupied.add(alias); + aliases.set(name, alias); + names.set(alias, name); + } + // Alias every occurrence of a corrected name, including plain spellings, + // so the decoder retains its last-value-wins behavior for duplicate keys. + const replacements = keys.filter(({ name }) => aliases.has(name)); + if (replacements.length === 0) return { substituted: text, names }; // xmldom uses one-based locators in document order. Scan forward only to // needed lines instead of retaining an offset for every source newline. @@ -341,7 +375,7 @@ const substitutePrototypeKeys = (text: string, document: Document) => { let lineStart = 0; const parts: string[] = []; let cursor = 0; - for (const node of omitted) { + for (const { node, name, empty } of replacements) { const nodeLine = node?.lineNumber; const column = node?.columnNumber; if (nodeLine === undefined || column === undefined) @@ -354,9 +388,27 @@ const substitutePrototypeKeys = (text: string, document: Document) => { line += 1; } const offset = lineStart + column - 1; - parts.push(text.slice(cursor, offset), ``); - cursor = offset; + const marker = ``; + parts.push(text.slice(cursor, offset)); + if (empty) { + // Empty keys have no child locator. Replace only their opening tag, + // consuming quoted attributes whole, and expand self-closing spellings. + const opening = /^<(?:[^>"']|"[^"]*"|'[^']*')*>/u.exec( + text.slice(offset), + )?.[0]; + if (opening === undefined) + throw new TypeError("XML plist key has an invalid opening tag"); + parts.push( + /\/\s*>$/u.test(opening) + ? `${opening.replace(/\/\s*>$/u, ">")}${marker}` + : `${opening}${marker}`, + ); + cursor = offset + opening.length; + } else { + parts.push(marker); + cursor = offset; + } } parts.push(text.slice(cursor)); - return { substituted: parts.join(""), placeholder }; + return { substituted: parts.join(""), names }; }; diff --git a/src/native/parsers/plist.test.ts b/src/native/parsers/plist.test.ts index db1cbf1352..8d6be612de 100644 --- a/src/native/parsers/plist.test.ts +++ b/src/native/parsers/plist.test.ts @@ -6,6 +6,20 @@ import { plistJsonNeedsNumberTypes, } from "./plist.js"; +it("retains complete segmented XML keys in native plist fallback evidence", () => { + const parsed = parsePlistXml( + "user42__proomittedBlobAA==", + ); + if (!parsed.ok) throw parsed.error; + expect(parsed.value.value).toEqual({ + userID: 42, + Blob: { $plist_type: "data", base64: "AA==" }, + }); + expect(parsed.value.limitations).toContain( + "1 dictionary entry keyed __proto__ was omitted because REA results cannot represent that key.", + ); +}); + it("keeps exact decimal text for JSON integers that XML shows are integers", () => { const parsed = parsePlistJson( '{"Max":18446744073709551615,"Min":-9223372036854775808,"Near":9007199254740993,"Edge":9007199254740992,"Safe":9007199254740991,"Real":1e21,"Ratio":0.5}', diff --git a/tests/acceptance/applications/interfaceBuilderPlistNumbers.test.ts b/tests/acceptance/applications/interfaceBuilderPlistNumbers.test.ts index a70f3e1aa2..3c7049e3b9 100644 --- a/tests/acceptance/applications/interfaceBuilderPlistNumbers.test.ts +++ b/tests/acceptance/applications/interfaceBuilderPlistNumbers.test.ts @@ -18,6 +18,7 @@ const scenarios = [ "integer-real collision", "real", "mixed XML", + "segmented keys", ] as const; cliTest.for(scenarios)( @@ -35,7 +36,9 @@ cliTest.for(scenarios)( ); await writeFile( join(contents, "Info.plist"), - build({ CFBundleExecutable: "Fixture" }), + scenario === "segmented keys" + ? "CFBundleFixture" + : build({ CFBundleExecutable: "Fixture" }), ); const source = { $archiver: "NSKeyedArchiver", @@ -54,7 +57,7 @@ cliTest.for(scenarios)( { $classname: "UIView", $classes: ["UIView", "NSObject"] }, ], }; - const bytes = + let bytes = scenario === "binary" ? Buffer.from(buildBinary(source)) : Buffer.from( @@ -86,6 +89,14 @@ cliTest.for(scenarios)( bytes[offset] = 0x13; bytes.writeBigInt64BE(9007199254740993n, offset + 1); } + if (scenario === "segmented keys") + bytes = Buffer.from( + bytes + .toString("utf8") + .replaceAll("$class", "$cl") + .replaceAll("children", "chil") + .replaceAll("safe", "sa"), + ); await writeFile(join(resources, "Main.nib"), bytes); const output = await cli.run({ arguments: ["decode-interface-builder", app, "--json"], @@ -109,7 +120,9 @@ cliTest.for(scenarios)( id: 'ib:"Contents/Resources/Main.nib":object:["1"]', attributes: { large: - scenario === "XML" || scenario === "binary" + scenario === "XML" || + scenario === "binary" || + scenario === "segmented keys" ? { $plist_type: "integer", decimal: "9007199254740993" } : 9007199254740992, safe: 42, @@ -126,7 +139,11 @@ cliTest.for(scenarios)( relation: "contains", }), ); - if (scenario === "XML" || scenario === "binary") + if ( + scenario === "XML" || + scenario === "binary" || + scenario === "segmented keys" + ) expect(result.limitations).toContainEqual( expect.stringContaining("1 unambiguous integer value"), ); diff --git a/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts b/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts index 79716b6cd1..a686d012f6 100644 --- a/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts +++ b/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts @@ -1,3 +1,7 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { buildBinary } from "plist"; import { expect } from "vitest"; import { parseEvidence } from "../../../src/domain/evidence.js"; @@ -6,6 +10,9 @@ import { keyedArchivePrototypeKeyFixture, } from "../../fixtures/keyedArchivePrototypeKeys.js"; import { cliTest } from "../../support/cli/cliFixture.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; cliTest( "compiled CLI decodes formatted XML prototype keys", @@ -22,3 +29,92 @@ cliTest( ); }, ); + +cliTest( + "preserves segmented XML root and field identities through inspection and root selection", + async ({ cli }) => { + const directory = await createTestTempDirectory( + "rea-keyed-character-data-", + ); + const { call } = await connectLocalToolsMcp({ + HOME: directory, + USERPROFILE: directory, + }); + const mixed = Buffer.from(` + +$archiverNSKeyedArchiver +$version100000 +$objects$null +user42 + +$topma +CF$UID1 +`); + const plain = Buffer.from( + mixed + .toString("utf8") + .replace("user", "userID") + .replace("ma", "main"), + ); + // Binary encoding is an independent control, not a decode of the XML fixture. + const binary = Buffer.from( + buildBinary({ + $archiver: "NSKeyedArchiver", + $version: 100000, + $objects: ["$null", { userID: 42 }], + $top: { main: { CF$UID: 1 } }, + }), + ); + for (const [name, bytes] of [ + ["mixed", mixed], + ["plain", plain], + ["binary", binary], + ] as const) { + const path = join(directory, `${name}.plist`); + await writeFile(path, bytes); + for (const rootArguments of [[], ["--root", "main"]]) { + const output = await cli.run({ + arguments: [ + "inspect-keyed-archive", + path, + ...rootArguments, + "--json", + ], + environment: { + REA_LOG_LEVEL: "silent", + HOME: directory, + USERPROFILE: directory, + }, + }); + expect(output.exitCode, output.stdout).toBe(0); + const result = keyedArchiveResultSchema.parse( + parseEvidence(output.json).normalized_result, + ); + expect(result.archive_sha256).toBe( + createHash("sha256").update(bytes).digest("hex"), + ); + expect(Object.keys(result.roots)).toEqual(["main"]); + expect(result.objects[1]).toMatchObject({ + id: 1, + kind: "dictionary", + status: "observed", + value: { userID: 42 }, + }); + expect(result.references).toMatchObject([ + { source: null, path: ["main"], target: 1, status: "resolved" }, + ]); + expect(result.truncated).toBe(false); + const opened = await call("open_binary", { path }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const mcp = await call("inspect_keyed_archive", (rootArguments.length === 0 ? {} : { root: "main" })); + expect(mcp.isError, JSON.stringify(mcp)).not.toBe(true); + expect( + keyedArchiveResultSchema.parse( + parseEvidence(mcp.structuredContent).normalized_result, + ), + ).toEqual(result); + } + expect(await readFile(path)).toEqual(bytes); + } + }, +);