diff --git a/src/artifacts/pyinstaller/MarshalSubset.ts b/src/artifacts/pyinstaller/MarshalSubset.ts index 12e7aa1695..5898aefc77 100644 --- a/src/artifacts/pyinstaller/MarshalSubset.ts +++ b/src/artifacts/pyinstaller/MarshalSubset.ts @@ -190,7 +190,7 @@ const readUtf8 = ( return err( pyInstallerFailure("format", "PYZ marshal string is not valid UTF-8"), ); - return value; + return ok(value.value.text); }; const readByteString = ( diff --git a/src/domain/python/marshalUtf8.test.ts b/src/domain/python/marshalUtf8.test.ts index 7166e6e11e..11ab46bd3b 100644 --- a/src/domain/python/marshalUtf8.test.ts +++ b/src/domain/python/marshalUtf8.test.ts @@ -13,11 +13,55 @@ it.each([ (hex, value) => { expect(decodeMarshalUtf8(Buffer.from(hex, "hex"))).toEqual({ ok: true, - value, + value: { text: value, prefixEnd: value.length }, }); }, ); +it("keeps source-point boundaries before UTF-16 merges adjacent surrogate encodings", () => { + const prefix = "a".repeat(79); + const scalar = Buffer.from(prefix + "\u{10000}z"); + const separate = Buffer.concat([ + Buffer.from(prefix), + Buffer.from("eda080edb080", "hex"), + Buffer.from("z"), + ]); + const a = decodeMarshalUtf8(scalar, 80); + const b = decodeMarshalUtf8(separate, 80); + expect(a).toEqual({ + ok: true, + value: { text: prefix + "\u{10000}z", prefixEnd: 79 }, + }); + expect(b).toEqual({ + ok: true, + value: { text: prefix + "\ud800\udc00z", prefixEnd: 80 }, + }); + if (a.ok && b.ok) { + expect(a.value.text.slice(0, a.value.prefixEnd)).toBe(prefix); + expect(b.value.text.slice(0, b.value.prefixEnd)).toBe(prefix + "\ud800"); + } +}); + +it("validates bytes after a closed preview and preserves whole boundary controls", () => { + expect(decodeMarshalUtf8(Buffer.from("\u{1f600}".repeat(40)), 80)).toEqual({ + ok: true, + value: { text: "\u{1f600}".repeat(40), prefixEnd: 80 }, + }); + expect(decodeMarshalUtf8(Buffer.from("abc"), 0)).toEqual({ + ok: true, + value: { text: "abc", prefixEnd: 0 }, + }); + expect( + decodeMarshalUtf8( + Buffer.concat([ + Buffer.from("a".repeat(100)), + Buffer.from("eda08080", "hex"), + ]), + 80, + ), + ).toEqual({ ok: false, error: { kind: "invalid-marshal-utf8" } }); +}); + it.each([ "c2eda080", "e282eda080", diff --git a/src/domain/python/marshalUtf8.ts b/src/domain/python/marshalUtf8.ts index d3c054d90d..d81b648e0e 100644 --- a/src/domain/python/marshalUtf8.ts +++ b/src/domain/python/marshalUtf8.ts @@ -2,12 +2,48 @@ import { err, ok, type Result } from "../result.js"; const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +export interface DecodedMarshalUtf8 { + readonly text: string; + /** UTF-16 prefix endpoint between source code points, before JS merges surrogates. */ + readonly prefixEnd: number; +} + +/** This span contains valid scalars, or one separately encoded surrogate. */ +const spanPrefixEnd = (text: string, limit: number): number => { + const end = Math.min(text.length, limit); + return end > 0 && + end < text.length && + text.charCodeAt(end - 1) >= 0xd800 && + text.charCodeAt(end - 1) <= 0xdbff && + text.charCodeAt(end) >= 0xdc00 && + text.charCodeAt(end) <= 0xdfff + ? end - 1 + : end; +}; + /** CPython marshal uses UTF-8 surrogatepass; all other malformed bytes fail. */ export const decodeMarshalUtf8 = ( bytes: Buffer, -): Result => { + maximumPrefixUnits = Infinity, +): Result => { + if ( + maximumPrefixUnits !== Infinity && + (!Number.isSafeInteger(maximumPrefixUnits) || maximumPrefixUnits < 0) + ) + throw new RangeError( + "Marshal prefix limit must be a nonnegative safe integer.", + ); const chunks: string[] = []; let start = 0; + let prefixEnd = 0; + let prefixOpen = true; + const append = (text: string): void => { + if (text.length !== 0) chunks.push(text); + if (!prefixOpen) return; + const end = spanPrefixEnd(text, maximumPrefixUnits - prefixEnd); + prefixEnd += end; + prefixOpen = end === text.length && prefixEnd < maximumPrefixUnits; + }; try { for ( let offset = bytes.indexOf(0xed); @@ -27,16 +63,19 @@ export const decodeMarshalUtf8 = ( continue; // Only ED A0..BF 80..BF encodes a surrogate. The fatal decoder owns // every unchanged span, including incomplete scalars on either side. - chunks.push( - UTF8.decode(bytes.subarray(start, offset)), + append(UTF8.decode(bytes.subarray(start, offset))); + append( String.fromCharCode(0xd000 | ((second & 0x3f) << 6) | (third & 0x3f)), ); start = offset + 3; offset += 2; } - if (start === 0) return ok(UTF8.decode(bytes)); - chunks.push(UTF8.decode(bytes.subarray(start))); - return ok(chunks.join("")); + if (start === 0) { + const text = UTF8.decode(bytes); + return ok({ text, prefixEnd: spanPrefixEnd(text, maximumPrefixUnits) }); + } + append(UTF8.decode(bytes.subarray(start))); + return ok({ text: chunks.join(""), prefixEnd }); } catch (cause: unknown) { if (cause instanceof TypeError) return err({ kind: "invalid-marshal-utf8" }); diff --git a/src/python/PythonPycParser.test.ts b/src/python/PythonPycParser.test.ts index cfb14fc335..da2fec700d 100644 --- a/src/python/PythonPycParser.test.ts +++ b/src/python/PythonPycParser.test.ts @@ -1625,3 +1625,21 @@ it("leaves 3.11 alpha magics with other code layouts unsupported", () => { parse(pythonPycFixture("py311", { magic: 3495 })).header, ).toMatchObject({ python_version: "3.11", code_layout: "py311" }); }); + +it.each([false, true])( + "preserves a source-encoded lone surrogate at the preview boundary (truncated=%s)", + (truncated) => { + const text = Buffer.concat([ + Buffer.from("a".repeat(79)), + Buffer.from([0xed, 0xa0, 0x80]), + ...(truncated ? [Buffer.from("z")] : []), + ]); + const report = parse(withConsts([marshalUnicode(text)])); + expect(root(report).consts[0]).toEqual({ + kind: "str", + value: "a".repeat(79) + "\ud800", + bytes: truncated ? 83 : 82, + truncated, + }); + }, +); diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index e68669f12c..af6b76394c 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -80,6 +80,7 @@ type MarshalValue = readonly bytes: number; /** Marshal bytes. JavaScript strings merge lone surrogates with pairs. */ readonly raw: Buffer; + readonly previewEnd: number; } | { readonly kind: "tuple"; readonly items: MarshalValue[] } | { readonly kind: "list"; readonly items: MarshalValue[] } @@ -1483,15 +1484,17 @@ class MarshalReader { return { kind: "str", value: bytes.toString("latin1"), + previewEnd: Math.min(length, PYTHON_PYC_CONST_PREVIEW_LIMIT), bytes: length, raw: bytes, }; } - const decoded = decodeMarshalUtf8(bytes); + const decoded = decodeMarshalUtf8(bytes, PYTHON_PYC_CONST_PREVIEW_LIMIT); if (!decoded.ok) return malformed("Marshal string is not valid UTF-8."); return { kind: "str", - value: decoded.value, + value: decoded.value.text, + previewEnd: decoded.value.prefixEnd, bytes: length, raw: bytes, }; @@ -1665,21 +1668,11 @@ const buildConstSummary = (value: MarshalValue): PythonPycConst => { return { kind: "float", value: jsonFloat(value.value) }; case "str": { const truncated = value.value.length > PYTHON_PYC_CONST_PREVIEW_LIMIT; - let end = PYTHON_PYC_CONST_PREVIEW_LIMIT; - // Preserve source lone surrogates, but do not split a source scalar. - if ( - truncated && - value.value.charCodeAt(end - 1) >= 0xd800 && - value.value.charCodeAt(end - 1) <= 0xdbff && - value.value.charCodeAt(end) >= 0xdc00 && - value.value.charCodeAt(end) <= 0xdfff - ) - end--; return { kind: "str", bytes: value.bytes, truncated, - value: truncated ? value.value.slice(0, end) : value.value, + value: truncated ? value.value.slice(0, value.previewEnd) : value.value, }; } case "bytes": diff --git a/tests/acceptance/pythonPycCli.test.ts b/tests/acceptance/pythonPycCli.test.ts index 5ff4ce8d8d..207755be5a 100644 --- a/tests/acceptance/pythonPycCli.test.ts +++ b/tests/acceptance/pythonPycCli.test.ts @@ -717,3 +717,123 @@ cliTest("reports a truncated pyc as invalid input", async ({ cli }) => { details: { operation: "inspect_python_pyc" }, }); }); + +cliTest( + "preserves whole Unicode scalars in bounded pyc previews through CLI and MCP", + async ({ cli }) => { + const cases = [ + { + input: "a" + "😀".repeat(40), + preview: "a" + "😀".repeat(39), + bytes: 161, + truncated: true, + }, + { + input: "a" + "😀".repeat(40) + "z", + preview: "a" + "😀".repeat(39), + bytes: 162, + truncated: true, + }, + { + input: "a" + "😀".repeat(39), + preview: "a" + "😀".repeat(39), + bytes: 157, + truncated: false, + }, + { + input: "😀".repeat(40), + preview: "😀".repeat(40), + bytes: 160, + truncated: false, + }, + { + input: "z".repeat(81), + preview: "z".repeat(80), + bytes: 81, + truncated: true, + }, + ]; + const consts = cases.map(({ input }) => { + const text = Buffer.from(input, "utf8"); + const header = Buffer.alloc(5); + header[0] = 0x75; + header.writeUInt32LE(text.length, 1); + return Buffer.concat([header, text]); + }); + const bytes = Buffer.concat([ + pythonPycHeader(3531), + marshalCode("py311", { consts }), + ]); + const root = await createTestTempDirectory("rea-python-pyc-preview-"); + const path = join(root, "unicode.pyc"); + await writeFile(path, bytes); + const response = await cli.run({ + arguments: ["inspect-python-pyc", path, "--json"], + }); + expect(response.exitCode, response.stderr).toBe(0); + const contract = toolContract("inspect_python_pyc"); + const report = contract.outputSchema.parse(response.json).normalized_result; + expect(report.coverage.status).toBe("complete"); + expect(report.code_objects[0]?.consts).toEqual( + cases.map((entry) => ({ + kind: "str", + value: entry.preview, + bytes: entry.bytes, + truncated: entry.truncated, + })), + ); + const call = await cli.sharedMcpCall?.(); + if (call === undefined) throw new Error("Expected file MCP session"); + const mcp = await call("inspect_python_pyc", { path }); + expect(mcp.isError).not.toBe(true); + expect( + contract.outputSchema.parse(mcp.structuredContent).normalized_result, + ).toEqual(report); + const toon = await cli.run({ + arguments: ["inspect-python-pyc", path, "--format", "toon"], + }); + expect(toon.exitCode, toon.stderr).toBe(0); + expect(toon.stdout).not.toContain("�"); + }, +); + +cliTest( + "distinguishes source surrogate code points from astral scalars at the preview cut", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-python-source-boundary-"); + const path = join(root, "source-boundaries.pyc"); + const prefix = "a".repeat(79); + const scalar = Buffer.from(prefix + "𐀀z"); + const separate = Buffer.concat([ + Buffer.from(prefix), + Buffer.from([0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80]), + Buffer.from("z"), + ]); + await writeFile( + path, + Buffer.concat([ + pythonPycHeader(3531), + marshalCode("py311", { + consts: [marshalUtf8(scalar), marshalUtf8(separate)], + }), + ]), + ); + const response = await cli.run({ + arguments: ["inspect-python-pyc", path, "--json"], + }); + expect(response.exitCode, response.stderr).toBe(0); + const schema = toolContract("inspect_python_pyc").outputSchema; + const report = schema.parse(response.json).normalized_result; + expect(report.code_objects[0]?.consts).toEqual([ + { kind: "str", value: prefix, bytes: 84, truncated: true }, + { kind: "str", value: prefix + "\ud800", bytes: 86, truncated: true }, + ]); + const call = await cli.sharedMcpCall?.(); + if (call === undefined) throw new Error("Expected file MCP session"); + const mcp = await call("inspect_python_pyc", { path }); + expect(mcp.isError).not.toBe(true); + expect(schema.parse(mcp.structuredContent).normalized_result).toEqual( + report, + ); + }, +);