fix(python): preserve source boundaries in pyc previews (#6492)

Preserve source code-point prefix endpoints in the shared marshal UTF-8 decoder, including separately encoded surrogate values. Keep native TextDecoder spans, full validation, byte identity and the existing 80-unit budget. Update the PyInstaller consumer without changing its text contract.

Validation: 207 relevant source tests;24 affected compiled CLI/MCP/filesystem CI tests; independent CPython3.11.2 compiler/marshal oracle and real argparse;18 equivalent performance tasks; build/check passed. Codex reviewed7c0f7abc6b without major findings.

CI bypass authorized for demonstrated baseline-only failures:101 source and179 built cases reproduced on actual tested main ba9c5bc68. Cancelled shard remainder is unverified. See PR body and discussion for scope and evidence.

Fixes #6476

src/python/PythonPycParser.test.ts

@@ -1625,3 +1625,21 @@ it("leaves 3.11 alpha magics with other code layouts unsupported", () => {
     parse(pythonPycFixture("py311", { magic: 3495 })).header,
   ).toMatchObject({ python_version: "3.11", code_layout: "py311" });
 });
+
+it.each([false, true])(
+  "preserves a source-encoded lone surrogate at the preview boundary (truncated=%s)",
+  (truncated) => {
+    const text = Buffer.concat([
+      Buffer.from("a".repeat(79)),
+      Buffer.from([0xed, 0xa0, 0x80]),
+      ...(truncated ? [Buffer.from("z")] : []),
+    ]);
+    const report = parse(withConsts([marshalUnicode(text)]));
+    expect(root(report).consts[0]).toEqual({
+      kind: "str",
+      value: "a".repeat(79) + "\ud800",
+      bytes: truncated ? 83 : 82,
+      truncated,
+    });
+  },
+);

src/python/PythonPycParser.ts

@@ -80,6 +80,7 @@ type MarshalValue =
       readonly bytes: number;
       /** Marshal bytes. JavaScript strings merge lone surrogates with pairs. */
       readonly raw: Buffer;
+      readonly previewEnd: number;
     }
   | { readonly kind: "tuple"; readonly items: MarshalValue[] }
   | { readonly kind: "list"; readonly items: MarshalValue[] }
@@ -1483,15 +1484,17 @@ class MarshalReader {
       return {
         kind: "str",
         value: bytes.toString("latin1"),
+        previewEnd: Math.min(length, PYTHON_PYC_CONST_PREVIEW_LIMIT),
         bytes: length,
         raw: bytes,
       };
     }
-    const decoded = decodeMarshalUtf8(bytes);
+    const decoded = decodeMarshalUtf8(bytes, PYTHON_PYC_CONST_PREVIEW_LIMIT);
     if (!decoded.ok) return malformed("Marshal string is not valid UTF-8.");
     return {
       kind: "str",
-      value: decoded.value,
+      value: decoded.value.text,
+      previewEnd: decoded.value.prefixEnd,
       bytes: length,
       raw: bytes,
     };
@@ -1665,21 +1668,11 @@ const buildConstSummary = (value: MarshalValue): PythonPycConst => {
       return { kind: "float", value: jsonFloat(value.value) };
     case "str": {
       const truncated = value.value.length > PYTHON_PYC_CONST_PREVIEW_LIMIT;
-      let end = PYTHON_PYC_CONST_PREVIEW_LIMIT;
-      // Preserve source lone surrogates, but do not split a source scalar.
-      if (
-        truncated &&
-        value.value.charCodeAt(end - 1) >= 0xd800 &&
-        value.value.charCodeAt(end - 1) <= 0xdbff &&
-        value.value.charCodeAt(end) >= 0xdc00 &&
-        value.value.charCodeAt(end) <= 0xdfff
-      )
-        end--;
       return {
         kind: "str",
         bytes: value.bytes,
         truncated,
-        value: truncated ? value.value.slice(0, end) : value.value,
+        value: truncated ? value.value.slice(0, value.previewEnd) : value.value,
       };
     }
     case "bytes":