mirror/rea
1
0
Fork 0
mirror of https://github.com/morluto/rea synced 2026-10-26 14:25:49 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(python): preserve marshal names and packaged bytecode

Browse Source 
Use one marshal surrogatepass decoder for PYZ directories and Python bytecode.
Validate ordinary UTF-8 spans with the native fatal decoder, preserving BOMs,
lone surrogates, raw bytes and marshal reference slots.

Distinguish legacy full-pyc CArchive modules from modern marshal-only modules
before reconstructing headers. Share typed source/codec failures between stream
output and completion while retaining native-read drain and cleanup ownership.

Validated focused producer, reader, lifecycle, CLI and MCP regressions, plus
public inventory/extraction of CPython 3.14-generated marshal tables and code.
No packaged application was executed. Document original versus derived bytes.

Fixes #6800
Fixes #6789
This commit is contained in:
morluto
2026-10-24 01:24:33 +08:00
parent 2443b949a4
commit f8f52f205a
18 changed files with 986 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/python/PythonPycParser.test.ts
+2 -2
View File
@@ -321,11 +321,11 @@ it("charges long digits to the string budget and keeps large previews bounded",
it("decodes marshal strings with CPython surrogatepass rules", () => {
const report = parse(
withConsts([marshalUnicode(Buffer.from([0x73, 0xed, 0xa0, 0x80]))]),
withConsts([marshalUnicode(Buffer.from("efbbbf73eda080efbbbf", "hex"))]),
);
expect(root(report).consts[0]).toMatchObject({
kind: "str",
value: "s\ud800",
value: "\ufeffs\ud800\ufeff",
});
expect(() =>
parse(withConsts([marshalUnicode(Buffer.from([0xf0, 0x90, 0x80]))])),
src/python/PythonPycParser.ts
+4 -59
View File
@@ -1,4 +1,5 @@
import { createHash } from "node:crypto";
import { decodeMarshalUtf8 } from "../domain/python/marshalUtf8.js";
import {
PYTHON_PYC_CHECK_SOURCE,
PYTHON_PYC_CONST_PREVIEW_LIMIT,
@@ -1463,9 +1464,11 @@ class MarshalReader {
raw: bytes,
};
}
const decoded = decodeMarshalUtf8(bytes);
if (!decoded.ok) return malformed("Marshal string is not valid UTF-8.");
return {
kind: "str",
value: decodeSurrogatePass(bytes),
value: decoded.value,
bytes: length,
raw: bytes,
};
@@ -1543,64 +1546,6 @@ class MarshalReader {
}
}
/**
* Decode marshal UTF-8 the way CPython does (`surrogatepass`): encoded lone
* surrogates are kept, every other ill-formed sequence is rejected.
*/
const decodeSurrogatePass = (bytes: Buffer): string => {
const units: number[] = [];
let text = "";
for (let index = 0; index < bytes.length;) {
const lead = bytes[index] ?? 0;
let codePoint: number;
let width: number;
if (lead < 0x80) {
codePoint = lead;
width = 1;
} else if (lead >= 0xc2 && lead <= 0xdf) {
codePoint = lead & 0x1f;
width = 2;
} else if (lead >= 0xe0 && lead <= 0xef) {
codePoint = lead & 0x0f;
width = 3;
} else if (lead >= 0xf0 && lead <= 0xf4) {
codePoint = lead & 0x07;
width = 4;
} else {
return malformed("Marshal string is not valid UTF-8.");
}
if (index + width > bytes.length)
return malformed("Marshal string is not valid UTF-8.");
for (let offset = 1; offset < width; offset += 1) {
const next = bytes[index + offset] ?? 0;
if ((next & 0xc0) !== 0x80)
return malformed("Marshal string is not valid UTF-8.");
codePoint = (codePoint << 6) | (next & 0x3f);
}
if (
(width === 3 && codePoint < 0x800) ||
(width === 4 && (codePoint < 0x10000 || codePoint > 0x10ffff))
)
return malformed("Marshal string is not valid UTF-8.");
units.push(codePoint);
index += width;
if (units.length === 4096) {
text += codePointsToString(units);
units.length = 0;
}
}
return text + codePointsToString(units);
};
const codePointsToString = (codePoints: readonly number[]): string =>
codePoints
.map((codePoint) =>
codePoint >= 0xd800 && codePoint <= 0xdfff
? String.fromCharCode(codePoint)
: String.fromCodePoint(codePoint),
)
.join("");
/** JSON has no non-finite numbers; keep CPython's repr for them. */
const jsonFloat = (value: number): number | "inf" | "-inf" | "nan" =>
Number.isNaN(value)