diff --git a/docs/python-packaged-applications.md b/docs/python-packaged-applications.md index e4d8d5aaef..fd561be399 100644 --- a/docs/python-packaged-applications.md +++ b/docs/python-packaged-applications.md @@ -96,8 +96,13 @@ returns a resource failure. Supported final Python bytecode magic values cover Python 3.7 through 3.14. Supported marshal directory values are lists, tuples, strings, integers and -bytes. An unsupported magic or directory stays an opaque member with a -limitation and `pyz_status: "opaque"`; its module contents remain unknown. +bytes. Directory values follow the identified CPython marshal semantics; for +example, a long integer with a zero most-significant digit is malformed and is +not normalized into a valid table value. An unsupported magic or malformed or +unsupported directory stays an opaque member with a limitation and +`pyz_status: "opaque"`; no child entries from that table are projected, and +their module contents remain unknown. The original PYZ member and its digest +remain available independently. Healthy CArchive siblings are still inventoried. Any opaque PYZ makes producer traversal coverage unknown, including retained summaries and exhausted queries. Per-archive reasons remain on the opaque occurrences; the coverage observation summarizes their count without repeating diff --git a/docs/python-pyc.md b/docs/python-pyc.md index 92e1bcc43d..4998b426be 100644 --- a/docs/python-pyc.md +++ b/docs/python-pyc.md @@ -1,10 +1,10 @@ # CPython pyc inspection -`inspect_python_pyc` / `inspect-python-pyc` answers which CPython version wrote -one selected local `.pyc` file, how its 16-byte header records source identity, -and which nested code objects marshal stored. It reads the file without running -bytecode and requires no Python installation, active binary target, Hopper, -Ghidra, or IDA. +`inspect_python_pyc` / `inspect-python-pyc` reports the supported CPython format +and version declared by one selected local `.pyc` file's magic, how its 16-byte +header records source identity, and which nested code objects its decoded +marshal graph contains. It reads the file without running bytecode and requires +no Python installation, active binary target, Hopper, Ghidra, or IDA. ```bash rea inspect-python-pyc /absolute/path/to/module.pyc --json @@ -33,10 +33,12 @@ The inline Evidence result contains the artifact path, SHA-256, and byte length. - `source_hash_hex` for hash-based pycs, or `source_mtime` and `source_size` for timestamp pycs. -`code_objects` lists each distinct marshal code object once, the top-level -module code first. `nested_code` holds the indexes of the distinct code objects -reachable from that object's constants, including code nested in tuples and -other containers and the owner itself when a constant container reaches it. +`code_objects` lists each code object reachable in the decoded marshal graph +once, the top-level module code first. This follows CPython container behavior; +it is not an inventory of raw serialized code-object records. `nested_code` +holds the indexes of the distinct code objects reachable from that object's +constants, including code nested in tuples and other containers and the owner +itself when a constant container reaches it. A function shared through a marshal reference is named by index once per owner, even when that owner reaches it again through the same container. Relationships are collected after the marshal graph is complete, so a container that a nested @@ -46,6 +48,13 @@ filename, the signed 32-bit `co_firstlineno` value, argument counts, flags, names, locals (or `localsplusnames` on 3.11+), const summaries, and bytecode length plus SHA-256. +Code-object equality used by sets and dictionaries follows the selected layout: +3.7–3.10 ignore line tables, while 3.11+ include line and exception tables. The +artifact SHA-256 identifies the exact selected `.pyc` bytes independently of +this decoded object graph. Reference slots follow each tag's CPython behavior: +singleton values and `TYPE_REF` do not reserve new slots even when their tag +has `FLAG_REF` set. + Bytecode is not decompiled. Const strings may be truncated in the summary while the marshal walk still consumed the original bytes; a string summary's `bytes` is that marshal length. Integer consts report `bits`; their decimal `value` is diff --git a/src/artifacts/pyinstaller/MarshalSubset.test.ts b/src/artifacts/pyinstaller/MarshalSubset.test.ts index bd0229ad0a..7b0c89105c 100644 --- a/src/artifacts/pyinstaller/MarshalSubset.test.ts +++ b/src/artifacts/pyinstaller/MarshalSubset.test.ts @@ -34,6 +34,13 @@ it("resolves a shared string through a reference reserved before the tuple", () ]); }); +it("does not allocate a new reference slot for a flagged TYPE_REF", () => { + const parsed = readMarshalSubset( + bytes("2904fa0161f200000000fa01627201000000"), + ); + expect(parsed).toEqual({ ok: true, value: ["a", "a", "b", "b"] }); +}); + it("reads a long integer that fits in a safe integer", () => { const parsed = readMarshalSubset( bytes( @@ -45,6 +52,33 @@ it("reads a long integer that fits in a safe integer", () => { expect(parsed.value).toEqual([["big", [0, 2 ** 40, 5]]]); }); +it.each(["6c0200000003000000", "6cfeffffff03000000"])( + "rejects marshal long %s with a zero most-significant digit as malformed", + (hex) => { + expect(readMarshalSubset(bytes(hex))).toMatchObject({ + ok: false, + error: { + reason: "format", + message: "PYZ marshal long is not normalized", + }, + }); + }, +); + +it("reads the zero-digit marshal long encoding as zero", () => { + expect(readMarshalSubset(bytes("6c00000000"))).toEqual({ + ok: true, + value: 0, + }); +}); + +it("keeps positive over-budget marshal long digit counts as resource refusals", () => { + expect(readMarshalSubset(bytes("6c09000000"))).toMatchObject({ + ok: false, + error: { reason: "limit" }, + }); +}); + it("reads a negative 32-bit integer", () => { const parsed = readMarshalSubset( bytes("5b01000000a902da016ea903e900000000e9ffffffffe901000000"), diff --git a/src/artifacts/pyinstaller/MarshalSubset.ts b/src/artifacts/pyinstaller/MarshalSubset.ts index b0290d0c85..566786de5d 100644 --- a/src/artifacts/pyinstaller/MarshalSubset.ts +++ b/src/artifacts/pyinstaller/MarshalSubset.ts @@ -67,6 +67,7 @@ const readObject = ( if (!code.ok) return code; const flag = (code.value & FLAG_REF) !== 0; const type = code.value & ~FLAG_REF; + if (type === 0x72) return readReference(cursor); if (type === 0x69) return storeScalar(cursor, flag, readInt32(cursor)); if (type === 0x6c) return storeScalar(cursor, flag, readLong(cursor)); if (type === 0x7a || type === 0x5a) @@ -78,7 +79,6 @@ const readObject = ( if (type === 0x28) return readSequence(cursor, flag, readInt32(cursor)); if (type === 0x29) return readSequence(cursor, flag, readByte(cursor)); if (type === 0x5b) return readSequence(cursor, flag, readInt32(cursor)); - if (type === 0x72) return readReference(cursor, flag); return err( pyInstallerFailure( "format", @@ -130,7 +130,6 @@ const readSequence = ( const readReference = ( cursor: MarshalCursor, - flag: boolean, ): Result => { const index = readInt32(cursor); if (!index.ok) return index; @@ -142,7 +141,6 @@ const readReference = ( "PYZ marshal reference does not point at a completed value", ), ); - if (flag) cursor.refs.push(value); return ok(value); }; @@ -219,6 +217,7 @@ const readLong = ( pyInstallerFailure("limit", "PYZ marshal integer exceeds the safe range"), ); let magnitude = 0n; + let mostSignificantDigit = 0; for (let index = 0; index < count; index += 1) { const digit = readUint16(cursor); if (!digit.ok) return digit; @@ -226,8 +225,13 @@ const readLong = ( return err( pyInstallerFailure("format", "PYZ marshal long digit is out of range"), ); + mostSignificantDigit = digit.value; magnitude += BigInt(digit.value) << BigInt(15 * index); } + if (count > 0 && mostSignificantDigit === 0) + return err( + pyInstallerFailure("format", "PYZ marshal long is not normalized"), + ); if (magnitude > BigInt(Number.MAX_SAFE_INTEGER)) return err( pyInstallerFailure("limit", "PYZ marshal integer exceeds the safe range"), diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index 54c7839a9a..83d53ac8aa 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -45,6 +45,14 @@ const TYPE_ASCII_INTERNED = 0x41; const TYPE_SMALL_TUPLE = 0x29; const TYPE_SHORT_ASCII = 0x7a; const TYPE_SHORT_ASCII_INTERNED = 0x5a; +const TYPES_WITHOUT_REFERENCE_SLOTS = new Set([ + TYPE_NULL, + TYPE_NONE, + TYPE_FALSE, + TYPE_TRUE, + TYPE_STOPITER, + TYPE_ELLIPSIS, +]); /** CPython `CO_VARARGS` / `CO_VARKEYWORDS`. Each consumes one local name. */ const CO_VARARGS = 0x04; const CO_VARKEYWORDS = 0x08; @@ -309,11 +317,13 @@ class MarshalReader { const raw = this.u8(); const flagged = (raw & FLAG_REF) !== 0; const type = raw & ~FLAG_REF; - if (type === TYPE_REF) { - if (flagged) malformed("TYPE_REF must not also set FLAG_REF."); - return this.readRef(); - } - const slot = flagged ? this.#intern.length : undefined; + // CPython masks FLAG_REF before dispatch; TYPE_REF returns its existing + // slot directly, and singleton tags bypass reference registration. + if (type === TYPE_REF) return this.readRef(); + const slot = + flagged && !TYPES_WITHOUT_REFERENCE_SLOTS.has(type) + ? this.#intern.length + : undefined; if (slot !== undefined) this.#intern.push(undefined); const value = this.readTyped(type, depth, slot); if (slot !== undefined && this.#intern[slot] === undefined) diff --git a/tests/acceptance/pyinstallerArchive.test.ts b/tests/acceptance/pyinstallerArchive.test.ts index d2926756bb..393ff64a05 100644 --- a/tests/acceptance/pyinstallerArchive.test.ts +++ b/tests/acceptance/pyinstallerArchive.test.ts @@ -88,6 +88,38 @@ const onefile = async ( return path; }; +const int32 = (value: number): Buffer => { + const bytes = Buffer.alloc(4); + bytes.writeInt32LE(value); + return bytes; +}; + +const marshalInt = (value: number): Buffer => + Buffer.concat([Buffer.from([0x69]), int32(value)]); + +const marshalLong = (digits: readonly number[]): Buffer => + Buffer.concat([ + Buffer.from([0x6c]), + int32(digits.length), + ...digits.map((digit) => { + const bytes = Buffer.alloc(2); + bytes.writeUInt16LE(digit); + return bytes; + }), + ]); + +const namespaceToc = (typeCode: Buffer): Buffer => + Buffer.concat([ + Buffer.from([0x5b]), + int32(1), + Buffer.from([0x29, 2, 0x7a, 3]), + Buffer.from("pkg"), + Buffer.from([0x29, 3]), + typeCode, + marshalInt(12), + marshalInt(0), + ]); + const namespacePathCollision = async (): Promise => { const directory = await createTestTempDirectory("rea-pyinstaller-namespace-"); const path = join(directory, "collision"); @@ -345,6 +377,67 @@ it("reports unknown traversal through the CLI for an opaque PYZ", async () => { }); }); +it.each([ + ["integer", marshalInt(3), true], + ["normalized long", marshalLong([3]), true], + ["unnormalized long", marshalLong([3, 0]), false], +])( + "keeps PYZ namespace projection consistent with CPython for %s type codes", + async (_encoding, typeCode, decoded) => { + const path = await onefile( + PYTHON_39_MAGIC, + false, + false, + namespaceToc(typeCode), + ); + const result = await execute(process.execPath, [ + cli, + "inspect-artifact", + path, + "--json", + ]); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(JSON.parse(result.stdout)).normalized_result, + ); + const inventory = inspection.substeps[0]?.evidence.normalized_result; + const pyz = inventory?.occurrences.find( + (row) => row.logical_path === "PYZ.pyz", + ); + const healthySibling = inventory?.occurrences.find( + (row) => row.logical_path === "healthy.txt", + ); + expect(healthySibling).toMatchObject({ hash_status: "verified" }); + expect( + inventory?.nodes.find( + (node) => node.artifact_id === healthySibling?.artifact_id, + ), + ).toMatchObject({ + sha256: createHash("sha256") + .update(Buffer.from("healthy sibling")) + .digest("hex"), + size: Buffer.byteLength("healthy sibling"), + }); + if (decoded) { + expect(inventory?.coverage.status).toBe("complete"); + expect(pyz?.format_metadata).toMatchObject({ pyz_status: "decoded" }); + expect( + inventory?.occurrences.some( + (row) => row.logical_path === "PYZ.pyz/pkg", + ), + ).toBe(true); + } else { + expect(inventory?.coverage.status).toBe("unknown"); + expect(pyz?.format_metadata).toMatchObject({ pyz_status: "opaque" }); + expect( + inventory?.occurrences.some( + (row) => row.logical_path === "PYZ.pyz/pkg", + ), + ).toBe(false); + expect(pyz?.limitations.join(" ")).toContain("long is not normalized"); + } + }, +); + it.each([ ["negative string length", "5b0100000073ffffffff"], ["negative list count", "5bffffffff"], diff --git a/tests/acceptance/pythonPycCli.test.ts b/tests/acceptance/pythonPycCli.test.ts index d83d8404a1..98ff658553 100644 --- a/tests/acceptance/pythonPycCli.test.ts +++ b/tests/acceptance/pythonPycCli.test.ts @@ -61,6 +61,56 @@ cliTest( }, ); +cliTest( + "matches CPython reference slots for singleton and reference tags", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-python-pyc-ref-slots-"); + const cases = [ + { + name: "flagged-none", + consts: "2903cefa01787200000000", + expected: [ + { kind: "none" }, + { kind: "str", value: "x" }, + { kind: "str", value: "x" }, + ], + }, + { + name: "flagged-ref", + consts: "2904fa0161f200000000fa01627201000000", + expected: [ + { kind: "str", value: "a" }, + { kind: "str", value: "a" }, + { kind: "str", value: "b" }, + { kind: "str", value: "b" }, + ], + }, + ]; + for (const testCase of cases) { + const path = join(root, `${testCase.name}.pyc`); + await writeFile( + path, + Buffer.concat([ + pythonPycHeader(3531), + marshalCode("py311", { + constsObject: Buffer.from(testCase.consts, "hex"), + }), + ]), + ); + const result = await cli.run({ + arguments: ["inspect-python-pyc", path, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const evidence = toolContract("inspect_python_pyc").outputSchema.parse( + result.json, + ); + expect(evidence.normalized_result.code_objects[0]?.consts).toMatchObject( + testCase.expected, + ); + } + }, +); + cliTest( "counts code keys using the selected CPython equality rules", async ({ cli }) => {