From 3784a8e547f006c3513fbc8948eaa72d8395718c Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:31:33 +0800 Subject: [PATCH] fix(python): compare code objects without inline caches Fixes #6597 --- docs/python-pyc.md | 12 +- src/python/PythonPycBytecodeEquality.ts | 468 ++++++++++++++++++++++++ src/python/PythonPycParser.ts | 17 +- tests/acceptance/pythonPycCli.test.ts | 65 ++++ 4 files changed, 552 insertions(+), 10 deletions(-) create mode 100644 src/python/PythonPycBytecodeEquality.ts diff --git a/docs/python-pyc.md b/docs/python-pyc.md index 614c87aa09..a33f200734 100644 --- a/docs/python-pyc.md +++ b/docs/python-pyc.md @@ -49,12 +49,16 @@ filename, the signed 32-bit `co_firstlineno` value, argument counts, flags, names, locals (or `localsplusnames` on 3.11+), const summaries, and bytecode length plus SHA-256. -Code-object equality used by sets and dictionaries follows the selected layout: +Code-object equality used by sets and dictionaries follows the selected CPython format: 3.7–3.10 derive `CO_NOFREE` from the free/cell tables for equality, while 3.11+ compare raw flags; raw `co_flags` remain in reported facts. The older -layouts ignore line tables, while 3.11+ include line and exception tables. The -artifact SHA-256 identifies the exact selected `.pyc` bytes independently of -this decoded object graph. Reference slots follow each tag's CPython behavior: +layouts ignore line tables, while 3.11+ include line and exception tables and +skip inline-cache code units according to the cache table for the exact magic. +Reported bytecode length and SHA-256 still describe the original code bytes. +This does not emulate CPython's deoptimization of specialized or instrumented +opcode bytes in handcrafted pyc files. +The artifact SHA-256 identifies the exact selected `.pyc` bytes independently +of this decoded object graph. Reference slots follow each tag's CPython behavior: singleton values and `TYPE_REF` do not reserve new slots even when their tag has `FLAG_REF` set. diff --git a/src/python/PythonPycBytecodeEquality.ts b/src/python/PythonPycBytecodeEquality.ts new file mode 100644 index 0000000000..9936d501b8 --- /dev/null +++ b/src/python/PythonPycBytecodeEquality.ts @@ -0,0 +1,468 @@ +import { createHash } from "node:crypto"; + +type InlineCacheEntry = readonly [opcode: number, codeUnits: number]; + +const inlineCacheProfiles = ( + magics: readonly number[], + entries: readonly InlineCacheEntry[], +): ReadonlyArray]> => { + const profile = new Map(entries); + return magics.map((magic) => [magic, profile]); +}; + +/** + * CPython equality skips cache code units after comparing original code size. + * These tables are pinned to CPython bytecode snapshots, not the shared marshal + * layout. The 3.11 profile is from v3.11.9; commit + * 616622cab7200bac476bf753d6cc98f4ee48808c adds magic 3495 with the same + * opcode/cache table. 3.12 magic/profile transitions + * follow CPython's v3.12.12 magic history and these cache-changing source + * commits: f012df706cf3a9e63625b8166fe3f419855e9a94 (3503), + * b083450f8896bb4a29ac522e4474d91c056b9f32 (3504), + * 5fcfdd87c9b5066a581d3ccb4b2fede938f343ec (3505), + * 6e4e14d98fe0868981f29701496d57a8223c5407 (3515), + * 7b14c2ef194b6eed79670aa9d7e29ab8e2256a56 (3516), + * 160f2fe2b90ed5ec7838cb4141dd35768422891f (3519), + * 08b67fb34f4519be1b0bb4673643a2c761c7ae92 (3521), + * 0444ae24875489dc290d8efdb9ee6440ee60dac8 (3523), + * 121057aa3600c4d4d392539aeb79e1b09fd5659d (3524), + * b4978ff872be5102117b4e25d93dbbb4e04c8292 (3525), + * ef25febcf2ede92a03c5ea00a13e167e0b5cb274 (3528), and + * 77262458fe3fe9f3f0266bbf578675a906b1e353 (3530). The 3.13 magic history + * is recorded in CPython's Lib/importlib/_bootstrap_external.py; opcode/cache sources include + * 1d857da7f0e4858e561223f319ae5afe737d5657 (3553/3554), + * 665a4391e10167dad1c854fb604c86f336fcd331 (3559), + * 3b2ecbc1275bd05534885cee9ac1389987238561 (3560), + * bcce5e271815c0bdbe894964e853210d2c75949b (3561), + * 8b55adfa8ff05477b4be7def36db7b66c73f181d (3562), + * 22e65eecaad3f5337862319687047afe9861e1ef (3563), and + * 52cc4af6ae9002f11605f91b672746c127494efd (3564). Adjacent magics share a + * profile only where their source changes do not alter cache widths or IDs. + */ +const INLINE_CACHE_ENTRIES_BY_MAGIC: ReadonlyMap< + number, + ReadonlyMap +> = new Map([ + ...inlineCacheProfiles( + [3494, 3495], + [ + [25, 4], + [60, 1], + [92, 1], + [95, 4], + [106, 4], + [107, 2], + [116, 5], + [122, 1], + [160, 10], + [166, 1], + [171, 4], + ], + ), + ...inlineCacheProfiles( + [3500, 3501, 3502], + [ + [25, 4], + [60, 1], + [92, 1], + [95, 4], + [106, 4], + [107, 2], + [116, 5], + [122, 1], + [160, 10], + [171, 4], + ], + ), + ...inlineCacheProfiles( + [3503], + [ + [25, 4], + [60, 1], + [92, 1], + [95, 4], + [106, 4], + [107, 2], + [116, 5], + [122, 1], + [160, 9], + [171, 4], + ], + ), + ...inlineCacheProfiles( + [3504], + [ + [25, 4], + [60, 1], + [92, 1], + [95, 4], + [106, 9], + [107, 2], + [116, 5], + [122, 1], + [171, 4], + ], + ), + ...inlineCacheProfiles( + [3505, 3506, 3507, 3508, 3509, 3510, 3511, 3512, 3513, 3514], + [ + [25, 4], + [60, 1], + [92, 1], + [93, 1], + [95, 4], + [106, 9], + [107, 2], + [116, 5], + [122, 1], + [171, 4], + ], + ), + ...inlineCacheProfiles( + [3515], + [ + [25, 4], + [60, 1], + [92, 1], + [93, 1], + [95, 4], + [106, 9], + [107, 1], + [116, 5], + [122, 1], + [171, 4], + ], + ), + ...inlineCacheProfiles( + [3516, 3517, 3518], + [ + [25, 4], + [60, 1], + [92, 1], + [93, 1], + [95, 4], + [106, 9], + [107, 1], + [116, 5], + [122, 1], + [141, 1], + [171, 4], + ], + ), + ...inlineCacheProfiles( + [3519, 3520], + [ + [25, 4], + [60, 1], + [92, 1], + [93, 1], + [95, 4], + [106, 9], + [107, 1], + [116, 5], + [122, 1], + [123, 1], + [141, 1], + [171, 4], + ], + ), + ...inlineCacheProfiles( + [3521, 3522], + [ + [25, 4], + [60, 1], + [92, 1], + [93, 1], + [95, 4], + [106, 9], + [107, 1], + [116, 4], + [122, 1], + [123, 1], + [141, 1], + [171, 4], + ], + ), + ...inlineCacheProfiles( + [3523], + [ + [25, 4], + [60, 1], + [92, 1], + [93, 1], + [95, 4], + [106, 9], + [107, 1], + [116, 4], + [122, 1], + [123, 1], + [171, 4], + ], + ), + ...inlineCacheProfiles( + [3524], + [ + [25, 1], + [60, 1], + [92, 1], + [93, 1], + [95, 4], + [106, 9], + [107, 1], + [116, 4], + [122, 1], + [123, 1], + [171, 4], + ], + ), + ...inlineCacheProfiles( + [3525, 3526, 3527], + [ + [25, 1], + [60, 1], + [92, 1], + [93, 1], + [95, 4], + [106, 9], + [107, 1], + [116, 4], + [122, 1], + [123, 1], + [171, 3], + ], + ), + ...inlineCacheProfiles( + [3528, 3529], + [ + [25, 1], + [60, 1], + [92, 1], + [93, 1], + [95, 4], + [106, 9], + [107, 1], + [116, 4], + [122, 1], + [123, 1], + [141, 9], + [171, 3], + ], + ), + ...inlineCacheProfiles( + [3530, 3531], + [ + [25, 1], + [60, 1], + [92, 1], + [93, 1], + [95, 4], + [106, 9], + [107, 1], + [116, 4], + [122, 1], + [123, 1], + [141, 1], + [171, 3], + ], + ), + ...inlineCacheProfiles( + [3550, 3551, 3552, 3553, 3554, 3555, 3556, 3557, 3558], + [ + [25, 1], + [60, 1], + [92, 1], + [93, 1], + [95, 4], + [106, 9], + [107, 1], + [116, 4], + [122, 1], + [123, 1], + [140, 1], + [141, 1], + [171, 3], + ], + ), + ...inlineCacheProfiles( + [3559], + [ + [12, 1], + [53, 1], + [56, 3], + [67, 1], + [75, 3], + [97, 1], + [114, 1], + [123, 1], + [129, 9], + [150, 4], + [154, 1], + [168, 1], + [173, 4], + [183, 1], + ], + ), + ...inlineCacheProfiles( + [3560], + [ + [12, 1], + [54, 1], + [57, 3], + [68, 1], + [76, 3], + [98, 1], + [115, 1], + [124, 1], + [130, 9], + [151, 4], + [155, 1], + [169, 1], + [174, 4], + [184, 1], + ], + ), + ...inlineCacheProfiles( + [3561], + [ + [12, 1], + [54, 1], + [57, 3], + [68, 1], + [76, 3], + [98, 1], + [115, 1], + [124, 1], + [130, 9], + [151, 4], + [155, 1], + [161, 1], + [162, 1], + [163, 1], + [164, 1], + [169, 1], + [174, 4], + [184, 1], + ], + ), + ...inlineCacheProfiles( + [3562], + [ + [5, 1], + [39, 1], + [40, 3], + [45, 1], + [53, 3], + [57, 1], + [71, 1], + [76, 1], + [82, 9], + [91, 4], + [93, 1], + [97, 1], + [98, 1], + [99, 1], + [100, 1], + [104, 1], + [108, 4], + [117, 1], + ], + ), + ...inlineCacheProfiles( + [3563, 3564, 3565, 3566, 3567, 3568], + [ + [5, 1], + [39, 1], + [40, 3], + [45, 1], + [53, 3], + [58, 1], + [72, 1], + [77, 1], + [82, 9], + [91, 4], + [93, 1], + [97, 1], + [98, 1], + [99, 1], + [100, 1], + [104, 1], + [108, 4], + [117, 1], + ], + ), + ...inlineCacheProfiles( + [3569, 3570], + [ + [5, 1], + [39, 1], + [40, 3], + [45, 1], + [53, 3], + [58, 1], + [59, 1], + [72, 1], + [77, 1], + [82, 9], + [91, 4], + [93, 1], + [97, 1], + [98, 1], + [99, 1], + [100, 1], + [104, 1], + [108, 4], + [117, 1], + ], + ), + ...inlineCacheProfiles( + [3571], + [ + [5, 1], + [38, 1], + [39, 3], + [44, 1], + [52, 3], + [57, 1], + [58, 1], + [71, 1], + [76, 1], + [81, 9], + [91, 4], + [93, 1], + [97, 1], + [98, 1], + [99, 1], + [100, 1], + [104, 1], + [108, 4], + [117, 1], + ], + ), +]); + +/** Preserve CPython's bytecode-size check and omit only cache code units. */ +export const pythonPycBytecodeIdentity = ( + bytecode: Buffer, + magic: number, + rawSha256: string, +): string => { + const caches = INLINE_CACHE_ENTRIES_BY_MAGIC.get(magic); + if (caches === undefined) return `${bytecode.length}:${rawSha256}`; + + const digest = createHash("sha256"); + let activeStart = 0; + for (let offset = 0; offset < bytecode.length;) { + const opcode = bytecode[offset] ?? 0; + const afterOpcode = offset + 2; + const cacheBytes = (caches.get(opcode) ?? 0) * 2; + offset = afterOpcode + cacheBytes; + if (cacheBytes > 0) { + digest.update(bytecode.subarray(activeStart, afterOpcode)); + activeStart = offset; + } + } + if (activeStart < bytecode.length) { + digest.update(bytecode.subarray(activeStart)); + } + return `${bytecode.length}:${digest.digest("hex")}`; +}; diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index 2c245241eb..1ab4ab1d5e 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -16,6 +16,7 @@ import { PythonPycFormatFailure, PythonPycResourceFailure, } from "./PythonPyc.errors.js"; +import { pythonPycBytecodeIdentity } from "./PythonPycBytecodeEquality.js"; const FLAG_REF = 0x80; const TYPE_NULL = 0x30; @@ -154,6 +155,7 @@ type CodeEquality = { /** The co_consts tuple itself, which may be a reference to an unfinished tuple. */ readonly constsTuple: MarshalValue; readonly effectiveFlags: number; + readonly bytecode: Buffer; readonly name: Buffer; readonly nameTables: readonly (readonly Buffer[])[]; readonly linetable: string; @@ -234,6 +236,7 @@ export const parsePythonPyc = ( const reader = new MarshalReader( marshal, mapped.code_layout, + magic, input, mapped.negative_code_fields === "rejected", ); @@ -328,6 +331,7 @@ class MarshalReader { constructor( readonly bytes: Buffer, readonly layout: PythonPycCodeLayout, + readonly magic: number, readonly input: InspectPythonPycInput, readonly rejectNegativeCodeFields: boolean, ) {} @@ -558,6 +562,7 @@ class MarshalReader { consts, constsTuple, effectiveFlags, + bytecode, name: codeName.raw, nameTables: localsplusnames !== null @@ -1148,7 +1153,7 @@ class MarshalReader { ":", ); case "str": - return this.#stringKey(value.raw); + return this.#marshalStringKey(value.raw); case "bytes": { this.#reserveKey(value.value.length * 2 + 8); const prefix = mode === "hash" ? "b" : "y"; @@ -1227,7 +1232,11 @@ class MarshalReader { node.nlocals === null ? "" : String(node.nlocals), String(material.effectiveFlags), String(node.firstlineno), - `${node.bytecode.bytes}:${node.bytecode.sha256}`, + pythonPycBytecodeIdentity( + material.bytecode, + this.magic, + node.bytecode.sha256, + ), this.#constKeyItems(material.consts), ...material.nameTables.map((items) => this.#nameTableKey(items)), // CPython started including line and exception tables in code equality @@ -1307,10 +1316,6 @@ class MarshalReader { } /** Python string identity is the marshal bytes, not the JavaScript string. */ - #stringKey(raw: Buffer): string { - return this.#marshalStringKey(raw); - } - #marshalStringKey(raw: Buffer): string { this.#reserveKey(this.#marshalStringKeyLength(raw.length)); return `s${raw.length}:${raw.toString("hex")}`; diff --git a/tests/acceptance/pythonPycCli.test.ts b/tests/acceptance/pythonPycCli.test.ts index d2ab1c7584..404cfdb567 100644 --- a/tests/acceptance/pythonPycCli.test.ts +++ b/tests/acceptance/pythonPycCli.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { mkdir, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; @@ -16,6 +17,70 @@ const marshalUtf8 = (bytes: Buffer): Buffer => { return Buffer.concat([Buffer.from([0x75]), length, bytes]); }; +cliTest( + "uses magic-specific cache tables for synthetic pyc bytecode equality", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-python-pyc-inline-cache-"); + const cases = [ + { magic: 3495, opcode: 166, cacheUnits: 1 }, + { magic: 3500, opcode: 25, cacheUnits: 4 }, + { magic: 3524, opcode: 25, cacheUnits: 1 }, + { magic: 3531, opcode: 106, cacheUnits: 9 }, + { magic: 3550, opcode: 25, cacheUnits: 1 }, + { magic: 3553, opcode: 25, cacheUnits: 1 }, + { magic: 3564, opcode: 58, cacheUnits: 1 }, + { magic: 3571, opcode: 81, cacheUnits: 9 }, + ]; + for (const { magic, opcode, cacheUnits } of cases) { + const cache = Buffer.alloc(cacheUnits * 2); + const bytecode = (arg: number, cachePayload: Buffer) => + Buffer.concat([ + Buffer.from([opcode, arg]), + cachePayload, + Buffer.from([83, 0]), + ]); + const changedCache = Buffer.from(cache); + changedCache[0] = 1; + const frozenSet = Buffer.concat([ + Buffer.from([0x3e, 4, 0, 0, 0]), + ...[ + bytecode(0, cache), + bytecode(0, changedCache), + bytecode(1, cache), + Buffer.concat([bytecode(0, cache), Buffer.from([0, 0])]), + ].map((code) => marshalCode("py311", { bytecode: code })), + ]); + const pyc = Buffer.concat([ + pythonPycHeader(magic), + marshalCode("py311", { consts: [frozenSet] }), + ]); + const path = join(root, `inline-caches-${magic}.pyc`); + await writeFile(path, pyc); + + const result = await cli.run({ + arguments: ["inspect-python-pyc", path, "--json"], + }); + expect( + result.exitCode, + `${result.stderr}\n${JSON.stringify(result.json)}`, + ).toBe(0); + const evidence = toolContract("inspect_python_pyc").outputSchema.parse( + result.json, + ); + expect(evidence.normalized_result.code_objects[0]?.consts).toEqual([ + expect.objectContaining({ kind: "frozenset", length: 3 }), + ]); + expect(evidence.normalized_result.code_objects[1]?.bytecode).toEqual({ + bytes: bytecode(0, cache).length, + sha256: createHash("sha256").update(bytecode(0, cache)).digest("hex"), + }); + expect(evidence.normalized_result.artifact.sha256).toBe( + createHash("sha256").update(pyc).digest("hex"), + ); + } + }, +); + cliTest( "keeps distinct Python code names distinct when JavaScript display strings collide", async ({ cli }) => {