diff --git a/docs/python-pyc.md b/docs/python-pyc.md index 33942187fa..e422cd9c14 100644 --- a/docs/python-pyc.md +++ b/docs/python-pyc.md @@ -65,8 +65,8 @@ is that marshal length. Integer consts report `bits`; their decimal `value` is `-inf`, and `nan` spellings. Dictionary and set lengths follow CPython: unhashable keys and members fail the walk, equal set members count once, equal dictionary keys keep the first key object and replace only its value, code -objects are valid keys and members, and a null dictionary value ends the dict -without that entry. +objects are valid keys and members only when all of their consts are hashable, +and a null dictionary value ends the dict without that entry. ## Limits and failures diff --git a/src/python/PythonPycParser.test.ts b/src/python/PythonPycParser.test.ts index d2cf9d594b..e7fe76707f 100644 --- a/src/python/PythonPycParser.test.ts +++ b/src/python/PythonPycParser.test.ts @@ -933,6 +933,79 @@ it("accepts code objects as set members and dictionary keys", () => { expect(pythonPycSchema.safeParse(report).success).toBe(true); }); +it("rejects code keys and members whose consts are unhashable", () => { + const list = Buffer.concat([Buffer.from([0x5b]), u32(0)]); + const int = Buffer.concat([Buffer.from([0x69]), i32(1)]); + const tuple = (...items: Buffer[]): Buffer => + Buffer.concat([Buffer.from([0x29, items.length]), ...items]); + const code = (...consts: Buffer[]): Buffer => + marshalCode("py311", { name: "u", qualname: "u", consts }); + const setOf = (kind: number, member: Buffer): Buffer => + Buffer.concat([Buffer.from([kind]), u32(1), member]); + const dictKey = (key: Buffer): Buffer => + Buffer.concat([Buffer.from([0x7b]), key, int, Buffer.from([0x30])]); + for (const key of [code(list), code(tuple(int, list)), code(code(list))]) { + expect(() => parse(withConsts([setOf(0x3c, key)]))).toThrow( + PythonPycFormatFailure, + ); + expect(() => parse(withConsts([setOf(0x3e, key)]))).toThrow( + PythonPycFormatFailure, + ); + expect(() => parse(withConsts([dictKey(key)]))).toThrow( + PythonPycFormatFailure, + ); + } + const cyclic = Buffer.concat([ + Buffer.from([0x29 | 0x80, 1]), + marshalCode("py311", { + name: "u", + qualname: "u", + flagged: true, + constsObject: tuple(marshalRef(0)), + }), + ]); + expect(() => parse(withConsts([cyclic, setOf(0x3c, marshalRef(1))]))).toThrow( + PythonPycFormatFailure, + ); + + // A set inside a tuple that reaches the unfinished tuple, directly, inside + // a code object's consts, or as those consts, before the list is read. + for (const member of [ + marshalRef(0), + marshalCode("py311", { + name: "u", + qualname: "u", + constsObject: tuple(marshalRef(0)), + }), + marshalCode("py311", { + name: "u", + qualname: "u", + constsObject: marshalRef(0), + }), + ]) + expect(() => + parse( + withConsts([ + Buffer.concat([ + Buffer.from([0x29 | 0x80, 2]), + setOf(0x3c, member), + list, + ]), + ]), + ), + ).toThrow(PythonPycFormatFailure); + + const unhashed = parse(withConsts([tuple(code(list))])); + expect(unhashed.coverage.status).toBe("complete"); + expect(unhashed.code_objects[1]?.consts[0]).toEqual({ + kind: "list", + length: 0, + }); + const hashable = parse(withConsts([setOf(0x3c, code(tuple(int)))])); + expect(root(hashable).consts[0]).toEqual({ kind: "set", length: 1 }); + expect(hashable.coverage.status).toBe("complete"); +}); + it("uses the code equality fields for each CPython layout", () => { const codeCollections = ( layout: "py38" | "py311", diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index f27a9d1819..c943885d21 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -151,6 +151,8 @@ const containerChildren = (value: MarshalValue): readonly MarshalValue[] => { */ type CodeEquality = { readonly consts: readonly MarshalValue[]; + /** The co_consts tuple itself, which may be a reference to an unfinished tuple. */ + readonly constsTuple: MarshalValue; readonly effectiveFlags: number; readonly linetable: string; readonly exceptiontable: string; @@ -295,6 +297,10 @@ class MarshalReader { readonly #codeKeys = new WeakMap(); readonly #codeIdentities = new WeakMap(); readonly #codeEquality = new WeakMap(); + /** Python hashability per code object. False while that check is in progress. */ + readonly #codeHashable = new WeakMap(); + /** Tuples whose items are still being read; CPython cannot hash them yet. */ + readonly #readingTuples = new WeakSet(); /** Code objects whose const summaries wait until the graph is fully decoded. */ readonly #codes: CodeNode[] = []; /** @@ -433,7 +439,8 @@ class MarshalReader { // 3.7–3.10 constructors do not, so an odd `co_code` stays readable there. if (this.layout === "py311" && bytecode.length % 2 !== 0) malformed("co_code is malformed."); - const consts = asTuple(this.readObject(depth + 1), "co_consts"); + const constsTuple = this.readObject(depth + 1); + const consts = asTuple(constsTuple, "co_consts"); this.#chargeConsts(consts.length); const names = this.#nameTable( asTuple(this.readObject(depth + 1), "co_names"), @@ -531,6 +538,7 @@ class MarshalReader { : flags & ~CO_NOFREE; this.#codeEquality.set(node, { consts, + constsTuple, effectiveFlags, linetable: `${linetable.length}:${sha256(linetable)}`, exceptiontable: `${exceptiontable.length}:${sha256(exceptiontable)}`, @@ -834,12 +842,14 @@ class MarshalReader { const value: MarshalValue = kind === "tuple" ? { kind: "tuple", items } : { kind: "list", items }; this.#publish(slot, value); + if (kind === "tuple") this.#readingTuples.add(value); for (let index = 0; index < count; index += 1) { const item = this.readObject(depth + 1); if (item.kind === "null") malformed("NULL object in marshal data for sequence."); items.push(item); } + this.#readingTuples.delete(value); return value; } @@ -953,6 +963,9 @@ class MarshalReader { this.#takeKeyStep(); const frame = frames[frames.length - 1]; if (frame === undefined) break; + // A tuple still being read has missing items; CPython cannot hash it. + if (mode === "hash" && this.#readingTuples.has(frame.value)) + frame.failed = true; const items = frame.value.kind === "tuple" || frame.value.kind === "frozenset" ? frame.value.items @@ -1107,6 +1120,7 @@ class MarshalReader { return `${prefix}${value.value.length}:${value.value.toString("hex")}`; } case "code": + if (mode === "hash" && !this.#isCodeHashable(value.value)) return null; return this.#codeKey(value.value); default: return mode === "const" ? this.#identity(value) : null; @@ -1207,6 +1221,23 @@ class MarshalReader { return key; } + /** + * CPython hashes the consts tuple itself, so one unhashable const makes the + * code unhashable. A const that leads back to the same code counts as + * unhashable instead of recursing. + */ + #isCodeHashable(node: CodeNode): boolean { + const known = this.#codeHashable.get(node); + if (known !== undefined) return known; + this.#codeHashable.set(node, false); + const material = + this.#codeEquality.get(node) ?? + malformed("Code object is missing equality material."); + const hashable = this.#equalityKey(material.constsTuple, "hash") !== null; + this.#codeHashable.set(node, hashable); + return hashable; + } + #codeIdentity(node: CodeNode): number { const existing = this.#codeIdentities.get(node); if (existing !== undefined) return existing;