From f8f52f205a22270c96ef7d6ebcb936e5aef40193 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 01:24:33 +0800 Subject: [PATCH] fix(python): preserve marshal names and packaged bytecode Use one marshal surrogatepass decoder for PYZ directories and Python bytecode. Validate ordinary UTF-8 spans with the native fatal decoder, preserving BOMs, lone surrogates, raw bytes and marshal reference slots. Distinguish legacy full-pyc CArchive modules from modern marshal-only modules before reconstructing headers. Share typed source/codec failures between stream output and completion while retaining native-read drain and cleanup ownership. Validated focused producer, reader, lifecycle, CLI and MCP regressions, plus public inventory/extraction of CPython 3.14-generated marshal tables and code. No packaged application was executed. Document original versus derived bytes. Fixes #6800 Fixes #6789 --- docs/python-packaged-applications.md | 19 ++- src/artifacts/PyInstallerArtifactReader.ts | 15 +- src/artifacts/pyinstaller/CArchive.fixture.ts | 94 ++++++++++- src/artifacts/pyinstaller/CArchiveMembers.ts | 12 +- .../pyinstaller/MarshalSubset.test.ts | 68 ++++++++ src/artifacts/pyinstaller/MarshalSubset.ts | 12 +- .../PyInstallerArtifactReader.test.ts | 51 ++++++ .../pyinstaller/PyInstallerBoundaries.test.ts | 139 +++++++++++++++- .../PyInstallerFormatBoundaries.test.ts | 154 ++++++++++++++++++ .../pyinstaller/PyInstallerMemberStream.ts | 78 +++++++-- .../pyinstaller/PyInstallerPyzStore.ts | 54 +++++- src/artifacts/pyinstaller/PythonBytecode.ts | 11 ++ src/domain/python/marshalUtf8.test.ts | 37 +++++ src/domain/python/marshalUtf8.ts | 45 +++++ src/python/PythonPycParser.test.ts | 4 +- src/python/PythonPycParser.ts | 63 +------ tests/acceptance/pyinstallerArchive.test.ts | 90 ++++++++++ .../mcp/pyinstallerArtifactMcp.test.ts | 149 +++++++++++++++++ 18 files changed, 986 insertions(+), 109 deletions(-) create mode 100644 src/domain/python/marshalUtf8.test.ts create mode 100644 src/domain/python/marshalUtf8.ts diff --git a/docs/python-packaged-applications.md b/docs/python-packaged-applications.md index ac3f394174..fd3e0feec9 100644 --- a/docs/python-packaged-applications.md +++ b/docs/python-packaged-applications.md @@ -115,8 +115,9 @@ returns a resource failure. Supported final Python bytecode magic values cover Python 3.7 through 3.14. Supported marshal directory values are lists, tuples, strings, integers and -bytes, including interned Unicode strings used for module names. Directory -values follow the identified CPython marshal semantics; for +bytes, including interned Unicode strings used for module names. Unicode strings +use CPython's UTF-8 `surrogatepass` encoding, preserving lone surrogate code +points in decoded module names. Directory values follow the identified CPython marshal semantics; for example, a long integer with a zero most-significant digit is malformed and is not normalized into a valid table value. An unsupported magic or malformed or unsupported directory stays an opaque member with a limitation and @@ -133,11 +134,15 @@ The directory declares compressed module lengths, so uncompressed declared sizes and source spans inside decoded data remain unknown. PYZ modules receive a derived 16-byte `.pyc` header from their own supported -magic. CArchive bytecode receives that header only when every sibling PYZ magic -and the cookie's Python version agree. Otherwise it retains its original bytes -and `bytecode_header: "unknown"`. `bytecode_magic` and graph derivation edges -identify synthesized headers; member digests cover the resulting derived files. -These headers do not establish successful execution or decompilation. +magic. CArchive scripts receive that header only when every sibling PYZ magic +and the cookie's Python version agree. Under the same agreement, CArchive +modules (`m`/`M`) receive a header only when their decompressed prefix identifies +headerless marshal code. Legacy modules that retain a `.pyc` header, and modules +whose prefix is unrecognized, preserve their original bytes and report +`bytecode_header: "unknown"`. A header prefix does not establish that the entire +file is valid. `bytecode_magic` and graph derivation edges identify synthesized +headers; member digests cover the resulting derived files. These headers do not +establish successful execution or decompilation. The nonempty CArchive Python module `pyimod00_crypto_key` (`m`/`M`) is recorded as an encryption marker; a same-named data file or script is not. Affected diff --git a/src/artifacts/PyInstallerArtifactReader.ts b/src/artifacts/PyInstallerArtifactReader.ts index b065139af5..43d2c2479d 100644 --- a/src/artifacts/PyInstallerArtifactReader.ts +++ b/src/artifacts/PyInstallerArtifactReader.ts @@ -530,6 +530,7 @@ export class PyInstallerArtifactReader implements ArtifactReader { pathDialect: CarrierPathDialect, signal: AbortSignal | undefined, ): Promise { + const handle = await this.#openHandle(signal); const members: PyInstallerMember[] = []; const append = (member: PyInstallerMember): void => { this.#reservePathComponents(member.entry.path); @@ -553,8 +554,20 @@ export class PyInstallerArtifactReader implements ArtifactReader { abortIfNeeded(signal); const image = entry.typeCode === "z" ? images.get(entry) : undefined; if (image === undefined) { + const bytecode = await this.#pyzStore.bytecodeHeader( + handle, + cookie.archiveStart, + entry, + magic, + signal, + ); append( - projectStoredMember(entry, cookie.archiveStart, magic, pathDialect), + projectStoredMember( + entry, + cookie.archiveStart, + bytecode, + pathDialect, + ), ); continue; } diff --git a/src/artifacts/pyinstaller/CArchive.fixture.ts b/src/artifacts/pyinstaller/CArchive.fixture.ts index 73623db4a8..2732420858 100644 --- a/src/artifacts/pyinstaller/CArchive.fixture.ts +++ b/src/artifacts/pyinstaller/CArchive.fixture.ts @@ -1,5 +1,9 @@ import { deflateSync } from "node:zlib"; import { pe } from "../../domain/binaryTarget.fixture.js"; +import { + pythonPycFixture, + pythonPycHeader, +} from "../../python/PythonPyc.fixture.js"; import { PYINSTALLER_COOKIE_MAGIC } from "./CArchiveLayout.js"; @@ -83,6 +87,88 @@ export const cArchiveExecutable = (input: { ]); }; +/** Inert code fixtures model the legacy full-pyc and modern marshal-only writers. */ +export const cArchiveBytecodeExecutable = () => { + const timestampPyc = pythonPycFixture("py38", { + magic: 3439, + header: { mtime: 123, sourceSize: 456 }, + }); + const hashPyc = pythonPycFixture("py38", { + magic: 3439, + header: { flags: 3, sourceHash: Buffer.from("0102030405060708", "hex") }, + }); + const modern = pythonPycFixture("py38", { magic: 3439, flaggedCode: true }); + const modules = [ + { + name: "legacy-stored", + typeCode: "m", + payload: timestampPyc, + compress: false, + bytes: timestampPyc, + derived: false, + }, + { + name: "legacy-compressed", + typeCode: "m", + payload: hashPyc, + compress: true, + bytes: hashPyc, + derived: false, + }, + { + name: "legacy-package", + typeCode: "M", + payload: timestampPyc, + compress: true, + bytes: timestampPyc, + derived: false, + }, + { + name: "modern-module", + typeCode: "m", + payload: modern.subarray(16), + compress: true, + bytes: modern, + derived: true, + }, + { + name: "modern-package", + typeCode: "M", + payload: timestampPyc.subarray(16), + compress: false, + bytes: Buffer.concat([pythonPycHeader(3439), timestampPyc.subarray(16)]), + derived: true, + }, + { + name: "script", + typeCode: "s", + payload: modern.subarray(16), + compress: true, + bytes: modern, + derived: true, + }, + ]; + return { + modules, + archive: cArchiveExecutable({ + layout: "2.1", + pythonVersion: 310, + members: [ + ...modules, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: timestampPyc.subarray(0, 4), + modules: [], + }), + compress: false, + }, + ], + }), + }; +}; + /** Cookie whose package range starts before the file. */ export const cookieRangeOutsideFile = (): Buffer => { const cookie = Buffer.alloc(88); @@ -134,7 +220,7 @@ export const pyzArchive = (input: { export interface PyzFixtureModule { readonly legacyNamespacePayload?: boolean; readonly compressedSuffix?: Buffer; - readonly name: string; + readonly name: string | Buffer; readonly typeCode: number; readonly body: Buffer; } @@ -186,7 +272,7 @@ const marshalList = (items: readonly Buffer[]): Buffer => { }; const marshalPair = ( - name: string, + name: string | Buffer, typeCode: number, offset: number, length: number, @@ -199,8 +285,8 @@ const marshalPair = ( const smallTuple = (items: readonly Buffer[]): Buffer => Buffer.concat([Buffer.from([0x29, items.length]), ...items]); -const marshalString = (text: string): Buffer => { - const body = Buffer.from(text, "utf8"); +const marshalString = (text: string | Buffer): Buffer => { + const body = Buffer.isBuffer(text) ? text : Buffer.from(text, "utf8"); if (body.length <= 255 && body.every((byte) => byte < 0x80)) return Buffer.concat([Buffer.from([0x7a, body.length]), body]); const length = Buffer.alloc(4); diff --git a/src/artifacts/pyinstaller/CArchiveMembers.ts b/src/artifacts/pyinstaller/CArchiveMembers.ts index b8cbd0aee9..687e99c845 100644 --- a/src/artifacts/pyinstaller/CArchiveMembers.ts +++ b/src/artifacts/pyinstaller/CArchiveMembers.ts @@ -5,6 +5,7 @@ import type { PyzArchive, PyzEntry } from "./PyzArchive.js"; import { RECONSTRUCTED_PYTHON_HEADER_LIMITATION, reconstructedPycHeader, + type CArchiveBytecodeHeader, } from "./PythonBytecode.js"; import { cArchiveEntryMetadata, @@ -60,7 +61,7 @@ export const hasEncryptionMarker = ( export const projectStoredMember = ( entry: CArchiveEntry, archiveStart: number, - pythonMagic: Buffer | undefined, + bytecode: CArchiveBytecodeHeader, pathDialect: CarrierPathDialect, ): PyInstallerMember => { if (isCArchiveDeclaration(entry)) @@ -82,16 +83,14 @@ export const projectStoredMember = ( }), read: undefined, }; - const header = - pythonMagic !== undefined && isBytecodeType(entry.typeCode) - ? reconstructedPycHeader(pythonMagic) - : undefined; + const { header } = bytecode; const path = canonicalMemberPath(entry.name, pathDialect); const symlink = entry.typeCode === "n"; const limitations: string[] = []; if (symlink) limitations.push("PyInstaller symlink target was not read."); if (header !== undefined) limitations.push(RECONSTRUCTED_PYTHON_HEADER_LIMITATION); + if (bytecode.limitation !== undefined) limitations.push(bytecode.limitation); const declaredSize = entry.uncompressedSize; return { entry: fileEntry({ @@ -322,9 +321,6 @@ const pyzMemberPath = ( return `${container}/${dotted}.pyc`; }; -const isBytecodeType = (typeCode: string): boolean => - typeCode === "s" || typeCode === "m" || typeCode === "M"; - const fileEntry = (input: { readonly path: string; readonly declaredSize: number | null; diff --git a/src/artifacts/pyinstaller/MarshalSubset.test.ts b/src/artifacts/pyinstaller/MarshalSubset.test.ts index 7b0c89105c..ff91f16c71 100644 --- a/src/artifacts/pyinstaller/MarshalSubset.test.ts +++ b/src/artifacts/pyinstaller/MarshalSubset.test.ts @@ -20,6 +20,74 @@ it("reads a Python 3.9 reference-flagged PYZ table", () => { ]); }); +it("decodes CPython marshal Unicode with surrogatepass", () => { + expect( + readMarshalSubset( + bytes( + "5b01000000290275080000006e6f64655fedb3bf29036900000000690c0000006901000000", + ), + ), + ).toEqual({ + ok: true, + value: [["node_\udcff", [0, 12, 1]]], + }); +}); + +it("retains a surrogate-bearing Unicode string in its marshal reference slot", () => { + expect(readMarshalSubset(bytes("2902f50400000061edb3bf7200000000"))).toEqual({ + ok: true, + value: ["a\udcff", "a\udcff"], + }); +}); + +it.each([ + ["", ""], + ["c2a2", "\u00a2"], + ["e282ac", "\u20ac"], + ["efbbbf", "\ufeff"], + ["f0908080", "\ud800\udc00"], + ["f48fbfbf", "\udbff\udfff"], + ["eda080", "\ud800"], + ["edbfbf", "\udfff"], + ["eda080edbfbf", "\ud800\udfff"], +])("preserves CPython Unicode code points for UTF-8 %s", (hex, value) => { + const body = bytes(hex); + const length = Buffer.alloc(4); + length.writeInt32LE(body.length); + // Unicode and reference-flagged interned Unicode use the same codec. + for (const type of [0x75, 0xf4]) + expect( + readMarshalSubset(Buffer.concat([Buffer.from([type]), length, body])), + ).toEqual({ ok: true, value }); +}); + +it.each([ + "80", + "c080", + "c2", + "c241", + "e08080", + "eda0", + "eda041", + "f0808080", + "f09080", + "f4908080", + "f5808080", +])("rejects malformed UTF-8 %s despite surrogatepass", (malformed) => { + const body = bytes(malformed); + const length = Buffer.alloc(4); + length.writeInt32LE(body.length); + expect( + readMarshalSubset(Buffer.concat([Buffer.from([0x75]), length, body])), + ).toMatchObject({ + ok: false, + error: { + reason: "format", + message: "PYZ marshal string is not valid UTF-8", + }, + }); +}); + it("resolves a shared string through a reference reserved before the tuple", () => { const parsed = readMarshalSubset( bytes( diff --git a/src/artifacts/pyinstaller/MarshalSubset.ts b/src/artifacts/pyinstaller/MarshalSubset.ts index ea5c7ff499..12e7aa1695 100644 --- a/src/artifacts/pyinstaller/MarshalSubset.ts +++ b/src/artifacts/pyinstaller/MarshalSubset.ts @@ -1,4 +1,5 @@ import { err, ok, type Result } from "../../domain/result.js"; +import { decodeMarshalUtf8 } from "../../domain/python/marshalUtf8.js"; import { pyInstallerFailure, type PyInstallerFailure, @@ -184,17 +185,12 @@ const readUtf8 = ( if (!length.ok) return length; const bytes = readBytes(cursor, length.value); if (!bytes.ok) return bytes; - try { - return ok( - new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes.value, - ), - ); - } catch { + const value = decodeMarshalUtf8(bytes.value); + if (!value.ok) return err( pyInstallerFailure("format", "PYZ marshal string is not valid UTF-8"), ); - } + return value; }; const readByteString = ( diff --git a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts index 6658045c1f..5a0506c2fa 100644 --- a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts @@ -12,9 +12,12 @@ import { scanCanonicalArtifactInventory } from "../../../tests/fixtures/artifact import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js"; import { reconstructedPycHeader } from "./PythonBytecode.js"; +import { parsePythonPyc } from "../../python/PythonPycParser.js"; +import { inspectPythonPycInputSchema } from "../../domain/python/pythonPyc.js"; import { PYTHON_39_MAGIC, cArchiveExecutable, + cArchiveBytecodeExecutable, cookieRangeOutsideFile, pyzArchive, type CArchiveFixtureMember, @@ -93,6 +96,54 @@ const sampleMembers = ( }, ]; +it("preserves legacy pyc bytes and derives only modern headerless CArchive bytecode", async () => { + const fixture = cArchiveBytecodeExecutable(); + const path = await writeArchive("bytecode-layouts", fixture.archive); + const reader = new PyInstallerArtifactReader(path); + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + for (const module of fixture.modules) { + const entry = entries.find((item) => item.path === module.name)!; + expect(entry.formatMetadata).toMatchObject({ + bytecode_header: module.derived ? "derived" : "unknown", + bytecode_magic: module.derived ? "6f0d0d0a" : null, + }); + const bytes = await buffer(await reader.open(entry)); + expect(bytes).toEqual(module.bytes); + expect( + parsePythonPyc( + bytes, + { path: module.name, bytes: bytes.length, sha256: sha256(bytes) }, + inspectPythonPycInputSchema.parse({ path: module.name }), + ).coverage.status, + ).toBe("complete"); + expect(await buffer(await reader.open(entry))).toEqual(module.bytes); + } + } finally { + await reader.close(); + } + const snapshot = await scanCanonicalArtifactInventory(path); + for (const module of fixture.modules) { + const occurrence = snapshot.occurrences.find( + (row) => row.logical_path === module.name, + )!; + expect( + snapshot.nodes.find( + (node) => node.artifact_id === occurrence.artifact_id, + ), + ).toMatchObject({ + sha256: sha256(module.bytes), + size: module.bytes.length, + }); + expect( + snapshot.edges.find( + (edge) => edge.occurrence_id === occurrence.occurrence_id, + )?.relation, + ).toBe(module.derived ? "derived-from" : "contains"); + } +}); + it("inventories a 2.1 archive, nested PYZ modules, and derived bytecode", async () => { const script = Buffer.from("print('hi')"); const stored = Buffer.from("stored-data"); diff --git a/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts b/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts index ae8b84204e..808b1f8bfc 100644 --- a/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts @@ -1,7 +1,7 @@ import { withPyInstallerReader } from "../../../tests/fixtures/pyinstallerReader.js"; import { join } from "node:path"; import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; -import { createHash } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import { writeFile, type FileHandle } from "node:fs/promises"; import { buffer } from "node:stream/consumers"; import { deflateSync } from "node:zlib"; @@ -10,6 +10,8 @@ import { expect, it, vi } from "vitest"; import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js"; import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; import { hashReadable } from "../ArtifactHash.js"; +import { marshalCode } from "../../python/PythonPyc.fixture.js"; +import { PyInstallerPyzStore } from "./PyInstallerPyzStore.js"; import { ArtifactReaderFailure, type ArtifactEntry, @@ -28,9 +30,12 @@ const archive = (body: Buffer) => pythonVersion: 309, members: [{ name: "data", typeCode: "x", payload: body, compress: false }], }); -const members = async (reader: PyInstallerArtifactReader) => { +const members = async ( + reader: PyInstallerArtifactReader, + signal?: AbortSignal, +) => { const entries: ArtifactEntry[] = []; - for await (const entry of reader.entries()) entries.push(entry); + for await (const entry of reader.entries(signal)) entries.push(entry); return entries; }; const deferred = () => { @@ -41,6 +46,134 @@ const deferred = () => { return { promise, resolve }; }; +it.each(["success", "io", "caller cancellation", "reader close"] as const)( + "joins the pending descriptor read after a module prefix stop (%s)", + async (outcome) => { + const payload = marshalCode("py38", { + bytecode: randomBytes(256 * 1024), + }); + const compressedBytes = deflateSync(payload).length; + expect(compressedBytes).toBeGreaterThan(128 * 1024); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "module", typeCode: "m", payload, compress: true }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }), + compress: false, + }, + ], + }), + async (reader, handle) => { + const entered = deferred(), + prefixStopped = deferred(), + release = deferred(); + const caller = new AbortController(); + const cancelled = new Error("caller leaves during prefix drain"); + const io = Object.assign(new Error("late module prefix read EIO"), { + code: "EIO", + }); + const memberSource = PyInstallerPyzStore.prototype.memberSource; + let prefixSignal: AbortSignal | undefined; + const source = vi + .spyOn(PyInstallerPyzStore.prototype, "memberSource") + .mockImplementation(function (this: PyInstallerPyzStore, ...args) { + prefixSignal = args[2]; + prefixSignal.addEventListener("abort", prefixStopped.resolve, { + once: true, + }); + return memberSource.apply(this, args); + }); + const read = handle.read.bind(handle); + let memberReads = 0; + let requestedMemberBytes = 0; + const descriptor = vi + .spyOn(handle, "read") + .mockImplementation(async (...args: unknown[]) => { + if (prefixSignal !== undefined) { + memberReads++; + // Inflate can deliver this prefix while the next native read is pending. + if (memberReads === 1) args[2] = 64; + requestedMemberBytes += Number(args[2]); + if (memberReads === 2) { + entered.resolve(); + await release.promise; + if (outcome === "io") throw io; + } + } + return Reflect.apply(read, handle, args); + }); + let settled = false; + const listing = members(reader, caller.signal).finally(() => { + settled = true; + }); + void listing.catch(() => undefined); + let closing: Promise | undefined; + try { + await Promise.all([entered.promise, prefixStopped.promise]); + expect(prefixSignal?.aborted).toBe(true); + expect(caller.signal.aborted).toBe(false); + expect(memberReads).toBe(2); + expect(settled).toBe(false); + if (outcome === "caller cancellation") { + caller.abort(cancelled); + await setImmediate(); + } + if (outcome === "reader close") { + let closed = false; + closing = reader.close().then(() => { + closed = true; + }); + await setImmediate(); + expect(closed).toBe(false); + expect(handle.fd).toBeGreaterThanOrEqual(0); + } + expect(settled).toBe(false); + release.resolve(); + if (outcome === "success") { + const module = (await listing).find( + (entry) => entry.path === "module", + ); + expect(module).toMatchObject({ + formatMetadata: { bytecode_header: "derived" }, + }); + } else { + const failure: unknown = await listing.catch( + (cause: unknown) => cause, + ); + expect(failure).toBeInstanceOf(ArtifactReaderFailure); + if (!(failure instanceof ArtifactReaderFailure)) + throw new Error("Expected module prefix failure"); + expect(failure.reason).toBe(outcome === "io" ? "io" : "cancelled"); + if (outcome !== "reader close") + expect(failure.cause).toBe(outcome === "io" ? io : cancelled); + else + expect(failure.cause).toMatchObject({ + reason: "unavailable", + message: "PyInstaller reader closed", + }); + } + await closing; + expect(memberReads).toBe(2); + expect(requestedMemberBytes).toBeLessThan(compressedBytes); + } finally { + release.resolve(); + await Promise.allSettled([ + listing, + ...(closing === undefined ? [] : [closing]), + ]); + descriptor.mockRestore(); + source.mockRestore(); + } + }, + ); + }, +); + it.each([false, true])( "settles independent private and carrier closes and retries only failed owners (carrier failure: %s)", async (failCarrier) => { diff --git a/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts b/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts index 80e2c53bb8..1432986d2a 100644 --- a/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts @@ -2,6 +2,7 @@ import { buffer } from "node:stream/consumers"; import { readFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { deflateSync } from "node:zlib"; +import { randomBytes } from "node:crypto"; import { expect, it, vi } from "vitest"; import { withPyInstallerReader } from "../../../tests/fixtures/pyinstallerReader.js"; import { @@ -11,10 +12,12 @@ import { } from "../../../tests/fixtures/artifactInventory.js"; import { cArchiveExecutable, + cArchiveBytecodeExecutable, pyzArchive, PYTHON_39_MAGIC, } from "./CArchive.fixture.js"; import { readPyzArchive } from "./PyzArchive.js"; +import { marshalCode } from "../../python/PythonPyc.fixture.js"; import { createEvidence } from "../../domain/evidence.js"; import { jsonValueSchema } from "../../domain/jsonValue.js"; import { parseArtifactInventoryEvidence } from "../../domain/artifactInventoryEvidence.js"; @@ -23,6 +26,157 @@ import { type ArtifactEntry, } from "../ArtifactReader.js"; +it.each([ + ["empty", Buffer.alloc(0)], + ["short", Buffer.from("630000", "hex")], + ["unrecognized", Buffer.from("not marshal")], + [ + "unknown magic resembling TYPE_CODE", + Buffer.from("e30d0d0a000000000000000000000000", "hex"), + ], +])( + "preserves a CArchive module with %s prefix despite known sibling magic", + async (_description, payload) => { + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "module", typeCode: "m", payload, compress: true }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }), + compress: false, + }, + ], + }), + async (reader) => { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + const module = entries.find((entry) => entry.path === "module")!; + expect(module.formatMetadata).toMatchObject({ + bytecode_header: "unknown", + bytecode_magic: null, + }); + expect(module.limitations.join(" ")).toContain( + "original bytes are preserved", + ); + expect(await buffer(await reader.open(module))).toEqual(payload); + }, + ); + }, +); + +it("fills short descriptor reads when distinguishing both CArchive module layouts", async () => { + const fixture = cArchiveBytecodeExecutable(); + await withPyInstallerReader(fixture.archive, async (reader, handle) => { + const read = handle.read; + const spy = vi.spyOn(handle, "read").mockImplementation(function ( + this: unknown, + ...args: unknown[] + ) { + if (typeof args[2] === "number" && args[2] > 1) args[2] = 1; + return Reflect.apply(read, this, args); + }); + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + for (const module of fixture.modules) { + const entry = entries.find((item) => item.path === module.name)!; + expect(await buffer(await reader.open(entry))).toEqual(module.bytes); + } + } finally { + spy.mockRestore(); + } + }); +}); + +it.each(["unused compressed input", "wrong declared size"])( + "keeps %s as an integrity failure after CArchive module prefix inspection", + async (fault) => { + const payload = marshalCode("py38", { + bytecode: randomBytes(256 * 1024), + }); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "module", + typeCode: "m", + payload, + compress: true, + ...(fault === "unused compressed input" + ? { compressedSuffix: Buffer.from("junk") } + : { declaredUncompressedSize: payload.length + 1 }), + }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }), + compress: false, + }, + ], + }), + async (reader) => { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + const module = entries.find((entry) => entry.path === "module")!; + expect(module.formatMetadata).toMatchObject({ + bytecode_header: "derived", + }); + await expect(buffer(await reader.open(module))).rejects.toMatchObject({ + reason: "integrity", + message: expect.stringContaining( + fault === "unused compressed input" + ? "complete declared source span" + : "size disagrees", + ), + }); + }, + ); + }, +); + +it.each(["malformed zlib", "declared plaintext overrun"] as const)( + "preserves %s discovered before a CArchive module prefix is accepted", + async (fault) => { + const payload = cArchiveBytecodeExecutable().modules[1]!.bytes; + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 310, + members: [ + { + name: "module", + typeCode: "m", + payload, + compress: true, + ...(fault === "declared plaintext overrun" + ? { declaredUncompressedSize: 2 } + : {}), + }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ magic: payload.subarray(0, 4), modules: [] }), + compress: false, + }, + ], + }); + if (fault === "malformed zlib") bytes[64] = 0; // First member follows the ELF prefix. + await withPyInstallerReader(bytes, async (reader) => { + const list = async () => { + for await (const entry of reader.entries()) void entry; + }; + await expect(list()).rejects.toMatchObject({ + reason: fault === "malformed zlib" ? "format" : "limit", + }); + }); + }, +); + it("keeps compressed source locations unknown when stored and decoded lengths coincide", async () => { const payload = Buffer.alloc(11, 0x61); expect(deflateSync(payload).length).toBe(payload.length); diff --git a/src/artifacts/pyinstaller/PyInstallerMemberStream.ts b/src/artifacts/pyinstaller/PyInstallerMemberStream.ts index 480df2d002..865086e1fb 100644 --- a/src/artifacts/pyinstaller/PyInstallerMemberStream.ts +++ b/src/artifacts/pyinstaller/PyInstallerMemberStream.ts @@ -15,6 +15,43 @@ interface MemberStreamOptions { readonly complete?: (bytes: number) => Promise; } +/** Observe a bounded plaintext prefix, then drain the existing stream owner. */ +export const readPyInstallerMemberPrefix = async ( + openSource: (signal: AbortSignal) => Promise, + options: Omit, + length: number, +): Promise => { + const stopped = new AbortController(); + const prefixComplete = new Error("PyInstaller member prefix observed"); + const signal = AbortSignal.any([options.signal, stopped.signal]); + const source = await openSource(signal); + const stream = pyInstallerMemberStream(source, { + ...options, + header: undefined, + signal, + }); + const prefix = Buffer.alloc(length); + let copied = 0; + stream.output.on("data", (chunk: Buffer) => { + copied += chunk.copy(prefix, copied, 0, length - copied); + if (copied === length) stopped.abort(prefixComplete); + }); + try { + await stream.completion; + } catch (cause: unknown) { + // Only our accepted early stop is successful. Native I/O, malformed zlib, + // declaration failures, and caller cancellation keep their original reason. + if ( + !(cause instanceof ArtifactReaderFailure) || + cause.reason !== "cancelled" || + cause.cause !== prefixComplete + ) + throw cause; + } + abortIfNeeded(options.signal); + return prefix.subarray(0, copied); +}; + /** Backpressure bounds plaintext independently of the untrusted declared size. */ export const pyInstallerMemberStream = ( source: Readable, @@ -23,6 +60,7 @@ export const pyInstallerMemberStream = ( let bytes = 0; let headerSent = false; let completing = false; + let failure: ArtifactReaderFailure | undefined; const prependHeader = (): void => { if (!headerSent && options.header !== undefined) output.push(options.header); @@ -104,35 +142,43 @@ export const pyInstallerMemberStream = ( }; options.signal.addEventListener("abort", onAbort, { once: true }); const onSourceFailure = (cause: unknown): void => { - output.destroy( + failure ??= cause instanceof ArtifactReaderFailure ? cause : new ArtifactReaderFailure( "io", "PyInstaller member source could not be read", { cause }, - ), - ); + ); + output.destroy(failure); }; source.on("error", onSourceFailure); const inflate = options.compressed ? createInflate() : undefined; - inflate?.on("error", (cause: unknown) => - output.destroy( - new ArtifactReaderFailure( - "format", - "Compressed PyInstaller member is not a complete zlib stream", - { cause }, - ), - ), - ); + inflate?.on("error", (cause: unknown) => { + failure ??= + cause instanceof ArtifactReaderFailure + ? cause + : new ArtifactReaderFailure( + "format", + "Compressed PyInstaller member is not a complete zlib stream", + { cause }, + ); + output.destroy(failure); + }); const completion = ( inflate === undefined ? pipeline(source, output) : pipeline(source, inflate, output) - ).finally(() => { - options.signal.removeEventListener("abort", onAbort); - source.removeListener("error", onSourceFailure); - }); + ) + .catch((cause: unknown) => { + // Pipeline can reject with the native error that caused a mapped output + // failure. Completion and output must expose the same boundary reason. + throw failure ?? cause; + }) + .finally(() => { + options.signal.removeEventListener("abort", onAbort); + source.removeListener("error", onSourceFailure); + }); // Preserve rejection for the owning scope without an early unhandled promise. void completion.catch(() => undefined); if (options.signal.aborted) onAbort(); diff --git a/src/artifacts/pyinstaller/PyInstallerPyzStore.ts b/src/artifacts/pyinstaller/PyInstallerPyzStore.ts index 04d17b17c1..4b5b053706 100644 --- a/src/artifacts/pyinstaller/PyInstallerPyzStore.ts +++ b/src/artifacts/pyinstaller/PyInstallerPyzStore.ts @@ -17,10 +17,18 @@ import { } from "../ArtifactResourceScope.js"; import { abortIfNeeded } from "../ArtifactHash.js"; import { readArtifactFileChunks } from "../readArtifactFileChunks.js"; -import { pyInstallerMemberStream } from "./PyInstallerMemberStream.js"; +import { + pyInstallerMemberStream, + readPyInstallerMemberPrefix, +} from "./PyInstallerMemberStream.js"; import type { CArchiveEntry } from "./CArchiveLayout.js"; import type { PyInstallerMember } from "./CArchiveMembers.js"; import { pyInstallerFileAdmissionFailure } from "./PyInstallerFileAdmission.js"; +import { + reconstructedPycHeader, + isHeaderlessPythonCode, + type CArchiveBytecodeHeader, +} from "./PythonBytecode.js"; // Each outer-compressed image retains one inode and its identity until close. // This is a per-reader filesystem capacity budget, independent of byte budgets. @@ -56,6 +64,50 @@ export class PyInstallerPyzStore { ); } + async bytecodeHeader( + handle: FileHandle, + archiveStart: number, + entry: CArchiveEntry, + magic: Buffer | undefined, + signal?: AbortSignal, + ): Promise { + const unchanged = { header: undefined, limitation: undefined }; + if (magic === undefined) return unchanged; + if (entry.typeCode === "s") + return { header: reconstructedPycHeader(magic), limitation: undefined }; + if (entry.typeCode !== "m" && entry.typeCode !== "M") return unchanged; + const prefix = await readPyInstallerMemberPrefix( + async (active) => + ( + await this.memberSource( + { + kind: "carchive", + archiveStart, + source: entry, + header: undefined, + }, + handle, + active, + ) + ).source, + { + sourceBytes: entry.compressedSize, + compressed: entry.compressionFlag === 1, + expectedBytes: entry.uncompressedSize, + maxBytes: entry.uncompressedSize, + signal: signal ?? new AbortController().signal, + }, + 4, + ); + return isHeaderlessPythonCode(prefix) + ? { header: reconstructedPycHeader(magic), limitation: undefined } + : { + header: undefined, + limitation: + "CArchive module is not identified as headerless marshal code; its original bytes are preserved without reconstructing a Python bytecode header.", + }; + } + async spool( handle: FileHandle, archiveStart: number, diff --git a/src/artifacts/pyinstaller/PythonBytecode.ts b/src/artifacts/pyinstaller/PythonBytecode.ts index cefcc35174..e24c637edc 100644 --- a/src/artifacts/pyinstaller/PythonBytecode.ts +++ b/src/artifacts/pyinstaller/PythonBytecode.ts @@ -31,3 +31,14 @@ export const reconstructedPycHeader = (magic: Buffer): Buffer => { magic.copy(header, 0, 0, 4); return header; }; + +/** TYPE_CODE with or without FLAG_REF; a pyc magic takes precedence. */ +export const isHeaderlessPythonCode = (prefix: Buffer): boolean => + prefix.length === 4 && + !(prefix[2] === 0x0d && prefix[3] === 0x0a) && + (prefix[0] === 0x63 || prefix[0] === 0xe3); + +export interface CArchiveBytecodeHeader { + readonly header: Buffer | undefined; + readonly limitation: string | undefined; +} diff --git a/src/domain/python/marshalUtf8.test.ts b/src/domain/python/marshalUtf8.test.ts new file mode 100644 index 0000000000..7166e6e11e --- /dev/null +++ b/src/domain/python/marshalUtf8.test.ts @@ -0,0 +1,37 @@ +import { expect, it } from "vitest"; + +import { decodeMarshalUtf8 } from "./marshalUtf8.js"; + +it.each([ + ["efbbbfeda080efbbbfedbfbfefbbbf", "\ufeff\ud800\ufeff\udfff\ufeff"], + ["ed9fbfeda080ee8080", "\ud7ff\ud800\ue000"], + ["f0908080eda080f48fbfbf", "\ud800\udc00\ud800\udbff\udfff"], + ["eda080edbfbf", "\ud800\udfff"], + ["616263eda080646566edbfbf676869", "abc\ud800def\udfffghi"], +])( + "preserves ordinary UTF-8 spans around marshal surrogates (%s)", + (hex, value) => { + expect(decodeMarshalUtf8(Buffer.from(hex, "hex"))).toEqual({ + ok: true, + value, + }); + }, +); + +it.each([ + "c2eda080", + "e282eda080", + "f09080eda080", + "80eda080", + "eda08080", + "eda080c2", + "eda080e08080", + "eda080f4908080", + "eda080eda0", + "eda0eda080", +])("does not hide malformed bytes beside a marshal surrogate (%s)", (hex) => { + expect(decodeMarshalUtf8(Buffer.from(hex, "hex"))).toEqual({ + ok: false, + error: { kind: "invalid-marshal-utf8" }, + }); +}); diff --git a/src/domain/python/marshalUtf8.ts b/src/domain/python/marshalUtf8.ts new file mode 100644 index 0000000000..d3c054d90d --- /dev/null +++ b/src/domain/python/marshalUtf8.ts @@ -0,0 +1,45 @@ +import { err, ok, type Result } from "../result.js"; + +const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + +/** CPython marshal uses UTF-8 surrogatepass; all other malformed bytes fail. */ +export const decodeMarshalUtf8 = ( + bytes: Buffer, +): Result => { + const chunks: string[] = []; + let start = 0; + try { + for ( + let offset = bytes.indexOf(0xed); + offset !== -1; + offset = bytes.indexOf(0xed, offset + 1) + ) { + const second = bytes[offset + 1]; + const third = bytes[offset + 2]; + if ( + second === undefined || + second < 0xa0 || + second > 0xbf || + third === undefined || + third < 0x80 || + third > 0xbf + ) + continue; + // Only ED A0..BF 80..BF encodes a surrogate. The fatal decoder owns + // every unchanged span, including incomplete scalars on either side. + chunks.push( + UTF8.decode(bytes.subarray(start, offset)), + String.fromCharCode(0xd000 | ((second & 0x3f) << 6) | (third & 0x3f)), + ); + start = offset + 3; + offset += 2; + } + if (start === 0) return ok(UTF8.decode(bytes)); + chunks.push(UTF8.decode(bytes.subarray(start))); + return ok(chunks.join("")); + } catch (cause: unknown) { + if (cause instanceof TypeError) + return err({ kind: "invalid-marshal-utf8" }); + throw cause; + } +}; diff --git a/src/python/PythonPycParser.test.ts b/src/python/PythonPycParser.test.ts index 4fea1c6ca7..01143495db 100644 --- a/src/python/PythonPycParser.test.ts +++ b/src/python/PythonPycParser.test.ts @@ -321,11 +321,11 @@ it("charges long digits to the string budget and keeps large previews bounded", it("decodes marshal strings with CPython surrogatepass rules", () => { const report = parse( - withConsts([marshalUnicode(Buffer.from([0x73, 0xed, 0xa0, 0x80]))]), + withConsts([marshalUnicode(Buffer.from("efbbbf73eda080efbbbf", "hex"))]), ); expect(root(report).consts[0]).toMatchObject({ kind: "str", - value: "s\ud800", + value: "\ufeffs\ud800\ufeff", }); expect(() => parse(withConsts([marshalUnicode(Buffer.from([0xf0, 0x90, 0x80]))])), diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index 8f257fd6ef..9e993eac8c 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { decodeMarshalUtf8 } from "../domain/python/marshalUtf8.js"; import { PYTHON_PYC_CHECK_SOURCE, PYTHON_PYC_CONST_PREVIEW_LIMIT, @@ -1463,9 +1464,11 @@ class MarshalReader { raw: bytes, }; } + const decoded = decodeMarshalUtf8(bytes); + if (!decoded.ok) return malformed("Marshal string is not valid UTF-8."); return { kind: "str", - value: decodeSurrogatePass(bytes), + value: decoded.value, bytes: length, raw: bytes, }; @@ -1543,64 +1546,6 @@ class MarshalReader { } } -/** - * Decode marshal UTF-8 the way CPython does (`surrogatepass`): encoded lone - * surrogates are kept, every other ill-formed sequence is rejected. - */ -const decodeSurrogatePass = (bytes: Buffer): string => { - const units: number[] = []; - let text = ""; - for (let index = 0; index < bytes.length;) { - const lead = bytes[index] ?? 0; - let codePoint: number; - let width: number; - if (lead < 0x80) { - codePoint = lead; - width = 1; - } else if (lead >= 0xc2 && lead <= 0xdf) { - codePoint = lead & 0x1f; - width = 2; - } else if (lead >= 0xe0 && lead <= 0xef) { - codePoint = lead & 0x0f; - width = 3; - } else if (lead >= 0xf0 && lead <= 0xf4) { - codePoint = lead & 0x07; - width = 4; - } else { - return malformed("Marshal string is not valid UTF-8."); - } - if (index + width > bytes.length) - return malformed("Marshal string is not valid UTF-8."); - for (let offset = 1; offset < width; offset += 1) { - const next = bytes[index + offset] ?? 0; - if ((next & 0xc0) !== 0x80) - return malformed("Marshal string is not valid UTF-8."); - codePoint = (codePoint << 6) | (next & 0x3f); - } - if ( - (width === 3 && codePoint < 0x800) || - (width === 4 && (codePoint < 0x10000 || codePoint > 0x10ffff)) - ) - return malformed("Marshal string is not valid UTF-8."); - units.push(codePoint); - index += width; - if (units.length === 4096) { - text += codePointsToString(units); - units.length = 0; - } - } - return text + codePointsToString(units); -}; - -const codePointsToString = (codePoints: readonly number[]): string => - codePoints - .map((codePoint) => - codePoint >= 0xd800 && codePoint <= 0xdfff - ? String.fromCharCode(codePoint) - : String.fromCodePoint(codePoint), - ) - .join(""); - /** JSON has no non-finite numbers; keep CPython's repr for them. */ const jsonFloat = (value: number): number | "inf" | "-inf" | "nan" => Number.isNaN(value) diff --git a/tests/acceptance/pyinstallerArchive.test.ts b/tests/acceptance/pyinstallerArchive.test.ts index bf074dcdff..e330de9d0d 100644 --- a/tests/acceptance/pyinstallerArchive.test.ts +++ b/tests/acceptance/pyinstallerArchive.test.ts @@ -10,6 +10,7 @@ import { expect, it } from "vitest"; import { PYTHON_39_MAGIC, cArchiveExecutable, + cArchiveBytecodeExecutable, pyzArchive, } from "../../src/artifacts/pyinstaller/CArchive.fixture.js"; import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; @@ -216,6 +217,94 @@ it("lists CArchive members and the Python version through inspect-artifact", asy }); }); +it("extracts legacy and modern CArchive modules with one pyc header through the CLI", async () => { + const fixture = cArchiveBytecodeExecutable(); + const directory = await createTestTempDirectory("rea-pyinstaller-bytecode-"); + const path = join(directory, "bytecode-layouts"); + await writeFile(path, fixture.archive); + const result = await execute(process.execPath, [ + cli, + "extract-artifact", + path, + "--json", + ]); + const extraction = artifactExtractionResultSchema.parse( + parseEvidence(JSON.parse(result.stdout)).normalized_result, + ); + try { + for (const module of fixture.modules) { + expect(await readFile(join(extraction.output_root, module.name))).toEqual( + module.bytes, + ); + expect( + extraction.artifacts.find((item) => item.relative_path === module.name), + ).toMatchObject({ + sha256: createHash("sha256").update(module.bytes).digest("hex"), + }); + } + } finally { + await rm(extraction.output_root, { recursive: true, force: true }); + } +}); + +it("keeps PYZ namespaces with CPython surrogatepass names through the CLI", async () => { + const directory = await createTestTempDirectory("rea-pyinstaller-surrogate-"); + const path = join(directory, "surrogate-name"); + const namespace = Buffer.concat([ + Buffer.from("node_"), + Buffer.from("edb3bf", "hex"), + ]); + await writeFile( + path, + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 39, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: namespace, typeCode: 3, body: Buffer.alloc(0) }, + { name: "healthy", typeCode: 0, body: Buffer.from("healthy") }, + ], + }), + compress: false, + }, + ], + }), + ); + + const result = await execute(process.execPath, [ + cli, + "inspect-artifact", + path, + "--json", + ]); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(JSON.parse(result.stdout)).normalized_result, + ); + const inventory = inspection.substeps[0]?.evidence.normalized_result; + expect(inventory?.coverage.status).toBe("complete"); + expect( + inventory?.occurrences.find( + (row) => row.logical_path === "PYZ.pyz/node_\udcff", + ), + ).toMatchObject({ + entry_kind: "metadata", + format_metadata: { source_name: "node_\udcff", type_code: 3 }, + }); + expect( + inventory?.occurrences.find( + (row) => row.logical_path === "PYZ.pyz/healthy.pyc", + ), + ).toMatchObject({ + entry_kind: "file", + format_metadata: { source_name: "healthy" }, + }); +}); + it("classifies a PyInstaller PE carrier when its signature straddles the probe prefix", async () => { const directory = await createTestTempDirectory("rea-pyinstaller-far-pe-"); const path = join(directory, "hello.exe"); @@ -487,6 +576,7 @@ it.each([ ["negative list count", "5bffffffff"], ["negative tuple count", "28ffffffff"], ["truncated string", "5b010000007301000000"], + ["malformed UTF-8 string", "5b010000007502000000eda0"], ["unsupported marshal type", "5b010000004e"], ])( "keeps healthy CArchive siblings when the PYZ table has %s", diff --git a/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts b/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts index 8dfd22fcf8..8ffa03ea38 100644 --- a/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts +++ b/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts @@ -1,4 +1,5 @@ import { writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; import { join } from "node:path"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; @@ -9,6 +10,7 @@ import { createArtifactBinarySession } from "../../../src/composition/binary.js" import { PYTHON_39_MAGIC, cArchiveExecutable, + cArchiveBytecodeExecutable, pyzArchive, } from "../../../src/artifacts/pyinstaller/CArchive.fixture.js"; import { artifactInspectionResultSchema } from "../../../src/domain/artifactInspection.js"; @@ -159,6 +161,39 @@ const writeExecutable = async (opaque: boolean): Promise => { return path; }; +const writeSurrogateNamespaceExecutable = async (): Promise => { + const directory = await createTestTempDirectory( + "rea-pyinstaller-mcp-surrogate-", + ); + const path = join(directory, "surrogate-name"); + const namespace = Buffer.concat([ + Buffer.from("node_"), + Buffer.from("edb3bf", "hex"), + ]); + await writeFile( + path, + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 39, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: namespace, typeCode: 3, body: Buffer.alloc(0) }, + { name: "healthy", typeCode: 0, body: Buffer.from("healthy") }, + ], + }), + compress: false, + }, + ], + }), + ); + return path; +}; + it.each([false, true])( "inventories a PyInstaller executable through inspect_artifact (opaque: %s)", async (opaque) => { @@ -275,3 +310,117 @@ it.each([false, true])( } }, ); + +it("preserves legacy CArchive identities and modern bytecode derivation through inspect_artifact", async () => { + const fixture = cArchiveBytecodeExecutable(); + const directory = await createTestTempDirectory( + "rea-pyinstaller-mcp-bytecode-", + ); + const path = join(directory, "bytecode-layouts"); + await writeFile(path, fixture.archive); + const session = await createArtifactBinarySession(process.env); + const server = createServer({ kind: "session", session }); + const client = new Client({ + name: "pyinstaller-bytecode-mcp-test", + version: "1", + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path }, + }); + expect(opened.isError).not.toBe(true); + const inspected = await client.callTool({ + name: "inspect_artifact", + arguments: {}, + }); + expect(inspected.isError).not.toBe(true); + const inventory = artifactInspectionResultSchema.parse( + parseEvidence(inspected.structuredContent).normalized_result, + ).substeps[0]?.evidence.normalized_result; + expect(inventory?.coverage.status).toBe("complete"); + for (const module of fixture.modules) { + const occurrence = inventory?.occurrences.find( + (row) => row.logical_path === module.name, + ); + expect(occurrence).toMatchObject({ + hash_status: "verified", + format_metadata: { + bytecode_header: module.derived ? "derived" : "unknown", + }, + }); + expect( + inventory?.nodes.find( + (node) => node.artifact_id === occurrence?.artifact_id, + ), + ).toMatchObject({ + sha256: createHash("sha256").update(module.bytes).digest("hex"), + size: module.bytes.length, + }); + expect( + inventory?.edges.find( + (edge) => edge.occurrence_id === occurrence?.occurrence_id, + )?.relation, + ).toBe(module.derived ? "derived-from" : "contains"); + } + } finally { + await client.close(); + await server.close(); + await session.close(); + } +}); + +it("preserves surrogatepass PYZ module names through inspect_artifact", async () => { + const path = await writeSurrogateNamespaceExecutable(); + const session = await createArtifactBinarySession(process.env); + const server = createServer({ kind: "session", session }); + const client = new Client({ + name: "pyinstaller-surrogate-mcp-test", + version: "1", + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path }, + }); + expect(opened.isError).not.toBe(true); + const inspected = await client.callTool({ + name: "inspect_artifact", + arguments: {}, + }); + expect(inspected.isError).not.toBe(true); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(inspected.structuredContent).normalized_result, + ); + const inventory = inspection.substeps[0]?.evidence.normalized_result; + expect(inventory?.coverage.status).toBe("complete"); + expect( + inventory?.occurrences.find( + (row) => row.logical_path === "PYZ.pyz/node_\udcff", + ), + ).toMatchObject({ + entry_kind: "metadata", + format_metadata: { source_name: "node_\udcff", type_code: 3 }, + }); + expect( + inventory?.occurrences.find( + (row) => row.logical_path === "PYZ.pyz/healthy.pyc", + ), + ).toMatchObject({ + entry_kind: "file", + format_metadata: { source_name: "healthy" }, + }); + } finally { + await client.close(); + await server.close(); + await session.close(); + } +});