diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 4487bb7412..933a7ca21a 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -44,12 +44,18 @@ root deltas. Missing graph members yield `truncated`; unknown traversal coverage or unavailable occurrence hashes retain uncertainty. Integrity mismatches remain contradictions. Static artifact equality does not establish unchanged runtime behavior. +Source-map relation deltas include the content identity of the mapped source, +so an unchanged map file is reported when its target source bytes change. Exhausted traversals with deliberately omitted filesystem entries retain `unknown` coverage and identify each omitted path/type in `limitations`; FIFO, socket and device entries are not opened. Case-distinct archive members remain separate observations. Their case-insensitive materialization diagnostic does not reduce traversal coverage when both spellings were read. +PyInstaller inventory also preserves distinct table rows that share a logical +path, including decoded children beneath a colliding file/directory path. +Inventory coverage describes observed archive facts; extraction still refuses +destinations where those paths cannot coexist. Current inventory Evidence requires scoped `coverage` with a `complete` or `unknown` status. Consumers use this producer-owned fact and reject records diff --git a/docs/python-packaged-applications.md b/docs/python-packaged-applications.md index ab0b7680e9..cde4eb98c1 100644 --- a/docs/python-packaged-applications.md +++ b/docs/python-packaged-applications.md @@ -18,6 +18,13 @@ path rules; the source name retains its spelling, including backslashes. the decoded PYZ image for PYZ rows. A CArchive directory record starts at `archive_start + toc_offset + directory_record_offset`; its size is `directory_record_size`. PYZ row directory extents remain unknown. +CArchive type `b` retains PyInstaller's executable-member declaration in the +generic `executable` fact; type `x` remains non-executable. This comes from the +archive type code, not the filename or member contents. +Distinct CArchive rows retain their table-index identities even when they +share a logical path. Inventory can preserve both a file and a decoded PYZ +directory at that path, including the PYZ children; extraction refuses a +destination conflict instead of silently omitting those children. Namespace packages remain metadata declarations at their projected query path; they do not claim a destination directory. Their `relative_offset` and @@ -105,7 +112,8 @@ returns a resource failure. Supported final Python bytecode magic values cover Python 3.7 through 3.14. Supported marshal directory values are lists, tuples, strings, integers and -bytes. Directory values follow the identified CPython marshal semantics; for +bytes, including interned Unicode strings used for module names. Directory +values follow the identified CPython marshal semantics; for example, a long integer with a zero most-significant digit is malformed and is not normalized into a valid table value. An unsupported magic or malformed or unsupported directory stays an opaque member with a limitation and diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index 3a39ebb990..d27e36b07a 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -123,6 +123,11 @@ const materializeArtifactInventoryInScope = async ( const regularPaths = new Set( selectedOccurrences.map(({ logical_path: path }) => path), ); + const directoryPaths = new Set( + snapshot.occurrences + .filter(({ entry_kind }) => entry_kind === "directory") + .map(({ logical_path }) => logical_path), + ); const activeOccurrences = selectedOccurrences.filter(({ logical_path }) => { // Nested members remain represented by their containing regular file; // only entries exposed by the active reader are materialized. @@ -133,7 +138,7 @@ const materializeArtifactInventoryInScope = async ( slash = path.lastIndexOf("/") ) { path = path.slice(0, slash); - if (regularPaths.has(path)) return false; + if (regularPaths.has(path) && !directoryPaths.has(path)) return false; } return true; }); diff --git a/src/artifacts/inventory/scanReader.ts b/src/artifacts/inventory/scanReader.ts index dc97a77d37..bc4a71f905 100644 --- a/src/artifacts/inventory/scanReader.ts +++ b/src/artifacts/inventory/scanReader.ts @@ -198,10 +198,14 @@ const visitArtifactEntries = async ( : `${frame.prefix}/${entry.path}`, ); const expandableAsar = isExpandableAsar(entry, logicalPath); - context.registry.add( - logicalPath, - expandableAsar ? "directory" : entry.kind, - ); + // PyInstaller TOC rows are source-distinct even when their logical + // destinations collide. Inventory retains those observations; the + // extraction path registry remains the authority for materialization. + if (frame.reader.format !== "pyinstaller") + context.registry.add( + logicalPath, + expandableAsar ? "directory" : entry.kind, + ); const duplicateOrdinal = context.occurrenceCountByPath.get(logicalPath) ?? 0; context.occurrenceCountByPath.set(logicalPath, duplicateOrdinal + 1); diff --git a/src/artifacts/pyinstaller/CArchiveLayout.ts b/src/artifacts/pyinstaller/CArchiveLayout.ts index b5bc1f4375..887fdea825 100644 --- a/src/artifacts/pyinstaller/CArchiveLayout.ts +++ b/src/artifacts/pyinstaller/CArchiveLayout.ts @@ -320,7 +320,6 @@ const validateCArchiveMembers = ( entries: readonly CArchiveEntry[], tocOffset: number, ): Result => { - const names = new Set(); for (const entry of entries) { if (isCArchiveDeclaration(entry)) { if ( @@ -334,15 +333,6 @@ const validateCArchiveMembers = ( `CArchive declaration at table index ${String(entry.index)} has local payload or compression`, ), ); - } else { - if (names.has(entry.name)) - return err( - pyInstallerFailure( - "integrity", - `Duplicate CArchive member name at table index ${String(entry.index)}`, - ), - ); - names.add(entry.name); } if ( entry.compressionFlag === 0 && diff --git a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts index 3220bbb4e0..e3500e854e 100644 --- a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts @@ -26,6 +26,44 @@ const sha256 = (bytes: Buffer): string => const derived = (payload: Buffer): Buffer => Buffer.concat([reconstructedPycHeader(PYTHON_39_MAGIC), payload]); +const internedString = (value: string): Buffer => { + const bytes = Buffer.from(value, "utf8"); + const length = Buffer.alloc(4); + length.writeInt32LE(bytes.length); + return Buffer.concat([Buffer.from([0x74]), length, bytes]); +}; + +const marshalInteger = (value: number): Buffer => { + const integer = Buffer.alloc(5); + integer[0] = 0x69; + integer.writeInt32LE(value, 1); + return integer; +}; + +const marshalTuple = (values: readonly Buffer[]): Buffer => + Buffer.concat([Buffer.from([0x29, values.length]), ...values]); + +const pyzWithInternedModuleName = (name: string, body: Buffer): Buffer => { + const ordinary = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "cafe", typeCode: 0, body }], + }); + const tableOffset = ordinary.readUInt32BE(8); + const table = Buffer.concat([ + Buffer.from([0x5b]), + marshalInteger(1), + marshalTuple([ + internedString(name), + marshalTuple([ + marshalInteger(0), + marshalInteger(12), + marshalInteger(tableOffset - 12), + ]), + ]), + ]); + return Buffer.concat([ordinary.subarray(0, tableOffset), table]); +}; + const writeArchive = async (name: string, bytes: Buffer): Promise => { const directory = await createTestTempDirectory("rea-pyinstaller-"); const path = join(directory, name); @@ -131,6 +169,81 @@ it("inventories a 2.1 archive, nested PYZ modules, and derived bytecode", async expect(scriptEdge?.relation).toBe("contains"); }); +it("decodes an interned Unicode module name from the PYZ table", async () => { + const path = await writeArchive( + "interned-unicode-module", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 313, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzWithInternedModuleName( + "café", + Buffer.from("value = 42\n"), + ), + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + const module = snapshot.occurrences.find( + ({ logical_path }) => logical_path === "PYZ.pyz/café.pyc", + ); + expect(snapshot.coverage.status).toBe("complete"); + expect(module).toMatchObject({ + entry_kind: "file", + artifact_id: expect.any(String), + format_metadata: { + kind: "pyinstaller-entry", + source_name: "café", + }, + }); +}); + +it("preserves the CArchive binary type as executable metadata", async () => { + const path = await writeArchive( + "executable-member", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 314, + members: [ + { + name: "helper.sh", + typeCode: "b", + payload: Buffer.from("#!/bin/sh\nexit 0\n"), + compress: false, + }, + { + name: "healthy.txt", + typeCode: "x", + payload: Buffer.from("healthy\n"), + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + expect( + snapshot.occurrences.find( + ({ logical_path }) => logical_path === "helper.sh", + ), + ).toMatchObject({ + executable: true, + format_metadata: { type_code: "b" }, + }); + expect( + snapshot.occurrences.find( + ({ logical_path }) => logical_path === "healthy.txt", + ), + ).toMatchObject({ + executable: false, + format_metadata: { type_code: "x" }, + }); +}); + it("records a 2.0 cookie without a Python library name", async () => { const path = await writeArchive( "legacy", @@ -1040,7 +1153,8 @@ it("uses known PYZ magic for separate CArchive scripts when the marshal table st expect(await readFile(join(output, "hello"))).toEqual(derived(script)); }); -it("rejects duplicate CArchive member names before projecting PYZ children", async () => { +it("retains colliding CArchive rows and PYZ children but refuses extraction", async () => { + const data = Buffer.from("same-name data member"); const image = pyzArchive({ magic: PYTHON_39_MAGIC, modules: [{ name: "child", typeCode: 0, body: Buffer.from("child") }], @@ -1055,16 +1169,78 @@ it("rejects duplicate CArchive member names before projecting PYZ children", asy { name: "PYZ.pyz", typeCode: "x", - payload: Buffer.alloc(0), + payload: data, + compress: false, + }, + { + name: "sibling.txt", + typeCode: "x", + payload: Buffer.from("sibling"), compress: false, }, ], }), ); - await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({ - reason: "integrity", - message: "Duplicate CArchive member name at table index 1", + const snapshot = await scanCanonicalArtifactInventory(path); + const samePath = snapshot.occurrences.filter( + ({ logical_path }) => logical_path === "PYZ.pyz", + ); + const pyzContainer = samePath.find( + ({ entry_kind }) => entry_kind === "directory", + ); + const dataMember = samePath.find(({ entry_kind }) => entry_kind === "file"); + const child = snapshot.occurrences.find( + ({ logical_path }) => logical_path === "PYZ.pyz/child.pyc", + ); + expect(snapshot.coverage.status).toBe("complete"); + expect(pyzContainer?.format_metadata).toMatchObject({ + kind: "pyinstaller-entry", + index: 0, + type_code: "z", + source_name: "PYZ.pyz", }); + expect(dataMember).toMatchObject({ + artifact_id: expect.any(String), + format_metadata: { + kind: "pyinstaller-entry", + index: 1, + type_code: "x", + source_name: "PYZ.pyz", + }, + }); + expect(child).toMatchObject({ + entry_kind: "file", + parent_occurrence_id: pyzContainer?.occurrence_id, + }); + expect( + snapshot.nodes.find( + ({ artifact_id }) => artifact_id === dataMember?.artifact_id, + )?.sha256, + ).toBe(sha256(data)); + expect( + snapshot.occurrences.find( + ({ logical_path }) => logical_path === "sibling.txt", + ), + ).toMatchObject({ artifact_id: expect.any(String) }); + + const root = await createTestTempDirectory( + "rea-pyinstaller-colliding-extract-", + ); + const output = join(root, "extracted"); + await expect( + materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + path, + snapshot, + ), + ).rejects.toMatchObject({ reason: "path" }); + await expect(stat(output)).rejects.toMatchObject({ code: "ENOENT" }); }); it("closes each spooled PYZ image after listing and reopens it for a requested module", async () => { diff --git a/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts b/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts index 5f92a3702a..72087f42a9 100644 --- a/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts @@ -198,7 +198,7 @@ it.each([true, false])( }, ); -it.each(["duplicate", "size", "offset", "overlap"])( +it.each(["size", "offset", "overlap"])( "bounds %s diagnostics for large nondeclaration names", async (mode) => { const name = "member".repeat(200_000); @@ -213,12 +213,7 @@ it.each(["duplicate", "size", "offset", "overlap"])( layout: "2.1", pythonVersion: 309, members: - mode === "duplicate" || mode === "overlap" - ? [ - first, - { ...first, name: mode === "duplicate" ? name : name + "-b" }, - ] - : [first], + mode === "overlap" ? [first, { ...first, name: name + "-b" }] : [first], }); const toc = 64 + bytes.readUInt32BE(bytes.length - 88 + 12); if (mode === "offset") @@ -239,7 +234,7 @@ it.each(["duplicate", "size", "offset", "overlap"])( expect(failure.reason).toBe(mode === "offset" ? "format" : "integrity"); expect(failure.message.length).toBeLessThan(128); expect(failure.message).toContain( - `table index ${mode === "duplicate" || mode === "overlap" ? 1 : 0}`, + `table index ${mode === "overlap" ? 1 : 0}`, ); }); },