diff --git a/docs/godot-pck.md b/docs/godot-pck.md index b81633adb2..efd464cf09 100644 --- a/docs/godot-pck.md +++ b/docs/godot-pck.md @@ -7,6 +7,9 @@ releases have not been verified. Format 1 is unsupported. Inspection retains the original header and directory facts, including engine version, flags, declared file count, source paths, offsets and declared MD5. +For a member declared as `res://scenes/main.gd`, the occurrence's +`source_path` retains that exact declaration while `logical_path` is +`scenes/main.gd`, the relative path used for lookup and extraction. Observed plaintext bytes receive content-addressed SHA-256 identities. MD5 is a container integrity claim, separate from the observed SHA-256. A mismatch fails by default; `--integrity-policy record-and-continue` retains the observed bytes diff --git a/docs/python-packaged-applications.md b/docs/python-packaged-applications.md index 7937ddd9f4..53c0d8c353 100644 --- a/docs/python-packaged-applications.md +++ b/docs/python-packaged-applications.md @@ -25,8 +25,12 @@ metadata retains the original name, table index, type, compression flag, relative offset and known directory record extent. Logical paths follow the carrier's path dialect: backslashes separate components for PE carriers, while they remain literal filename characters for POSIX carriers. The source name -always retains its stored spelling. If the carrier dialect is unconfirmed, REA -preserves backslashes verbatim instead of inferring POSIX semantics. +always retains its stored spelling in occurrence `source_path` and +`format_metadata.source_name`. For PYZ rows, that source is the exact module or +data key in the PYZ table, not a reconstructed filesystem name; +`format_metadata.container_name` identifies the containing PYZ image. If the +carrier dialect is unconfirmed, REA preserves backslashes verbatim instead of +inferring POSIX semantics. `relative_offset` is relative to the CArchive start for CArchive rows, or to the decoded PYZ image for PYZ rows. A CArchive directory record starts at `archive_start + toc_offset + directory_record_offset`; its size is diff --git a/src/artifacts/GodotPckArtifactReader.test.ts b/src/artifacts/GodotPckArtifactReader.test.ts index f79a6d2129..c3e82036e8 100644 --- a/src/artifacts/GodotPckArtifactReader.test.ts +++ b/src/artifacts/GodotPckArtifactReader.test.ts @@ -72,6 +72,9 @@ describe("GodotPckArtifactReader", () => { "project.binary", "data/é.json", ]); + expect(entries.map(({ sourcePath }) => sourcePath)).toEqual( + members.map(({ path }) => path), + ); expect(reader.formatMetadata()).toMatchObject({ embedded, engine_major: 4, diff --git a/src/artifacts/GodotPckArtifactReader.ts b/src/artifacts/GodotPckArtifactReader.ts index 7233553864..a0d63f59d4 100644 --- a/src/artifacts/GodotPckArtifactReader.ts +++ b/src/artifacts/GodotPckArtifactReader.ts @@ -190,8 +190,9 @@ export class GodotPckArtifactReader implements ArtifactReader { assertArtifactReadActive(signal); const pathBytes = await cursor.uint32(); // Check worst-case JSON escaping before decoding or allocating the path. - // Its source bytes, decoded string and retained metadata share one budget. - if (pathBytes * 12 > MAX_DIRECTORY_METADATA_BYTES - retainedBytes) + // The logical path, raw source path, and metadata source path can each + // need six JSON bytes per input byte before retained facts are charged. + if (pathBytes * 18 > MAX_DIRECTORY_METADATA_BYTES - retainedBytes) throw metadataLimit(); const rawPath = await cursor.bytes(pathBytes); const offset = await cursor.uint64(); @@ -226,6 +227,7 @@ export class GodotPckArtifactReader implements ArtifactReader { path: sourcePath.startsWith("res://") ? sourcePath.slice(6) : sourcePath, + sourcePath, kind: "file" as const, declaredSize: Number(size), compressedSize: null, @@ -386,7 +388,7 @@ class PckCursor { async bytes(length: number): Promise { if (length > this.end - this.position) throw formatFailure("directory entry exceeds the pack"); - if (length > MAX_DIRECTORY_METADATA_BYTES / 12) throw metadataLimit(); + if (length > MAX_DIRECTORY_METADATA_BYTES / 18) throw metadataLimit(); if (length > DIRECTORY_CHUNK_BYTES) { // A large valid virtual path is bounded by retained representation, not // the host filesystem's PATH_MAX. Only this path receives a larger read. diff --git a/src/artifacts/PyInstallerArtifactReader.ts b/src/artifacts/PyInstallerArtifactReader.ts index 43d2c2479d..0607105bd7 100644 --- a/src/artifacts/PyInstallerArtifactReader.ts +++ b/src/artifacts/PyInstallerArtifactReader.ts @@ -541,6 +541,7 @@ export class PyInstallerArtifactReader implements ArtifactReader { this.#reserveMetadata( { path: entry.path, + source_path: entry.sourcePath ?? null, adapter_key: entry.adapterKey, format_metadata: entry.formatMetadata ?? null, limitations: [...entry.limitations], diff --git a/src/artifacts/pyinstaller/CArchiveMembers.ts b/src/artifacts/pyinstaller/CArchiveMembers.ts index 687e99c845..d64a61c65a 100644 --- a/src/artifacts/pyinstaller/CArchiveMembers.ts +++ b/src/artifacts/pyinstaller/CArchiveMembers.ts @@ -66,7 +66,7 @@ export const projectStoredMember = ( ): PyInstallerMember => { if (isCArchiveDeclaration(entry)) return { - entry: fileEntry({ + entry: projectEntry({ path: `__pyinstaller__/${entry.typeCode === "o" ? "runtime-options" : "dependencies"}/${String(entry.index)}`, kind: "metadata", declaredSize: entry.uncompressedSize, @@ -93,7 +93,7 @@ export const projectStoredMember = ( if (bytecode.limitation !== undefined) limitations.push(bytecode.limitation); const declaredSize = entry.uncompressedSize; return { - entry: fileEntry({ + entry: projectEntry({ path, kind: symlink ? "symlink" : "file", declaredSize, @@ -126,7 +126,7 @@ export const projectPyzContainer = ( const limitations: string[] = []; if (archive.kind === "opaque") return { - entry: fileEntry({ + entry: projectEntry({ path, declaredSize: entry.uncompressedSize, compressedSize: entry.compressedSize, @@ -167,16 +167,12 @@ export const projectPyzContainer = ( }, }; return { - entry: { + entry: projectEntry({ path, kind: "directory", declaredSize: entry.uncompressedSize, compressedSize: entry.compressedSize, - executable: false, - encrypted: false, byteOffset, - declaredIntegrity: null, - unpacked: false, limitations: [ ...limitations, `PyInstaller PYZ sha256 ${archive.sha256}.`, @@ -191,7 +187,7 @@ export const projectPyzContainer = ( archive, encrypted, ), - }, + }), read: undefined, }; }; @@ -218,16 +214,13 @@ export const projectPyzMember = ( const encrypted = pyzEntryEncrypted(item, encryptionMarker); if (item.namespace) return { - entry: { - path: path, + entry: projectEntry({ + path, kind: "metadata", declaredSize: 0, compressedSize: item.length, - executable: false, encrypted, byteOffset: null, - declaredIntegrity: null, - unpacked: false, limitations: [ "Namespace package has no importable bytecode; its declared payload is not materialized.", ...(encrypted @@ -243,7 +236,7 @@ export const projectPyzMember = ( undefined, encryptionMarker, ), - }, + }), read: undefined, }; const header = @@ -261,7 +254,7 @@ export const projectPyzMember = ( "PYZ tables declare compressed lengths; the uncompressed size is unknown until decoding.", ); return { - entry: fileEntry({ + entry: projectEntry({ path: path, declaredSize: null, compressedSize: item.length, @@ -321,7 +314,7 @@ const pyzMemberPath = ( return `${container}/${dotted}.pyc`; }; -const fileEntry = (input: { +const projectEntry = (input: { readonly path: string; readonly declaredSize: number | null; readonly compressedSize: number; @@ -331,9 +324,10 @@ const fileEntry = (input: { readonly formatMetadata: PyInstallerEntryMetadata; readonly executable?: boolean; readonly encrypted?: boolean; - readonly kind?: "file" | "symlink" | "metadata"; + readonly kind?: "file" | "directory" | "symlink" | "metadata"; }): ArtifactEntry => ({ path: input.path, + sourcePath: input.formatMetadata.source_name, kind: input.kind ?? "file", declaredSize: input.declaredSize, compressedSize: input.compressedSize, diff --git a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts index 5a0506c2fa..c3d759f57a 100644 --- a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts @@ -218,8 +218,13 @@ it("inventories a 2.1 archive, nested PYZ modules, and derived bytecode", async expect( snapshot.occurrences.find( (occurrence) => occurrence.logical_path === "PYZ.pyz", - )?.entry_kind, - ).toBe("directory"); + ), + ).toMatchObject({ entry_kind: "directory", source_path: "PYZ.pyz" }); + expect( + snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "PYZ.pyz/ns", + ), + ).toMatchObject({ entry_kind: "metadata", source_path: "ns" }); const scriptEdge = snapshot.edges.find( (edge) => edge.occurrence_id === hello?.occurrence_id, ); @@ -254,6 +259,7 @@ it("decodes an interned Unicode module name from the PYZ table", async () => { ); expect(module).toMatchObject({ entry_kind: "file", + source_path: "café", artifact_id: expect.any(String), format_metadata: { kind: "pyinstaller-entry", @@ -927,6 +933,8 @@ it("inventories and extracts a Windows member name under its canonical path", as ); expect(member?.format_metadata).toMatchObject({ source_name: storedName }); expect(resource?.format_metadata).toMatchObject({ source_name: pyzName }); + expect(member?.source_path).toBe(storedName); + expect(resource?.source_path).toBe(pyzName); const directory = await createTestTempDirectory("rea-pyinstaller-out-"); const output = join(directory, "extracted"); await materializeArtifactInventory( diff --git a/tests/acceptance/godotPck.test.ts b/tests/acceptance/godotPck.test.ts index 8746344df7..cbe60ab632 100644 --- a/tests/acceptance/godotPck.test.ts +++ b/tests/acceptance/godotPck.test.ts @@ -67,6 +67,7 @@ cliTest.for([2, 3, 4] as const)( source_path: member.path, flags: 0, }); + expect(row?.source_path).toBe(member.path); expect(row?.hash_status).toBe("verified"); } const extracted = await cli.run({ diff --git a/tests/acceptance/pyinstallerArchive.test.ts b/tests/acceptance/pyinstallerArchive.test.ts index 4cf529c266..0fdad05fe0 100644 --- a/tests/acceptance/pyinstallerArchive.test.ts +++ b/tests/acceptance/pyinstallerArchive.test.ts @@ -17,6 +17,7 @@ import { artifactInspectionResultSchema } from "../../src/domain/artifactInspect import { artifactExtractionResultSchema } from "../../src/domain/artifactGraph.js"; import { parseEvidence } from "../../src/domain/evidence.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; const execute = promisify(execFile); const cli = fileURLToPath(new URL("../../scripts/rea.mjs", import.meta.url)); @@ -192,6 +193,7 @@ it("lists CArchive members and the Python version through inspect-artifact", asy ), ).toMatchObject({ artifact_id: null, + source_path: "pyi-contents-directory _internal", format_metadata: { source_name: "pyi-contents-directory _internal", type_code: "o", @@ -217,6 +219,92 @@ it("lists CArchive members and the Python version through inspect-artifact", asy }); }); +cliTest( + "preserves Windows PyInstaller source names", + async ({ cli: driver }) => { + const directory = await createTestTempDirectory( + "rea-pyinstaller-windows-names-", + ); + const path = join(directory, "windows.exe"); + const storedName = String.raw`pkg\data.bin`; + const dataKey = String.raw`res\icon.bin`; + const stored = Buffer.from("stored bytes"); + const icon = Buffer.from("icon bytes"); + const module = Buffer.from("module bytes"); + await writeFile( + path, + cArchiveExecutable({ + carrier: "pe", + layout: "2.1", + pythonVersion: 309, + members: [ + { name: storedName, typeCode: "x", payload: stored, compress: false }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: dataKey, typeCode: 2, body: icon }, + { name: "pkg.mod", typeCode: 0, body: module }, + ], + }), + compress: false, + }, + ], + }), + ); + const inspected = await driver.run({ + arguments: ["inspect-artifact", path, "--json"], + }); + expect(inspected.exitCode, inspected.stdout + inspected.stderr).toBe(0); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(inspected.json).normalized_result, + ); + const rows = inspection.substeps[0]?.evidence.normalized_result.occurrences; + expect( + rows?.find((row) => row.logical_path === "pkg/data.bin"), + ).toMatchObject({ + source_path: storedName, + format_metadata: { source_name: storedName }, + }); + expect( + rows?.find((row) => row.logical_path === "PYZ.pyz/res/icon.bin"), + ).toMatchObject({ + source_path: dataKey, + format_metadata: { container_name: "PYZ.pyz", source_name: dataKey }, + }); + expect( + rows?.find((row) => row.logical_path === "PYZ.pyz/pkg/mod.pyc"), + ).toMatchObject({ + source_path: "pkg.mod", + format_metadata: { container_name: "PYZ.pyz", source_name: "pkg.mod" }, + }); + const extracted = await driver.run({ + arguments: ["extract-artifact", path, "--json"], + }); + expect(extracted.exitCode, extracted.stdout + extracted.stderr).toBe(0); + const result = artifactExtractionResultSchema.parse( + parseEvidence(extracted.json).normalized_result, + ); + try { + expect( + await readFile(join(result.output_root, "pkg", "data.bin")), + ).toEqual(stored); + expect( + await readFile(join(result.output_root, "PYZ.pyz", "res", "icon.bin")), + ).toEqual(icon); + expect( + ( + await readFile(join(result.output_root, "PYZ.pyz", "pkg", "mod.pyc")) + ).subarray(16), + ).toEqual(module); + } finally { + await rm(result.output_root, { recursive: true, force: true }); + } + }, +); + it("extracts legacy and modern CArchive modules with one pyc header through the CLI", async () => { const fixture = cArchiveBytecodeExecutable(); const directory = await createTestTempDirectory("rea-pyinstaller-bytecode-"); diff --git a/tests/boundary/mcp/godotPckMcp.test.ts b/tests/boundary/mcp/godotPckMcp.test.ts index b7b2a7dc85..ec67806d81 100644 --- a/tests/boundary/mcp/godotPckMcp.test.ts +++ b/tests/boundary/mcp/godotPckMcp.test.ts @@ -38,6 +38,37 @@ const viewValidators = async (client: Client) => { return { validateInput, validateOutput }; }; +const retainedInventoryView = + ( + client: Client, + evidenceId: string, + validators: Awaited>, + ) => + async (query: Record) => { + const arguments_ = { + source: { kind: "retained-evidence", evidence_id: evidenceId }, + view: { kind: "inventory", ...query }, + }; + expect( + validators.validateInput(arguments_), + JSON.stringify(validators.validateInput.errors), + ).toBe(true); + const reply = await client.callTool({ + name: "inspect_analysis_view", + arguments: arguments_, + }); + expect(reply.isError, JSON.stringify(reply.structuredContent)).not.toBe( + true, + ); + expect( + validators.validateOutput(reply.structuredContent), + JSON.stringify(validators.validateOutput.errors), + ).toBe(true); + const evidence = parseEvidence(reply.structuredContent); + expect(evidence.evidence_links).toContain(evidenceId); + return analysisViewResultSchema.parse(evidence.normalized_result); + }; + const assertInvalidNodeField = async ( client: Client, evidenceId: string, @@ -82,7 +113,7 @@ it.each(profiles)("reuses retained PCK inventory (%s)", async (profile) => { try { await server.connect(serverTransport); await client.connect(clientTransport); - const { validateInput, validateOutput } = await viewValidators(client); + const validators = await viewValidators(client); expect( (await client.callTool({ name: "open_binary", arguments: { path } })) .isError, @@ -116,41 +147,22 @@ it.each(profiles)("reuses retained PCK inventory (%s)", async (profile) => { session.evidenceForAnalysis(nested.evidence_id)?.normalized_result, ).toBe(nested.normalized_result); const graph = nested.normalized_result; - const source = { - kind: "retained-evidence", - evidence_id: parent.evidence_id, - }; - const view = async (query: Record) => { - const arguments_ = { source, view: { kind: "inventory", ...query } }; - expect( - validateInput(arguments_), - JSON.stringify(validateInput.errors), - ).toBe(true); - const reply = await client.callTool({ - name: "inspect_analysis_view", - arguments: arguments_, - }); - expect(reply.isError, JSON.stringify(reply.structuredContent)).not.toBe( - true, - ); - expect( - validateOutput(reply.structuredContent), - JSON.stringify(validateOutput.errors), - ).toBe(true); - const evidence = parseEvidence(reply.structuredContent); - expect(evidence.evidence_links).toContain(parent.evidence_id); - return analysisViewResultSchema.parse(evidence.normalized_result); - }; + const view = retainedInventoryView(client, parent.evidence_id, validators); await assertInvalidNodeField( client, parent.evidence_id, - profile === "full" ? validateInput : undefined, + profile === "full" ? validators.validateInput : undefined, ); const wanted = graph.occurrences.filter((row) => row.logical_path !== "."); const occurrences = await view({ collection: "occurrences", logical_paths: wanted.map((row) => row.logical_path), - fields: ["logical_path", "declared_integrity", "format_metadata"], + fields: [ + "logical_path", + "source_path", + "declared_integrity", + "format_metadata", + ], }); expect(occurrences.kind).toBe("inventory"); if (occurrences.kind !== "inventory") @@ -161,6 +173,7 @@ it.each(profiles)("reuses retained PCK inventory (%s)", async (profile) => { artifact_id: row.artifact_id, parent_occurrence_id: row.parent_occurrence_id, logical_path: row.logical_path, + source_path: row.source_path, declared_integrity: row.declared_integrity, format_metadata: row.format_metadata, })), diff --git a/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts b/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts index 8ffa03ea38..4401226c77 100644 --- a/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts +++ b/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts @@ -245,6 +245,18 @@ it.each([false, true])( ...(opaque ? [] : ["PYZ.pyz/app.pyc"]), ]), ); + if (!opaque) + expect( + inspection.substeps[0]?.evidence.normalized_result.occurrences.find( + (row) => row.logical_path === "PYZ.pyz/app.pyc", + ), + ).toMatchObject({ + source_path: "app", + format_metadata: { + container_name: "PYZ.pyz", + source_name: "app", + }, + }); const parent = parseEvidence(inspected.structuredContent); const compact = await client.callTool({ name: "inspect_artifact",