From 7fe5e5ff5e80f4aa918a5a911140dae249a9757c Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:30:22 +0800 Subject: [PATCH] fix(artifacts): retain raw archive keys across entry projections Publish Godot res:// declarations and PyInstaller CArchive/PYZ keys through shared occurrence source_path while preserving safe logical extraction paths. Consolidate PyInstaller file, directory and namespace projection through one owner, and account for the additional retained source spelling before allocation. Cover producer inventory, Windows separator conversion, PYZ module/data keys, CLI extraction bytes and retained MCP inventory views. Format fixtures are synthetic; real Godot/PyInstaller producer lanes were not run. --- docs/godot-pck.md | 3 + docs/python-packaged-applications.md | 8 +- src/artifacts/GodotPckArtifactReader.test.ts | 3 + src/artifacts/GodotPckArtifactReader.ts | 8 +- src/artifacts/PyInstallerArtifactReader.ts | 1 + src/artifacts/pyinstaller/CArchiveMembers.ts | 30 +++---- .../PyInstallerArtifactReader.test.ts | 12 ++- tests/acceptance/godotPck.test.ts | 1 + tests/acceptance/pyinstallerArchive.test.ts | 88 +++++++++++++++++++ tests/boundary/mcp/godotPckMcp.test.ts | 69 +++++++++------ .../mcp/pyinstallerArtifactMcp.test.ts | 12 +++ 11 files changed, 182 insertions(+), 53 deletions(-) diff --git a/docs/godot-pck.md b/docs/godot-pck.md index b81633adb2..efd464cf09 100644 --- a/docs/godot-pck.md +++ b/docs/godot-pck.md @@ -7,6 +7,9 @@ releases have not been verified. Format 1 is unsupported. Inspection retains the original header and directory facts, including engine version, flags, declared file count, source paths, offsets and declared MD5. +For a member declared as `res://scenes/main.gd`, the occurrence's +`source_path` retains that exact declaration while `logical_path` is +`scenes/main.gd`, the relative path used for lookup and extraction. Observed plaintext bytes receive content-addressed SHA-256 identities. MD5 is a container integrity claim, separate from the observed SHA-256. A mismatch fails by default; `--integrity-policy record-and-continue` retains the observed bytes diff --git a/docs/python-packaged-applications.md b/docs/python-packaged-applications.md index 7937ddd9f4..53c0d8c353 100644 --- a/docs/python-packaged-applications.md +++ b/docs/python-packaged-applications.md @@ -25,8 +25,12 @@ metadata retains the original name, table index, type, compression flag, relative offset and known directory record extent. Logical paths follow the carrier's path dialect: backslashes separate components for PE carriers, while they remain literal filename characters for POSIX carriers. The source name -always retains its stored spelling. If the carrier dialect is unconfirmed, REA -preserves backslashes verbatim instead of inferring POSIX semantics. +always retains its stored spelling in occurrence `source_path` and +`format_metadata.source_name`. For PYZ rows, that source is the exact module or +data key in the PYZ table, not a reconstructed filesystem name; +`format_metadata.container_name` identifies the containing PYZ image. If the +carrier dialect is unconfirmed, REA preserves backslashes verbatim instead of +inferring POSIX semantics. `relative_offset` is relative to the CArchive start for CArchive rows, or to the decoded PYZ image for PYZ rows. A CArchive directory record starts at `archive_start + toc_offset + directory_record_offset`; its size is diff --git a/src/artifacts/GodotPckArtifactReader.test.ts b/src/artifacts/GodotPckArtifactReader.test.ts index f79a6d2129..c3e82036e8 100644 --- a/src/artifacts/GodotPckArtifactReader.test.ts +++ b/src/artifacts/GodotPckArtifactReader.test.ts @@ -72,6 +72,9 @@ describe("GodotPckArtifactReader", () => { "project.binary", "data/é.json", ]); + expect(entries.map(({ sourcePath }) => sourcePath)).toEqual( + members.map(({ path }) => path), + ); expect(reader.formatMetadata()).toMatchObject({ embedded, engine_major: 4, diff --git a/src/artifacts/GodotPckArtifactReader.ts b/src/artifacts/GodotPckArtifactReader.ts index 7233553864..a0d63f59d4 100644 --- a/src/artifacts/GodotPckArtifactReader.ts +++ b/src/artifacts/GodotPckArtifactReader.ts @@ -190,8 +190,9 @@ export class GodotPckArtifactReader implements ArtifactReader { assertArtifactReadActive(signal); const pathBytes = await cursor.uint32(); // Check worst-case JSON escaping before decoding or allocating the path. - // Its source bytes, decoded string and retained metadata share one budget. - if (pathBytes * 12 > MAX_DIRECTORY_METADATA_BYTES - retainedBytes) + // The logical path, raw source path, and metadata source path can each + // need six JSON bytes per input byte before retained facts are charged. + if (pathBytes * 18 > MAX_DIRECTORY_METADATA_BYTES - retainedBytes) throw metadataLimit(); const rawPath = await cursor.bytes(pathBytes); const offset = await cursor.uint64(); @@ -226,6 +227,7 @@ export class GodotPckArtifactReader implements ArtifactReader { path: sourcePath.startsWith("res://") ? sourcePath.slice(6) : sourcePath, + sourcePath, kind: "file" as const, declaredSize: Number(size), compressedSize: null, @@ -386,7 +388,7 @@ class PckCursor { async bytes(length: number): Promise { if (length > this.end - this.position) throw formatFailure("directory entry exceeds the pack"); - if (length > MAX_DIRECTORY_METADATA_BYTES / 12) throw metadataLimit(); + if (length > MAX_DIRECTORY_METADATA_BYTES / 18) throw metadataLimit(); if (length > DIRECTORY_CHUNK_BYTES) { // A large valid virtual path is bounded by retained representation, not // the host filesystem's PATH_MAX. Only this path receives a larger read. diff --git a/src/artifacts/PyInstallerArtifactReader.ts b/src/artifacts/PyInstallerArtifactReader.ts index 43d2c2479d..0607105bd7 100644 --- a/src/artifacts/PyInstallerArtifactReader.ts +++ b/src/artifacts/PyInstallerArtifactReader.ts @@ -541,6 +541,7 @@ export class PyInstallerArtifactReader implements ArtifactReader { this.#reserveMetadata( { path: entry.path, + source_path: entry.sourcePath ?? null, adapter_key: entry.adapterKey, format_metadata: entry.formatMetadata ?? null, limitations: [...entry.limitations], diff --git a/src/artifacts/pyinstaller/CArchiveMembers.ts b/src/artifacts/pyinstaller/CArchiveMembers.ts index 687e99c845..d64a61c65a 100644 --- a/src/artifacts/pyinstaller/CArchiveMembers.ts +++ b/src/artifacts/pyinstaller/CArchiveMembers.ts @@ -66,7 +66,7 @@ export const projectStoredMember = ( ): PyInstallerMember => { if (isCArchiveDeclaration(entry)) return { - entry: fileEntry({ + entry: projectEntry({ path: `__pyinstaller__/${entry.typeCode === "o" ? "runtime-options" : "dependencies"}/${String(entry.index)}`, kind: "metadata", declaredSize: entry.uncompressedSize, @@ -93,7 +93,7 @@ export const projectStoredMember = ( if (bytecode.limitation !== undefined) limitations.push(bytecode.limitation); const declaredSize = entry.uncompressedSize; return { - entry: fileEntry({ + entry: projectEntry({ path, kind: symlink ? "symlink" : "file", declaredSize, @@ -126,7 +126,7 @@ export const projectPyzContainer = ( const limitations: string[] = []; if (archive.kind === "opaque") return { - entry: fileEntry({ + entry: projectEntry({ path, declaredSize: entry.uncompressedSize, compressedSize: entry.compressedSize, @@ -167,16 +167,12 @@ export const projectPyzContainer = ( }, }; return { - entry: { + entry: projectEntry({ path, kind: "directory", declaredSize: entry.uncompressedSize, compressedSize: entry.compressedSize, - executable: false, - encrypted: false, byteOffset, - declaredIntegrity: null, - unpacked: false, limitations: [ ...limitations, `PyInstaller PYZ sha256 ${archive.sha256}.`, @@ -191,7 +187,7 @@ export const projectPyzContainer = ( archive, encrypted, ), - }, + }), read: undefined, }; }; @@ -218,16 +214,13 @@ export const projectPyzMember = ( const encrypted = pyzEntryEncrypted(item, encryptionMarker); if (item.namespace) return { - entry: { - path: path, + entry: projectEntry({ + path, kind: "metadata", declaredSize: 0, compressedSize: item.length, - executable: false, encrypted, byteOffset: null, - declaredIntegrity: null, - unpacked: false, limitations: [ "Namespace package has no importable bytecode; its declared payload is not materialized.", ...(encrypted @@ -243,7 +236,7 @@ export const projectPyzMember = ( undefined, encryptionMarker, ), - }, + }), read: undefined, }; const header = @@ -261,7 +254,7 @@ export const projectPyzMember = ( "PYZ tables declare compressed lengths; the uncompressed size is unknown until decoding.", ); return { - entry: fileEntry({ + entry: projectEntry({ path: path, declaredSize: null, compressedSize: item.length, @@ -321,7 +314,7 @@ const pyzMemberPath = ( return `${container}/${dotted}.pyc`; }; -const fileEntry = (input: { +const projectEntry = (input: { readonly path: string; readonly declaredSize: number | null; readonly compressedSize: number; @@ -331,9 +324,10 @@ const fileEntry = (input: { readonly formatMetadata: PyInstallerEntryMetadata; readonly executable?: boolean; readonly encrypted?: boolean; - readonly kind?: "file" | "symlink" | "metadata"; + readonly kind?: "file" | "directory" | "symlink" | "metadata"; }): ArtifactEntry => ({ path: input.path, + sourcePath: input.formatMetadata.source_name, kind: input.kind ?? "file", declaredSize: input.declaredSize, compressedSize: input.compressedSize, diff --git a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts index 5a0506c2fa..c3d759f57a 100644 --- a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts @@ -218,8 +218,13 @@ it("inventories a 2.1 archive, nested PYZ modules, and derived bytecode", async expect( snapshot.occurrences.find( (occurrence) => occurrence.logical_path === "PYZ.pyz", - )?.entry_kind, - ).toBe("directory"); + ), + ).toMatchObject({ entry_kind: "directory", source_path: "PYZ.pyz" }); + expect( + snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "PYZ.pyz/ns", + ), + ).toMatchObject({ entry_kind: "metadata", source_path: "ns" }); const scriptEdge = snapshot.edges.find( (edge) => edge.occurrence_id === hello?.occurrence_id, ); @@ -254,6 +259,7 @@ it("decodes an interned Unicode module name from the PYZ table", async () => { ); expect(module).toMatchObject({ entry_kind: "file", + source_path: "café", artifact_id: expect.any(String), format_metadata: { kind: "pyinstaller-entry", @@ -927,6 +933,8 @@ it("inventories and extracts a Windows member name under its canonical path", as ); expect(member?.format_metadata).toMatchObject({ source_name: storedName }); expect(resource?.format_metadata).toMatchObject({ source_name: pyzName }); + expect(member?.source_path).toBe(storedName); + expect(resource?.source_path).toBe(pyzName); const directory = await createTestTempDirectory("rea-pyinstaller-out-"); const output = join(directory, "extracted"); await materializeArtifactInventory( diff --git a/tests/acceptance/godotPck.test.ts b/tests/acceptance/godotPck.test.ts index 8746344df7..cbe60ab632 100644 --- a/tests/acceptance/godotPck.test.ts +++ b/tests/acceptance/godotPck.test.ts @@ -67,6 +67,7 @@ cliTest.for([2, 3, 4] as const)( source_path: member.path, flags: 0, }); + expect(row?.source_path).toBe(member.path); expect(row?.hash_status).toBe("verified"); } const extracted = await cli.run({ diff --git a/tests/acceptance/pyinstallerArchive.test.ts b/tests/acceptance/pyinstallerArchive.test.ts index 4cf529c266..0fdad05fe0 100644 --- a/tests/acceptance/pyinstallerArchive.test.ts +++ b/tests/acceptance/pyinstallerArchive.test.ts @@ -17,6 +17,7 @@ import { artifactInspectionResultSchema } from "../../src/domain/artifactInspect import { artifactExtractionResultSchema } from "../../src/domain/artifactGraph.js"; import { parseEvidence } from "../../src/domain/evidence.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; const execute = promisify(execFile); const cli = fileURLToPath(new URL("../../scripts/rea.mjs", import.meta.url)); @@ -192,6 +193,7 @@ it("lists CArchive members and the Python version through inspect-artifact", asy ), ).toMatchObject({ artifact_id: null, + source_path: "pyi-contents-directory _internal", format_metadata: { source_name: "pyi-contents-directory _internal", type_code: "o", @@ -217,6 +219,92 @@ it("lists CArchive members and the Python version through inspect-artifact", asy }); }); +cliTest( + "preserves Windows PyInstaller source names", + async ({ cli: driver }) => { + const directory = await createTestTempDirectory( + "rea-pyinstaller-windows-names-", + ); + const path = join(directory, "windows.exe"); + const storedName = String.raw`pkg\data.bin`; + const dataKey = String.raw`res\icon.bin`; + const stored = Buffer.from("stored bytes"); + const icon = Buffer.from("icon bytes"); + const module = Buffer.from("module bytes"); + await writeFile( + path, + cArchiveExecutable({ + carrier: "pe", + layout: "2.1", + pythonVersion: 309, + members: [ + { name: storedName, typeCode: "x", payload: stored, compress: false }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: dataKey, typeCode: 2, body: icon }, + { name: "pkg.mod", typeCode: 0, body: module }, + ], + }), + compress: false, + }, + ], + }), + ); + const inspected = await driver.run({ + arguments: ["inspect-artifact", path, "--json"], + }); + expect(inspected.exitCode, inspected.stdout + inspected.stderr).toBe(0); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(inspected.json).normalized_result, + ); + const rows = inspection.substeps[0]?.evidence.normalized_result.occurrences; + expect( + rows?.find((row) => row.logical_path === "pkg/data.bin"), + ).toMatchObject({ + source_path: storedName, + format_metadata: { source_name: storedName }, + }); + expect( + rows?.find((row) => row.logical_path === "PYZ.pyz/res/icon.bin"), + ).toMatchObject({ + source_path: dataKey, + format_metadata: { container_name: "PYZ.pyz", source_name: dataKey }, + }); + expect( + rows?.find((row) => row.logical_path === "PYZ.pyz/pkg/mod.pyc"), + ).toMatchObject({ + source_path: "pkg.mod", + format_metadata: { container_name: "PYZ.pyz", source_name: "pkg.mod" }, + }); + const extracted = await driver.run({ + arguments: ["extract-artifact", path, "--json"], + }); + expect(extracted.exitCode, extracted.stdout + extracted.stderr).toBe(0); + const result = artifactExtractionResultSchema.parse( + parseEvidence(extracted.json).normalized_result, + ); + try { + expect( + await readFile(join(result.output_root, "pkg", "data.bin")), + ).toEqual(stored); + expect( + await readFile(join(result.output_root, "PYZ.pyz", "res", "icon.bin")), + ).toEqual(icon); + expect( + ( + await readFile(join(result.output_root, "PYZ.pyz", "pkg", "mod.pyc")) + ).subarray(16), + ).toEqual(module); + } finally { + await rm(result.output_root, { recursive: true, force: true }); + } + }, +); + it("extracts legacy and modern CArchive modules with one pyc header through the CLI", async () => { const fixture = cArchiveBytecodeExecutable(); const directory = await createTestTempDirectory("rea-pyinstaller-bytecode-"); diff --git a/tests/boundary/mcp/godotPckMcp.test.ts b/tests/boundary/mcp/godotPckMcp.test.ts index b7b2a7dc85..ec67806d81 100644 --- a/tests/boundary/mcp/godotPckMcp.test.ts +++ b/tests/boundary/mcp/godotPckMcp.test.ts @@ -38,6 +38,37 @@ const viewValidators = async (client: Client) => { return { validateInput, validateOutput }; }; +const retainedInventoryView = + ( + client: Client, + evidenceId: string, + validators: Awaited>, + ) => + async (query: Record) => { + const arguments_ = { + source: { kind: "retained-evidence", evidence_id: evidenceId }, + view: { kind: "inventory", ...query }, + }; + expect( + validators.validateInput(arguments_), + JSON.stringify(validators.validateInput.errors), + ).toBe(true); + const reply = await client.callTool({ + name: "inspect_analysis_view", + arguments: arguments_, + }); + expect(reply.isError, JSON.stringify(reply.structuredContent)).not.toBe( + true, + ); + expect( + validators.validateOutput(reply.structuredContent), + JSON.stringify(validators.validateOutput.errors), + ).toBe(true); + const evidence = parseEvidence(reply.structuredContent); + expect(evidence.evidence_links).toContain(evidenceId); + return analysisViewResultSchema.parse(evidence.normalized_result); + }; + const assertInvalidNodeField = async ( client: Client, evidenceId: string, @@ -82,7 +113,7 @@ it.each(profiles)("reuses retained PCK inventory (%s)", async (profile) => { try { await server.connect(serverTransport); await client.connect(clientTransport); - const { validateInput, validateOutput } = await viewValidators(client); + const validators = await viewValidators(client); expect( (await client.callTool({ name: "open_binary", arguments: { path } })) .isError, @@ -116,41 +147,22 @@ it.each(profiles)("reuses retained PCK inventory (%s)", async (profile) => { session.evidenceForAnalysis(nested.evidence_id)?.normalized_result, ).toBe(nested.normalized_result); const graph = nested.normalized_result; - const source = { - kind: "retained-evidence", - evidence_id: parent.evidence_id, - }; - const view = async (query: Record) => { - const arguments_ = { source, view: { kind: "inventory", ...query } }; - expect( - validateInput(arguments_), - JSON.stringify(validateInput.errors), - ).toBe(true); - const reply = await client.callTool({ - name: "inspect_analysis_view", - arguments: arguments_, - }); - expect(reply.isError, JSON.stringify(reply.structuredContent)).not.toBe( - true, - ); - expect( - validateOutput(reply.structuredContent), - JSON.stringify(validateOutput.errors), - ).toBe(true); - const evidence = parseEvidence(reply.structuredContent); - expect(evidence.evidence_links).toContain(parent.evidence_id); - return analysisViewResultSchema.parse(evidence.normalized_result); - }; + const view = retainedInventoryView(client, parent.evidence_id, validators); await assertInvalidNodeField( client, parent.evidence_id, - profile === "full" ? validateInput : undefined, + profile === "full" ? validators.validateInput : undefined, ); const wanted = graph.occurrences.filter((row) => row.logical_path !== "."); const occurrences = await view({ collection: "occurrences", logical_paths: wanted.map((row) => row.logical_path), - fields: ["logical_path", "declared_integrity", "format_metadata"], + fields: [ + "logical_path", + "source_path", + "declared_integrity", + "format_metadata", + ], }); expect(occurrences.kind).toBe("inventory"); if (occurrences.kind !== "inventory") @@ -161,6 +173,7 @@ it.each(profiles)("reuses retained PCK inventory (%s)", async (profile) => { artifact_id: row.artifact_id, parent_occurrence_id: row.parent_occurrence_id, logical_path: row.logical_path, + source_path: row.source_path, declared_integrity: row.declared_integrity, format_metadata: row.format_metadata, })), diff --git a/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts b/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts index 8ffa03ea38..4401226c77 100644 --- a/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts +++ b/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts @@ -245,6 +245,18 @@ it.each([false, true])( ...(opaque ? [] : ["PYZ.pyz/app.pyc"]), ]), ); + if (!opaque) + expect( + inspection.substeps[0]?.evidence.normalized_result.occurrences.find( + (row) => row.logical_path === "PYZ.pyz/app.pyc", + ), + ).toMatchObject({ + source_path: "app", + format_metadata: { + container_name: "PYZ.pyz", + source_name: "app", + }, + }); const parent = parseEvidence(inspected.structuredContent); const compact = await client.callTool({ name: "inspect_artifact",