From afa5da745e0b139e68029a96b776ae30e48ea2b2 Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Tue, 20 Oct 2026 21:48:29 -0700 Subject: [PATCH] feat(artifacts): inspect and extract PyInstaller archives --- .../skills/reverse-engineer-anything/SKILL.md | 12 +- README.md | 59 +- docs/.vitepress/config.ts | 4 + docs/index.md | 26 +- docs/python-packaged-applications.md | 128 ++ docs/readme/README_ar.md | 1 + docs/readme/README_de.md | 1 + docs/readme/README_el.md | 1 + docs/readme/README_es.md | 1 + docs/readme/README_fa.md | 1 + docs/readme/README_fr.md | 1 + docs/readme/README_id.md | 1 + docs/readme/README_it.md | 1 + docs/readme/README_ja.md | 1 + docs/readme/README_ko.md | 1 + docs/readme/README_ml.md | 1 + docs/readme/README_pl.md | 1 + docs/readme/README_pt-BR.md | 1 + docs/readme/README_ru.md | 1 + docs/readme/README_th.md | 1 + docs/readme/README_tr.md | 1 + docs/readme/README_uk.md | 1 + docs/readme/README_vi.md | 1 + docs/readme/README_zh-TW.md | 1 + docs/readme/README_zh.md | 1 + src/artifacts/ArtifactPaths.test.ts | 31 + src/artifacts/ArtifactPaths.ts | 40 +- src/artifacts/ArtifactReader.ts | 3 +- src/artifacts/PyInstallerArtifactReader.ts | 608 +++++++++ .../extraction/ArtifactExtraction.test.ts | 54 + .../extraction/ArtifactExtraction.ts | 34 +- .../ArtifactGraphConstruction.test.ts | 6 +- .../inventory/ArtifactGraphConstruction.ts | 48 +- src/artifacts/inventory/classify.ts | 16 +- src/artifacts/inventory/reader.ts | 3 + src/artifacts/inventory/scanReader.ts | 9 +- src/artifacts/pyinstaller/CArchive.fixture.ts | 211 +++ src/artifacts/pyinstaller/CArchiveLayout.ts | 389 ++++++ src/artifacts/pyinstaller/CArchiveMembers.ts | 340 +++++ .../pyinstaller/MarshalSubset.test.ts | 62 + src/artifacts/pyinstaller/MarshalSubset.ts | 282 ++++ .../PyInstallerAdmissionBoundaries.test.ts | 139 ++ .../PyInstallerArtifactReader.test.ts | 1178 ++++++++++++++++ .../pyinstaller/PyInstallerBoundaries.test.ts | 673 +++++++++ src/artifacts/pyinstaller/PyInstallerBytes.ts | 60 + .../pyinstaller/PyInstallerFailure.ts | 10 + .../pyinstaller/PyInstallerFileAdmission.ts | 36 + .../PyInstallerFormatBoundaries.test.ts | 1213 +++++++++++++++++ .../pyinstaller/PyInstallerMemberStream.ts | 147 ++ .../pyinstaller/PyInstallerMetadata.ts | 179 +++ .../pyinstaller/PyInstallerPyzStore.ts | 255 ++++ src/artifacts/pyinstaller/PythonBytecode.ts | 33 + src/artifacts/pyinstaller/PyzArchive.test.ts | 137 ++ src/artifacts/pyinstaller/PyzArchive.ts | 266 ++++ src/domain/analysisSnapshot.ts | 1 + .../analysisView/artifactInventoryView.ts | 55 +- src/domain/artifactFormatMetadata.ts | 61 + src/domain/artifactGraph.ts | 3 +- src/domain/artifactInventoryEvidence.ts | 25 +- src/domain/evidence.ts | 1 + tests/acceptance/pyinstallerArchive.test.ts | 203 +++ .../mcp/pyinstallerArtifactMcp.test.ts | 277 ++++ tests/fixtures/pyinstallerReader.ts | 31 + 63 files changed, 7254 insertions(+), 114 deletions(-) create mode 100644 docs/python-packaged-applications.md create mode 100644 src/artifacts/PyInstallerArtifactReader.ts create mode 100644 src/artifacts/extraction/ArtifactExtraction.test.ts create mode 100644 src/artifacts/pyinstaller/CArchive.fixture.ts create mode 100644 src/artifacts/pyinstaller/CArchiveLayout.ts create mode 100644 src/artifacts/pyinstaller/CArchiveMembers.ts create mode 100644 src/artifacts/pyinstaller/MarshalSubset.test.ts create mode 100644 src/artifacts/pyinstaller/MarshalSubset.ts create mode 100644 src/artifacts/pyinstaller/PyInstallerAdmissionBoundaries.test.ts create mode 100644 src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts create mode 100644 src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts create mode 100644 src/artifacts/pyinstaller/PyInstallerBytes.ts create mode 100644 src/artifacts/pyinstaller/PyInstallerFailure.ts create mode 100644 src/artifacts/pyinstaller/PyInstallerFileAdmission.ts create mode 100644 src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts create mode 100644 src/artifacts/pyinstaller/PyInstallerMemberStream.ts create mode 100644 src/artifacts/pyinstaller/PyInstallerMetadata.ts create mode 100644 src/artifacts/pyinstaller/PyInstallerPyzStore.ts create mode 100644 src/artifacts/pyinstaller/PythonBytecode.ts create mode 100644 src/artifacts/pyinstaller/PyzArchive.test.ts create mode 100644 src/artifacts/pyinstaller/PyzArchive.ts create mode 100644 tests/acceptance/pyinstallerArchive.test.ts create mode 100644 tests/boundary/mcp/pyinstallerArtifactMcp.test.ts create mode 100644 tests/fixtures/pyinstallerReader.ts diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index e283e35b62..ef06652078 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -91,9 +91,15 @@ explicit path or endpoint and do not need it. - ASAR or extracted JavaScript/Electron tree: `analyze_javascript_application`. -- Archive/package member inventory (ZIP/APK/IPA/MSIX/AppX or DMG): - `open_binary` with the supplied local path; use `inspect_artifact` when its - graph and findings help answer the question. +- Archive/package member inventory (ZIP/APK/IPA/MSIX/AppX, DMG, or a PyInstaller + executable): `open_binary` with the supplied local path; use `inspect_artifact` + when its graph and findings help answer the question. A PyInstaller CArchive + is the application layer; the bootloader is only the native prefix. Start with + `detail: "summary"` for large inventories, then use retained inventory views + for selected member fields/pages. A complete small inventory can cost less + than extra calls and schemas. Runtime options are metadata, not files. + Compact delivery still inventories and hashes members. See the + [Python packaged applications guide](https://github.com/morluto/rea/blob/main/docs/python-packaged-applications.md). - Apple app plist, Interface Builder, asset catalog, or keyed archive resources: `open_bundle_resources`, then the selected resource tool. This does not require `CFBundleExecutable` or a deep provider. Resource sessions have no diff --git a/README.md b/README.md index a4da4bb818..ff4777a55b 100644 --- a/README.md +++ b/README.md @@ -174,35 +174,36 @@ CLI commands use the same workflows. REA requires Node.js 22.x (>=22.19), 24.x (>=24.11), or 26+, plus npm. Additional tools and host support depend on the target: -| Target | What REA returns | Requirements and guide | -| ---------------------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| Desktop installations | Runtime candidates, supporting files, component markers and next probes | Built-in bounded directory/header inspection; [desktop applications](docs/desktop-applications.md) | -| Native binaries | Pseudocode, assembly, strings, symbols, calls and references | Hopper, Ghidra or IDA; [native analysis](docs/native-investigation.md) | -| Offline ELF layout | Sections, segments, original symbols/relocations and static mitigation candidates | Caller-supplied pwntools on Linux x64; [binary diagnostics](docs/binary-diagnostics.md) | -| Go binaries | Embedded compiler version, modules, replacements and build settings | Built-in static ELF/PE/thin Mach-O inspection; no Go installation required | -| Model artifacts | GGUF / safetensors metadata, tensor declarations and derived totals | Built-in header inspection; optional full-file hash; [model files](docs/model-artifacts.md) | -| IL2CPP metadata | Unity global-metadata.dat assembly images, type and method names and index ranges | Built-in static decoding of versions 24, 27, 29, 31 and 39; no Unity required | -| Godot PCK | Archive inventory and extraction, original header/directory facts and selected content identities | Built-in static formats 2, 3 and 4; no Godot required | -| Rust binaries | Embedded cargo-auditable crate names, versions, sources and dependency edges | Built-in static ELF/PE/thin Mach-O inspection; [Rust binaries](docs/rust-binaries.md) | -| WASM artifact | Validation, sections, exact import/export WAT forms and decoded WAT | Caller-supplied WABT 1.0.42; `inspect_wasm_artifact` | -| EVM bytecode | Dispatch selectors, byte offsets, inferred arguments and mutability | Local raw/hex carrier on Linux x64; [offline EVM guide](docs/evm-bytecode.md) | -| SQLite databases | Tables, columns, indexes, views, triggers and selected ordinary-table rows | Local database/WAL snapshot; Node.js 24.x >=24.15 or 26+ with native SQLite limits | -| Recorded Linux crashes | Raw notes, every recorded thread's registers/signals and optional mapping candidates | Caller-supplied pwntools on Linux x64; optional GDB/pwndbg; [recorded crashes](docs/recorded-crashes.md) | -| JavaScript / Electron | Modules, imports, source maps, routes, IPC and native add-on relationships | Node.js and npm; Linux: procps-compatible `ps`; [application analysis](docs/javascript-application-workflows.md) | -| Websites | Page structure, scripts, network observations and requested screenshots | A Chrome-family browser; [browser analysis](docs/browser-observation.md) | -| Saved network captures | Requests, responses, exposed payloads and source locations | HAR or Fiddler Everywhere SAZ; mitmdump on Linux for native mitmproxy; [capture guide](docs/web-network-captures.md) | -| .NET assemblies | Metadata, CIL instructions, declared native dependencies and build comparisons | None (static only); [managed-code guide](docs/managed-code-analysis.md) | -| Android APKs | Manifest declarations, classes, decompiled methods and references | Headless JADX and a full JDK on Linux/macOS/Windows x64; [Android guide](docs/android-analysis.md) | -| Android devices | Devices, packages, processes, logs, dumps, screen captures, and APK/file transfers | Caller-supplied adb (emulator or hardware); [ADB guide](docs/adb-device-analysis.md) | -| JEB projects | Engine-backed project units, artifact digests and type/method pseudo-code | A caller-started JEB client serving MCP, selected with REA_JEB_MCP_URL; [JEB guide](docs/jeb-analysis.md) | -| Android resources | Decoded manifest, string tables, locales and version facts | Caller-supplied Apktool launcher; [resource guide](docs/apktool-resource-analysis.md) | -| Flutter apps | Dart snapshot hash, per-ABI library digests and engine build provenance | None (static APK parsing); [Flutter guide](docs/flutter-analysis.md) | -| Firmware | Regions, extraction results and native-analysis handoffs | Binwalk / Unblob on Linux; [firmware guide](docs/firmware-analysis.md) | -| Recorded packets | Selected fields, repeated values, filters and decode-as rules | Linux: TShark 4.x + prlimit; [capture guide](docs/packet-captures.md) | -| Packet summaries | Endpoints, DNS, TLS SNI, protocol distribution and HTTP requests | Caller-supplied tshark and capinfos (Linux-verified); [pcap guide](docs/pcap-analysis.md) | -| Recorded logic signals | Decoder annotations, sample ranges and diagnostics | Linux: sigrok-cli 0.7.2 + prlimit; [signal recordings](docs/signal-recordings.md) | -| Packages and resources | File inventories, digests, plists, Apple bundle anatomy and extracted resources | [Artifact and JavaScript guide](docs/javascript-artifact-reconstruction.md), [Apple applications](docs/apple-application-analysis.md) | -| Process behavior | Terminal output, interactions, exit and filesystem observations, and run comparisons | Linux/macOS with a native PTY; [process capture](docs/process-capture.md) | +| Target | What REA returns | Requirements and guide | +| ----------------------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| Desktop installations | Runtime candidates, supporting files, component markers and next probes | Built-in bounded directory/header inspection; [desktop applications](docs/desktop-applications.md) | +| Native binaries | Pseudocode, assembly, strings, symbols, calls and references | Hopper, Ghidra or IDA; [native analysis](docs/native-investigation.md) | +| Offline ELF layout | Sections, segments, original symbols/relocations and static mitigation candidates | Caller-supplied pwntools on Linux x64; [binary diagnostics](docs/binary-diagnostics.md) | +| Go binaries | Embedded compiler version, modules, replacements and build settings | Built-in static ELF/PE/thin Mach-O inspection; no Go installation required | +| Model artifacts | GGUF / safetensors metadata, tensor declarations and derived totals | Built-in header inspection; optional full-file hash; [model files](docs/model-artifacts.md) | +| IL2CPP metadata | Unity global-metadata.dat assembly images, type and method names and index ranges | Built-in static decoding of versions 24, 27, 29, 31 and 39; no Unity required | +| Godot PCK | Archive inventory and extraction, original header/directory facts and selected content identities | Built-in static formats 2, 3 and 4; no Godot required | +| Rust binaries | Embedded cargo-auditable crate names, versions, sources and dependency edges | Built-in static ELF/PE/thin Mach-O inspection; [Rust binaries](docs/rust-binaries.md) | +| WASM artifact | Validation, sections, exact import/export WAT forms and decoded WAT | Caller-supplied WABT 1.0.42; `inspect_wasm_artifact` | +| EVM bytecode | Dispatch selectors, byte offsets, inferred arguments and mutability | Local raw/hex carrier on Linux x64; [offline EVM guide](docs/evm-bytecode.md) | +| SQLite databases | Tables, columns, indexes, views, triggers and selected ordinary-table rows | Local database/WAL snapshot; Node.js 24.x >=24.15 or 26+ with native SQLite limits | +| Recorded Linux crashes | Raw notes, every recorded thread's registers/signals and optional mapping candidates | Caller-supplied pwntools on Linux x64; optional GDB/pwndbg; [recorded crashes](docs/recorded-crashes.md) | +| JavaScript / Electron | Modules, imports, source maps, routes, IPC and native add-on relationships | Node.js and npm; Linux: procps-compatible `ps`; [application analysis](docs/javascript-application-workflows.md) | +| Websites | Page structure, scripts, network observations and requested screenshots | A Chrome-family browser; [browser analysis](docs/browser-observation.md) | +| Saved network captures | Requests, responses, exposed payloads and source locations | HAR or Fiddler Everywhere SAZ; mitmdump on Linux for native mitmproxy; [capture guide](docs/web-network-captures.md) | +| .NET assemblies | Metadata, CIL instructions, declared native dependencies and build comparisons | None (static only); [managed-code guide](docs/managed-code-analysis.md) | +| Android APKs | Manifest declarations, classes, decompiled methods and references | Headless JADX and a full JDK on Linux/macOS/Windows x64; [Android guide](docs/android-analysis.md) | +| Android devices | Devices, packages, processes, logs, dumps, screen captures, and APK/file transfers | Caller-supplied adb (emulator or hardware); [ADB guide](docs/adb-device-analysis.md) | +| JEB projects | Engine-backed project units, artifact digests and type/method pseudo-code | A caller-started JEB client serving MCP, selected with REA_JEB_MCP_URL; [JEB guide](docs/jeb-analysis.md) | +| Android resources | Decoded manifest, string tables, locales and version facts | Caller-supplied Apktool launcher; [resource guide](docs/apktool-resource-analysis.md) | +| Flutter apps | Dart snapshot hash, per-ABI library digests and engine build provenance | None (static APK parsing); [Flutter guide](docs/flutter-analysis.md) | +| Firmware | Regions, extraction results and native-analysis handoffs | Binwalk / Unblob on Linux; [firmware guide](docs/firmware-analysis.md) | +| Recorded packets | Selected fields, repeated values, filters and decode-as rules | Linux: TShark 4.x + prlimit; [capture guide](docs/packet-captures.md) | +| Packet summaries | Endpoints, DNS, TLS SNI, protocol distribution and HTTP requests | Caller-supplied tshark and capinfos (Linux-verified); [pcap guide](docs/pcap-analysis.md) | +| Recorded logic signals | Decoder annotations, sample ranges and diagnostics | Linux: sigrok-cli 0.7.2 + prlimit; [signal recordings](docs/signal-recordings.md) | +| Packages and resources | File inventories, digests, plists, Apple bundle anatomy and extracted resources | [Artifact and JavaScript guide](docs/javascript-artifact-reconstruction.md), [Apple applications](docs/apple-application-analysis.md) | +| Process behavior | Terminal output, interactions, exit and filesystem observations, and run comparisons | Linux/macOS with a native PTY; [process capture](docs/process-capture.md) | +| PyInstaller executables | CArchive members, Python version, entry scripts and extracted bytecode | None (static only); [Python packaged applications](docs/python-packaged-applications.md) | Static JavaScript and .NET inspection read the supplied files without running the application. Runtime capture runs or interacts with the selected target diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 561b9f2213..86d825c13d 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -50,6 +50,10 @@ export default defineConfig({ { text: "Electron fuses", link: "/electron-fuses" }, { text: "Process capture", link: "/process-capture" }, { text: ".NET assemblies", link: "/managed-code-analysis" }, + { + text: "Python packaged applications", + link: "/python-packaged-applications", + }, { text: "Android applications", link: "/android-analysis" }, { text: "Apple applications", link: "/apple-application-analysis" }, { text: "Desktop installations", link: "/desktop-applications" }, diff --git a/docs/index.md b/docs/index.md index 74c72ef059..070b8aad51 100644 --- a/docs/index.md +++ b/docs/index.md @@ -47,19 +47,19 @@ For runtime instrumentation, see the [Frida guide](./frida-instrumentation.md). ## Choose a guide -| Task | Guides | -| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Identify a desktop installation | [Runtime candidates, evidence and follow-up files](desktop-applications.md) | -| Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) | -| Select a native provider or import format | [Ghidra semantics](ghidra-provider.md), [Windows](windows-ghidra-p0.md), [DOS](ghidra-dos.md), [raw bytes](raw-binary.md), [MIPS](ghidra-mips.md), [PSP](ghidra-psp.md), [NativeAOT](ghidra-nativeaot.md), [IDA](ida-provider.md), [Kuna](kuna-provider.md), [JEB](jeb-analysis.md) | -| Analyze JavaScript or Electron artifacts | [Static analysis](javascript-artifact-reconstruction.md), [tracing and comparisons](javascript-application-workflows.md), [application graph](javascript-application-graph.md), [source recovery](javascript-recovery.md) | -| Compare static JavaScript with runtime observations | [Runtime attribution](javascript-runtime-observation.md), [reconciliation](javascript-runtime-reconciliation.md), [Electron observation](electron-observation.md), [Electron fuses](electron-fuses.md) | -| Inspect assemblies, packages, or firmware | [Godot PCK](godot-pck.md), [.NET](managed-code-analysis.md), [Android](android-analysis.md), [Apktool resources](apktool-resource-analysis.md), [ADB acquisition](adb-device-analysis.md), [Apple](apple-application-analysis.md), [Harmony](harmony-analysis.md), [firmware](firmware-analysis.md), [WASM](wasm-artifacts.md), [EVM](evm-bytecode.md) | -| Observe or compare a browser scenario | [CDP observation](browser-observation.md), [scenarios](browser-scenario-contract.md), [runtime evidence](web-runtime.md), [historical captures](web-network-captures.md) | -| Serve recorded behavior as an agent environment | [Agent environments](agent-environments.md) | -| Trace captured website source | [Source locations](web-source-location.md), [module traces](website-module-trace.md), [script export](website-script-export.md) | -| Capture and compare a command | [Process capture](process-capture.md) | -| Use CLI/MCP and retain evidence | [CLI](cli.md), [MCP contracts](mcp-contracts.md), [agent prompts](mcp-prompts.md), [obligation ledgers](reconstruction-obligation-ledgers.md) | +| Task | Guides | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Identify a desktop installation | [Runtime candidates, evidence and follow-up files](desktop-applications.md) | +| Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) | +| Select a native provider or import format | [Ghidra semantics](ghidra-provider.md), [Windows](windows-ghidra-p0.md), [DOS](ghidra-dos.md), [raw bytes](raw-binary.md), [MIPS](ghidra-mips.md), [PSP](ghidra-psp.md), [NativeAOT](ghidra-nativeaot.md), [IDA](ida-provider.md), [Kuna](kuna-provider.md), [JEB](jeb-analysis.md) | +| Analyze JavaScript or Electron artifacts | [Static analysis](javascript-artifact-reconstruction.md), [tracing and comparisons](javascript-application-workflows.md), [application graph](javascript-application-graph.md), [source recovery](javascript-recovery.md) | +| Compare static JavaScript with runtime observations | [Runtime attribution](javascript-runtime-observation.md), [reconciliation](javascript-runtime-reconciliation.md), [Electron observation](electron-observation.md), [Electron fuses](electron-fuses.md) | +| Inspect assemblies, packages, or firmware | [Godot PCK](godot-pck.md), [.NET](managed-code-analysis.md), [Python packages](python-packaged-applications.md), [Android](android-analysis.md), [Apktool resources](apktool-resource-analysis.md), [ADB acquisition](adb-device-analysis.md), [Apple](apple-application-analysis.md), [Harmony](harmony-analysis.md), [firmware](firmware-analysis.md), [WASM](wasm-artifacts.md), [EVM](evm-bytecode.md) | +| Observe or compare a browser scenario | [CDP observation](browser-observation.md), [scenarios](browser-scenario-contract.md), [runtime evidence](web-runtime.md), [historical captures](web-network-captures.md) | +| Serve recorded behavior as an agent environment | [Agent environments](agent-environments.md) | +| Trace captured website source | [Source locations](web-source-location.md), [module traces](website-module-trace.md), [script export](website-script-export.md) | +| Capture and compare a command | [Process capture](process-capture.md) | +| Use CLI/MCP and retain evidence | [CLI](cli.md), [MCP contracts](mcp-contracts.md), [agent prompts](mcp-prompts.md), [obligation ledgers](reconstruction-obligation-ledgers.md) | For development, use [testing](testing.md), [client verification](client-verification.md), [tool design](tool-design.md), and [releasing](releasing.md). diff --git a/docs/python-packaged-applications.md b/docs/python-packaged-applications.md new file mode 100644 index 0000000000..5785def1fb --- /dev/null +++ b/docs/python-packaged-applications.md @@ -0,0 +1,128 @@ +# Python packaged applications + +REA statically inventories PyInstaller executables with an appended CArchive. +`inspect_artifact` and `extract_artifact` share the same reader through CLI and +MCP. They read archive bytes without launching the application. Bytecode +source decompilation, encrypted PYZ recovery, zipapp and PEX analysis are outside +this capability. + +## Facts and focused queries + +The root occurrence's `format_metadata` records the CArchive layout (2.0 or +2.1), archive and cookie offsets, table extent, entry count, raw Python version, +normalized major/minor when known, library name and encryption marker. Member +metadata retains the original name, table index, type, compression flag, +relative offset and known directory record extent. Logical paths use archive +path rules; the source name retains its spelling, including backslashes. +`relative_offset` is relative to the CArchive start for CArchive rows, or to +the decoded PYZ image for PYZ rows. A CArchive directory record starts at +`archive_start + toc_offset + directory_record_offset`; its size is +`directory_record_size`. PYZ row directory extents remain unknown. + +Namespace packages remain directories. Their `relative_offset` and +`compressed_size` preserve any payload declared by older producers, while +`declared_size` is zero because no bytecode is materialized for a namespace. +Every declared span must end before the PYZ table of contents. +PYZ `compression_flag` is derived from storage: nonempty payloads are compressed, +including legacy namespaces; an empty namespace has no compressed payload. + +CArchive type `o` entries are runtime options. They are metadata occurrences, +with no content artifact or extractable file. Their query paths are +`__pyinstaller__/runtime-options/`; the option text is +`format_metadata.source_name`. Repeated declarations retain distinct occurrence +identities. These paths do not claim output filesystem names. + +Type `d` entries are multipackage dependencies. They are also metadata, under +`__pyinstaller__/dependencies/
`, and never materialize an empty file. +The reader preserves the raw reference and splits its first colon into +`dependency_archive` and `dependency_member`. `dependency_status` remains +`unresolved`, or `invalid-reference` with unknown parts for malformed references. +`external_dependency_count` summarizes these declarations. REA does not open +another executable or claim its contents are available. A real file with the +same logical path as metadata remains independently extractable. +Both declaration types require empty payloads and no compression. Their query +paths do not participate in filesystem collision checks, directory ancestry or +source-map target lookup; their parent identifies the declaring container. +Complete authenticated inventory page sets preserve producer identities in any +merge order, including metadata/file aliases. Malformed TOC diagnostics name the +table index; original names remain available in the supplied artifact. +Path failures use bounded escaped previews and report the original length; +accepted paths and retained source names are not shortened. + +Request `detail: "summary"` for a compact inspection, then query retained +Evidence with `inspect_analysis_view`. An inventory view can select +`collection: "occurrences"`, `entry_kinds: ["metadata"]`, exact logical paths or +occurrence IDs, requested `fields`, and `offset`/`limit`. Filters intersect; +identities, parent Evidence and coverage survive projection. Full inventory +Evidence remains available for export. A compact reply reduces returned context; +it still performs the complete member inventory and hashing work. +Focused views add calls and tool-schema costs. A complete small inventory can +cost less overall; prefer compact delivery when the full member graph would +overwhelm the answer. + +```bash +rea inspect-artifact /absolute/path/hello --detail summary --evidence_output /absolute/new/evidence.json --json +rea extract-artifact /absolute/path/hello --json +``` + +## Decoding and evidence limits + +Inventory hashes members individually through bounded streams. Stored PYZ +images use descriptor ranges; outer-compressed images stream to private files, +which the reader owns through cleanup and any cleanup retry. Extraction binds +the selected carrier to its inventoried digest and checks source identity and +state before and after reading. It does not retain expanded module bodies. +Opaque outer-compressed images reuse that verified private image when opened, +instead of inflating the carrier again. Direct source spans describe stored +CArchive bytes; decompressed content has no direct span even when its encoded +and decoded sizes coincide. Original offsets and compression remain metadata. + +Only the PYZ header and marshal directory are loaded for listing. PYZ tables +have a 16 MiB budget; CArchive tables have a 32 MiB budget. Parsed facts have a +32 MiB serialized budget. Projected entries share a 64 MiB retained budget, +including metadata, bookkeeping and conservative reservations for namespace +objects and possible case-collision annotations at each path component. These +reservations happen before normalization or trie allocation. +Outer-compressed PYZ images have a separate per-reader capacity of 128 private +files, bounding retained inodes and image identities. The complete image count +is admitted before any temporary file is created; excess images return a +capacity failure. This is a resource policy, not a PyInstaller format limit. +Only a zero-byte read establishes EOF; short successful descriptor reads fill +the bounded range. Compressed members must consume their complete declared +input span; trailing junk and concatenated streams are refused. +Plaintext expansion has an aggregate archive-derived budget. Exceeding a budget +returns a resource failure. + +Supported final Python bytecode magic values cover Python 3.7 through 3.14. +Supported marshal directory values are lists, tuples, strings, integers and +bytes. An unsupported magic or directory stays an opaque member with a +limitation and `pyz_status: "opaque"`; its module contents remain unknown. +Any opaque PYZ makes producer traversal coverage unknown, including retained +summaries and exhausted queries. Per-archive reasons remain on the opaque +occurrences; the coverage observation summarizes their count without repeating +every limitation. An empty result does not establish module absence. +Observed `pyz_python_magic` bytes remain available even when their Python version +is unknown. +The directory declares compressed module lengths, so uncompressed declared +sizes and source spans inside decoded data remain unknown. + +PYZ modules receive a derived 16-byte `.pyc` header from their own supported +magic. CArchive bytecode receives that header only when every sibling PYZ magic +and the cookie's Python version agree. Otherwise it retains its original bytes +and `bytecode_header: "unknown"`. `bytecode_magic` and graph derivation edges +identify synthesized headers; member digests cover the resulting derived files. +These headers do not establish successful execution or decompilation. + +The nonempty CArchive Python module `pyimod00_crypto_key` (`m`/`M`) is recorded +as an encryption marker; a same-named data file or script is not. Affected +PYZ blobs are reported without decoding; extraction refuses encrypted entries. +The raw PYZ container remains readable and hashable, including when its module +table is opaque. Its marker stays in metadata and opaque child coverage stays +unknown; extracting the container does not decrypt its payloads. +The marker also applies to nonempty legacy namespace payloads. Empty namespaces +have no payload to encrypt and retain a null encryption source. + +Detection searches the last 16 MiB for a cookie, allowing a trailing macOS code +signature within that bound. Member offsets are relative to the archive start: +cookie position plus cookie size minus package length. Native bootloader +analysis remains available through the native tools. diff --git a/docs/readme/README_ar.md b/docs/readme/README_ar.md index 99e7614769..86d7943131 100644 --- a/docs/readme/README_ar.md +++ b/docs/readme/README_ar.md @@ -152,6 +152,7 @@ npx rea-agents@latest setup | تسجيلات الشبكة المحفوظة | الطلبات، والاستجابات، ومحتويات الحمولة المتاحة، ومواضعها في المصدر | HAR؛ وmitmdump على Linux لتسجيلات mitmproxy بصيغتها الأصلية؛ [دليل تسجيلات الشبكة](../web-network-captures.md) | | التقاط الحزم | نقاط النهاية وDNS وTLS SNI وتوزيع البروتوكولات وطلبات HTTP | tshark وcapinfos يوفّرهما المُستدعي؛ [دليل pcap](../pcap-analysis.md) | | تجميعات .NET | البيانات الوصفية، وتعليمات CIL، واعتماديات الشيفرة الأصلية المعلنة، ومقارنات البناء | فحص ثابت؛ [دليل الشيفرة المُدارة](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | حزم Android APK | تصريحات ملف manifest، والأصناف، والدوال الناتجة عن فك الترجمة، والمراجع | JADX دون واجهة رسومية وJDK كامل على Linux/macOS/Windows x64؛ [دليل Android](../android-analysis.md) | | أجهزة Android | الأجهزة والحزم والعمليات والسجلات والدامبات ولقطات الشاشة ونقل APK/الملفات | adb يوفّره المُستدعي (محاكي أو جهاز)؛ [دليل ADB](../adb-device-analysis.md) | | مشاريع JEB | وحدات المشروع من المحرك وبصمات المصنّفات والكود الزائف للأنواع/الدوال | عميل JEB يبدأه المُستدعي ويقدّم MCP، ويُختار عبر REA_JEB_MCP_URL؛ [دليل JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_de.md b/docs/readme/README_de.md index 42cb9eaec1..13c5a56592 100644 --- a/docs/readme/README_de.md +++ b/docs/readme/README_de.md @@ -152,6 +152,7 @@ REA benötigt Node.js 22.x (>=22.19), 24.x (>=24.11) oder 26+ sowie npm. Zusätz | Gespeicherte Netzwerkaufzeichnungen | Anfragen, Antworten, zugängliche Nutzdaten und Quellpositionen | HAR; mitmdump unter Linux für Aufzeichnungen im nativen mitmproxy-Format; [Leitfaden zu Netzwerkaufzeichnungen](../web-network-captures.md) | | Paketerfassungen | Endpunkte, DNS, TLS SNI, Protokollverteilung und HTTP-Anfragen | Vom Aufrufer bereitgestellte tshark- und capinfos-Binaries; [pcap-Leitfaden](../pcap-analysis.md) | | .NET-Assemblies | Metadaten, CIL-Anweisungen, deklarierte native Abhängigkeiten und Build-Vergleiche | Statische Untersuchung; [Leitfaden für verwalteten Code](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | Android-APKs | Manifest-Deklarationen, Klassen, dekompilierte Methoden und Verweise | JADX ohne grafische Oberfläche und ein vollständiges JDK unter Linux/macOS/Windows x64; [Android-Leitfaden](../android-analysis.md) | | Android-Geräte | Geräte, Pakete, Prozesse, Logs, Dumps, Bildschirmaufnahmen sowie APK- und Dateiübertragungen | Vom Aufrufer bereitgestelltes adb (Emulator oder Gerät); [ADB-Leitfaden](../adb-device-analysis.md) | | JEB-Projekte | Engine-gestützte Projekteinheiten, Artefakt-Prüfsummen und Pseudokode von Typen/Methoden | Ein vom Aufrufer gestarteter JEB-Client mit MCP, ausgewählt über REA_JEB_MCP_URL; [JEB-Leitfaden](../jeb-analysis.md) | diff --git a/docs/readme/README_el.md b/docs/readme/README_el.md index 9912a8fea9..b7e9e19f7b 100644 --- a/docs/readme/README_el.md +++ b/docs/readme/README_el.md @@ -158,6 +158,7 @@ npx rea-agents@latest setup | Ιστότοποι | Δομή σελίδων, scripts, παρατηρήσεις δικτύου και ζητούμενα στιγμιότυπα οθόνης | Πρόγραμμα περιήγησης της οικογένειας Chrome· [ανάλυση μέσω προγράμματος περιήγησης](../browser-observation.md) | | Αποθηκευμένες καταγραφές δικτύου | Αιτήματα, αποκρίσεις, διαθέσιμα δεδομένα φορτίου και θέσεις στην πηγή | HAR ή SAZ του Fiddler Everywhere· mitmdump σε Linux για την εγγενή μορφή mitmproxy· [οδηγός καταγραφών](../web-network-captures.md) | | Assemblies .NET | Μεταδεδομένα, εντολές CIL, δηλωμένες εξαρτήσεις από κώδικα μηχανής και συγκρίσεις builds | Καμία πρόσθετη απαίτηση (μόνο στατική ανάλυση)· [οδηγός διαχειριζόμενου κώδικα](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | APK Android | Δηλώσεις manifest, κλάσεις, απομεταγλωττισμένες μέθοδοι και αναφορές | JADX χωρίς γραφικό περιβάλλον και πλήρες JDK σε Linux/macOS/Windows x64· [οδηγός Android](../android-analysis.md) | | Συσκευές Android | Συσκευές, πακέτα, διεργασίες, logs, dumps, στιγμιότυπα οθόνης και μεταφορές APK/αρχείων | adb που παρέχει ο χρήστης (εξομοιωτής ή φυσική συσκευή)· [οδηγός ADB](../adb-device-analysis.md) | | Έργα JEB | Μονάδες έργου από τη μηχανή ανάλυσης, ψηφιακά αποτυπώματα αρχείων και ψευδοκώδικα τύπων/μεθόδων | Client JEB που έχει ξεκινήσει ο χρήστης και παρέχει MCP, επιλεγμένος μέσω REA_JEB_MCP_URL· [οδηγός JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_es.md b/docs/readme/README_es.md index d7a79ade22..26f197ff62 100644 --- a/docs/readme/README_es.md +++ b/docs/readme/README_es.md @@ -152,6 +152,7 @@ REA requiere Node.js 22.x (>=22.19), 24.x (>=24.11) o 26+, además de npm. Las h | Capturas de red guardadas | Solicitudes, respuestas, contenido accesible de las cargas útiles y ubicaciones de origen | HAR; mitmdump en Linux para capturas en formato nativo de mitmproxy; [guía de capturas](../web-network-captures.md) | | Capturas de paquetes | Endpoints, DNS, TLS SNI, distribución de protocolos y solicitudes HTTP | tshark y capinfos suministrados por el autor de la llamada; [guía pcap](../pcap-analysis.md) | | Ensamblados .NET | Metadatos, instrucciones CIL, dependencias nativas declaradas y comparaciones de compilaciones | Inspección estática; [guía de código administrado](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | APK de Android | Declaraciones del manifiesto, clases, métodos descompilados y referencias | JADX sin interfaz gráfica y un JDK completo en Linux/macOS/Windows x64; [guía de Android](../android-analysis.md) | | Dispositivos Android | Dispositivos, paquetes, procesos, registros, volcados, capturas de pantalla y transferencias de APK/archivos | adb suministrado por el autor de la llamada (emulador o dispositivo); [guía de ADB](../adb-device-analysis.md) | | Proyectos JEB | Unidades de proyecto del motor, resúmenes de artefactos y pseudocódigo de tipos/métodos | Un cliente JEB iniciado por el autor de la llamada que sirve MCP, seleccionado con REA_JEB_MCP_URL; [guía de JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_fa.md b/docs/readme/README_fa.md index 8f88b2e4e1..fa23c15949 100644 --- a/docs/readme/README_fa.md +++ b/docs/readme/README_fa.md @@ -143,6 +143,7 @@ REA به Node.js 22.x (>=22.19)، 24.x (>=24.11) یا 26+، به‌همراه np | وب‌سایت‌ها | ساختار صفحه، اسکریپت‌ها، مشاهدات شبکه و اسکرین‌شات‌های درخواستی | مرورگر مبتنی بر Chrome؛ [تحلیل مرورگر](../browser-observation.md) | | داده‌های ضبط‌شدهٔ شبکه | درخواست‌ها، پاسخ‌ها، payloadهای آشکارشده و مکان آن‌ها در منبع | HAR؛ ابزار mitmdump در لینوکس برای ضبط‌های بومی mitmproxy؛ [راهنمای ضبط شبکه](../web-network-captures.md) | | اسمبلی‌های .NET | فراداده، دستورالعمل‌های CIL، وابستگی‌های بومی اعلام‌شده و مقایسهٔ buildها | بررسی ایستا؛ [راهنمای کد مدیریت‌شده](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | فایل‌های APK اندروید | تعاریف Manifest، کلاس‌ها، متدهای دیکامپایل‌شده و ارجاعات | JADX بدون رابط گرافیکی و JDK کامل در Linux/macOS/Windows x64؛ [راهنمای اندروید](../android-analysis.md) | | دستگاه‌های اندروید | دستگاه‌ها، بسته‌ها، فرایندها، لاگ‌ها، دامپ‌ها، اسکرین‌شات‌ها و انتقال APK/فایل | adb تأمین‌شده توسط فراخوان‌کننده (شبیه‌ساز یا دستگاه)؛ [راهنمای ADB](../adb-device-analysis.md) | | پروژه‌های JEB | واحدهای پروژه از موتور، هش‌های مصنوع و شبه‌کدِ نوع/متد | یک کلاینت JEB که توسط فراخوان‌کننده اجرا شده و MCP ارائه می‌دهد و با REA_JEB_MCP_URL انتخاب می‌شود؛ [راهنمای JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_fr.md b/docs/readme/README_fr.md index c912ffb02a..db2a7051bb 100644 --- a/docs/readme/README_fr.md +++ b/docs/readme/README_fr.md @@ -156,6 +156,7 @@ REA nécessite Node.js 22.x (>=22.19), 24.x (>=24.11) ou 26+, ainsi que npm. Les | Captures réseau enregistrées | Requêtes, réponses, payloads exposés et emplacements dans le code source | HAR ; mitmdump sous Linux pour les captures mitmproxy natives ; [guide des captures](../web-network-captures.md) | | Captures de paquets | Points de terminaison, DNS, TLS SNI, distribution des protocoles et requêtes HTTP | tshark et capinfos fournis par l'appelant ; [guide pcap](../pcap-analysis.md) | | Assemblys .NET | Métadonnées, instructions CIL, dépendances natives déclarées et comparaisons de builds | Inspection statique ; [guide du code managé](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | APK Android | Déclarations du manifeste, classes, méthodes décompilées et références | JADX headless et un JDK complet sous Linux/macOS/Windows x64 ; [guide Android](../android-analysis.md) | | Appareils Android | Appareils, paquets, processus, journaux, dumps, captures d'écran et transferts d'APK/fichiers | adb fourni par l'appelant (émulateur ou appareil) ; [guide ADB](../adb-device-analysis.md) | | Projets JEB | Unités de projet fournies par le moteur, condensés d'artefacts et pseudo-code de types/méthodes | Un client JEB démarré par l'appelant servant MCP, sélectionné via REA_JEB_MCP_URL ; [guide JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_id.md b/docs/readme/README_id.md index d4eafc7881..d824477757 100644 --- a/docs/readme/README_id.md +++ b/docs/readme/README_id.md @@ -159,6 +159,7 @@ Alat tambahan dan dukungan sistem host bergantung pada target yang dianalisis: | Rekaman jaringan tersimpan | Permintaan, respons, payload yang terekspos, dan lokasi sumber | HAR; mitmdump di Linux untuk rekaman native mitmproxy; [panduan rekaman](../web-network-captures.md) | | Capturan paket | Endpoint, DNS, TLS SNI, distribusi protokol, dan permintaan HTTP | tshark dan capinfos buatan pemanggil; [panduan pcap](../pcap-analysis.md) | | Assembly .NET | Metadata, instruksi CIL, dependensi native yang dideklarasikan, dan perbandingan build | Tidak ada (hanya analisis statis); [panduan kode terkelola](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | APK Android | Deklarasi manifest, kelas, metode hasil dekompilasi, dan referensi | JADX mode headless dan JDK lengkap di Linux/macOS/Windows x64; [panduan Android](../android-analysis.md) | | Perangkat Android | Perangkat, paket, proses, log, dump, tangkapan layar, serta transfer APK/berkas | adb buatan pemanggil (emulator atau perangkat); [panduan ADB](../adb-device-analysis.md) | | Proyek JEB | Unit proyek bertenaga mesin, hash artefak, dan pseudokode tipe/metode | Klien JEB yang dijalankan pemanggil dan menyajikan MCP, dipilih dengan REA_JEB_MCP_URL; [panduan JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_it.md b/docs/readme/README_it.md index 5678c813c1..7f951545a2 100644 --- a/docs/readme/README_it.md +++ b/docs/readme/README_it.md @@ -156,6 +156,7 @@ REA richiede Node.js 22.x (>=22.19), 24.x (>=24.11) o 26+, oltre a npm. Strument | Acquisizioni di rete salvate | Richieste, risposte, payload esposti e posizioni sorgente | HAR; mitmdump su Linux per acquisizioni mitmproxy native; [guida alle acquisizioni](../web-network-captures.md) | | Captured di pacchetti | Endpoint, DNS, TLS SNI, distribuzione dei protocolli e richieste HTTP | tshark e capinfos forniti dal chiamante; [guida pcap](../pcap-analysis.md) | | Assembly .NET | Metadati, istruzioni CIL, dipendenze native dichiarate e confronti di build | Nessuno (solo statico); [guida managed-code](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | APK Android | Dichiarazioni manifest, classi, metodi decompilati e riferimenti | JADX headless e un JDK completo su Linux/macOS/Windows x64; [guida Android](../android-analysis.md) | | Dispositivi Android | Dispositivi, pacchetti, processi, log, dump, screenshot e trasferimenti APK/file | adb fornito dal chiamante (emulatore o hardware); [guida ADB](../adb-device-analysis.md) | | Progetti JEB | Unità di progetto dal motore, digest degli artifact e pseudocodice di tipi/metodi | Client JEB avviato dal chiamante via MCP, selezionato con REA_JEB_MCP_URL; [guida JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_ja.md b/docs/readme/README_ja.md index 2022f088f0..f4595b6824 100644 --- a/docs/readme/README_ja.md +++ b/docs/readme/README_ja.md @@ -152,6 +152,7 @@ REA には Node.js 22.x（>=22.19）、24.x（>=24.11）、または 26+ と npm | 保存済みのネットワークキャプチャ | リクエスト、レスポンス、取得可能なペイロード、ソース内の位置 | HAR；mitmproxy ネイティブ形式のキャプチャには Linux 上の mitmdump；[キャプチャガイド](../web-network-captures.md) | | パケットキャプチャ | エンドポイント、DNS、TLS SNI、プロトコル分布、HTTPリクエスト | 呼び出し側が用意した tshark と capinfos；[pcapガイド](../pcap-analysis.md) | | .NET アセンブリ | メタデータ、CIL 命令、宣言されたネイティブ依存関係、ビルドの比較 | 静的な検査；[マネージドコードのガイド](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | Android APK | マニフェストの宣言、クラス、逆コンパイルされたメソッド、参照 | Linux/macOS/Windows x64 上のヘッドレス JADX と完全な JDK；[Android ガイド](../android-analysis.md) | | Android デバイス | デバイス、パッケージ、プロセス、ログ、ダンプ、スクリーンキャプチャ、APK/ファイル転送 | 呼び出し側が用意した adb（エミュレーターまたは実機）；[ADB ガイド](../adb-device-analysis.md) | | JEBプロジェクト | エンジンによるプロジェクトユニット、アーティファクトのダイジェスト、型/メソッドの擬似コード | 呼び出し側が起動し MCP を提供する JEB クライアント（REA_JEB_MCP_URL で選択）；[JEB ガイド](../jeb-analysis.md) | diff --git a/docs/readme/README_ko.md b/docs/readme/README_ko.md index 14ad6cbb96..3fa5f2569a 100644 --- a/docs/readme/README_ko.md +++ b/docs/readme/README_ko.md @@ -173,6 +173,7 @@ REA에는 Node.js 22.x(>=22.19), 24.x(>=24.11) 또는 26+와 npm이 필요합니 | 기록된 논리 신호 | 디코더 주석, 샘플 범위, 진단 정보 | Linux: sigrok-cli 0.7.2 + prlimit; [신호 기록 가이드](../signal-recordings.md) | | 패키지 및 리소스 | 파일 목록, 다이제스트, plist, Apple 번들 구조, 추출한 리소스 | [아티팩트 및 JavaScript 가이드](../javascript-artifact-reconstruction.md), [Apple 애플리케이션](../apple-application-analysis.md) | | 프로세스 동작 | 터미널 출력, 상호작용, 종료 및 파일 시스템 관찰 결과, 실행 비교 | 네이티브 PTY를 지원하는 Linux/macOS; [프로세스 캡처](../process-capture.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | 정적 JavaScript 및 .NET 검사는 제공된 파일을 읽으며 애플리케이션을 실행하지 않습니다. 런타임 캡처는 사용자의 권한으로 선택한 대상을 실행하거나 대상과 상호작용합니다. 각 런타임 가이드에서 그 영향을 설명합니다. diff --git a/docs/readme/README_ml.md b/docs/readme/README_ml.md index 1d715eb2b8..5e07f3663f 100644 --- a/docs/readme/README_ml.md +++ b/docs/readme/README_ml.md @@ -160,6 +160,7 @@ REA-യ്ക്ക് Node.js 22.x (>=22.19), 24.x (>=24.11), അല്ലെ | Websites | Page structure, scripts, network observations, ആവശ്യപ്പെട്ട screenshots | Chrome-family browser; [browser analysis](../browser-observation.md) | | Saved network captures | Requests, responses, പുറത്തുകാണുന്ന payload-ുകൾ, source locations | HAR അല്ലെങ്കിൽ Fiddler Everywhere SAZ; native mitmproxy-നായി Linux-ൽ mitmdump; [capture guide](../web-network-captures.md) | | .NET assemblies | Metadata, CIL instructions, പ്രഖ്യാപിച്ച native dependencies, build comparisons | ഒന്നും ആവശ്യമില്ല (static മാത്രം); [managed-code guide](../managed-code-analysis.md) | +| Python packaged apps | CArchive/PYZ entries, Python version, bytecode, resources and runtime options | Static PyInstaller parsing; [Python guide](../python-packaged-applications.md) | | Android APKs | Manifest declarations, classes, decompiled methods, references | Linux/macOS/Windows x64-ൽ Headless JADX-ഉം full JDK-ഉം; [Android guide](../android-analysis.md) | | Android devices | Devices, packages, processes, logs, dumps, screen captures, APK/file transfers | Caller നൽകുന്ന adb (emulator അല്ലെങ്കിൽ hardware); [ADB guide](../adb-device-analysis.md) | | JEB projects | Engine-backed project units, artifact digests, type/method pseudo-code | MCP സേവനം നൽകുന്ന caller ആരംഭിച്ച JEB client; REA_JEB_MCP_URL ഉപയോഗിച്ച് തിരഞ്ഞെടുക്കുക; [JEB guide](../jeb-analysis.md) | diff --git a/docs/readme/README_pl.md b/docs/readme/README_pl.md index e19413fd87..4c692d2364 100644 --- a/docs/readme/README_pl.md +++ b/docs/readme/README_pl.md @@ -152,6 +152,7 @@ REA wymaga Node.js 22.x (>=22.19), 24.x (>=24.11) lub 26+ oraz npm. Dodatkowe na | Zapisane przechwycenia ruchu sieciowego | Żądania, odpowiedzi, dostępne dane przesyłane i lokalizacje źródłowe | HAR; mitmdump w systemie Linux dla natywnych przechwyceń mitmproxy; [przewodnik przechwytywania](../web-network-captures.md) | | Przechwytywanie pakietów | Punkty końcowe, DNS, TLS SNI, dystrybucja protokołów i żądania HTTP | Dostarczone przez wywołującego tshark i capinfos; [przewodnik pcap](../pcap-analysis.md) | | Zestawy .NET | Metadane, instrukcje CIL, zadeklarowane zależności natywne i porównania kompilacji | Analiza statyczna; [przewodnik kodu zarządzanego](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | Pakiety APK dla Androida | Deklaracje manifestu, klasy, zdekompilowane metody i odwołania | JADX bez interfejsu graficznego i pełny JDK w systemach Linux/macOS/Windows x64; [przewodnik Androida](../android-analysis.md) | | Urządzenia z Androidem | Urządzenia, pakiety, procesy, logi, zrzuty, zrzuty ekranu oraz transfery APK/plików | adb dostarczony przez wywołującego (emulator lub urządzenie); [przewodnik ADB](../adb-device-analysis.md) | | Projekty JEB | Jednostki projektu obsługiwane przez silnik, skróty artefaktów i pseudokod typów/metod | Uruchomiony przez wywołującego klient JEB udostępniający MCP, wybrany przez REA_JEB_MCP_URL; [przewodnik JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_pt-BR.md b/docs/readme/README_pt-BR.md index efbd4e9b41..019bba1129 100644 --- a/docs/readme/README_pt-BR.md +++ b/docs/readme/README_pt-BR.md @@ -152,6 +152,7 @@ O REA requer Node.js 22.x (>=22.19), 24.x (>=24.11) ou 26+, além do npm. As fer | Capturas de rede salvas | Requisições, respostas, payloads acessíveis e localizações de origem | HAR; mitmdump no Linux para capturas no formato nativo do mitmproxy; [guia de capturas](../web-network-captures.md) | | Capturas de pacotes | Endpoints, DNS, TLS SNI, distribuição de protocolos e requisições HTTP | tshark e capinfos fornecidos pelo chamador; [guia pcap](../pcap-analysis.md) | | Assemblies .NET | Metadados, instruções CIL, dependências nativas declaradas e comparações de builds | Inspeção estática; [guia de código gerenciado](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | APKs Android | Declarações do manifesto, classes, métodos descompilados e referências | JADX sem interface gráfica e um JDK completo no Linux/macOS/Windows x64; [guia de Android](../android-analysis.md) | | Dispositivos Android | Dispositivos, pacotes, processos, logs, despejos, capturas de tela e transferências de APK/arquivos | adb fornecido pelo chamador (emulador ou aparelho); [guia de ADB](../adb-device-analysis.md) | | Projetos JEB | Unidades de projeto do motor, hashes de artefatos e pseudocódigo de tipos/métodos | Um cliente JEB iniciado pelo chamador servindo MCP, selecionado com REA_JEB_MCP_URL; [guia de JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_ru.md b/docs/readme/README_ru.md index 12f06df7bf..22ff426b8a 100644 --- a/docs/readme/README_ru.md +++ b/docs/readme/README_ru.md @@ -162,6 +162,7 @@ REA требует Node.js 22.x (>=22.19), 24.x (>=24.11) или 26+, а так | Записанные логические сигналы | Аннотации декодера, диапазоны отсчётов и диагностика | Linux: sigrok-cli 0.7.2 + prlimit; [руководство по сигналам](../signal-recordings.md) | | Пакеты и ресурсы | Списки файлов, хеши, plist, структура пакетов Apple и извлечённые ресурсы | [Руководство по артефактам и JavaScript](../javascript-artifact-reconstruction.md), [приложения Apple](../apple-application-analysis.md) | | Поведение процессов | Вывод терминала, взаимодействия, наблюдения за завершением и файловой системой, сравнения запусков | Linux/macOS с нативным PTY; [захват процессов](../process-capture.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | Статическое исследование JavaScript и .NET читает предоставленные файлы, не запуская приложение. Захват во время выполнения запускает выбранную цель или взаимодействует с ней с правами вашего пользователя; соответствующие руководства описывают последствия. diff --git a/docs/readme/README_th.md b/docs/readme/README_th.md index 304e0d5437..fb8c3ff604 100644 --- a/docs/readme/README_th.md +++ b/docs/readme/README_th.md @@ -152,6 +152,7 @@ REA ต้องใช้ Node.js 22.x (>=22.19), 24.x (>=24.11) หรือ 26 | ข้อมูลเครือข่ายที่บันทึกไว้ | คำขอ คำตอบ เพย์โหลดที่ปรากฏ และตำแหน่งต้นทาง | HAR; mitmdump บน Linux สำหรับข้อมูลที่บันทึกในรูปแบบเนทีฟของ mitmproxy; [คู่มือการบันทึกข้อมูล](../web-network-captures.md) | | การจับแพ็กเก็ต | จุดเชื่อมต่อ, DNS, TLS SNI, การกระจายโปรโตคอล และคำขอ HTTP | tshark และ capinfos ที่ผู้เรียกจัดเตรียม; [คู่มือ pcap](../pcap-analysis.md) | | แอสเซมบลี .NET | เมทาดาทา คำสั่ง CIL การพึ่งพาเนทีฟที่ประกาศไว้ และการเปรียบเทียบผลการบิลด์ | การตรวจสอบแบบสถิต; [คู่มือโค้ดแบบจัดการ](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | APK ของ Android | การประกาศใน manifest คลาส เมธอดที่ดีคอมไพล์ และการอ้างอิง | JADX แบบไม่มี GUI และ JDK แบบเต็มบน Linux/macOS/Windows x64; [คู่มือ Android](../android-analysis.md) | | อุปกรณ์ Android | อุปกรณ์ แพ็กเกจ โพรเซส บันทึก ดัมพ์ ภาพหน้าจอ และการถ่ายโอน APK/ไฟล์ | adb ที่ผู้เรียกจัดเตรียม (โปรแกรมจำลองหรืออุปกรณ์จริง); [คู่มือ ADB](../adb-device-analysis.md) | | โปรเจกต์ JEB | หน่วยโปรเจกต์จากเอนจิน, แฮชของอาร์ติแฟกต์ และซูโดโค้ดของไทป์/เมท็อด | ไคลเอนต์ JEB ที่ผู้เรียกเริ่มและให้บริการ MCP เลือกด้วย REA_JEB_MCP_URL; [คู่มือ JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_tr.md b/docs/readme/README_tr.md index b0ba8f343b..2cb89a5496 100644 --- a/docs/readme/README_tr.md +++ b/docs/readme/README_tr.md @@ -156,6 +156,7 @@ REA; Node.js 22.x (>=22.19), 24.x (>=24.11) veya 26+ ve npm gerektirir. Ek araç | Web siteleri | Sayfa yapısı, betikler, ağ gözlemleri ve istenen ekran görüntüleri | Chrome ailesinden bir tarayıcı; [tarayıcı analizi](../browser-observation.md) | | Kaydedilmiş ağ yakalamaları | İstekler, yanıtlar, erişilebilir yük içerikleri ve kaynak konumları | HAR veya Fiddler Everywhere SAZ; Linux'ta native mitmproxy için mitmdump; [ağ yakalama rehberi](../web-network-captures.md) | | .NET derlemeleri | Meta veriler, CIL talimatları, bildirilmiş yerel bağımlılıklar ve derleme karşılaştırmaları | Statik inceleme; [yönetilen kod rehberi](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | Android APK'leri | Manifest bildirimleri, sınıflar, tersine derlenmiş metotlar ve referanslar | Linux/macOS/Windows x64 üzerinde arayüzsüz JADX ve tam bir JDK; [Android rehberi](../android-analysis.md) | | Android cihazları | Cihazlar, paketler, süreçler, günlükler, dökümler, ekran görüntüleri ve APK/dosya aktarımları | Çağıranın sağladığı adb (emülatör veya cihaz); [ADB rehberi](../adb-device-analysis.md) | | JEB projeleri | Motor destekli proje birimleri, artefakt özetleri ve tür/metot sözdekodu | Çağıranın başlattığı, MCP sunan bir JEB istemcisi, REA_JEB_MCP_URL ile seçilir; [JEB rehberi](../jeb-analysis.md) | diff --git a/docs/readme/README_uk.md b/docs/readme/README_uk.md index 4a33ac96d4..eac65f273e 100644 --- a/docs/readme/README_uk.md +++ b/docs/readme/README_uk.md @@ -152,6 +152,7 @@ REA потребує Node.js 22.x (>=22.19), 24.x (>=24.11) або 26+, а та | Збережені мережеві захоплення | Запити, відповіді, доступні корисні дані та місця в джерелі | HAR; mitmdump у Linux для захоплень у власному форматі mitmproxy; [посібник мережевих захоплень](../web-network-captures.md) | | Захоплення пакетів | Кінцеві точки, DNS, TLS SNI, розподіл протоколів та HTTP-запити | Надані викликаючою стороною tshark та capinfos; [підручник pcap](../pcap-analysis.md) | | Збірки .NET | Метадані, інструкції CIL, оголошені нативні залежності й порівняння збірок | Статична перевірка; [посібник керованого коду](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | Android APK | Оголошення маніфесту, класи, декомпільовані методи й посилання | JADX без графічного інтерфейсу й повний JDK у Linux/macOS/Windows x64; [посібник Android](../android-analysis.md) | | Пристрої Android | Пристрої, пакети, процеси, журнали, дампи, знімки екрана та передавання APK/файлів | adb, наданий викликаючою стороною (емулятор або пристрій); [підручник ADB](../adb-device-analysis.md) | | Проєкти JEB | Єдиниці проєкту від рушія, дайджести артефактів і псевдокод типів/методів | Запущений викликаною стороною клієнт JEB з MCP, обирається через REA_JEB_MCP_URL; [посібник JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_vi.md b/docs/readme/README_vi.md index 19f18c8ca2..cfba9f3c47 100644 --- a/docs/readme/README_vi.md +++ b/docs/readme/README_vi.md @@ -162,6 +162,7 @@ REA yêu cầu Node.js 22.x (>=22.19), 24.x (>=24.11) hoặc 26+, cùng với np | Tín hiệu logic đã ghi | Chú thích bộ giải mã, phạm vi mẫu và chẩn đoán | Linux: sigrok-cli 0.7.2 + prlimit; [hướng dẫn tín hiệu](../signal-recordings.md) | | Gói và tài nguyên | Danh sách tệp, giá trị băm, plist, cấu trúc bundle Apple và tài nguyên được trích xuất | [Hướng dẫn hiện vật phần mềm và JavaScript](../javascript-artifact-reconstruction.md), [ứng dụng Apple](../apple-application-analysis.md) | | Hành vi tiến trình | Đầu ra terminal, tương tác, quan sát khi thoát và hệ thống tệp, cùng so sánh các lần chạy | Linux/macOS có PTY gốc; [ghi lại tiến trình](../process-capture.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | Kiểm tra tĩnh JavaScript và .NET đọc các tệp được cung cấp mà không chạy ứng dụng. Ghi lại khi chạy sẽ khởi chạy hoặc tương tác với mục tiêu đã chọn bằng quyền người dùng của bạn; từng hướng dẫn mô tả tác động của nó. diff --git a/docs/readme/README_zh-TW.md b/docs/readme/README_zh-TW.md index 403cf3d2a4..635bcbc9af 100644 --- a/docs/readme/README_zh-TW.md +++ b/docs/readme/README_zh-TW.md @@ -156,6 +156,7 @@ REA 需要 Node.js 22.x（>=22.19）、24.x（>=24.11）或 26+，以及 npm。 | 已保存的網路擷取 | 請求、回應、可訪問的酬載和來源位置 | HAR；原生 mitmproxy 擷取需要 Linux 上的 mitmdump；[擷取指南](../web-network-captures.md) | | 封包擷取 | 端點、DNS、TLS SNI、通訊協定分佈和 HTTP 請求 | 呼叫端提供的 tshark 和 capinfos；[pcap 指南](../pcap-analysis.md) | | .NET 程式集 | 中繼資料、CIL 指令、聲明的原生依賴和建置對比 | 靜態檢查；[托管程式碼指南](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | Android APK | 清單聲明、類別、反編譯的方法和引用 | Linux/macOS/Windows x64 上的無界面 JADX 和完整 JDK；[Android 指南](../android-analysis.md) | | Android 裝置 | 裝置、套件、程序、日誌、傾印、螢幕擷取以及 APK/檔案傳輸 | 呼叫端提供的 adb（模擬器或實體裝置）；[ADB 指南](../adb-device-analysis.md) | | JEB 專案 | 引擎支援的專案單元、成品摘要和型別/方法的虛擬碼 | 由呼叫端啟動、提供 MCP 的 JEB 用戶端，透過 REA_JEB_MCP_URL 選擇；[JEB 指南](../jeb-analysis.md) | diff --git a/docs/readme/README_zh.md b/docs/readme/README_zh.md index 51275f4c08..313262ae53 100644 --- a/docs/readme/README_zh.md +++ b/docs/readme/README_zh.md @@ -156,6 +156,7 @@ REA 需要 Node.js 22.x（>=22.19）、24.x（>=24.11）或 26+，以及 npm。 | 已保存的网络捕获 | 请求、响应、可访问的载荷和来源位置 | HAR；原生 mitmproxy 捕获需要 Linux 上的 mitmdump；[捕获指南](../web-network-captures.md) | | 数据包捕获 | 端点、DNS、TLS SNI、协议分布和 HTTP 请求 | 调用方提供的 tshark 和 capinfos；[pcap 指南](../pcap-analysis.md) | | .NET 程序集 | 元数据、CIL 指令、声明的原生依赖和构建对比 | 静态检查；[托管代码指南](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | Android APK | 清单声明、类、反编译的方法和引用 | Linux/macOS/Windows x64 上的无界面 JADX 和完整 JDK；[Android 指南](../android-analysis.md) | | Android 设备 | 设备、软件包、进程、日志、转储、屏幕截图以及 APK/文件传输 | 调用方提供的 adb（模拟器或实体设备）；[ADB 指南](../adb-device-analysis.md) | | JEB 项目 | 引擎支持的项目单元、工件摘要和类型/方法的伪代码 | 由调用方启动、提供 MCP 的 JEB 客户端，通过 REA_JEB_MCP_URL 选择；[JEB 指南](../jeb-analysis.md) | diff --git a/src/artifacts/ArtifactPaths.test.ts b/src/artifacts/ArtifactPaths.test.ts index 5957bb825e..b57a37b3f1 100644 --- a/src/artifacts/ArtifactPaths.test.ts +++ b/src/artifacts/ArtifactPaths.test.ts @@ -4,13 +4,44 @@ import { ArtifactPathRegistry, annotateEnglishUnicodeCaseCollisions, ENGLISH_UNICODE_CASE_INVENTORY_LIMITATION, + normalizeArtifactPath, } from "./ArtifactPaths.js"; const NOTE = "a case-insensitive destination cannot store both. This comparison is not a filesystem observation."; +it.each(["drive", "escaped", "normalization", "prefix", "collision"])( + "bounds path failure diagnostics at the owning boundary: %s", + (kind) => { + const name = "x".repeat(1024 * 1024); + const registry = new ArtifactPathRegistry(); + const fail = () => { + if (kind === "drive") normalizeArtifactPath("C:" + name); + else if (kind === "escaped") + normalizeArtifactPath("\0".repeat(name.length)); + else if (kind === "normalization") + normalizeArtifactPath(name + "/../child"); + else { + registry.add(name, "file"); + registry.add(kind === "prefix" ? name + "/child" : name, "directory"); + } + }; + const failure = (() => { + try { + fail(); + } catch (cause: unknown) { + return cause; + } + return undefined; + })(); + expect(failure).toMatchObject({ reason: "path" }); + expect((failure as Error).message.length).toBeLessThan(1024); + }, +); + const annotate = (paths: readonly string[]) => { const occurrences = paths.map((logical_path) => ({ + entry_kind: "file" as const, logical_path, limitations: [] as string[], })); diff --git a/src/artifacts/ArtifactPaths.ts b/src/artifacts/ArtifactPaths.ts index 2bc22f7642..f29c1faa9f 100644 --- a/src/artifacts/ArtifactPaths.ts +++ b/src/artifacts/ArtifactPaths.ts @@ -2,10 +2,25 @@ import { posix } from "node:path"; import { logicalPathNfc } from "../domain/artifactIdentity.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; -import { ArtifactReaderFailure } from "./ArtifactReader.js"; +import { ArtifactReaderFailure, type ArtifactEntry } from "./ArtifactReader.js"; const DRIVE_PREFIX = /^[A-Za-z]:/u; +// Bound diagnostic JSON expansion to 768 characters before formatting. This is +// a presentation budget, never a limit on accepted paths or retained facts. +const PATH_DIAGNOSTIC_CODE_UNITS = 128; +const pathDiagnostic = (path: string): string => + path.length <= PATH_DIAGNOSTIC_CODE_UNITS + ? JSON.stringify(path) + : `${JSON.stringify(path.slice(0, PATH_DIAGNOSTIC_CODE_UNITS))}… (${String(path.length)} UTF-16 code units)`; + +type DestinationEntryKind = Exclude; + +/** Declarations have query identities, but never claim a destination path. */ +export const hasDestinationPath = ( + kind: ArtifactEntry["kind"], +): kind is DestinationEntryKind => kind !== "metadata"; + /** * English Unicode case folding used only to describe logical names a * case-insensitive destination might collapse. It is not a filesystem case table. @@ -25,7 +40,7 @@ export const normalizeArtifactPath = (input: string): string => { if (input.includes("\0") || input.startsWith("/") || DRIVE_PREFIX.test(input)) throw new ArtifactReaderFailure( "path", - `Artifact path is absolute or unsafe: ${JSON.stringify(input)}`, + `Artifact path is absolute or unsafe: ${pathDiagnostic(input)}`, ); const normalized = logicalPathNfc(input).replace(/\/+$/u, ""); const parts = normalized.split("/"); @@ -36,7 +51,7 @@ export const normalizeArtifactPath = (input: string): string => { ) throw new ArtifactReaderFailure( "path", - `Artifact path is not normalized: ${JSON.stringify(input)}`, + `Artifact path is not normalized: ${pathDiagnostic(input)}`, ); return normalized; }; @@ -45,14 +60,15 @@ export const normalizeArtifactPath = (input: string): string => { export class ArtifactPathRegistry { readonly #root: PathTrieNode = { kind: undefined, children: new Map() }; - add(path: string, kind: "file" | "directory" | "symlink" | "slice"): void { + add(path: string, kind: ArtifactEntry["kind"]): void { + if (!hasDestinationPath(kind)) return; const parts = path.split("/"); let node = this.#root; for (const [index, part] of parts.entries()) { if (node.kind !== undefined && node.kind !== "directory") throw new ArtifactReaderFailure( "path", - `Artifact prefix conflict: ${path}`, + `Artifact prefix conflict: ${pathDiagnostic(path)}`, ); let child = node.children.get(part); if (child === undefined) { @@ -64,12 +80,12 @@ export class ArtifactPathRegistry { if (node.kind !== undefined && node.kind !== kind) throw new ArtifactReaderFailure( "path", - `Artifact path collision: ${path}`, + `Artifact path collision: ${pathDiagnostic(path)}`, ); if (kind !== "directory" && node.children.size > 0) throw new ArtifactReaderFailure( "path", - `Artifact prefix conflict: ${path}`, + `Artifact prefix conflict: ${pathDiagnostic(path)}`, ); } } @@ -78,7 +94,7 @@ export class ArtifactPathRegistry { } interface PathTrieNode { - kind: "file" | "directory" | "symlink" | "slice" | undefined; + kind: DestinationEntryKind | undefined; readonly children: Map; } @@ -103,15 +119,19 @@ const englishCaseFold = (part: string): string => */ export const annotateEnglishUnicodeCaseCollisions = ( occurrences: readonly { + readonly entry_kind: ArtifactEntry["kind"]; readonly logical_path: string; limitations: string[]; }[], ): string | undefined => { + const destinations = occurrences.filter(({ entry_kind }) => + hasDestinationPath(entry_kind), + ); const collisions = englishUnicodeCaseCollisions( - occurrences.map(({ logical_path }) => logical_path), + destinations.map(({ logical_path }) => logical_path), ); if (collisions.size === 0) return undefined; - for (const occurrence of occurrences) + for (const occurrence of destinations) for (const collision of collisions.get(occurrence.logical_path) ?? []) occurrence.limitations.push( caseCollisionLimitation(occurrence.logical_path, collision), diff --git a/src/artifacts/ArtifactReader.ts b/src/artifacts/ArtifactReader.ts index 1a2a5a6147..33cb771ea2 100644 --- a/src/artifacts/ArtifactReader.ts +++ b/src/artifacts/ArtifactReader.ts @@ -33,7 +33,7 @@ export interface ArtifactReaderFailureOptions extends ErrorOptions { /** Archive-neutral entry metadata. Reader adapters never choose output paths. */ export interface ArtifactEntry { readonly path: string; - readonly kind: "file" | "directory" | "symlink" | "slice"; + readonly kind: "file" | "directory" | "symlink" | "slice" | "metadata"; readonly declaredSize: number | null; readonly compressedSize: number | null; readonly executable: boolean; @@ -167,6 +167,7 @@ export interface ArtifactReader { | ZipPackageFormat | "asar" | "godot-pck" + | "pyinstaller" | "file"; entries( signal?: AbortSignal, diff --git a/src/artifacts/PyInstallerArtifactReader.ts b/src/artifacts/PyInstallerArtifactReader.ts new file mode 100644 index 0000000000..9f79d5f513 --- /dev/null +++ b/src/artifacts/PyInstallerArtifactReader.ts @@ -0,0 +1,608 @@ +import type { Readable } from "node:stream"; +import { finished } from "node:stream/promises"; +import { setImmediate } from "node:timers/promises"; +import { jsonValueByteLength } from "../domain/jsonValueByteLength.js"; +import type { JsonValue } from "../domain/jsonValue.js"; +import { normalizeArtifactPath } from "./ArtifactPaths.js"; +import { lstat, type FileHandle } from "node:fs/promises"; + +import type { PyInstallerMetadata } from "../domain/artifactFormatMetadata.js"; +import { + cookieMetadata, + scriptPythonMagic, +} from "./pyinstaller/PyInstallerMetadata.js"; +import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { + RegularFileAdmissionFailure, + openRegularFile, + sameRegularFileState, + type StableRegularFileDescriptor, +} from "../filesystem/RegularFile.js"; +import { + ArtifactReaderFailure, + sameArtifactEntry, + type ArtifactEntry, + type ArtifactReader, + type ArtifactReaderTraversal, + type ArtifactTraversalOptions, +} from "./ArtifactReader.js"; +import { abortIfNeeded, hashReadable } from "./ArtifactHash.js"; +import { readArtifactFileChunks } from "./readArtifactFileChunks.js"; +import { ArtifactResourceScope } from "./ArtifactResourceScope.js"; +import { artifactIntegrityDetails } from "../domain/artifactIntegrity.js"; +import { PyInstallerPyzStore } from "./pyinstaller/PyInstallerPyzStore.js"; +import { pyInstallerMemberStream } from "./pyinstaller/PyInstallerMemberStream.js"; +import { + findPyInstallerCookie, + readCArchiveToc, + type CArchiveEntry, + type PyInstallerCookie, +} from "./pyinstaller/CArchiveLayout.js"; +import { + hasEncryptionMarker, + projectPyzContainer, + projectPyzMember, + projectStoredMember, + sealMember, + type PyInstallerMember, + type PyzImage, +} from "./pyinstaller/CArchiveMembers.js"; +import { readPyzArchive } from "./pyinstaller/PyzArchive.js"; +import type { PyInstallerFailure } from "./pyinstaller/PyInstallerFailure.js"; +import { pyInstallerFileAdmissionFailure } from "./pyinstaller/PyInstallerFileAdmission.js"; + +const MAX_PARSED_FACT_BYTES = 32 * 1024 * 1024; +const MAX_RETAINED_ENTRY_BYTES = 64 * 1024 * 1024; +const RECORD_OWNERSHIP_RESERVATION = 256; +// Conservative allowance for registry/case trie nodes, maps and traversal state. +// Repeated prefixes are charged again rather than underestimating retained work. +const PATH_COMPONENT_RESERVATION = 1024; + +interface LoadedArchive { + readonly members: readonly PyInstallerMember[]; + readonly traversal: ArtifactReaderTraversal; +} + +/** + * Read-only inventory of a PyInstaller CArchive appended to a native executable. + * Encrypted PYZ entries are reported and not decoded. Code members are not decompiled. + */ +export class PyInstallerArtifactReader implements ArtifactReader { + readonly format = "pyinstaller" as const; + #source: StableRegularFileDescriptor | undefined; + #opening: Promise | undefined; + #closed = false; + readonly #scope = new ArtifactResourceScope(); + readonly #cleanup = new ArtifactResourceScope(); + readonly #lifecycle = new AbortController(); + #boundSha256: string | undefined; + #loaded: Promise | undefined; + #budget = 0; + #charged = 0; + #parsedFactBytes = 0; + #retainedEntryBytes = 0; + #metadata: PyInstallerMetadata | undefined; + readonly #pyzStore = new PyInstallerPyzStore(); + readonly #members = new Map(); + readonly #openedPyzSizes = new Map(); + + constructor( + private readonly path: string, + source?: StableRegularFileDescriptor, + ) { + this.#source = source; + this.#cleanup.retain({ + kind: "lease", + resource: "PyInstaller private PYZ images", + lease: { release: () => this.#pyzStore.close() }, + }); + } + + formatMetadata(): PyInstallerMetadata | undefined { + return this.#metadata === undefined ? undefined : { ...this.#metadata }; + } + + async prepareContainer( + expectedSha256: string, + signal?: AbortSignal, + ): Promise { + const active = this.#signal(signal); + await this.#scope.run(async () => { + const handle = await this.#openHandle(active); + await this.#verifySource(); + const observed = + this.#boundSha256 ?? + ( + await hashReadable( + readArtifactFileChunks(handle, { signal: active }), + active, + ) + ).sha256; + await this.#verifySource(); + abortIfNeeded(active); + if (observed !== expectedSha256) + throw new ArtifactReaderFailure( + "integrity", + "PyInstaller carrier changed after inventory", + undefined, + artifactIntegrityDetails( + ".", + { algorithm: "sha256", digest: expectedSha256 }, + observed, + false, + ), + ); + this.#boundSha256 = observed; + }); + } + + async *entries( + signal?: AbortSignal, + options?: ArtifactTraversalOptions, + ): AsyncIterable { + const active = this.#signal(signal); + const loaded = await this.#scope.run(async () => { + const value = await this.#load(active); + await this.#verifySource(); + this.#members.clear(); + return value; + }); + options?.onCoverage?.(loaded.traversal); + for (const member of loaded.members) { + abortIfNeeded(active); + const sealed = sealMember(member); + this.#members.set(sealed.entry.adapterKey, sealed); + yield sealed.entry; + } + } + + async open(entry: ArtifactEntry, signal?: AbortSignal): Promise { + const active = this.#signal(signal); + return this.#scope.run(async () => { + const stored = this.#members.get(entry.adapterKey); + if (stored === undefined || !sameArtifactEntry(entry, stored.entry)) + throw new ArtifactReaderFailure( + "integrity", + `PyInstaller entry was not produced by this reader: ${entry.path}`, + ); + if (entry.kind !== "file") + throw new ArtifactReaderFailure( + "format", + `PyInstaller entry is not a regular file: ${entry.path}`, + ); + if (entry.encrypted) + throw new ArtifactReaderFailure( + "unavailable", + `PYZ entry is encrypted and was not decoded: ${entry.path}`, + ); + const read = stored.read; + if (read === undefined) + throw new ArtifactReaderFailure( + "format", + `PyInstaller entry has no readable bytes: ${entry.path}`, + ); + await this.#verifySource(); + const knownSize = + read.kind === "pyz" + ? (read.uncompressedSize ?? + this.#openedPyzSizes.get(entry.adapterKey)) + : undefined; + if (read.kind === "pyz" && knownSize === undefined) + this.#charge(read.header?.length ?? 0); + const parts = await this.#pyzStore.memberSource( + read, + this.#source?.handle, + active, + ); + if (active.aborted) { + parts.source.destroy(); + await finished(parts.source, { cleanup: true }).catch(() => undefined); + abortIfNeeded(active); + } + // Keep one close attempt in this read. Failed owners stay in the store + // for reader.close() to retry, without losing the original failure. + let closeAttempt: Promise | undefined; + const closeSource = (primary?: unknown) => + (closeAttempt ??= parts.close?.(primary)); + const stream = pyInstallerMemberStream(parts.source, { + sourceBytes: + read.kind === "carchive" ? read.source.compressedSize : read.length, + compressed: + read.kind === "pyz" + ? read.compressed + : read.source.compressionFlag === 1, + header: read.header, + expectedBytes: + read.kind === "carchive" ? read.source.uncompressedSize : knownSize, + maxBytes: + read.kind === "carchive" + ? read.source.uncompressedSize + : (knownSize ?? this.#budget - this.#charged), + signal: active, + ...(read.kind === "pyz" && knownSize === undefined + ? { charge: (bytes: number) => this.#charge(bytes) } + : {}), + complete: async (bytes) => { + try { + await parts.verify?.(); + await this.#verifySource(); + abortIfNeeded(active); + } catch (cause: unknown) { + await closeSource(cause); + throw cause; + } + await closeSource(); + abortIfNeeded(active); + if (read.kind === "pyz" && knownSize === undefined) + this.#openedPyzSizes.set(entry.adapterKey, bytes); + }, + }); + void this.#scope + .run(async () => { + let primary: unknown; + try { + await stream.completion; + } catch (cause: unknown) { + primary = cause; + } + await closeSource(primary); + if (primary !== undefined) throw primary; + }) + .catch((cause: unknown) => + stream.output.destroy( + cause instanceof Error + ? cause + : new Error("PyInstaller stream failed", { cause }), + ), + ); + return stream.output; + }); + } + + provenance(): readonly [] { + return []; + } + + async close(): Promise { + this.#closed = true; + const closing = new ArtifactReaderFailure( + "unavailable", + "PyInstaller reader closed", + ); + this.#lifecycle.abort(closing); + await this.#scope.close(); + await this.#cleanup.close(); + this.#source = undefined; + this.#closed = true; + this.#loaded = undefined; + this.#members.clear(); + this.#openedPyzSizes.clear(); + } + + #signal(signal?: AbortSignal): AbortSignal { + abortIfNeeded(signal); + if (this.#closed) + throw new ArtifactReaderFailure( + "unavailable", + "PyInstaller archive is closed", + ); + return signal === undefined + ? this.#lifecycle.signal + : AbortSignal.any([signal, this.#lifecycle.signal]); + } + + async #load(signal?: AbortSignal): Promise { + this.#loaded ??= this.#parse(signal); + return this.#loaded; + } + + async #parse(signal?: AbortSignal): Promise { + const handle = await this.#openHandle(signal); + await this.#verifySource(); + const size = (await handle.stat()).size; + const cookie = await this.#cookie(handle, size, signal); + this.#budget = artifactExpansionBudgetBytes(cookie.packageLength); + const toc = await this.#toc(handle, cookie, signal); + for (const entry of toc) { + this.#charge(entry.uncompressedSize); + this.#reserveMetadata({ ...entry }, false); + } + const images = await this.#pyzImages(handle, cookie, toc, signal); + const magic = scriptPythonMagic( + cookie, + [...images.values()].map((image) => image.archive), + ); + const encrypted = hasEncryptionMarker(toc); + const members = await this.#project( + toc, + cookie, + images, + magic, + encrypted, + signal, + ); + for (const member of members) + if (member.read?.kind === "carchive") + this.#charge(member.read.header?.length ?? 0); + await this.#verifySource(); + this.#metadata = cookieMetadata(cookie, toc, encrypted); + let opaqueImages = 0; + for (const image of images.values()) + if (image.archive.kind === "opaque") opaqueImages++; + return { + members, + traversal: { + status: opaqueImages === 0 ? "complete" : "unknown", + limitations: + opaqueImages === 0 + ? [] + : [ + `${String(opaqueImages)} PYZ archives remain opaque; module traversal is unknown. See opaque PYZ occurrences for source magic and per-archive limitations.`, + ], + }, + }; + } + + async #openHandle(signal?: AbortSignal): Promise { + abortIfNeeded(signal); + if (this.#closed) + throw new ArtifactReaderFailure( + "unavailable", + "PyInstaller archive is closed", + ); + if (this.#source !== undefined) return this.#source.handle; + // Admission belongs to the reader. A departing caller cannot cancel a peer + // or replace a late descriptor owner; every caller joins the same admission. + this.#opening ??= this.#admit(); + let handle: FileHandle; + try { + handle = await this.#opening; + } catch (cause: unknown) { + if ( + cause instanceof ArtifactReaderFailure && + cause.reason === "cancelled" && + signal?.aborted + ) + throw new ArtifactReaderFailure("cancelled", cause.message, { + cause: signal.reason, + cleanup: cause.cleanup, + cleanupLeases: cause.cleanupLeases, + }); + throw cause; + } + abortIfNeeded(signal); + return handle; + } + + async #admit(): Promise { + try { + const handle = await openRegularFile(this.path, { + symlinks: "reject", + signal: this.#lifecycle.signal, + }); + this.#retainCarrier(new OwnedFileHandle(handle)); + this.#source = { handle, initial: await handle.stat() }; + return handle; + } catch (cause: unknown) { + if (cause instanceof RegularFileAdmissionFailure) + this.#retainCarrier(cause.owner); + throw pyInstallerFileAdmissionFailure(this.path, cause); + } + } + + #retainCarrier(handle: OwnedFileHandle): void { + this.#cleanup.retain({ kind: "file-handle", resource: this.path, handle }); + } + + async #cookie( + handle: FileHandle, + size: number, + signal?: AbortSignal, + ): Promise { + const found = await findPyInstallerCookie(handle, size, signal); + if (found === undefined) + throw new ArtifactReaderFailure( + "format", + "PyInstaller CArchive cookie was not found", + ); + if (!found.ok) throw failure(found.error); + return found.value; + } + + async #toc( + handle: FileHandle, + cookie: PyInstallerCookie, + signal?: AbortSignal, + ): Promise { + const toc = await readCArchiveToc(handle, cookie, signal); + if (!toc.ok) throw failure(toc.error); + return toc.value; + } + + async #pyzImages( + handle: FileHandle, + cookie: PyInstallerCookie, + toc: readonly CArchiveEntry[], + signal: AbortSignal | undefined, + ): Promise> { + const images = new Map(); + // Admit the complete filesystem footprint before the first private write. + this.#pyzStore.reserveImages( + toc.reduce( + (count, entry) => + count + + (entry.typeCode === "z" && entry.compressionFlag === 1 ? 1 : 0), + 0, + ), + ); + for (const entry of toc) { + if (entry.typeCode !== "z") continue; + abortIfNeeded(signal); + const spooled = + entry.compressionFlag === 0 + ? undefined + : await this.#pyzStore.spool( + handle, + cookie.archiveStart, + entry, + signal, + ); + const source = spooled ?? { + handle, + start: cookie.archiveStart + entry.offset, + }; + const parsed = await readPyzArchive( + source.handle, + source.start, + entry.uncompressedSize, + signal, + ); + if (!parsed.ok) throw failure(parsed.error); + if (parsed.value.kind === "entries") + for (const item of parsed.value.entries) + this.#reserveMetadata({ ...item, container: entry.name }, false); + if (spooled !== undefined) await this.#pyzStore.release(spooled.handle); + images.set(entry, { + archive: parsed.value, + handle: source.handle, + start: source.start, + }); + } + return images; + } + + async #project( + toc: readonly CArchiveEntry[], + cookie: PyInstallerCookie, + images: ReadonlyMap, + magic: Buffer | undefined, + encrypted: boolean, + signal: AbortSignal | undefined, + ): Promise { + const members: PyInstallerMember[] = []; + const append = (member: PyInstallerMember): void => { + this.#reservePathComponents(member.entry.path); + const entry = { + ...member.entry, + path: normalizeArtifactPath(member.entry.path), + }; + this.#reserveMetadata( + { + path: entry.path, + adapter_key: entry.adapterKey, + format_metadata: entry.formatMetadata ?? null, + limitations: [...entry.limitations], + }, + true, + ); + members.push({ ...member, entry }); + }; + for (const entry of toc) { + if (members.length % 256 === 0) await setImmediate(); + abortIfNeeded(signal); + const image = entry.typeCode === "z" ? images.get(entry) : undefined; + if (image === undefined) { + append(projectStoredMember(entry, cookie.archiveStart, magic)); + continue; + } + append(projectPyzContainer(entry, cookie.archiveStart, image, encrypted)); + if (image.archive.kind !== "entries") continue; + for (const item of image.archive.entries) { + if (members.length % 256 === 0) await setImmediate(); + abortIfNeeded(signal); + append( + projectPyzMember( + entry, + item, + encrypted, + image.archive.pythonMagic, + item.namespace || encrypted + ? undefined + : { + handle: image.handle, + offset: image.start + item.offset, + length: item.length, + }, + ), + ); + } + } + return members; + } + + async #verifySource(): Promise { + const source = this.#source; + if (source === undefined) + throw new ArtifactReaderFailure( + "unavailable", + "PyInstaller archive is closed", + ); + const [opened, currentPath] = await Promise.all([ + source.handle.stat(), + lstat(this.path), + ]); + if ( + !sameRegularFileState(source.initial, opened) || + !sameRegularFileState(source.initial, currentPath) + ) + throw new ArtifactReaderFailure( + "integrity", + `PyInstaller source changed during reading: ${this.path}`, + ); + } + + #reservePathComponents(path: string): void { + const remaining = MAX_RETAINED_ENTRY_BYTES - this.#retainedEntryBytes; + // Case annotations can carry the path and two prefixes at every component. + // Reserve escaped text growth as well as registry/case trie nodes and maps. + const componentCost = + PATH_COMPONENT_RESERVATION + 6 * jsonValueByteLength(path); + let reservation = componentCost; + for ( + let index = 0; + index < path.length && reservation <= remaining; + index++ + ) + if (path.charCodeAt(index) === 47) reservation += componentCost; + if (reservation > remaining) + throw new ArtifactReaderFailure( + "limit", + "PyInstaller paths exceed their retained namespace budget", + { boundary: "memory" }, + ); + this.#retainedEntryBytes += reservation; + } + + #reserveMetadata(value: JsonValue, projected: boolean): void { + const used = projected ? this.#retainedEntryBytes : this.#parsedFactBytes; + const limit = projected ? MAX_RETAINED_ENTRY_BYTES : MAX_PARSED_FACT_BYTES; + const remaining = limit - used - RECORD_OWNERSHIP_RESERVATION; + const bytes = remaining < 0 ? 0 : jsonValueByteLength(value, remaining); + if (remaining < 0 || bytes > remaining) + throw new ArtifactReaderFailure( + "limit", + "PyInstaller directory facts exceed their retained metadata budget", + { boundary: "memory" }, + ); + if (projected) + this.#retainedEntryBytes += bytes + RECORD_OWNERSHIP_RESERVATION; + else this.#parsedFactBytes += bytes + RECORD_OWNERSHIP_RESERVATION; + } + + #charge(bytes: number): void { + if ( + !Number.isSafeInteger(bytes) || + bytes < 0 || + this.#charged + bytes > this.#budget + ) + throw new ArtifactReaderFailure( + "limit", + `PyInstaller members declare ${String(this.#charged + Math.max(0, bytes))} uncompressed bytes, exceeding the decompression budget`, + ); + this.#charged += bytes; + } +} + +const failure = (error: PyInstallerFailure): ArtifactReaderFailure => + new ArtifactReaderFailure(error.reason, error.message); diff --git a/src/artifacts/extraction/ArtifactExtraction.test.ts b/src/artifacts/extraction/ArtifactExtraction.test.ts new file mode 100644 index 0000000000..d22855d6e6 --- /dev/null +++ b/src/artifacts/extraction/ArtifactExtraction.test.ts @@ -0,0 +1,54 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it, vi } from "vitest"; + +import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { ArtifactResourceScope } from "../ArtifactResourceScope.js"; +import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { extractArtifact } from "./ArtifactExtraction.js"; + +vi.mock("../pyinstaller/CArchiveLayout.js", async (importOriginal) => ({ + ...(await importOriginal()), + containsPyInstallerArchive: () => + Promise.reject(new ArtifactReaderFailure("format", "detection failed")), +})); + +it("keeps the detection error primary and reports a failed detection close", async () => { + const directory = await createTestTempDirectory("rea-extraction-detect-"); + const path = join(directory, "target.elf"); + await writeFile(path, Buffer.from("not an archive")); + const resourceScope = new ArtifactResourceScope(); + const originalClose = OwnedFileHandle.prototype.close; + const closeFailure = new Error("detection close failed"); + let attempts = 0; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + attempts += 1; + if (attempts === 1) throw closeFailure; + await originalClose.call(this); + }); + try { + await expect( + extractArtifact({ + inputPath: path, + inputFormat: "elf", + outputRoot: join(directory, "out"), + environment: process.env, + integrityPolicy: "fail", + resourceScope, + }), + ).rejects.toMatchObject({ + reason: "format", + message: "detection failed", + cleanup: { reason: closeFailure.message }, + }); + await resourceScope.close(); + expect(attempts).toBe(2); + } finally { + close.mockRestore(); + await resourceScope.close().catch(() => undefined); + } +}); diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index 8a3a4cb0bc..2b5b591f1b 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -23,6 +23,8 @@ import { SafeOutputTreeCreationFailure } from "../SafeOutputTreeCreationFailure. import { ZipArtifactReader } from "../ZipArtifactReader.js"; import { isZipPackageFormat } from "../../domain/zipPackageFormat.js"; import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js"; +import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js"; +import { containsPyInstallerArchive } from "../pyinstaller/CArchiveLayout.js"; import { artifactExtractionResultSchema, type ArtifactExtractionResult, @@ -244,6 +246,7 @@ const materializeSelection = async ({ sourcePath, input, inventory.manifest.root_format, + signal, ); const readerOwner: ArtifactResourceOwner = { kind: "reader" as const, @@ -314,14 +317,15 @@ const writeSelectedEntries = async ({ for await (const entry of reader.entries(signal)) { const path = normalizeArtifactPath(entry.path); registry.add(path, entry.kind); + // Logical declarations may share a query path with a real file. They do + // not consume that file's selection or participate in materialization. + if (entry.kind !== "file" && entry.kind !== "slice") continue; const selectedItem = byPath.get(path); if (selectedItem === undefined) { - if (entry.kind === "file" || entry.kind === "slice") - throw new ArtifactReaderFailure( - "integrity", - `Regular artifact entry is missing from inventory: ${path}`, - ); - continue; + throw new ArtifactReaderFailure( + "integrity", + `Regular artifact entry is missing from inventory: ${path}`, + ); } preflight(entry); const stream = await reader.open(entry, signal); @@ -444,10 +448,17 @@ const requireExtractableFormat = async ( path, input.resourceScope, signal, - async (handle) => - locateGodotPck(handle, (await handle.stat()).size, signal), + async (handle) => { + const size = (await handle.stat()).size; + if (await locateGodotPck(handle, size, signal)) return true; + if (format !== "elf" && format !== "pe") return false; + abortIfNeeded(signal); + const packaged = await containsPyInstallerArchive(handle, size, signal); + abortIfNeeded(signal); + return packaged; + }, ); - if (pack === undefined) + if (!pack) throw new AnalysisUnsupportedTargetError( "extract_artifact", input.inputPath, @@ -464,10 +475,13 @@ const createReader = async ( "inputPath" | "inputFormat" | "environment" | "resourceScope" >, containerFormat: ArtifactGraphManifest["root_format"], + signal?: AbortSignal, ): Promise => { const format = input.inputFormat; if (containerFormat === "godot-pck") return new GodotPckArtifactReader(path); - if (await requireExtractableFormat(path, input)) + if (containerFormat === "pyinstaller") + return new PyInstallerArtifactReader(path); + if (await requireExtractableFormat(path, input, signal)) return new DirectoryArtifactReader(path); if (format === "asar") return new AsarArtifactReader(path); if (isZipPackageFormat(format)) return new ZipArtifactReader(path, format); diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts index 7432e7751c..1c5fd3edc8 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts @@ -10,7 +10,7 @@ import { createArtifactEdges, classifyArtifactContent, createOccurrence, - indexOccurrencesByPath, + indexDestinationOccurrencesByPath, resolveOccurrenceParents, } from "./ArtifactGraphConstruction.js"; import { createArtifactClassificationProbe } from "../ArtifactClassificationProbe.js"; @@ -42,7 +42,7 @@ describe("occurrence parent ownership", () => { resolveOccurrenceParents( occurrences, - indexOccurrencesByPath(occurrences), + indexDestinationOccurrencesByPath(occurrences), new Set(), ); @@ -57,7 +57,7 @@ describe("occurrence parent ownership", () => { const resolution = resolveOccurrenceParents( occurrences, - indexOccurrencesByPath(occurrences), + indexDestinationOccurrencesByPath(occurrences), new Set(), ); diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.ts b/src/artifacts/inventory/ArtifactGraphConstruction.ts index ad828bf97d..3cf2c9caee 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.ts @@ -1,6 +1,7 @@ import { hasGodotPckHeader } from "../../domain/godotPckFormat.js"; import type { ArtifactClassificationEvidence } from "../ArtifactClassificationProbe.js"; import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { hasDestinationPath } from "../ArtifactPaths.js"; import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; import { isJavaScriptSourcePath } from "../../domain/javascript/javascriptArtifactFiles.js"; import { digestCanonicalValue } from "../../domain/canonicalDigest.js"; @@ -56,12 +57,13 @@ export interface MutableOccurrence { limitations: string[]; } -/** Index every occurrence by canonical logical path without choosing an owner. */ -export const indexOccurrencesByPath = ( +/** Index destination paths without mixing in declaration query names or choosing an owner. */ +export const indexDestinationOccurrencesByPath = ( occurrences: readonly MutableOccurrence[], ): ReadonlyMap => { const byPath = new Map(); for (const occurrence of occurrences) { + if (!hasDestinationPath(occurrence.entry_kind)) continue; const matches = byPath.get(occurrence.logical_path); if (matches === undefined) byPath.set(occurrence.logical_path, [occurrence]); @@ -82,16 +84,18 @@ export const createOccurrence = ( duplicateOrdinal = 0, ): MutableOccurrence => { const role = - entry.kind === "directory" - ? { - kind: path.toLowerCase().endsWith(".framework") - ? ("framework" as const) - : ("container" as const), - format: "directory" as const, - } - : containerSuffixFamily(path) === undefined - ? classifyArtifactPath(path) - : ({ kind: "unknown", format: "unknown" } as const); + entry.kind === "metadata" + ? { kind: "package-metadata" as const, format: "unknown" as const } + : entry.kind === "directory" + ? { + kind: path.toLowerCase().endsWith(".framework") + ? ("framework" as const) + : ("container" as const), + format: "directory" as const, + } + : containerSuffixFamily(path) === undefined + ? classifyArtifactPath(path) + : ({ kind: "unknown", format: "unknown" } as const); return { occurrence_id: `occ_${digestCanonicalValue( duplicateOrdinal === 0 @@ -263,7 +267,7 @@ export const rootOccurrenceFor = ( artifact_kind: metadata.format === "directory" || isZipPackageFormat(metadata.format) || - ["asar", "godot-pck", "dmg", "pkg"].includes(metadata.format) + ["asar", "godot-pck", "dmg", "pkg", "pyinstaller"].includes(metadata.format) ? "container" : artifactRoleForFormat(metadata.path, metadata.format).kind, artifact_format: metadata.format, @@ -316,7 +320,7 @@ export const createArtifactEdges = ( producer?: ArtifactCommand, ): { readonly edges: ArtifactEdge[]; readonly limitations: string[] } => { const byId = new Map(occurrences.map((item) => [item.occurrence_id, item])); - const byPath = indexOccurrencesByPath(occurrences); + const byPath = indexDestinationOccurrencesByPath(occurrences); const edges: ArtifactEdge[] = []; const limitations: string[] = []; for (const occurrence of occurrences) { @@ -364,11 +368,14 @@ export const createArtifactEdges = ( relation: occurrence.entry_kind === "slice" ? ("slice-of" as const) - : isSourceMap - ? mappedSource === undefined - ? ("contains" as const) - : ("maps-source" as const) - : relationFor(occurrence.logical_path), + : occurrence.format_metadata?.kind === "pyinstaller-entry" && + occurrence.format_metadata.bytecode_header === "derived" + ? ("derived-from" as const) + : isSourceMap + ? mappedSource === undefined + ? ("contains" as const) + : ("maps-source" as const) + : relationFor(occurrence.logical_path), occurrence_id: occurrence.occurrence_id, }; edges.push({ @@ -428,6 +435,9 @@ export const resolveOccurrenceParents = ( const limitations = new Set(); const unknownParentOccurrences = new Set(); for (const occurrence of occurrences) { + // Query paths do not imply filesystem containment. The scanner records + // the declaring container for metadata instead. + if (occurrence.entry_kind === "metadata") continue; const path = occurrence.logical_path; let branch = root; let parent: MutableOccurrence | undefined; diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.ts index 06e75b3899..c36d6e43df 100644 --- a/src/artifacts/inventory/classify.ts +++ b/src/artifacts/inventory/classify.ts @@ -5,6 +5,7 @@ import type { Stats } from "node:fs"; import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js"; import type { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; +import { containsPyInstallerArchive } from "../pyinstaller/CArchiveLayout.js"; import { classifyArtifactContent } from "./ArtifactGraphConstruction.js"; import { abortIfNeeded } from "../ArtifactHash.js"; import { createArtifactClassificationProbe } from "../ArtifactClassificationProbe.js"; @@ -114,7 +115,8 @@ export const classifyAndHashRootForInventory = async ( const retainSource = isZipPackageFormat(format) || format === "mach-o-universal" || - format === "godot-pck"; + format === "godot-pck" || + format === "pyinstaller"; outcome = { kind: "completed", value: { @@ -178,7 +180,7 @@ const classifyRootFormat = async ( evidence.fatSliceHeaders, signal, ); - return classifyArtifactContent( + const format = classifyArtifactContent( path, { ...evidence, @@ -186,6 +188,16 @@ const classifyRootFormat = async ( }, archiveBytes, ).format; + if (["elf", "pe", "mach-o", "mach-o-universal"].includes(format)) { + const packaged = await containsPyInstallerArchive( + handle, + archiveBytes, + signal, + ); + abortIfNeeded(signal); + if (packaged) return "pyinstaller"; + } + return format; }; /** Read exact FAT header windows missed by the bounded prefix probe. */ diff --git a/src/artifacts/inventory/reader.ts b/src/artifacts/inventory/reader.ts index 03b91077f6..8b9956a89e 100644 --- a/src/artifacts/inventory/reader.ts +++ b/src/artifacts/inventory/reader.ts @@ -7,6 +7,7 @@ import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js"; import { GodotPckArtifactReader } from "../GodotPckArtifactReader.js"; import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js"; import { NativeDmgArtifactReader } from "../NativeDmgArtifactReader.js"; +import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js"; import { ZipArtifactReader } from "../ZipArtifactReader.js"; import type { ArtifactInventoryCoverage, @@ -31,6 +32,8 @@ export const createReader = ( return new DirectoryArtifactReader(path); case "asar": return new AsarArtifactReader(path); + case "pyinstaller": + return new PyInstallerArtifactReader(path, rootSource); case "godot-pck": return new GodotPckArtifactReader(path, rootSource); case "mach-o-universal": diff --git a/src/artifacts/inventory/scanReader.ts b/src/artifacts/inventory/scanReader.ts index 616eafc0a0..9d0be9d0b8 100644 --- a/src/artifacts/inventory/scanReader.ts +++ b/src/artifacts/inventory/scanReader.ts @@ -26,7 +26,7 @@ import { createArtifactNode, createOccurrence, noteUnreadContainerSuffix, - indexOccurrencesByPath, + indexDestinationOccurrencesByPath, resolveOccurrenceParents, type MutableOccurrence, } from "./ArtifactGraphConstruction.js"; @@ -107,7 +107,7 @@ export const scanReader = async ( await visitArtifactEntries(context, reader, ""); // Archive directories may appear after their children. Resolve containment // against the complete index before directory identities are materialized. - const byPath = indexOccurrencesByPath(occurrences); + const byPath = indexDestinationOccurrencesByPath(occurrences); const parentResolution = resolveOccurrenceParents( occurrences, byPath, @@ -167,11 +167,13 @@ const visitArtifactEntries = async ( const stack: Array<{ readonly reader: ArtifactReader; readonly prefix: string; + readonly containerId: string | null; readonly iterator: AsyncIterator; }> = [ { reader: currentReader, prefix, + containerId: null, iterator: inventoryEntries(context, currentReader, prefix), }, ]; @@ -202,7 +204,7 @@ const visitArtifactEntries = async ( const occurrence = createOccurrence( entry, logicalPath, - null, + entry.kind === "metadata" ? frame.containerId : null, duplicateOrdinal, ); let digested: @@ -254,6 +256,7 @@ const visitArtifactEntries = async ( stack.push({ reader: nested, prefix: logicalPath, + containerId: occurrence.occurrence_id, iterator: inventoryEntries(context, nested, logicalPath), }); } diff --git a/src/artifacts/pyinstaller/CArchive.fixture.ts b/src/artifacts/pyinstaller/CArchive.fixture.ts new file mode 100644 index 0000000000..de4691ac18 --- /dev/null +++ b/src/artifacts/pyinstaller/CArchive.fixture.ts @@ -0,0 +1,211 @@ +import { deflateSync } from "node:zlib"; + +import { PYINSTALLER_COOKIE_MAGIC } from "./CArchiveLayout.js"; + +/** Final CPython 3.9 bytecode magic, little-endian with the CR LF trailer. */ +export const PYTHON_39_MAGIC = Buffer.from("610d0d0a", "hex"); + +const ELF_PREFIX = (() => { + const bytes = Buffer.alloc(64); + bytes.writeUInt32LE(0x464c457f, 0); + bytes[4] = 2; + bytes[5] = 1; + bytes[6] = 1; + bytes.writeUInt16LE(2, 16); + bytes.writeUInt16LE(62, 18); + bytes.writeUInt32LE(1, 20); + bytes.writeUInt16LE(64, 52); + return bytes; +})(); + +/** One CArchive member supplied as uncompressed bytes. */ +export interface CArchiveFixtureMember { + readonly name: string; + readonly typeCode: string; + readonly payload: Buffer; + readonly compress: boolean; + readonly compressedSuffix?: Buffer; + readonly declaredUncompressedSize?: number; +} + +/** Build a tiny ELF with an appended CArchive, optionally followed by a trailer. */ +export const cArchiveExecutable = (input: { + readonly layout: "2.0" | "2.1"; + readonly pythonVersion: number; + readonly pythonLibrary?: string; + readonly members: readonly CArchiveFixtureMember[]; + readonly trailer?: Buffer; + readonly tocBytes?: Buffer; +}): Buffer => { + const payloads: Buffer[] = []; + const entries: Buffer[] = []; + let dataLength = 0; + for (const member of input.members) { + const stored = member.compress + ? Buffer.concat([ + deflateSync(member.payload), + member.compressedSuffix ?? Buffer.alloc(0), + ]) + : member.payload; + entries.push( + tocEntry({ + name: member.name, + typeCode: member.typeCode, + offset: dataLength, + compressedSize: stored.length, + uncompressedSize: + member.declaredUncompressedSize ?? member.payload.length, + flag: member.compress ? 1 : 0, + }), + ); + payloads.push(stored); + dataLength += stored.length; + } + const toc = input.tocBytes ?? Buffer.concat(entries); + const cookie = cookieBytes({ + layout: input.layout, + pythonVersion: input.pythonVersion, + pythonLibrary: input.pythonLibrary, + tocOffset: dataLength, + tocLength: toc.length, + }); + return Buffer.concat([ + ELF_PREFIX, + ...payloads, + toc, + cookie, + input.trailer ?? Buffer.alloc(0), + ]); +}; + +/** Cookie whose package range starts before the file. */ +export const cookieRangeOutsideFile = (): Buffer => { + const cookie = Buffer.alloc(88); + PYINSTALLER_COOKIE_MAGIC.copy(cookie, 0); + cookie.writeUInt32BE(200, 8); + cookie.writeUInt32BE(80, 12); + cookie.writeUInt32BE(32, 16); + cookie.writeUInt32BE(314, 20); + return Buffer.concat([ELF_PREFIX, cookie]); +}; + +/** PYZ image. Module bodies are zlib-compressed. Namespace entries have no body. */ +export const pyzArchive = (input: { + readonly magic: Buffer; + readonly modules: readonly PyzFixtureModule[]; + readonly toc?: Buffer; +}): Buffer => { + const blobs: Buffer[] = []; + const entries: Buffer[] = []; + let offset = 12; + for (const module of input.modules) { + const compressed = + module.typeCode === 3 && module.legacyNamespacePayload !== true + ? Buffer.alloc(0) + : Buffer.concat([ + deflateSync(module.body), + module.compressedSuffix ?? Buffer.alloc(0), + ]); + entries.push( + marshalPair(module.name, module.typeCode, offset, compressed.length), + ); + if (compressed.length > 0) { + blobs.push(compressed); + offset += compressed.length; + } + } + const toc = input.toc ?? marshalList(entries); + const header = Buffer.alloc(12); + Buffer.from("PYZ\0").copy(header, 0); + input.magic.copy(header, 4, 0, 4); + header.writeUInt32BE( + 12 + blobs.reduce((sum, blob) => sum + blob.length, 0), + 8, + ); + return Buffer.concat([header, ...blobs, toc]); +}; + +/** Type 3 normally stores no body; older producers can write a namespace payload. */ +export interface PyzFixtureModule { + readonly legacyNamespacePayload?: boolean; + readonly compressedSuffix?: Buffer; + readonly name: string; + readonly typeCode: number; + readonly body: Buffer; +} + +const tocEntry = (input: { + readonly name: string; + readonly typeCode: string; + readonly offset: number; + readonly compressedSize: number; + readonly uncompressedSize: number; + readonly flag: number; +}): Buffer => { + const name = Buffer.from(input.name, "utf8"); + const length = Math.ceil((18 + name.length + 1) / 16) * 16; + const bytes = Buffer.alloc(length); + bytes.writeUInt32BE(length, 0); + bytes.writeUInt32BE(input.offset, 4); + bytes.writeUInt32BE(input.compressedSize, 8); + bytes.writeUInt32BE(input.uncompressedSize, 12); + bytes[16] = input.flag; + bytes[17] = input.typeCode.charCodeAt(0); + name.copy(bytes, 18); + return bytes; +}; + +const cookieBytes = (input: { + readonly layout: "2.0" | "2.1"; + readonly pythonVersion: number; + readonly pythonLibrary: string | undefined; + readonly tocOffset: number; + readonly tocLength: number; +}): Buffer => { + const size = input.layout === "2.1" ? 88 : 24; + const bytes = Buffer.alloc(size); + PYINSTALLER_COOKIE_MAGIC.copy(bytes, 0); + bytes.writeUInt32BE(input.tocOffset + input.tocLength + size, 8); + bytes.writeUInt32BE(input.tocOffset, 12); + bytes.writeUInt32BE(input.tocLength, 16); + bytes.writeUInt32BE(input.pythonVersion, 20); + if (input.pythonLibrary !== undefined) + Buffer.from(input.pythonLibrary, "utf8").copy(bytes, 24); + return bytes; +}; + +const marshalList = (items: readonly Buffer[]): Buffer => { + const count = Buffer.alloc(4); + count.writeInt32LE(items.length, 0); + return Buffer.concat([Buffer.from([0x5b]), count, ...items]); +}; + +const marshalPair = ( + name: string, + typeCode: number, + offset: number, + length: number, +): Buffer => + smallTuple([ + marshalString(name), + smallTuple([int32(typeCode), int32(offset), int32(length)]), + ]); + +const smallTuple = (items: readonly Buffer[]): Buffer => + Buffer.concat([Buffer.from([0x29, items.length]), ...items]); + +const marshalString = (text: string): Buffer => { + const body = Buffer.from(text, "utf8"); + if (body.length <= 255 && body.every((byte) => byte < 0x80)) + return Buffer.concat([Buffer.from([0x7a, body.length]), body]); + const length = Buffer.alloc(4); + length.writeUInt32LE(body.length); + return Buffer.concat([Buffer.from([0x75]), length, body]); +}; + +const int32 = (value: number): Buffer => { + const bytes = Buffer.alloc(5); + bytes[0] = 0x69; + bytes.writeInt32LE(value, 1); + return bytes; +}; diff --git a/src/artifacts/pyinstaller/CArchiveLayout.ts b/src/artifacts/pyinstaller/CArchiveLayout.ts new file mode 100644 index 0000000000..b5bc1f4375 --- /dev/null +++ b/src/artifacts/pyinstaller/CArchiveLayout.ts @@ -0,0 +1,389 @@ +import type { FileHandle } from "node:fs/promises"; +import { readPyInstallerBytes } from "./PyInstallerBytes.js"; + +import { err, ok, type Result } from "../../domain/result.js"; +import { + pyInstallerFailure, + type PyInstallerFailure, +} from "./PyInstallerFailure.js"; + +/** Magic at the start of a CArchive cookie. */ +export const PYINSTALLER_COOKIE_MAGIC = Buffer.from("MEI\x0c\x0b\x0a\x0b\x0e"); + +/** PyInstaller 2.1+ cookie size, including the 64-byte library name. */ +export const PYINSTALLER_COOKIE_V21_BYTES = 88; + +/** PyInstaller 2.0 cookie size, without the library name. */ +export const PYINSTALLER_COOKIE_V20_BYTES = 24; + +/** + * How far before EOF the cookie may sit. macOS code signatures are appended + * after the cookie, and a large binary's signature hashes are larger than the + * bootloader's own 8 KiB scan chunk, so detection uses one bounded tail. + */ +export const PYINSTALLER_COOKIE_SEARCH_BYTES = 16 * 1024 * 1024; + +const MAX_TOC_BYTES = 32 * 1024 * 1024; +const MAX_TOC_ENTRIES = 100_000; +const TOC_HEADER_BYTES = 18; + +/** Observed CArchive cookie. Offsets are relative to the archive start. */ +export interface PyInstallerCookie { + readonly layout: "2.0" | "2.1"; + readonly archiveStart: number; + readonly packageLength: number; + readonly tocOffset: number; + readonly tocLength: number; + readonly pythonVersion: number; + readonly pythonLibrary: string | null; + readonly cookieOffset: number; +} + +/** One CArchive table entry. */ +export interface CArchiveEntry { + readonly index: number; + readonly tocRecordOffset: number; + readonly tocRecordLength: number; + readonly name: string; + readonly typeCode: string; + readonly offset: number; + readonly compressedSize: number; + readonly uncompressedSize: number; + readonly compressionFlag: number; +} + +/** Runtime options and external dependencies are declarations, never local blobs. */ +export const isCArchiveDeclaration = ( + entry: CArchiveEntry, +): entry is CArchiveEntry & { readonly typeCode: "o" | "d" } => + entry.typeCode === "o" || entry.typeCode === "d"; + +/** True when a native executable has a structurally valid CArchive cookie. */ +export const containsPyInstallerArchive = async ( + handle: FileHandle, + fileSize: number, + signal?: AbortSignal, +): Promise => { + const found = await findPyInstallerCookie(handle, fileSize, signal); + return found?.ok === true; +}; + +/** Locate a structurally valid cookie. Invalid magic copies are skipped. */ +export const findPyInstallerCookie = async ( + handle: FileHandle, + fileSize: number, + signal?: AbortSignal, +): Promise | undefined> => { + if ( + !Number.isSafeInteger(fileSize) || + fileSize < PYINSTALLER_COOKIE_V20_BYTES + ) + return undefined; + const tailSize = Math.min(fileSize, PYINSTALLER_COOKIE_SEARCH_BYTES); + const read = await readPyInstallerBytes(handle, { + position: fileSize - tailSize, + length: tailSize, + label: "CArchive tail", + signal, + }); + if (!read.ok) return read; + const observed = read.value; + let failure: PyInstallerFailure | undefined; + let from = observed.length; + while (from >= PYINSTALLER_COOKIE_MAGIC.length) { + const relative = observed + .subarray(0, from) + .lastIndexOf(PYINSTALLER_COOKIE_MAGIC); + if (relative < 0) break; + const cookieOffset = fileSize - tailSize + relative; + const parsed = parseCookieAt(observed, relative, cookieOffset, fileSize); + if (parsed.ok) return parsed; + failure = parsed.error; + from = relative; + } + return failure === undefined ? undefined : err(failure); +}; + +/** Read and parse the TOC that a validated cookie names. */ +export const readCArchiveToc = async ( + handle: FileHandle, + cookie: PyInstallerCookie, + signal?: AbortSignal, +): Promise> => { + if (cookie.tocLength > MAX_TOC_BYTES) + return err( + pyInstallerFailure( + "limit", + "CArchive table of contents exceeds its bound", + ), + ); + const position = cookie.archiveStart + cookie.tocOffset; + const read = await readPyInstallerBytes(handle, { + position, + length: cookie.tocLength, + label: "CArchive table of contents", + signal, + }); + if (!read.ok) return read; + const parsed = parseCArchiveToc(read.value); + if (!parsed.ok) return parsed; + const valid = validateCArchiveMembers(parsed.value, cookie.tocOffset); + return valid.ok ? parsed : valid; +}; + +/** Parse TOC bytes. Each entry is a big-endian header plus a NUL-padded name. */ +export const parseCArchiveToc = ( + bytes: Buffer, +): Result => { + const entries: CArchiveEntry[] = []; + let offset = 0; + while (offset < bytes.length) { + if (entries.length >= MAX_TOC_ENTRIES) + return err( + pyInstallerFailure( + "limit", + "CArchive table of contents exceeds its entry bound", + ), + ); + if (offset + TOC_HEADER_BYTES > bytes.length) + return err( + pyInstallerFailure( + "format", + "CArchive table of contents entry is truncated", + ), + ); + const entryLength = bytes.readUInt32BE(offset); + if (entryLength < TOC_HEADER_BYTES || offset + entryLength > bytes.length) + return err( + pyInstallerFailure( + "format", + "CArchive table of contents entry extends outside the table", + ), + ); + const nameBytes = bytes.subarray( + offset + TOC_HEADER_BYTES, + offset + entryLength, + ); + const name = tocName(nameBytes); + if (!name.ok) return name; + const compressionFlag = bytes[offset + 16] ?? 0; + if (compressionFlag !== 0 && compressionFlag !== 1) + return err( + pyInstallerFailure( + "format", + `CArchive compression flag ${String(compressionFlag)} is not stored or zlib`, + ), + ); + const typeCode = bytes[offset + 17] ?? 0; + if (typeCode < 0x20 || typeCode > 0x7e) + return err( + pyInstallerFailure( + "format", + "CArchive type code is not a printable character", + ), + ); + entries.push({ + index: entries.length, + tocRecordOffset: offset, + tocRecordLength: entryLength, + name: name.value, + typeCode: String.fromCharCode(typeCode), + offset: bytes.readUInt32BE(offset + 4), + compressedSize: bytes.readUInt32BE(offset + 8), + uncompressedSize: bytes.readUInt32BE(offset + 12), + compressionFlag, + }); + offset += entryLength; + } + return ok(entries); +}; + +const parseCookieAt = ( + tail: Buffer, + relative: number, + cookieOffset: number, + fileSize: number, +): Result => { + const modern = parseCookie(tail, relative, cookieOffset, fileSize, "2.1"); + if (modern.ok) return modern; + const legacy = parseCookie(tail, relative, cookieOffset, fileSize, "2.0"); + return legacy.ok ? legacy : modern; +}; + +const parseCookie = ( + tail: Buffer, + relative: number, + cookieOffset: number, + fileSize: number, + layout: "2.0" | "2.1", +): Result => { + const cookieBytes = + layout === "2.1" + ? PYINSTALLER_COOKIE_V21_BYTES + : PYINSTALLER_COOKIE_V20_BYTES; + if (relative + cookieBytes > tail.length) + return err( + pyInstallerFailure( + "format", + "CArchive cookie extends past the end of the file", + ), + ); + const view = tail.subarray(relative, relative + cookieBytes); + const packageLength = view.readUInt32BE(8); + const tocOffset = view.readUInt32BE(12); + const tocLength = view.readUInt32BE(16); + const pythonVersion = view.readUInt32BE(20); + if ( + !safeSum(tocOffset, tocLength, cookieBytes) || + packageLength !== tocOffset + tocLength + cookieBytes + ) + return err( + pyInstallerFailure( + "format", + "CArchive cookie does not describe a table of contents inside the package", + ), + ); + const archiveStart = cookieOffset + cookieBytes - packageLength; + if (archiveStart < 0 || archiveStart + packageLength > fileSize) + return err( + pyInstallerFailure("format", "CArchive cookie range is outside the file"), + ); + const library = + layout === "2.1" ? libraryName(view.subarray(24, 88)) : ok(null); + if (!library.ok) return library; + return ok({ + layout, + archiveStart, + packageLength, + tocOffset, + tocLength, + pythonVersion, + pythonLibrary: library.value, + cookieOffset, + }); +}; + +const libraryName = ( + bytes: Buffer, +): Result => { + const end = bytes.indexOf(0); + const text = end < 0 ? bytes : bytes.subarray(0, end); + if (end >= 0 && bytes.subarray(end).some((byte) => byte !== 0)) + return err( + pyInstallerFailure( + "format", + "CArchive Python library name is not NUL-padded", + ), + ); + if (text.length === 0) return ok(null); + try { + return ok( + new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text), + ); + } catch { + return err( + pyInstallerFailure( + "format", + "CArchive Python library name is not valid UTF-8", + ), + ); + } +}; + +const tocName = (bytes: Buffer): Result => { + const end = bytes.indexOf(0); + const text = end < 0 ? bytes : bytes.subarray(0, end); + if (text.length === 0) + return err(pyInstallerFailure("format", "CArchive entry name is empty")); + try { + return ok( + new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text), + ); + } catch { + return err( + pyInstallerFailure("format", "CArchive entry name is not valid UTF-8"), + ); + } +}; + +const safeSum = (left: number, middle: number, right: number): boolean => + Number.isSafeInteger(left) && + Number.isSafeInteger(middle) && + Number.isSafeInteger(right) && + left >= 0 && + middle >= 0 && + right >= 0 && + left + middle + right <= Number.MAX_SAFE_INTEGER; + +/** Source directory semantics are validated once before any consumer projects paths. */ +const validateCArchiveMembers = ( + entries: readonly CArchiveEntry[], + tocOffset: number, +): Result => { + const names = new Set(); + for (const entry of entries) { + if (isCArchiveDeclaration(entry)) { + if ( + entry.compressedSize !== 0 || + entry.uncompressedSize !== 0 || + entry.compressionFlag !== 0 + ) + return err( + pyInstallerFailure( + "integrity", + `CArchive declaration at table index ${String(entry.index)} has local payload or compression`, + ), + ); + } else { + if (names.has(entry.name)) + return err( + pyInstallerFailure( + "integrity", + `Duplicate CArchive member name at table index ${String(entry.index)}`, + ), + ); + names.add(entry.name); + } + if ( + entry.compressionFlag === 0 && + entry.compressedSize !== entry.uncompressedSize + ) + return err( + pyInstallerFailure( + "integrity", + `Stored CArchive member size disagrees with its table entry at table index ${String(entry.index)}`, + ), + ); + if (entry.offset > tocOffset) + return err( + pyInstallerFailure( + "format", + `CArchive member offset is outside the archive at table index ${String(entry.index)}`, + ), + ); + } + const ranges = entries + .filter((entry) => entry.compressedSize > 0) + .map((entry) => ({ + index: entry.index, + start: entry.offset, + end: entry.offset + entry.compressedSize, + })) + .sort((left, right) => left.start - right.start); + let occupied = 0; + for (const range of ranges) { + if ( + !Number.isSafeInteger(range.end) || + range.end > tocOffset || + range.start < occupied + ) + return err( + pyInstallerFailure( + "integrity", + `CArchive member extends outside the archive or overlaps another member at table index ${String(range.index)}`, + ), + ); + occupied = range.end; + } + return ok(null); +}; diff --git a/src/artifacts/pyinstaller/CArchiveMembers.ts b/src/artifacts/pyinstaller/CArchiveMembers.ts new file mode 100644 index 0000000000..2047f6e006 --- /dev/null +++ b/src/artifacts/pyinstaller/CArchiveMembers.ts @@ -0,0 +1,340 @@ +import type { FileHandle } from "node:fs/promises"; +import { copyArtifactEntry, type ArtifactEntry } from "../ArtifactReader.js"; +import { isCArchiveDeclaration, type CArchiveEntry } from "./CArchiveLayout.js"; +import type { PyzArchive, PyzEntry } from "./PyzArchive.js"; +import { + RECONSTRUCTED_PYTHON_HEADER_LIMITATION, + reconstructedPycHeader, +} from "./PythonBytecode.js"; +import { + cArchiveEntryMetadata, + pyzEntryMetadata, + pyzEntryEncrypted, +} from "./PyInstallerMetadata.js"; +import type { PyInstallerEntryMetadata } from "../../domain/artifactFormatMetadata.js"; + +const CRYPTO_KEY_NAME = "pyimod00_crypto_key"; + +/** Parsed facts and the descriptor identity used to locate a retained image. */ +export interface PyzImage { + readonly archive: PyzArchive; + readonly handle: FileHandle; + readonly start: number; +} + +/** Stable entry descriptions and bounded read plans; expanded member bytes are never retained. */ +export interface PyInstallerMember { + readonly entry: ArtifactEntry; + readonly read: + | { + readonly kind: "carchive"; + readonly archiveStart: number; + readonly source: CArchiveEntry; + readonly header: Buffer | undefined; + } + | { + readonly kind: "pyz"; + readonly handle: FileHandle; + readonly offset: number; + readonly length: number; + readonly compressed: boolean; + readonly uncompressedSize: number | undefined; + readonly header: Buffer | undefined; + } + | undefined; +} + +/** The producer's key is an importable, nonempty Python module, not a data filename. */ +export const hasEncryptionMarker = ( + entries: readonly CArchiveEntry[], +): boolean => + entries.some( + (entry) => + entry.name === CRYPTO_KEY_NAME && + (entry.typeCode === "m" || entry.typeCode === "M") && + entry.uncompressedSize > 0, + ); + +export const projectStoredMember = ( + entry: CArchiveEntry, + archiveStart: number, + pythonMagic: Buffer | undefined, +): PyInstallerMember => { + if (isCArchiveDeclaration(entry)) + return { + entry: fileEntry({ + path: `__pyinstaller__/${entry.typeCode === "o" ? "runtime-options" : "dependencies"}/${String(entry.index)}`, + kind: "metadata", + declaredSize: entry.uncompressedSize, + compressedSize: entry.compressedSize, + byteOffset: null, + limitations: + entry.typeCode === "d" + ? [ + "Multipackage dependency bytes belong to another executable and remain unresolved.", + ] + : [], + adapterKey: `carchive:${String(entry.index)}:${entry.name}`, + formatMetadata: cArchiveEntryMetadata(entry), + }), + read: undefined, + }; + const header = + pythonMagic !== undefined && isBytecodeType(entry.typeCode) + ? reconstructedPycHeader(pythonMagic) + : undefined; + const path = canonicalMemberPath(entry.name); + const symlink = entry.typeCode === "n"; + const limitations: string[] = []; + if (symlink) limitations.push("PyInstaller symlink target was not read."); + if (header !== undefined) + limitations.push(RECONSTRUCTED_PYTHON_HEADER_LIMITATION); + const declaredSize = entry.uncompressedSize; + return { + entry: fileEntry({ + path, + kind: symlink ? "symlink" : "file", + declaredSize, + compressedSize: entry.compressedSize, + byteOffset: sourceByteOffset(entry, archiveStart), + limitations, + adapterKey: `carchive:${String(entry.index)}:${entry.name}`, + formatMetadata: cArchiveEntryMetadata(entry, header), + }), + read: { + kind: "carchive", + archiveStart, + source: entry, + header, + }, + }; +}; + +export const projectPyzContainer = ( + entry: CArchiveEntry, + archiveStart: number, + image: PyzImage, + encrypted: boolean, +): PyInstallerMember => { + const { archive } = image; + const path = canonicalMemberPath(entry.name); + const byteOffset = sourceByteOffset(entry, archiveStart); + const limitations: string[] = []; + if (archive.kind === "opaque") + return { + entry: fileEntry({ + path, + declaredSize: entry.uncompressedSize, + compressedSize: entry.compressedSize, + byteOffset, + // Legacy encryption applies to inner payloads, never the PYZ image. + encrypted: false, + limitations: [ + ...limitations, + archive.limitation, + ...(encrypted + ? ["PYZ entries are encrypted and were not decoded."] + : []), + ], + adapterKey: `carchive:${String(entry.index)}:${entry.name}`, + formatMetadata: cArchiveEntryMetadata( + entry, + undefined, + archive, + encrypted, + ), + }), + read: + entry.compressionFlag === 1 + ? { + kind: "pyz", + handle: image.handle, + offset: image.start, + length: entry.uncompressedSize, + compressed: false, + uncompressedSize: entry.uncompressedSize, + header: undefined, + } + : { + kind: "carchive", + archiveStart, + source: entry, + header: undefined, + }, + }; + return { + entry: { + path, + kind: "directory", + declaredSize: entry.uncompressedSize, + compressedSize: entry.compressedSize, + executable: false, + encrypted: false, + byteOffset, + declaredIntegrity: null, + unpacked: false, + limitations: [ + ...limitations, + `PyInstaller PYZ sha256 ${archive.sha256}.`, + ...(encrypted + ? ["PYZ entries are encrypted and were not decoded."] + : []), + ], + adapterKey: `carchive:${String(entry.index)}:${entry.name}`, + formatMetadata: cArchiveEntryMetadata( + entry, + undefined, + archive, + encrypted, + ), + }, + read: undefined, + }; +}; + +export const projectPyzMember = ( + container: CArchiveEntry, + item: PyzEntry, + encryptionMarker: boolean, + pythonMagic: Buffer | undefined, + range: + | { + readonly handle: FileHandle; + readonly offset: number; + readonly length: number; + } + | undefined, +): PyInstallerMember => { + const path = pyzMemberPath(canonicalMemberPath(container.name), item); + const encrypted = pyzEntryEncrypted(item, encryptionMarker); + if (item.namespace) + return { + entry: { + path: path, + kind: "directory", + declaredSize: 0, + compressedSize: item.length, + executable: false, + encrypted, + byteOffset: null, + declaredIntegrity: null, + unpacked: false, + limitations: [ + "Namespace package has no importable bytecode; its declared payload is not materialized.", + ...(encrypted + ? [ + "PYZ namespace payload is declared encrypted and is not materialized.", + ] + : []), + ], + adapterKey: `pyz:${String(container.index)}:${String(item.index)}:${path}`, + formatMetadata: pyzEntryMetadata( + container, + item, + undefined, + encryptionMarker, + ), + }, + read: undefined, + }; + const header = + !encrypted && + pythonMagic !== undefined && + (item.typeCode === 0 || item.typeCode === 1) + ? reconstructedPycHeader(pythonMagic) + : undefined; + const limitations: string[] = []; + if (encrypted) + limitations.push("PYZ entry is encrypted and was not decoded."); + if (header !== undefined) + limitations.push(RECONSTRUCTED_PYTHON_HEADER_LIMITATION); + limitations.push( + "PYZ tables declare compressed lengths; the uncompressed size is unknown until decoding.", + ); + return { + entry: fileEntry({ + path: path, + declaredSize: null, + compressedSize: item.length, + byteOffset: null, + encrypted, + limitations, + adapterKey: `pyz:${String(container.index)}:${String(item.index)}:${path}`, + formatMetadata: pyzEntryMetadata( + container, + item, + header, + encryptionMarker, + ), + }), + read: + range === undefined + ? undefined + : { + kind: "pyz", + ...range, + compressed: true, + uncompressedSize: undefined, + header, + }, + }; +}; + +/** + * CArchive and PYZ data names treat `\` as a separator. The logical path is + * canonical; the stored spelling is kept when that conversion changes it. + * `source_location` is an offset plus declared size, so a compressed member + * publishes a file span only when that size is the physical extent. Otherwise + * the span stays unknown, as ZIP does for compressed entries. + */ +const canonicalMemberPath = (storedName: string): string => + storedName.replaceAll("\\", "/"); + +const sourceByteOffset = ( + entry: CArchiveEntry, + archiveStart: number, +): number | null => + entry.compressionFlag === 0 ? archiveStart + entry.offset : null; + +const pyzMemberPath = (container: string, item: PyzEntry): string => { + if (item.typeCode === 2) + return `${container}/${canonicalMemberPath(item.name)}`; + const dotted = item.name.replaceAll(".", "/"); + if (item.namespace) return `${container}/${dotted}`; + if (item.package) return `${container}/${dotted}/__init__.pyc`; + return `${container}/${dotted}.pyc`; +}; + +const isBytecodeType = (typeCode: string): boolean => + typeCode === "s" || typeCode === "m" || typeCode === "M"; + +const fileEntry = (input: { + readonly path: string; + readonly declaredSize: number | null; + readonly compressedSize: number; + readonly byteOffset: number | null; + readonly limitations: readonly string[]; + readonly adapterKey: string; + readonly formatMetadata: PyInstallerEntryMetadata; + readonly encrypted?: boolean; + readonly kind?: "file" | "symlink" | "metadata"; +}): ArtifactEntry => ({ + path: input.path, + kind: input.kind ?? "file", + declaredSize: input.declaredSize, + compressedSize: input.compressedSize, + executable: false, + encrypted: input.encrypted === true, + byteOffset: input.byteOffset, + declaredIntegrity: null, + unpacked: false, + limitations: input.limitations, + adapterKey: input.adapterKey, + formatMetadata: input.formatMetadata, +}); + +/** Stable entry stored for a later open(). */ +export const sealMember = (member: PyInstallerMember): PyInstallerMember => ({ + ...member, + entry: copyArtifactEntry(member.entry), +}); diff --git a/src/artifacts/pyinstaller/MarshalSubset.test.ts b/src/artifacts/pyinstaller/MarshalSubset.test.ts new file mode 100644 index 0000000000..c4dee6781e --- /dev/null +++ b/src/artifacts/pyinstaller/MarshalSubset.test.ts @@ -0,0 +1,62 @@ +import { expect, it } from "vitest"; + +import { readMarshalSubset } from "./MarshalSubset.js"; + +const bytes = (hex: string): Buffer => Buffer.from(hex, "hex"); + +it("reads a Python 3.9 reference-flagged PYZ table", () => { + const parsed = readMarshalSubset( + bytes( + "db04000000a902da0568656c6c6fa903e900000000e90a000000e914000000a902da03706b67a903e901000000e91e000000e928000000a902fa0c6a736f6e2e6465636f646572a9037204000000e964000000e932000000a902da026e73a903e90300000072040000007204000000", + ), + ); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value).toEqual([ + ["hello", [0, 10, 20]], + ["pkg", [1, 30, 40]], + ["json.decoder", [0, 100, 50]], + ["ns", [3, 0, 0]], + ]); +}); + +it("resolves a shared string through a reference reserved before the tuple", () => { + const parsed = readMarshalSubset( + bytes( + "db020000002902da06736861726564a903e900000000e901000000e90200000029027201000000a9037204000000e903000000e904000000", + ), + ); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value).toEqual([ + ["shared", [0, 1, 2]], + ["shared", [1, 3, 4]], + ]); +}); + +it("reads a long integer that fits in a safe integer", () => { + const parsed = readMarshalSubset( + bytes( + "5b01000000a902da03626967a903e900000000ec03000000000000000004e905000000", + ), + ); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value).toEqual([["big", [0, 2 ** 40, 5]]]); +}); + +it("reads a negative 32-bit integer", () => { + const parsed = readMarshalSubset( + bytes("5b01000000a902da016ea903e900000000e9ffffffffe901000000"), + ); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value).toEqual([["n", [0, -1, 1]]]); +}); + +it("rejects a marshal type outside the subset", () => { + const parsed = readMarshalSubset(Buffer.from([0x4e])); + expect(parsed.ok).toBe(false); + if (parsed.ok) return; + expect(parsed.error.reason).toBe("format"); +}); diff --git a/src/artifacts/pyinstaller/MarshalSubset.ts b/src/artifacts/pyinstaller/MarshalSubset.ts new file mode 100644 index 0000000000..665d203c70 --- /dev/null +++ b/src/artifacts/pyinstaller/MarshalSubset.ts @@ -0,0 +1,282 @@ +import { err, ok, type Result } from "../../domain/result.js"; +import { + pyInstallerFailure, + type PyInstallerFailure, +} from "./PyInstallerFailure.js"; + +const MAX_DEPTH = 16; +const MAX_NODES = 100_000; +const MAX_SEQUENCE = 100_000; +const MAX_STRING_BYTES = 65_536; +const FLAG_REF = 0x80; + +/** Values the PYZ table of contents is allowed to contain. */ +export type MarshalValue = + | number + | string + | Uint8Array + | readonly MarshalValue[]; + +interface MarshalCursor { + readonly bytes: Buffer; + offset: number; + readonly refs: Array; + depth: number; + nodes: number; +} + +/** + * Read one marshal value using only list, tuple, str, int, and bytes. + * Container slots are reserved before their elements so later references match + * CPython. Any other type fails the whole value. + */ +export const readMarshalSubset = ( + bytes: Buffer, +): Result => { + const cursor: MarshalCursor = { + bytes, + offset: 0, + refs: [], + depth: 0, + nodes: 0, + }; + const value = readObject(cursor); + if (!value.ok) return value; + if (cursor.offset !== bytes.length) + return err( + pyInstallerFailure( + "format", + "PYZ marshal value did not consume its table of contents", + ), + ); + return value; +}; + +const readObject = ( + cursor: MarshalCursor, +): Result => { + if (cursor.depth > MAX_DEPTH || cursor.nodes >= MAX_NODES) + return err( + pyInstallerFailure( + "limit", + "PYZ marshal value exceeds the depth or object bound", + ), + ); + cursor.nodes += 1; + const code = readByte(cursor); + if (!code.ok) return code; + const flag = (code.value & FLAG_REF) !== 0; + const type = code.value & ~FLAG_REF; + if (type === 0x69) return storeScalar(cursor, flag, readInt32(cursor)); + if (type === 0x6c) return storeScalar(cursor, flag, readLong(cursor)); + if (type === 0x7a || type === 0x5a) + return storeScalar(cursor, flag, readShortAscii(cursor)); + if (type === 0x61 || type === 0x41) + return storeScalar(cursor, flag, readCountedAscii(cursor)); + if (type === 0x75) return storeScalar(cursor, flag, readUtf8(cursor)); + if (type === 0x73) return storeScalar(cursor, flag, readByteString(cursor)); + if (type === 0x28) return readSequence(cursor, flag, readInt32(cursor)); + if (type === 0x29) return readSequence(cursor, flag, readByte(cursor)); + if (type === 0x5b) return readSequence(cursor, flag, readInt32(cursor)); + if (type === 0x72) return readReference(cursor, flag); + return err( + pyInstallerFailure( + "format", + "PYZ marshal value uses a type outside the supported subset", + ), + ); +}; + +const storeScalar = ( + cursor: MarshalCursor, + flag: boolean, + value: Result, +): Result => { + if (!value.ok) return value; + if (flag) cursor.refs.push(value.value); + return value; +}; + +const readSequence = ( + cursor: MarshalCursor, + flag: boolean, + count: Result, +): Result => { + if (!count.ok) return count; + if (count.value < 0 || count.value > MAX_SEQUENCE) + return err( + pyInstallerFailure( + "limit", + "PYZ marshal sequence exceeds the supported count", + ), + ); + const slot = flag ? cursor.refs.length : undefined; + if (slot !== undefined) cursor.refs.push(undefined); + const items: MarshalValue[] = []; + cursor.depth += 1; + for (let index = 0; index < count.value; index += 1) { + const item = readObject(cursor); + if (!item.ok) return item; + items.push(item.value); + } + cursor.depth -= 1; + if (slot !== undefined) cursor.refs[slot] = items; + return ok(items); +}; + +const readReference = ( + cursor: MarshalCursor, + flag: boolean, +): Result => { + const index = readInt32(cursor); + if (!index.ok) return index; + const value = cursor.refs[index.value]; + if (value === undefined) + return err( + pyInstallerFailure( + "format", + "PYZ marshal reference does not point at a completed value", + ), + ); + if (flag) cursor.refs.push(value); + return ok(value); +}; + +const readShortAscii = ( + cursor: MarshalCursor, +): Result => { + const length = readByte(cursor); + if (!length.ok) return length; + return readAsciiBytes(cursor, length.value); +}; + +const readCountedAscii = ( + cursor: MarshalCursor, +): Result => { + const length = readInt32(cursor); + if (!length.ok) return length; + return readAsciiBytes(cursor, length.value); +}; + +const readAsciiBytes = ( + cursor: MarshalCursor, + length: number, +): Result => { + const bytes = readBytes(cursor, length); + if (!bytes.ok) return bytes; + if (bytes.value.some((byte) => byte > 0x7f)) + return err( + pyInstallerFailure( + "format", + "PYZ marshal ASCII string contains a non-ASCII byte", + ), + ); + return ok(bytes.value.toString("ascii")); +}; + +const readUtf8 = ( + cursor: MarshalCursor, +): Result => { + const length = readInt32(cursor); + if (!length.ok) return length; + const bytes = readBytes(cursor, length.value); + if (!bytes.ok) return bytes; + try { + return ok( + new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( + bytes.value, + ), + ); + } catch { + return err( + pyInstallerFailure("format", "PYZ marshal string is not valid UTF-8"), + ); + } +}; + +const readByteString = ( + cursor: MarshalCursor, +): Result => { + const length = readInt32(cursor); + if (!length.ok) return length; + const bytes = readBytes(cursor, length.value); + if (!bytes.ok) return bytes; + return ok(bytes.value); +}; + +const readLong = ( + cursor: MarshalCursor, +): Result => { + const digits = readInt32(cursor); + if (!digits.ok) return digits; + const count = Math.abs(digits.value); + if (count > 8) + return err( + pyInstallerFailure("limit", "PYZ marshal integer exceeds the safe range"), + ); + let magnitude = 0n; + for (let index = 0; index < count; index += 1) { + const digit = readUint16(cursor); + if (!digit.ok) return digit; + if (digit.value > 0x7fff) + return err( + pyInstallerFailure("format", "PYZ marshal long digit is out of range"), + ); + magnitude += BigInt(digit.value) << BigInt(15 * index); + } + if (magnitude > BigInt(Number.MAX_SAFE_INTEGER)) + return err( + pyInstallerFailure("limit", "PYZ marshal integer exceeds the safe range"), + ); + const signed = digits.value < 0 ? -Number(magnitude) : Number(magnitude); + return ok(signed); +}; + +const readBytes = ( + cursor: MarshalCursor, + length: number, +): Result => { + if (!Number.isSafeInteger(length) || length < 0 || length > MAX_STRING_BYTES) + return err( + pyInstallerFailure( + "limit", + "PYZ marshal string exceeds the supported length", + ), + ); + const end = cursor.offset + length; + if (end > cursor.bytes.length) + return err(pyInstallerFailure("format", "PYZ marshal value is truncated")); + const bytes = cursor.bytes.subarray(cursor.offset, end); + cursor.offset = end; + return ok(bytes); +}; + +const readByte = ( + cursor: MarshalCursor, +): Result => { + if (cursor.offset >= cursor.bytes.length) + return err(pyInstallerFailure("format", "PYZ marshal value is truncated")); + const value = cursor.bytes[cursor.offset] ?? 0; + cursor.offset += 1; + return ok(value); +}; + +const readInt32 = ( + cursor: MarshalCursor, +): Result => { + if (cursor.offset + 4 > cursor.bytes.length) + return err(pyInstallerFailure("format", "PYZ marshal value is truncated")); + const value = cursor.bytes.readInt32LE(cursor.offset); + cursor.offset += 4; + return ok(value); +}; + +const readUint16 = ( + cursor: MarshalCursor, +): Result => { + if (cursor.offset + 2 > cursor.bytes.length) + return err(pyInstallerFailure("format", "PYZ marshal value is truncated")); + const value = cursor.bytes.readUInt16LE(cursor.offset); + cursor.offset += 2; + return ok(value); +}; diff --git a/src/artifacts/pyinstaller/PyInstallerAdmissionBoundaries.test.ts b/src/artifacts/pyinstaller/PyInstallerAdmissionBoundaries.test.ts new file mode 100644 index 0000000000..4242b390f8 --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerAdmissionBoundaries.test.ts @@ -0,0 +1,139 @@ +import type { FileHandle } from "node:fs/promises"; +import { expect, it, vi } from "vitest"; +import { withPyInstallerReader } from "../../../tests/fixtures/pyinstallerReader.js"; +import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; +import type { ArtifactEntry } from "../ArtifactReader.js"; +import { PyInstallerPyzStore } from "./PyInstallerPyzStore.js"; +import { + cArchiveExecutable, + pyzArchive, + PYTHON_39_MAGIC, +} from "./CArchive.fixture.js"; + +const compressedImages = (count: number, modules = false) => + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: Array.from({ length: count }, (_, index) => ({ + name: `PYZ-${index}.pyz`, + typeCode: "z", + compress: true, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: modules + ? [{ name: "app", typeCode: 0, body: Buffer.from("body") }] + : [], + }), + })), + }); + +it("refuses excess private images before creating any temporary file", async () => { + const spool = vi + .spyOn(PyInstallerPyzStore.prototype, "spool") + .mockRejectedValue(new Error("unexpected private file creation")); + try { + await withPyInstallerReader(compressedImages(129), async (reader) => { + await expect( + reader.entries()[Symbol.asyncIterator]().next(), + ).rejects.toMatchObject({ + reason: "limit", + boundary: "capacity", + }); + expect(spool).not.toHaveBeenCalled(); + }); + } finally { + spool.mockRestore(); + } +}); + +it("accepts the private-image capacity and retires all image owners", async () => { + const spool = vi.spyOn(PyInstallerPyzStore.prototype, "spool"); + try { + await withPyInstallerReader(compressedImages(128), async (reader, root) => { + const listed: ArtifactEntry[] = []; + for await (const entry of reader.entries()) listed.push(entry); + expect(listed).toHaveLength(128); + expect(spool).toHaveBeenCalledTimes(128); + await reader.close(); + expect(root.fd).toBeGreaterThanOrEqual(0); + }); + } finally { + spool.mockRestore(); + } +}); + +it.each([false, true])( + "preserves private-image admission cancellation (failed cleanup: %s)", + async (failClose) => { + await withPyInstallerReader( + compressedImages(1, true), + async (reader, root) => { + let selected: ArtifactEntry | undefined; + for await (const entry of reader.entries()) + if (entry.path.endsWith("app.pyc")) selected = entry; + if (selected === undefined) throw new Error("Missing child"); + let entered!: () => void, release!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); + const prototype = Object.getPrototypeOf(root) as FileHandle; + const nativeStat = prototype.stat, + nativeClose = OwnedFileHandle.prototype.close; + let admitted: FileHandle | undefined, + retained: OwnedFileHandle | undefined, + attempts = 0; + const cancelled = new Error("cancel private-image admission"), + cleanup = new Error("admission cleanup EIO"); + const signal = new AbortController(); + const stat = vi + .spyOn(prototype, "stat") + .mockImplementation(async function ( + this: FileHandle, + ...args: Parameters + ) { + if (this !== root && admitted === undefined) { + admitted = this; + entered(); + await gate; + } + return Reflect.apply(nativeStat, this, args); + }); + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + if (this.handle === admitted) { + retained = this; + attempts++; + if (failClose && attempts === 1) throw cleanup; + } + await nativeClose.call(this); + }); + const opening = reader.open(selected, signal.signal); + void opening.catch(() => undefined); + try { + await started; + signal.abort(cancelled); + release(); + await expect(opening).rejects.toMatchObject({ + reason: "cancelled", + cancellation: { reason: cancelled }, + ...(failClose ? { cleanup: { reason: cleanup.message } } : {}), + }); + if (failClose) expect(retained?.handle.fd).toBeGreaterThanOrEqual(0); + await reader.close(); + expect(attempts).toBe(failClose ? 2 : 1); + expect(admitted?.fd).toBe(-1); + expect(root.fd).toBeGreaterThanOrEqual(0); + } finally { + release(); + await opening.catch(() => undefined); + stat.mockRestore(); + close.mockRestore(); + } + }, + ); + }, +); diff --git a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts new file mode 100644 index 0000000000..3220bbb4e0 --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts @@ -0,0 +1,1178 @@ +import { createHash } from "node:crypto"; +import { open, readFile, stat, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { buffer } from "node:stream/consumers"; + +import { expect, it, vi } from "vitest"; + +import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; +import { materializeArtifactInventory } from "../../../tests/fixtures/artifactInventory.js"; +import { scanCanonicalArtifactInventory } from "../../../tests/fixtures/artifactInventory.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js"; +import { reconstructedPycHeader } from "./PythonBytecode.js"; +import { + PYTHON_39_MAGIC, + cArchiveExecutable, + cookieRangeOutsideFile, + pyzArchive, + type CArchiveFixtureMember, +} from "./CArchive.fixture.js"; + +const sha256 = (bytes: Buffer): string => + createHash("sha256").update(bytes).digest("hex"); + +const derived = (payload: Buffer): Buffer => + Buffer.concat([reconstructedPycHeader(PYTHON_39_MAGIC), payload]); + +const writeArchive = async (name: string, bytes: Buffer): Promise => { + const directory = await createTestTempDirectory("rea-pyinstaller-"); + const path = join(directory, name); + await writeFile(path, bytes); + return path; +}; + +const sampleMembers = ( + script: Buffer, + stored: Buffer, + pyz: Buffer, +): CArchiveFixtureMember[] => [ + { name: "hello", typeCode: "s", payload: script, compress: true }, + { name: "data.bin", typeCode: "x", payload: stored, compress: false }, + { name: "PYZ.pyz", typeCode: "z", payload: pyz, compress: false }, + { + name: "pyi-contents-directory _internal", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, +]; + +it("inventories a 2.1 archive, nested PYZ modules, and derived bytecode", async () => { + const script = Buffer.from("print('hi')"); + const stored = Buffer.from("stored-data"); + const moduleBody = Buffer.from("module-body"); + const path = await writeArchive( + "hello", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 314, + pythonLibrary: "libpython3.14.dylib", + trailer: Buffer.from("signature"), + members: sampleMembers( + script, + stored, + pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "json.decoder", typeCode: 0, body: moduleBody }, + { name: "pkg", typeCode: 1, body: moduleBody }, + { name: "ns", typeCode: 3, body: Buffer.alloc(0) }, + ], + }), + ), + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + expect(snapshot.manifest.root_format).toBe("pyinstaller"); + expect(snapshot.coverage.status).toBe("complete"); + const root = snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === ".", + ); + expect(root).toMatchObject({ artifact_kind: "container" }); + expect(root?.format_metadata).toMatchObject({ + kind: "pyinstaller", + layout_version: "2.1", + python_version_raw: 314, + python_major: 3, + python_minor: 14, + python_library: "libpython3.14.dylib", + runtime_option_count: 1, + }); + const hello = snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "hello", + ); + expect(hello?.format_metadata).toMatchObject({ + kind: "pyinstaller-entry", + container: "carchive", + type_code: "s", + compression_flag: 1, + bytecode_header: "unknown", + bytecode_magic: null, + }); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(script)), + ).toMatchObject({ size: script.length }); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(stored)), + ).toBeDefined(); + expect( + snapshot.occurrences.map((occurrence) => occurrence.logical_path), + ).toEqual( + expect.arrayContaining([ + "hello", + "data.bin", + "PYZ.pyz", + "PYZ.pyz/json/decoder.pyc", + "PYZ.pyz/pkg/__init__.pyc", + "PYZ.pyz/ns", + "__pyinstaller__/runtime-options/3", + ]), + ); + expect( + snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "PYZ.pyz", + )?.entry_kind, + ).toBe("directory"); + const scriptEdge = snapshot.edges.find( + (edge) => edge.occurrence_id === hello?.occurrence_id, + ); + expect(scriptEdge?.relation).toBe("contains"); +}); + +it("records a 2.0 cookie without a Python library name", async () => { + const path = await writeArchive( + "legacy", + cArchiveExecutable({ + layout: "2.0", + pythonVersion: 307, + members: [ + { + name: "hello", + typeCode: "s", + payload: Buffer.from("print"), + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + const root = snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === ".", + ); + expect(root?.format_metadata).toMatchObject({ + kind: "pyinstaller", + layout_version: "2.0", + python_version_raw: 307, + python_major: 3, + python_minor: 7, + python_library: null, + }); +}); + +it("extracts derived members into the safe output tree", async () => { + const script = Buffer.from("print('hi')"); + const path = await writeArchive( + "hello", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: sampleMembers( + script, + Buffer.from("stored"), + pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "app", typeCode: 0, body: Buffer.from("app-body") }, + ], + }), + ), + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + const directory = await createTestTempDirectory("rea-pyinstaller-out-"); + const output = join(directory, "extracted"); + const extracted = await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + path, + snapshot, + ); + expect(await readFile(join(output, "hello"))).toEqual(derived(script)); + expect( + extracted.artifacts.find((artifact) => artifact.relative_path === "hello") + ?.sha256, + ).toBe(sha256(derived(script))); + expect(await readFile(join(output, "PYZ.pyz", "app.pyc"))).toEqual( + derived(Buffer.from("app-body")), + ); +}); + +it("reports encrypted PYZ entries and refuses to extract them", async () => { + const path = await writeArchive( + "encrypted", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: [ + { + name: "pyimod00_crypto_key", + typeCode: "m", + payload: Buffer.from("key"), + compress: false, + }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "secret", typeCode: 0, body: Buffer.from("hidden") }, + ], + }), + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + expect( + snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "PYZ.pyz/secret.pyc", + ), + ).toMatchObject({ + encrypted: true, + hash_status: "unavailable", + }); + const directory = await createTestTempDirectory("rea-pyinstaller-out-"); + await expect( + materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: join(directory, "extracted"), + environment: process.env, + integrityPolicy: "fail", + }, + path, + snapshot, + ), + ).rejects.toThrow(/encrypted/u); +}); + +it.each([false, true])( + "leaves an unsupported PYZ magic as one opaque member (marker: %s)", + async (marker) => { + const payload = pyzArchive({ + magic: Buffer.alloc(4), + modules: [{ name: "app", typeCode: 0, body: Buffer.from("app") }], + }); + const path = await writeArchive( + "opaque", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 314, + pythonLibrary: "libpython3.14.dylib", + members: [ + ...(marker + ? [ + { + name: "pyimod00_crypto_key", + typeCode: "m", + payload: Buffer.from("key"), + compress: false, + }, + ] + : []), + { + name: "PYZ.pyz", + typeCode: "z", + payload, + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + const pyz = snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "PYZ.pyz", + ); + expect(pyz).toMatchObject({ + entry_kind: "file", + encrypted: false, + hash_status: "verified", + }); + expect(snapshot.coverage.status).toBe("unknown"); + expect( + snapshot.nodes.find((node) => node.artifact_id === pyz?.artifact_id), + ).toMatchObject({ sha256: sha256(payload), size: payload.length }); + expect(pyz?.format_metadata).toMatchObject({ + pyz_python_magic: "00000000", + pyz_python_version: null, + pyz_status: "opaque", + encryption_source: marker ? "carchive-marker" : null, + }); + expect(pyz?.limitations.join(" ")).toContain("left opaque"); + expect( + snapshot.occurrences.some((occurrence) => + occurrence.logical_path.endsWith(".pyc"), + ), + ).toBe(false); + const outputRoot = join( + await createTestTempDirectory("rea-opaque-out-"), + "extracted", + ); + await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot, + environment: process.env, + integrityPolicy: "fail", + }, + path, + snapshot, + ); + expect(await readFile(join(outputRoot, "PYZ.pyz"))).toEqual(payload); + }, +); + +it("reads children of a zlib-compressed PYZ", async () => { + const body = Buffer.from("compressed-child"); + const path = await writeArchive( + "compressed-pyz", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "child", typeCode: 0, body }], + }), + compress: true, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(derived(body))), + ).toMatchObject({ size: derived(body).length }); +}); + +it("rejects a truncated table of contents", async () => { + const path = await writeArchive( + "truncated", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 314, + pythonLibrary: "libpython3.14.dylib", + members: [], + tocBytes: Buffer.concat([Buffer.from([0, 0, 0, 64]), Buffer.alloc(16)]), + }), + ); + await expect(scanCanonicalArtifactInventory(path)).rejects.toBeInstanceOf( + ArtifactReaderFailure, + ); + await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({ + reason: "format", + }); +}); + +it("rejects a cookie whose package range is outside the file", async () => { + const path = await writeArchive("outside", cookieRangeOutsideFile()); + const reader = new PyInstallerArtifactReader(path); + await expect(async () => { + for await (const entry of reader.entries()) void entry; + }).rejects.toMatchObject({ + reason: "format", + message: expect.stringContaining("outside the file"), + }); + await reader.close(); +}); + +it("rejects a member whose declared size exceeds the decompression budget", async () => { + const path = await writeArchive( + "bomb", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 314, + pythonLibrary: "libpython3.14.dylib", + members: [ + { + name: "bomb.bin", + typeCode: "b", + payload: Buffer.alloc(0), + compress: true, + declaredUncompressedSize: 300 * 1024 * 1024, + }, + ], + }), + ); + await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({ + reason: "limit", + }); +}); + +it("rejects a zlib member that expands past its declared size", async () => { + const path = await writeArchive( + "expand", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 314, + pythonLibrary: "libpython3.14.dylib", + members: [ + { + name: "expand.bin", + typeCode: "b", + payload: Buffer.from("hello world hello world"), + compress: true, + declaredUncompressedSize: 4, + }, + ], + }), + ); + await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({ + reason: "limit", + }); +}); + +it("reads a compressed member whose uncompressed size is zero", async () => { + const path = await writeArchive( + "empty-compressed", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: [ + { + name: "empty.bin", + typeCode: "x", + payload: Buffer.alloc(0), + compress: true, + }, + { + name: "empty.py", + typeCode: "s", + payload: Buffer.alloc(0), + compress: true, + }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "app", typeCode: 0, body: Buffer.from("app") }], + }), + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(Buffer.alloc(0))), + ).toMatchObject({ size: 0 }); + expect( + snapshot.nodes.find( + (node) => node.sha256 === sha256(derived(Buffer.alloc(0))), + ), + ).toMatchObject({ size: derived(Buffer.alloc(0)).length }); + + const expanded = await writeArchive( + "zero-declared", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: [ + { + name: "expand.bin", + typeCode: "x", + payload: Buffer.from("x"), + compress: true, + declaredUncompressedSize: 0, + }, + ], + }), + ); + await expect(scanCanonicalArtifactInventory(expanded)).rejects.toMatchObject({ + reason: "limit", + }); +}); + +it("preserves an uppercase Z ZIP member as a readable file", async () => { + const zip = Buffer.from("PK\u0003\u0004zip-member"); + const path = await writeArchive( + "zip-member", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: [ + { + name: "payload.zip", + typeCode: "Z", + payload: zip, + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + expect( + snapshot.occurrences.map((occurrence) => occurrence.logical_path).sort(), + ).toEqual([".", "payload.zip"]); + expect( + snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "payload.zip", + ), + ).toMatchObject({ + entry_kind: "file", + format_metadata: { kind: "pyinstaller-entry", type_code: "Z" }, + }); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(zip)), + ).toMatchObject({ size: zip.length }); +}); + +it("keeps a PYZ data entry resource path", async () => { + const body = Buffer.from('{"enabled":true}'); + const path = await writeArchive( + "pyz-data", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "pkg/config.json", typeCode: 2, body }], + }), + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + expect( + snapshot.occurrences.map((occurrence) => occurrence.logical_path).sort(), + ).toEqual([".", "PYZ.pyz", "PYZ.pyz/pkg/config.json"]); + expect( + snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "PYZ.pyz/pkg/config.json", + ), + ).toMatchObject({ entry_kind: "file" }); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(body)), + ).toMatchObject({ size: body.length }); +}); + +it("projects CArchive type n entries as symlinks", async () => { + const target = Buffer.from("libfoo.so.1\0"); + const payload = Buffer.from("stored-data"); + const path = await writeArchive( + "symlink", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: [ + { + name: "libfoo.so", + typeCode: "n", + payload: target, + compress: false, + }, + { name: "data.bin", typeCode: "x", payload, compress: false }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + expect( + snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "libfoo.so", + ), + ).toMatchObject({ + entry_kind: "symlink", + artifact_id: null, + hash_status: "not-hashed", + format_metadata: { + kind: "pyinstaller-entry", + type_code: "n", + compression_flag: 0, + }, + limitations: expect.arrayContaining([ + "PyInstaller symlink target was not read.", + ]), + }); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(target)), + ).toBeUndefined(); + const directory = await createTestTempDirectory("rea-pyinstaller-out-"); + const output = join(directory, "extracted"); + await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + path, + snapshot, + ); + await expect(readFile(join(output, "libfoo.so"))).rejects.toMatchObject({ + code: "ENOENT", + }); + expect(await readFile(join(output, "data.bin"))).toEqual(payload); +}); + +it.each([false, true])( + "retains the file owner after a failed close (temporary PYZ: %s)", + async (temporaryPyz) => { + const path = await writeArchive( + "close-owner", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: [ + { + name: temporaryPyz ? "PYZ.pyz" : "data.bin", + typeCode: temporaryPyz ? "z" : "x", + payload: temporaryPyz + ? pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }) + : Buffer.from("data"), + compress: temporaryPyz, + }, + ], + }), + ); + const reader = new PyInstallerArtifactReader(path); + for await (const entry of reader.entries()) void entry; + const originalClose = OwnedFileHandle.prototype.close; + const owners: OwnedFileHandle[] = []; + const closeFailure = new Error("close failed"); + let attempts = 0; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + owners.push(this); + attempts += 1; + if (attempts === 1) throw closeFailure; + await originalClose.call(this); + }); + try { + await expect(reader.close()).rejects.toMatchObject({ + cleanup: { reason: closeFailure.message, resources: [path] }, + }); + await reader.close(); + // Spooled PYZ images close after listing, so only the source owner remains. + expect(attempts).toBe(2); + expect(owners[0]).toBe(owners[1]); + } finally { + close.mockRestore(); + await reader.close(); + } + }, +); + +it("retains the descriptor owner when admission cleanup fails", async () => { + const directory = await createTestTempDirectory("rea-pyinstaller-admission-"); + const reader = new PyInstallerArtifactReader(directory); + const originalClose = OwnedFileHandle.prototype.close; + const owners: OwnedFileHandle[] = []; + const closeFailure = new Error("admission close failed"); + let attempts = 0; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + owners.push(this); + attempts += 1; + if (attempts === 1) throw closeFailure; + await originalClose.call(this); + }); + try { + await expect( + reader.entries()[Symbol.asyncIterator]().next(), + ).rejects.toMatchObject({ + reason: "format", + message: `PyInstaller source is not a regular file: ${directory}`, + cleanup: { + reason: closeFailure.message, + resources: [directory], + }, + }); + expect(owners).toHaveLength(1); + await reader.close(); + expect(attempts).toBe(2); + expect(owners[0]).toBe(owners[1]); + } finally { + close.mockRestore(); + await reader.close().catch(() => undefined); + await owners[0]?.close().catch(() => undefined); + } +}); + +it("inventories and extracts a Windows member name under its canonical path", async () => { + const storedName = String.raw`pkg\data.bin`; + const pyzName = String.raw`res\icon.bin`; + const payload = Buffer.from("nested-bytes"); + const icon = Buffer.from("icon-bytes"); + const path = await writeArchive( + "windows", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "python39.dll", + members: [ + { name: storedName, typeCode: "x", payload, compress: false }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: pyzName, typeCode: 2, body: icon }], + }), + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + const member = snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "pkg/data.bin", + ); + const resource = snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "PYZ.pyz/res/icon.bin", + ); + expect(member?.format_metadata).toMatchObject({ source_name: storedName }); + expect(resource?.format_metadata).toMatchObject({ source_name: pyzName }); + const directory = await createTestTempDirectory("rea-pyinstaller-out-"); + const output = join(directory, "extracted"); + await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + path, + snapshot, + ); + expect(await readFile(join(output, "pkg", "data.bin"))).toEqual(payload); + expect(await readFile(join(output, "PYZ.pyz", "res", "icon.bin"))).toEqual( + icon, + ); +}); + +it("rejects a stored member whose declared size disagrees with its bytes", async () => { + const path = await writeArchive( + "stored-size", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "python39.dll", + members: [ + { + name: "data.bin", + typeCode: "x", + payload: Buffer.from("abcd"), + compress: false, + declaredUncompressedSize: 0, + }, + ], + }), + ); + await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({ + reason: "integrity", + message: expect.stringContaining( + "Stored CArchive member size disagrees with its table entry", + ), + }); +}); + +it("keeps compressed member spans on the physical extent or unknown", async () => { + const payload = Buffer.alloc(10_000, 0x61); + const path = await writeArchive( + "compressed-span", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "python39.dll", + members: [ + { name: "payload.bin", typeCode: "x", payload, compress: true }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "app", typeCode: 0, body: payload }], + }), + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + const fileSize = (await stat(path)).size; + expect(fileSize).toBeLessThan(payload.length); + for (const logicalPath of ["payload.bin", "PYZ.pyz/app.pyc"]) { + const occurrence = snapshot.occurrences.find( + (candidate) => candidate.logical_path === logicalPath, + ); + if (logicalPath === "payload.bin") { + expect(occurrence?.declared_size).toBeGreaterThan(fileSize); + expect(occurrence?.compressed_size ?? fileSize).toBeLessThan( + occurrence?.declared_size ?? 0, + ); + } else { + expect(occurrence?.declared_size).toBeNull(); + expect( + snapshot.nodes.find( + (node) => node.artifact_id === occurrence?.artifact_id, + )?.size, + ).toBeGreaterThan(fileSize); + } + const span = occurrence?.source_location; + if (span === null || span === undefined) { + expect(span).toBeNull(); + continue; + } + expect(span.length).toBe(occurrence?.compressed_size); + expect(span.offset + span.length).toBeLessThanOrEqual(fileSize); + } + for (const occurrence of snapshot.occurrences) { + const span = occurrence.source_location; + if (span === null) continue; + expect(span.offset + span.length).toBeLessThanOrEqual(fileSize); + } +}); + +it("reconstructs each PYZ image's bytecode with that image's own magic", async () => { + const python314Magic = Buffer.from("2b0e0d0a", "hex"); + const first = Buffer.from("first-module"); + const second = Buffer.from("second-module"); + const path = await writeArchive( + "two-pyz", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 314, + pythonLibrary: "libpython3.14.dylib", + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "first", typeCode: 0, body: first }], + }), + }, + { + name: "second.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: python314Magic, + modules: [{ name: "second", typeCode: 0, body: second }], + }), + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + const withHeader = (magic: Buffer, body: Buffer): Buffer => + Buffer.concat([reconstructedPycHeader(magic), body]); + const firstBytes = withHeader(PYTHON_39_MAGIC, first); + const secondBytes = withHeader(python314Magic, second); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(firstBytes)), + ).toMatchObject({ size: firstBytes.length }); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(secondBytes)), + ).toMatchObject({ size: secondBytes.length }); + expect( + snapshot.nodes.find( + (node) => node.sha256 === sha256(withHeader(PYTHON_39_MAGIC, second)), + ), + ).toBeUndefined(); +}); + +it("opens selected PYZ modules lazily and permits repeat reads without retaining their expanded bytes", async () => { + const good = Buffer.from("selected module"); + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "damaged", typeCode: 0, body: Buffer.from("damaged module") }, + { name: "selected", typeCode: 0, body: good }, + ], + }); + image[12] = 0; // Damage the first zlib header without changing the valid PYZ table. + const path = await writeArchive( + "lazy-pyz", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "PYZ.pyz", typeCode: "z", payload: image, compress: false }, + ], + }), + ); + const reader = new PyInstallerArtifactReader(path); + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + const selected = entries.find( + (entry) => entry.path === "PYZ.pyz/selected.pyc", + ); + const damaged = entries.find( + (entry) => entry.path === "PYZ.pyz/damaged.pyc", + ); + if (selected === undefined || damaged === undefined) + throw new Error("Missing PYZ entries"); + expect(selected.declaredSize).toBeNull(); + for (let attempt = 0; attempt < 2; attempt += 1) + expect(await buffer(await reader.open(selected))).toEqual(derived(good)); + await expect(buffer(await reader.open(damaged))).rejects.toMatchObject({ + reason: "format", + }); + } finally { + await reader.close(); + } +}); + +it.each([false, true])( + "lists a large file-backed PYZ and reads a selected module (outer compression: %s)", + async (compress) => { + const original = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "selected", typeCode: 0, body: Buffer.from("selected module") }, + ], + }); + const toc = original.readUInt32BE(8); + // Unreferenced image padding is legal and must not become retained module bytes. + const image = Buffer.concat([ + original.subarray(0, toc), + Buffer.alloc(64 * 1024 * 1024), + original.subarray(toc), + ]); + image.writeUInt32BE(toc + 64 * 1024 * 1024, 8); + const path = await writeArchive( + "large-pyz", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [{ name: "PYZ.pyz", typeCode: "z", payload: image, compress }], + }), + ); + const reader = new PyInstallerArtifactReader(path); + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + const selected = entries.find( + (entry) => entry.path === "PYZ.pyz/selected.pyc", + ); + if (selected === undefined) throw new Error("Missing selected module"); + expect(await buffer(await reader.open(selected))).toEqual( + derived(Buffer.from("selected module")), + ); + } finally { + await reader.close(); + } + }, +); + +it("uses known PYZ magic for separate CArchive scripts when the marshal table stays opaque", async () => { + const script = Buffer.from("entry script"); + const path = await writeArchive( + "opaque-table", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "hello", typeCode: "s", payload: script, compress: true }, + { + name: "PYZ.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [], + toc: Buffer.from([0x4e]), + }), + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + const opaque = snapshot.occurrences.find( + (entry) => entry.logical_path === "PYZ.pyz", + ); + expect(opaque).toMatchObject({ entry_kind: "file" }); + expect(opaque?.limitations.join(" ")).toContain("left opaque"); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(derived(script))), + ).toBeDefined(); + const output = join( + await createTestTempDirectory("rea-opaque-script-"), + "extracted", + ); + await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + path, + snapshot, + ); + expect(await readFile(join(output, "hello"))).toEqual(derived(script)); +}); + +it("rejects duplicate CArchive member names before projecting PYZ children", async () => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "child", typeCode: 0, body: Buffer.from("child") }], + }); + const path = await writeArchive( + "duplicate-names", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "PYZ.pyz", typeCode: "z", payload: image, compress: false }, + { + name: "PYZ.pyz", + typeCode: "x", + payload: Buffer.alloc(0), + compress: false, + }, + ], + }), + ); + await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({ + reason: "integrity", + message: "Duplicate CArchive member name at table index 1", + }); +}); + +it("closes each spooled PYZ image after listing and reopens it for a requested module", async () => { + const bodies = [0, 1, 2].map((index) => Buffer.from(`module ${index}`)); + const path = await writeArchive( + "spooled-pyz", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: bodies.map((body, index) => ({ + name: `PYZ-${index}.pyz`, + typeCode: "z" as const, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "child", typeCode: 0, body }], + }), + compress: true, + })), + }), + ); + const closed = new Set(); + const originalClose = OwnedFileHandle.prototype.close; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + await originalClose.call(this); + closed.add(this); + }); + const rootHandle = await open(path, "r"); + const reader = new PyInstallerArtifactReader(path, { + handle: rootHandle, + initial: await rootHandle.stat(), + }); + try { + const iterator = reader.entries()[Symbol.asyncIterator](); + const first = await iterator.next(); + // Every spooled image is closed once its table is read, before any entry is yielded. + expect(closed.size).toBeGreaterThanOrEqual(bodies.length); + for (const owner of closed) expect(owner.handle.fd).toBe(-1); + const listedHandles = closed.size; + const entries = first.done === true ? [] : [first.value]; + for ( + let next = await iterator.next(); + next.done !== true; + next = await iterator.next() + ) + entries.push(next.value); + const selected = entries.find( + (entry) => entry.path === "PYZ-1.pyz/child.pyc", + ); + if (selected === undefined) throw new Error("Missing PYZ child"); + expect(await buffer(await reader.open(selected))).toEqual( + derived(bodies[1] as Buffer), + ); + await reader.close(); + expect(closed.size).toBe(listedHandles + 1); + for (const owner of closed) expect(owner.handle.fd).toBe(-1); + expect(rootHandle.fd).toBeGreaterThanOrEqual(0); + } finally { + try { + await reader.close(); + } finally { + close.mockRestore(); + await rootHandle.close(); + } + } +}); + +it.each([1, 2])( + "retains a private PYZ owner whose close fails at stage %s", + async (failedStage) => { + const path = await writeArchive( + "spool-close-owner", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }), + compress: true, + }, + ], + }), + ); + const originalClose = OwnedFileHandle.prototype.close; + const owners: OwnedFileHandle[] = []; + const closeFailure = new Error("spool close failed"); + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + owners.push(this); + if (owners.length === failedStage) throw closeFailure; + await originalClose.call(this); + }); + const reader = new PyInstallerArtifactReader(path); + try { + await expect( + reader.entries()[Symbol.asyncIterator]().next(), + ).rejects.toBe(closeFailure); + await reader.close(); + expect( + owners.filter((owner) => owner === owners[failedStage - 1]), + ).toHaveLength(2); + } finally { + close.mockRestore(); + await reader.close().catch(() => undefined); + } + }, +); diff --git a/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts b/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts new file mode 100644 index 0000000000..ae8b84204e --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts @@ -0,0 +1,673 @@ +import { withPyInstallerReader } from "../../../tests/fixtures/pyinstallerReader.js"; +import { join } from "node:path"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { createHash } from "node:crypto"; +import { writeFile, type FileHandle } from "node:fs/promises"; +import { buffer } from "node:stream/consumers"; +import { deflateSync } from "node:zlib"; +import { setImmediate } from "node:timers/promises"; +import { expect, it, vi } from "vitest"; +import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js"; +import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; +import { hashReadable } from "../ArtifactHash.js"; +import { + ArtifactReaderFailure, + type ArtifactEntry, +} from "../ArtifactReader.js"; +import { + cArchiveExecutable, + pyzArchive, + PYTHON_39_MAGIC, +} from "./CArchive.fixture.js"; + +const hash = (bytes: Buffer): string => + createHash("sha256").update(bytes).digest("hex"); +const archive = (body: Buffer) => + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [{ name: "data", typeCode: "x", payload: body, compress: false }], + }); +const members = async (reader: PyInstallerArtifactReader) => { + const entries: ArtifactEntry[] = []; + for await (const entry of reader.entries()) entries.push(entry); + return entries; +}; +const deferred = () => { + let resolve: () => void = () => undefined; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +}; + +it.each([false, true])( + "settles independent private and carrier closes and retries only failed owners (carrier failure: %s)", + async (failCarrier) => { + const path = join(await createTestTempDirectory("rea-pyi-cleanup-"), "app"); + await writeFile( + path, + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: ["one", "two"].map((name) => ({ + name: `${name}.pyz`, + typeCode: "z", + compress: true, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "app", typeCode: 0, body: Buffer.from(name) }], + }), + })), + }), + ); + const reader = new PyInstallerArtifactReader(path); + const entries = (await members(reader)).filter((entry) => + entry.path.endsWith("app.pyc"), + ); + expect(entries).toHaveLength(2); + const nativeClose = OwnedFileHandle.prototype.close; + const privateOwners = new Set(); + const attempts: OwnedFileHandle[] = []; + let collecting = true; + let failing = true; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + attempts.push(this); + if (collecting) privateOwners.add(this); + if (failing && (privateOwners.has(this) || failCarrier)) + throw new Error( + privateOwners.has(this) ? "private close EIO" : "carrier close EIO", + ); + await nativeClose.call(this); + }); + try { + for (const entry of entries) + await expect(buffer(await reader.open(entry))).rejects.toMatchObject({ + cleanup: { reason: "private close EIO" }, + }); + expect(privateOwners.size).toBe(2); + collecting = false; + attempts.length = 0; + const failure = await reader.close().then( + () => undefined, + (cause: unknown) => cause, + ); + expect(attempts).toHaveLength(3); + expect(new Set(attempts).size).toBe(3); + expect(failure).toMatchObject({ + cleanup: { + resources: expect.arrayContaining(failCarrier ? [path] : []), + }, + }); + expect( + (failure as ArtifactReaderFailure).cleanup?.resources, + ).toHaveLength(failCarrier ? 3 : 2); + expect((failure as ArtifactReaderFailure).cleanup?.reason).toContain( + "private close EIO", + ); + if (failCarrier) + expect((failure as ArtifactReaderFailure).cleanup?.reason).toContain( + "carrier close EIO", + ); + const carrier = attempts.find((owner) => !privateOwners.has(owner))!; + expect(carrier.handle.fd >= 0).toBe(failCarrier); + for (const owner of privateOwners) + expect(owner.handle.fd).toBeGreaterThanOrEqual(0); + failing = false; + attempts.length = 0; + await reader.close(); + expect(attempts).toHaveLength(failCarrier ? 3 : 2); + for (const owner of [...privateOwners, carrier]) + expect(owner.handle.fd).toBe(-1); + attempts.length = 0; + await reader.close(); + expect(attempts).toEqual([]); + } finally { + close.mockRestore(); + await reader.close(); + } + }, +); + +it("binds materialization to the inventoried digest and checks cached carrier freshness", async () => { + const bytes = archive(Buffer.from("first")); + await withPyInstallerReader(bytes, async (reader, _handle, path) => { + await expect(reader.prepareContainer("0".repeat(64))).rejects.toMatchObject( + { reason: "integrity" }, + ); + await reader.prepareContainer(hash(bytes)); + await reader.prepareContainer(hash(bytes)); + await writeFile(path, archive(Buffer.from("other"))); + await expect(reader.prepareContainer(hash(bytes))).rejects.toMatchObject({ + reason: "integrity", + }); + }); +}); + +it("bounds descriptor reads while hashing a selected stored member", async () => { + const payload = Buffer.alloc(2 * 1024 * 1024, 0x61); + await withPyInstallerReader(archive(payload), async (reader, handle) => { + const entry = (await members(reader))[0]!; + const read = handle.read.bind(handle); + let maximum = 0; + const spy = vi + .spyOn(handle, "read") + .mockImplementation((...args: unknown[]) => { + if (typeof args[2] === "number") maximum = Math.max(maximum, args[2]); + return Reflect.apply(read, handle, args); + }); + try { + const observed = await hashReadable(await reader.open(entry)); + expect(observed.sha256).toBe(hash(payload)); + expect(observed.bytes).toBe(payload.length); + expect(maximum).toBeGreaterThan(0); + expect(maximum).toBeLessThanOrEqual(64 * 1024); + } finally { + spy.mockRestore(); + } + }); +}); + +it.each([false, true])( + "joins pending native reads during cancellation/close (I/O failure: %s)", + async (fail) => { + await withPyInstallerReader( + archive(Buffer.alloc(256 * 1024, 0x61)), + async (reader, handle) => { + const entry = (await members(reader))[0]!; + const entered = deferred(), + release = deferred(); + const read = handle.read.bind(handle); + const io = Object.assign(new Error("native read failed"), { + code: "EIO", + }); + const caller = new Error("caller stopped reading"); + const controller = new AbortController(); + let held = false; + const spy = vi + .spyOn(handle, "read") + .mockImplementation(async (...args: unknown[]) => { + if (!held) { + held = true; + entered.resolve(); + await release.promise; + if (fail) throw io; + } + return Reflect.apply(read, handle, args); + }); + const consumed = reader + .open(entry, controller.signal) + .then((stream) => buffer(stream)); + void consumed.catch(() => undefined); + let closing: Promise | undefined; + try { + await entered.promise; + controller.abort(caller); + let closed = false; + closing = reader.close().then(() => { + closed = true; + }); + await setImmediate(); + expect(closed).toBe(false); + expect(handle.fd).toBeGreaterThanOrEqual(0); + release.resolve(); + await expect(consumed).rejects.toMatchObject({ + reason: fail ? "io" : "cancelled", + cause: fail ? io : caller, + }); + await closing; + expect(handle.fd).toBeGreaterThanOrEqual(0); + } finally { + release.resolve(); + await Promise.allSettled([ + consumed, + ...(closing === undefined ? [] : [closing]), + ]); + spy.mockRestore(); + } + }, + ); + }, +); + +it("preserves BOM characters in length-delimited UTF-8 fields", async () => { + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "\ufefflibpython.so", + members: [ + { + name: "\ufeffasset", + typeCode: "x", + payload: Buffer.from("data"), + compress: false, + }, + { + name: "PYZ.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "\ufeffmodule", typeCode: 0, body: Buffer.from("module") }, + ], + }), + }, + ], + }); + await withPyInstallerReader(bytes, async (reader) => { + const entries = await members(reader); + expect(reader.formatMetadata()).toMatchObject({ + python_library: "\ufefflibpython.so", + }); + expect(entries.map((entry) => entry.path)).toContain("\ufeffasset"); + expect( + entries.find((entry) => entry.path === "\ufeffasset")?.formatMetadata, + ).toMatchObject({ source_name: "\ufeffasset" }); + expect( + entries.find((entry) => entry.path === "PYZ.pyz/\ufeffmodule.pyc") + ?.formatMetadata, + ).toMatchObject({ source_name: "\ufeffmodule" }); + }); +}); + +it("keeps runtime options as declarations with literal names and distinct source indices", async () => { + const option = "pyi-runtime-tmpdir /tmp/../chosen"; + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: option, + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: option, + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: "data", + typeCode: "x", + payload: Buffer.from("data"), + compress: false, + }, + ], + }), + async (reader) => { + const entries = await members(reader); + const options = entries.filter((entry) => entry.kind === "metadata"); + expect(options.map((entry) => entry.path)).toEqual([ + "__pyinstaller__/runtime-options/0", + "__pyinstaller__/runtime-options/1", + ]); + expect(options.map((entry) => entry.formatMetadata)).toMatchObject([ + { source_name: option, index: 0 }, + { source_name: option, index: 1 }, + ]); + expect(reader.formatMetadata()).toMatchObject({ + runtime_option_count: 2, + entry_count: 3, + }); + await expect(reader.open(options[0]!)).rejects.toMatchObject({ + reason: "format", + }); + }, + ); +}); + +it("rejects metadata expansion before retaining every repeated long container path", async () => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: Array.from({ length: 3000 }, (_, index) => ({ + name: `module${String(index)}`, + typeCode: 0, + body: Buffer.from("x"), + })), + }); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "a".repeat(40000), + typeCode: "z", + payload: image, + compress: false, + }, + ], + }), + async (reader) => { + let failure: unknown; + try { + for await (const entry of reader.entries()) { + void entry; + } + } catch (cause: unknown) { + failure = cause; + } + expect(failure).toBeInstanceOf(ArtifactReaderFailure); + expect(failure).toMatchObject({ reason: "limit", boundary: "memory" }); + }, + ); +}); + +it.each([false, true])( + "validates compressed member bounds through the stream (truncated: %s)", + async (truncated) => { + const payload = Buffer.from("complete module payload"); + const encoded = deflateSync(payload); + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "data", + typeCode: "x", + payload, + compress: true, + declaredUncompressedSize: truncated ? payload.length : 4, + }, + ], + }); + if (truncated) + bytes.writeUInt32BE(encoded.length - 2, 64 + encoded.length + 8); + await withPyInstallerReader(bytes, async (reader) => { + const entry = (await members(reader))[0]!; + await expect(buffer(await reader.open(entry))).rejects.toMatchObject({ + reason: truncated ? "format" : "limit", + }); + }); + }, +); + +it("joins a late private-file close and retries its failed owner after cancellation", async () => { + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }), + compress: true, + }, + ], + }); + await withPyInstallerReader(bytes, async (reader, borrowed) => { + const entered = deferred(), + release = deferred(); + const nativeClose = OwnedFileHandle.prototype.close; + const failure = new Error("private final close failed"); + let owner: OwnedFileHandle | undefined; + let attempts = 0; + const spy = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + if (owner === undefined) { + owner = this; + entered.resolve(); + await release.promise; + throw failure; + } + if (this === owner) attempts += 1; + await nativeClose.call(this); + }); + const signal = new AbortController(); + const listing = reader + .entries(signal.signal) + [Symbol.asyncIterator]() + .next() + .catch((cause: unknown) => cause); + try { + await entered.promise; + let settled = false; + const closing = reader.close().then(() => { + settled = true; + }); + signal.abort(new Error("cancel during final cleanup")); + await setImmediate(); + expect(settled).toBe(false); + expect(owner?.handle.fd).toBeGreaterThanOrEqual(0); + release.resolve(); + expect(await listing).toBe(failure); + await closing; + expect(attempts).toBe(1); + expect(owner?.handle.fd).toBe(-1); + expect(borrowed.fd).toBeGreaterThanOrEqual(0); + } finally { + release.resolve(); + spy.mockRestore(); + await listing; + await reader.close(); + } + }); +}); + +for (const cancel of [false, true]) { + it.each([false, true])( + `awaits private member close before completion (cancel: ${cancel}, close failure: %s)`, + async (failClose) => { + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + compress: true, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "app", typeCode: 0, body: Buffer.from("body") }, + ], + }), + }, + ], + }); + await withPyInstallerReader(bytes, async (reader, borrowed) => { + const entry = (await members(reader)).find((item) => + item.path.endsWith("app.pyc"), + )!; + const entered = deferred(), + release = deferred(); + const signal = new AbortController(); + const failure = new Error("private member final close failed"); + const nativeClose = OwnedFileHandle.prototype.close; + let owner: OwnedFileHandle | undefined; + let attempts = 0; + const spy = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + if (owner === undefined) { + owner = this; + attempts += 1; + entered.resolve(); + await release.promise; + if (failClose) throw failure; + } else if (this === owner) attempts += 1; + await nativeClose.call(this); + }); + const output = await reader.open(entry, signal.signal); + const reading = buffer(output).then( + (value) => ({ ok: true as const, value }), + (cause: unknown) => ({ ok: false as const, cause }), + ); + try { + await entered.promise; + expect(output.readableEnded).toBe(false); + expect(owner?.handle.fd).toBeGreaterThanOrEqual(0); + if (cancel) { + signal.abort(new Error("cancel during member cleanup")); + await setImmediate(); + expect(output.destroyed).toBe(false); + } + release.resolve(); + const result = await reading; + if (failClose || cancel) { + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected read failure"); + expect(result.cause).toMatchObject( + failClose + ? { reason: "io", message: failure.message } + : { reason: "cancelled" }, + ); + if (failClose) expect(owner?.handle.fd).toBeGreaterThanOrEqual(0); + } else { + expect(result.ok).toBe(true); + if (!result.ok) throw result.cause; + expect(result.value.subarray(16)).toEqual(Buffer.from("body")); + } + await reader.close(); + expect(attempts).toBe(failClose ? 2 : 1); + expect(owner?.handle.fd).toBe(-1); + expect(borrowed.fd).toBeGreaterThanOrEqual(0); + } finally { + release.resolve(); + await reading; + spy.mockRestore(); + await reader.close(); + } + }); + }, + ); +} + +it("preserves a private-image read failure when its descriptor close also fails", async () => { + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "app", typeCode: 0, body: Buffer.from("body") }], + }), + compress: true, + }, + ], + }); + await withPyInstallerReader(bytes, async (reader, borrowed) => { + const entry = (await members(reader)).find((item) => + item.path.endsWith("app.pyc"), + )!; + const nativeClose = OwnedFileHandle.prototype.close; + const nativeRead = borrowed.read; + const io = Object.assign(new Error("private read EIO"), { code: "EIO" }); + let failedOwner: OwnedFileHandle | undefined; + const read = vi + .spyOn(Object.getPrototypeOf(borrowed), "read") + .mockImplementation(function (this: FileHandle, ...args: unknown[]) { + return this === borrowed + ? Reflect.apply(nativeRead, this, args) + : Promise.reject(io); + }); + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + if (this.handle !== borrowed) { + failedOwner = this; + throw new Error("private close EIO"); + } + await nativeClose.call(this); + }); + try { + await expect(buffer(await reader.open(entry))).rejects.toMatchObject({ + reason: "io", + cause: io, + }); + await expect(reader.close()).rejects.toMatchObject({ + cleanup: { reason: "private close EIO" }, + }); + expect(failedOwner?.handle.fd).toBeGreaterThanOrEqual(0); + } finally { + read.mockRestore(); + close.mockRestore(); + await reader.close(); + } + expect(failedOwner?.handle.fd).toBe(-1); + expect(borrowed.fd).toBeGreaterThanOrEqual(0); + }); +}); + +it.each([false, true])( + "shares owned carrier admission across callers and joins late admission (owner closes: %s)", + async (ownerCloses) => { + const bytes = archive(Buffer.from("shared admission")); + await withPyInstallerReader( + bytes, + async (_borrowedReader, borrowed, path) => { + const reader = new PyInstallerArtifactReader(path); + const entered = deferred(), + release = deferred(); + const handles = new Set(); + const stat = borrowed.stat; + const spy = vi + .spyOn(Object.getPrototypeOf(borrowed), "stat") + .mockImplementation(async function ( + this: FileHandle, + ...args: unknown[] + ) { + if (this !== borrowed) { + handles.add(this); + entered.resolve(); + await release.promise; + } + return Reflect.apply(stat, this, args); + }); + const caller = new AbortController(), + reason = new Error("one admission waiter leaves"); + const first = reader + .prepareContainer(hash(bytes), caller.signal) + .catch((cause: unknown) => cause); + const peer = reader + .prepareContainer(hash(bytes)) + .catch((cause: unknown) => cause); + let closing: Promise | undefined; + try { + await entered.promise; + await setImmediate(); + expect(handles.size).toBe(1); + caller.abort(reason); + if (ownerCloses) { + let closed = false; + closing = reader.close().then(() => { + closed = true; + }); + await setImmediate(); + expect(closed).toBe(false); + } + release.resolve(); + expect(await first).toMatchObject({ + reason: "cancelled", + cause: reason, + }); + if (ownerCloses) { + expect(await peer).toMatchObject({ reason: "cancelled" }); + await closing; + } else expect(await peer).toBeUndefined(); + await reader.close(); + for (const handle of handles) expect(handle.fd).toBe(-1); + } finally { + release.resolve(); + await Promise.allSettled([first, peer]); + spy.mockRestore(); + await reader.close(); + await closing; + for (const handle of handles) { + await handle.close(); + } + } + }, + ); + }, +); diff --git a/src/artifacts/pyinstaller/PyInstallerBytes.ts b/src/artifacts/pyinstaller/PyInstallerBytes.ts new file mode 100644 index 0000000000..4dc61eef1b --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerBytes.ts @@ -0,0 +1,60 @@ +import type { FileHandle } from "node:fs/promises"; +import { err, ok, type Result } from "../../domain/result.js"; +import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { abortIfNeeded } from "../ArtifactHash.js"; +import { + pyInstallerFailure, + type PyInstallerFailure, +} from "./PyInstallerFailure.js"; + +/** Fill one already-budgeted format range; only zero bytes establish EOF. */ +export const readPyInstallerBytes = async ( + handle: Pick, + input: { + readonly position: number; + readonly length: number; + readonly label: string; + readonly signal?: AbortSignal | undefined; + }, +): Promise> => { + abortIfNeeded(input.signal); + if ( + !Number.isSafeInteger(input.position) || + !Number.isSafeInteger(input.length) || + input.position < 0 || + input.length < 0 || + !Number.isSafeInteger(input.position + input.length) + ) + return err( + pyInstallerFailure( + "format", + `${input.label} has an invalid source range`, + ), + ); + const bytes = Buffer.alloc(input.length); + let offset = 0; + while (offset < bytes.length) { + abortIfNeeded(input.signal); + let count: number; + try { + const read = await handle.read( + bytes, + offset, + bytes.length - offset, + input.position + offset, + ); + count = read.bytesRead; + } catch (cause: unknown) { + throw new ArtifactReaderFailure( + "io", + `${input.label} could not be read`, + { cause }, + ); + } + abortIfNeeded(input.signal); + if (count === 0) + return err(pyInstallerFailure("format", `${input.label} is truncated`)); + offset += count; + } + return ok(bytes); +}; diff --git a/src/artifacts/pyinstaller/PyInstallerFailure.ts b/src/artifacts/pyinstaller/PyInstallerFailure.ts new file mode 100644 index 0000000000..8ea582a055 --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerFailure.ts @@ -0,0 +1,10 @@ +/** Typed failure from a pure PyInstaller parser before the reader adapts it. */ +export interface PyInstallerFailure { + readonly reason: "format" | "integrity" | "limit"; + readonly message: string; +} + +export const pyInstallerFailure = ( + reason: PyInstallerFailure["reason"], + message: string, +): PyInstallerFailure => ({ reason, message }); diff --git a/src/artifacts/pyinstaller/PyInstallerFileAdmission.ts b/src/artifacts/pyinstaller/PyInstallerFileAdmission.ts new file mode 100644 index 0000000000..a472459a43 --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerFileAdmission.ts @@ -0,0 +1,36 @@ +import { + NonRegularFileReadError, + RegularFileAdmissionFailure, + RegularFileOperationCancelled, +} from "../../filesystem/RegularFile.js"; +import { ArtifactReaderFailure } from "../ArtifactReader.js"; + +/** Both carrier and private-image admission preserve cancellation and cleanup. */ +export const pyInstallerFileAdmissionFailure = ( + path: string, + cause: unknown, +): ArtifactReaderFailure => { + if (cause instanceof ArtifactReaderFailure) return cause; + if (cause instanceof RegularFileOperationCancelled) + return new ArtifactReaderFailure( + "cancelled", + "PyInstaller file admission cancelled", + { cause: cause.reason }, + ); + if (cause instanceof RegularFileAdmissionFailure) + return ArtifactReaderFailure.withCleanup( + pyInstallerFileAdmissionFailure(path, cause.cause), + ArtifactReaderFailure.cleanupObservation(cause.cleanupCause, path), + ); + if (cause instanceof NonRegularFileReadError) + return new ArtifactReaderFailure( + "format", + `PyInstaller source is not a regular file: ${path}`, + { cause }, + ); + return new ArtifactReaderFailure( + "io", + "PyInstaller archive could not be opened", + { cause }, + ); +}; diff --git a/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts b/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts new file mode 100644 index 0000000000..08cfb4f884 --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts @@ -0,0 +1,1213 @@ +import { buffer } from "node:stream/consumers"; +import { readFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { deflateSync } from "node:zlib"; +import { expect, it, vi } from "vitest"; +import { withPyInstallerReader } from "../../../tests/fixtures/pyinstallerReader.js"; +import { + scanCanonicalArtifactInventory, + inventoryArtifact, + materializeArtifactInventory, +} from "../../../tests/fixtures/artifactInventory.js"; +import { + cArchiveExecutable, + pyzArchive, + PYTHON_39_MAGIC, +} from "./CArchive.fixture.js"; +import { readPyzArchive } from "./PyzArchive.js"; +import { createEvidence } from "../../domain/evidence.js"; +import { jsonValueSchema } from "../../domain/jsonValue.js"; +import { parseArtifactInventoryEvidence } from "../../domain/artifactInventoryEvidence.js"; +import { + ArtifactReaderFailure, + type ArtifactEntry, +} from "../ArtifactReader.js"; + +it("keeps compressed source locations unknown when stored and decoded lengths coincide", async () => { + const payload = Buffer.alloc(11, 0x61); + expect(deflateSync(payload).length).toBe(payload.length); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [{ name: "data", typeCode: "x", payload, compress: true }], + }), + async (_reader, _carrier, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + expect( + inventory.occurrences.find((row) => row.logical_path === "data"), + ).toMatchObject({ + hash_status: "verified", + source_location: null, + format_metadata: { compression_flag: 1 }, + }); + }, + ); +}); + +it("bounds an admitted drive-prefixed CArchive path before diagnostic serialization", async () => { + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "C:" + "x".repeat(10 * 1024 * 1024), + typeCode: "x", + payload: Buffer.from("x"), + compress: false, + }, + ], + }), + async (reader) => { + const failure = await reader + .entries() + [Symbol.asyncIterator]() + .next() + .then( + () => undefined, + (cause: unknown) => cause, + ); + expect(failure).toMatchObject({ reason: "path" }); + expect((failure as Error).message.length).toBeLessThan(1024); + }, + ); +}); + +it.each(["magic", "marshal"])( + "reuses the private bytes of an opaque outer-compressed PYZ (%s)", + async (kind) => { + const payload = pyzArchive({ + magic: kind === "magic" ? Buffer.alloc(4) : PYTHON_39_MAGIC, + modules: [], + ...(kind === "marshal" ? { toc: Buffer.from([0x4e]) } : {}), + }); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [{ name: "PYZ.pyz", typeCode: "z", payload, compress: true }], + }), + async (reader, carrier) => { + const entries: ArtifactEntry[] = []; + for await (const entry of reader.entries()) entries.push(entry); + const entry = entries[0]!; + expect(entry.kind).toBe("file"); + const read = vi + .spyOn(carrier, "read") + .mockRejectedValue( + new Error("opaque image must not be reinflated from carrier"), + ); + try { + for (let run = 0; run < 2; run++) + expect(await buffer(await reader.open(entry))).toEqual(payload); + } finally { + read.mockRestore(); + } + }, + ); + }, +); + +it.each([true, false])( + "authenticates mixed-kind duplicate pages in any merge order (option first: %s)", + async (optionFirst) => { + const logical = `__pyinstaller__/runtime-options/${optionFirst ? 0 : 1}`; + const option = { + name: "pyi-option", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }; + const file = { + name: logical, + typeCode: "x", + payload: Buffer.from("first"), + compress: false, + }; + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + ...(optionFirst ? [option, file] : [file, option]), + { + ...file, + name: logical.replaceAll("/", "\\"), + payload: Buffer.from("second"), + }, + ], + }), + async (_reader, _handle, path) => { + const inventory = await inventoryArtifact(path); + const group = inventory.occurrences.filter( + (row) => row.logical_path === logical, + ); + expect(group).toHaveLength(3); + const pages = group.map((row, index) => + createEvidence( + { + path, + sha256: inventory.manifest.root_sha256, + format: "pyinstaller", + }, + { + id: "rea-artifact-graph", + name: "artifact inventory", + version: "1", + }, + { + operation: "inventory_artifact", + parameters: { page: index }, + confidence: "observed", + authority: "shipped-artifact", + result: jsonValueSchema.parse({ + ...inventory, + nodes: index === 0 ? inventory.nodes : [], + edges: index === 0 ? inventory.edges : [], + occurrences: + index === 0 + ? [ + ...inventory.occurrences.filter( + (item) => item.logical_path !== logical, + ), + row, + ] + : [row], + }), + }, + ), + ); + for (const order of [ + [0, 1, 2], + [0, 2, 1], + [1, 0, 2], + [1, 2, 0], + [2, 0, 1], + [2, 1, 0], + ]) { + const merged = parseArtifactInventoryEvidence( + order.map((index) => pages[index]), + ).inventory; + expect(merged.complete).toBe(true); + expect(merged.manifest).toEqual(inventory.manifest); + expect(merged.occurrences).toEqual(inventory.occurrences); + } + }, + ); + }, +); + +it.each(["duplicate", "size", "offset", "overlap"])( + "bounds %s diagnostics for large nondeclaration names", + async (mode) => { + const name = "member".repeat(200_000); + const first = { + name: mode === "overlap" ? name + "-a" : name, + typeCode: "x", + payload: Buffer.from("body"), + compress: false, + ...(mode === "size" ? { declaredUncompressedSize: 5 } : {}), + }; + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: + mode === "duplicate" || mode === "overlap" + ? [ + first, + { ...first, name: mode === "duplicate" ? name : name + "-b" }, + ] + : [first], + }); + const toc = 64 + bytes.readUInt32BE(bytes.length - 88 + 12); + if (mode === "offset") + bytes.writeUInt32BE( + bytes.readUInt32BE(bytes.length - 88 + 12) + 1, + toc + 4, + ); + if (mode === "overlap") + bytes.writeUInt32BE(0, toc + bytes.readUInt32BE(toc) + 4); + await withPyInstallerReader(bytes, async (_reader, _handle, path) => { + const failure: unknown = await scanCanonicalArtifactInventory(path).then( + () => undefined, + (cause: unknown) => cause, + ); + expect(failure).toBeInstanceOf(ArtifactReaderFailure); + if (!(failure instanceof ArtifactReaderFailure)) + throw new Error("Expected parser failure"); + expect(failure.reason).toBe(mode === "offset" ? "format" : "integrity"); + expect(failure.message.length).toBeLessThan(128); + expect(failure.message).toContain( + `table index ${mode === "duplicate" || mode === "overlap" ? 1 : 0}`, + ); + }); + }, +); + +it.each(["empty", "span"])( + "bounds PYZ %s diagnostics for a maximum-size marshal name", + async (mode) => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "n".repeat(65_536), typeCode: 0, body: Buffer.from("body") }, + ], + }); + image.writeInt32LE( + mode === "empty" ? 0 : image.length + 1, + image.length - 4, + ); + await withPyInstallerReader(image, async (_reader, handle) => { + const parsed = await readPyzArchive(handle, 0, image.length); + const message = !parsed.ok + ? parsed.error.message + : parsed.value.kind === "opaque" + ? parsed.value.limitation + : undefined; + expect(message).toBeDefined(); + expect(message?.length).toBeLessThan(256); + expect(message).toContain("table index 0"); + }); + }, +); + +it("keeps malformed-declaration diagnostics bounded while identifying the TOC row", async () => { + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "option".repeat(200_000), + typeCode: "o", + payload: Buffer.from("unexpected"), + compress: false, + }, + ], + }), + async (_reader, _handle, path) => { + const failure: unknown = await scanCanonicalArtifactInventory(path).then( + () => undefined, + (cause: unknown) => cause, + ); + expect(failure).toBeInstanceOf(ArtifactReaderFailure); + if (!(failure instanceof ArtifactReaderFailure)) + throw new Error("Expected a parser failure"); + expect(failure.reason).toBe("integrity"); + expect(failure.message.length).toBeLessThan(128); + expect(failure.message).toContain("table index 0"); + }, + ); +}); + +for (const typeCode of ["o", "d"] as const) { + it.each(["stored", "compressed", "flag-only"])( + `rejects local payload/compression on ${typeCode} declarations (%s)`, + async (mode) => { + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: typeCode === "d" ? "../owner:member" : "pyi-option", + typeCode, + payload: + mode === "flag-only" + ? Buffer.alloc(0) + : Buffer.from("local payload"), + compress: mode === "compressed", + }, + ], + }); + if (mode === "flag-only") { + const archiveStart = 64; + const toc = bytes.readUInt32BE(bytes.length - 88 + 12); + bytes[archiveStart + toc + 16] = 1; + } + await withPyInstallerReader(bytes, async (_reader, _handle, path) => { + await expect( + scanCanonicalArtifactInventory(path), + ).rejects.toMatchObject({ + reason: "integrity", + message: expect.stringContaining("declaration"), + }); + }); + }, + ); + it(`does not claim a destination case collision for a ${typeCode} query path`, async () => { + const logical = `__pyinstaller__/${typeCode === "o" ? "runtime-options" : "dependencies"}/0`; + const realPath = logical.toUpperCase(); + const payload = Buffer.from("real file"); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: typeCode === "d" ? "../owner:member" : "pyi-option", + typeCode, + payload: Buffer.alloc(0), + compress: false, + }, + { name: realPath, typeCode: "x", payload, compress: false }, + ], + }), + async (_reader, _handle, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + expect( + [ + ...inventory.limitations, + ...inventory.occurrences.flatMap((row) => row.limitations), + ].some((note) => note.includes("case-insensitive destination")), + ).toBe(false); + const output = join(dirname(path), "extracted"); + await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + path, + inventory, + ); + expect(await readFile(join(output, realPath))).toEqual(payload); + }, + ); + }); +} + +it("keeps genuine destination collisions on files without annotating same-path metadata", async () => { + const logical = "__pyinstaller__/runtime-options/0"; + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "pyi-option", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, + ...[logical, logical.toUpperCase()].map((name) => ({ + name, + typeCode: "x", + payload: Buffer.from(name), + compress: false, + })), + ], + }), + async (_reader, _handle, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + expect( + inventory.limitations.some((note) => + note.includes("case-insensitive destination"), + ), + ).toBe(true); + const metadata = inventory.occurrences.find( + (row) => row.entry_kind === "metadata", + ); + expect( + metadata?.limitations.some((note) => + note.includes("case-insensitive destination"), + ), + ).toBe(false); + expect( + inventory.occurrences + .filter( + (row) => row.entry_kind === "file" && row.logical_path !== ".", + ) + .every((row) => + row.limitations.some((note) => + note.includes("case-insensitive destination"), + ), + ), + ).toBe(true); + }, + ); +}); + +it("anchors declarations to their source container rather than a same-named directory", async () => { + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "../owner:member", + typeCode: "d", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: "__pyinstaller__", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "dependencies", typeCode: 3, body: Buffer.alloc(0) }, + ], + }), + }, + ], + }), + async (_reader, _handle, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + const root = inventory.occurrences.find( + (row) => row.logical_path === ".", + ); + const directory = inventory.occurrences.find( + (row) => row.logical_path === "__pyinstaller__/dependencies", + ); + const metadata = inventory.occurrences.find( + (row) => row.entry_kind === "metadata", + ); + expect(directory).toBeDefined(); + expect(root).toBeDefined(); + expect(metadata).toBeDefined(); + expect(metadata?.parent_occurrence_id).toBe(root?.occurrence_id); + expect(metadata?.parent_occurrence_id).not.toBe(directory?.occurrence_id); + }, + ); +}); + +it("resolves a unique source-map file despite a metadata query-path alias", async () => { + const logical = "__pyinstaller__/runtime-options/0"; + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "pyi-option", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: logical, + typeCode: "x", + payload: Buffer.from("let value = 1;"), + compress: false, + }, + { + name: logical + ".map", + typeCode: "x", + payload: Buffer.from( + JSON.stringify({ + version: 3, + sources: ["source.js"], + names: [], + mappings: "AAAA", + }), + ), + compress: false, + }, + ], + }), + async (_reader, _handle, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + const source = inventory.occurrences.find( + (row) => row.logical_path === logical && row.entry_kind === "file", + ); + const map = inventory.occurrences.find( + (row) => row.logical_path === logical + ".map", + ); + expect(source?.artifact_id).toBeTruthy(); + expect(map).toBeDefined(); + expect( + inventory.edges.find( + (edge) => edge.occurrence_id === map?.occurrence_id, + ), + ).toMatchObject({ + relation: "maps-source", + parent_artifact_id: source?.artifact_id, + }); + expect( + map?.limitations.some((note) => note.includes("multiple occurrences")), + ).toBe(false); + }, + ); +}); + +it.each(["x", "s", "empty-module"])( + "does not treat a same-named %s as a crypto-key module", + async (kind) => { + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "pyimod00_crypto_key", + typeCode: kind === "empty-module" ? "m" : kind, + payload: + kind === "empty-module" + ? Buffer.alloc(0) + : Buffer.from("ordinary content"), + compress: false, + }, + { + name: "PYZ.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { + name: "app", + typeCode: 0, + body: Buffer.from("readable module"), + }, + ], + }), + }, + ], + }), + async (_reader, _handle, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + expect( + inventory.occurrences.find((row) => row.logical_path === ".") + ?.format_metadata, + ).toMatchObject({ encryption_marker_present: false }); + expect( + inventory.occurrences.find( + (row) => row.logical_path === "PYZ.pyz/app.pyc", + ), + ).toMatchObject({ + encrypted: false, + hash_status: "verified", + format_metadata: { encryption_source: null }, + }); + const output = join(dirname(path), "extracted"); + await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + path, + inventory, + ); + expect( + (await readFile(join(output, "PYZ.pyz/app.pyc"))).subarray(16), + ).toEqual(Buffer.from("readable module")); + }, + ); + }, +); + +it.each([ + "module-in-toc", + "module-crossing-toc", + "namespace-in-toc", + "namespace-past-image", +])("rejects PYZ positions outside the payload region (%s)", async (kind) => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { + name: "app", + typeCode: kind.startsWith("namespace") ? 3 : 0, + body: Buffer.from("payload"), + }, + ], + }); + const toc = image.readUInt32BE(8); + if (kind === "module-crossing-toc") + image.writeInt32LE(toc - 12 + 1, image.length - 4); + else + image.writeInt32LE( + kind === "namespace-past-image" ? image.length + 1 : toc + 1, + image.length - 9, + ); + await withPyInstallerReader(image, async (_reader, handle) => { + await expect( + readPyzArchive(handle, 0, image.length), + ).resolves.toMatchObject({ ok: false, error: { reason: "integrity" } }); + }); +}); + +it("retains legacy namespace payload extents without materializing them", async () => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { + name: "ns", + typeCode: 3, + legacyNamespacePayload: true, + body: Buffer.from("legacy namespace code object"), + }, + ], + }); + const length = image.readInt32LE(image.length - 4); + expect(length).toBeGreaterThan(0); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "PYZ.pyz", typeCode: "z", compress: false, payload: image }, + ], + }), + async (_reader, _handle, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + expect( + inventory.occurrences.find((row) => row.logical_path === "PYZ.pyz/ns"), + ).toMatchObject({ + entry_kind: "directory", + declared_size: 0, + compressed_size: length, + format_metadata: { + relative_offset: 12, + type_code: 3, + compression_flag: 1, + }, + limitations: expect.arrayContaining([ + expect.stringContaining("not materialized"), + ]), + }); + const result = await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: join(dirname(path), "out"), + environment: process.env, + integrityPolicy: "fail", + }, + path, + inventory, + ); + expect(result.artifacts).toEqual([]); + }, + ); +}); + +it.each(["magic", "marshal"])( + "retains unknown traversal coverage for an opaque PYZ %s", + async (kind) => { + const known = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "known", typeCode: 0, body: Buffer.from("known") }], + }); + const opaque = pyzArchive({ + magic: kind === "magic" ? Buffer.alloc(4) : PYTHON_39_MAGIC, + modules: [], + ...(kind === "marshal" ? { toc: Buffer.from([0x4e]) } : {}), + }); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "known.pyz", typeCode: "z", payload: known, compress: false }, + { + name: "opaque.pyz", + typeCode: "z", + payload: opaque, + compress: false, + }, + ], + }), + async (reader, _root, path) => { + for (let run = 0; run < 2; run++) { + const observed: unknown[] = []; + const listed: ArtifactEntry[] = []; + for await (const entry of reader.entries(undefined, { + onCoverage: (coverage) => observed.push(coverage), + })) + listed.push(entry); + expect(listed).toHaveLength(3); + expect(observed).toEqual([ + expect.objectContaining({ status: "unknown" }), + ]); + } + const inventory = await scanCanonicalArtifactInventory(path); + expect(inventory.coverage).toMatchObject({ status: "unknown" }); + expect(inventory.limitations.join(" ")).toContain("opaque"); + expect( + inventory.occurrences.find( + (row) => row.logical_path === "known.pyz/known.pyc", + ), + ).toMatchObject({ hash_status: "verified" }); + expect( + inventory.occurrences.find((row) => row.logical_path === "opaque.pyz") + ?.format_metadata, + ).toMatchObject({ pyz_status: "opaque" }); + }, + ); + }, +); + +it.each([false, true])( + "derives namespace encryption from stored payload and the marker (%s)", + async (marker) => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { + name: "legacy", + typeCode: 3, + legacyNamespacePayload: true, + body: Buffer.from("declared namespace payload"), + }, + { name: "empty", typeCode: 3, body: Buffer.alloc(0) }, + ], + }); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + ...(marker + ? [ + { + name: "pyimod00_crypto_key", + typeCode: "m", + payload: Buffer.from("key module"), + compress: false, + }, + ] + : []), + { name: "PYZ.pyz", typeCode: "z", payload: image, compress: false }, + ], + }), + async (_reader, _root, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + expect( + inventory.occurrences.find( + (row) => row.logical_path === "PYZ.pyz/legacy", + ), + ).toMatchObject({ + entry_kind: "directory", + encrypted: marker, + compressed_size: expect.any(Number), + format_metadata: { + encryption_source: marker ? "carchive-marker" : null, + }, + }); + expect( + inventory.occurrences.find( + (row) => row.logical_path === "PYZ.pyz/empty", + ), + ).toMatchObject({ + entry_kind: "directory", + encrypted: false, + compressed_size: 0, + format_metadata: { encryption_source: null }, + }); + }, + ); + }, +); + +it.each([ + ["sequence", Buffer.from("5ba1860100", "hex")], + ["string", Buffer.from("7501000100", "hex")], + [ + "depth", + Buffer.concat([ + ...Array.from({ length: 18 }, () => Buffer.from([0x29, 1])), + Buffer.from("6900000000", "hex"), + ]), + ], + [ + "nodes", + Buffer.concat([ + Buffer.from("5ba0860100", "hex"), + Buffer.alloc(100_000 * 5).fill(Buffer.from("6900000000", "hex")), + ]), + ], +])( + "preserves the marshal %s resource failure through inventory", + async (_guard, toc) => { + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [], toc }), + }, + ], + }), + async (_reader, _handle, path) => { + await expect( + scanCanonicalArtifactInventory(path), + ).rejects.toMatchObject({ + reason: "limit", + message: expect.stringContaining("PYZ marshal"), + }); + }, + ); + }, +); + +it.each([1, 2])( + "rejects namespace data extending into the table (length %s)", + async (length) => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "ns", typeCode: 3, body: Buffer.alloc(0) }], + }); + // The fixture's last two marshal integers are the namespace offset/length. + image.writeInt32LE(12, image.length - 9); + image.writeInt32LE(length, image.length - 4); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "PYZ.pyz", typeCode: "z", compress: false, payload: image }, + ], + }), + async (_reader, _handle, path) => { + await expect( + scanCanonicalArtifactInventory(path), + ).rejects.toMatchObject({ + reason: "integrity", + message: expect.stringContaining("payload region"), + }); + }, + ); + }, +); + +it("retains the producer's nonzero namespace offset without inventing data", async () => { + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "ns", typeCode: 3, body: Buffer.alloc(0) }], + }), + }, + ], + }), + async (_reader, _handle, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + expect( + inventory.occurrences.find((row) => row.logical_path === "PYZ.pyz/ns"), + ).toMatchObject({ + entry_kind: "directory", + declared_size: 0, + compressed_size: 0, + format_metadata: { + relative_offset: 12, + type_code: 3, + compression_flag: 0, + }, + }); + }, + ); +}); + +it("retains multipackage references as distinct unresolved declarations", async () => { + const names = [ + "../owner/app:assets/x:y", + "../owner/app:assets/x:y", + "missing-colon", + ":empty-owner", + ]; + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: names.map((name) => ({ + name, + typeCode: "d", + payload: Buffer.alloc(0), + compress: false, + })), + }), + async (reader) => { + const entries: ArtifactEntry[] = []; + for await (const entry of reader.entries()) entries.push(entry); + expect(reader.formatMetadata()).toMatchObject({ + external_dependency_count: names.length, + runtime_option_count: 0, + }); + expect(new Set(entries.map((entry) => entry.adapterKey)).size).toBe( + names.length, + ); + for (const [index, entry] of entries.entries()) { + expect(entry.kind).toBe("metadata"); + expect(entry.path).toBe( + `__pyinstaller__/dependencies/${String(index)}`, + ); + expect(entry.formatMetadata).toMatchObject({ + source_name: names[index], + type_code: "d", + dependency_status: index < 2 ? "unresolved" : "invalid-reference", + dependency_archive: index < 2 ? "../owner/app" : null, + dependency_member: index < 2 ? "assets/x:y" : null, + }); + await expect(reader.open(entry)).rejects.toMatchObject({ + reason: "format", + }); + } + }, + ); +}); + +for (const container of ["carchive", "outer-pyz", "inner-pyz"] as const) { + it.each(["junk", "second-stream"])( + `rejects unused compressed input in ${container} (%s)`, + async (suffixKind) => { + const suffix = + suffixKind === "junk" + ? Buffer.from("trailing bytes") + : deflateSync(Buffer.from("another stream")); + const body = Buffer.from("first stream"); + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { + name: "app", + typeCode: 0, + body, + ...(container === "inner-pyz" ? { compressedSuffix: suffix } : {}), + }, + ], + }); + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: container === "carchive" ? "data" : "PYZ.pyz", + typeCode: container === "carchive" ? "x" : "z", + payload: container === "carchive" ? body : image, + compress: container !== "inner-pyz", + ...(container === "inner-pyz" ? {} : { compressedSuffix: suffix }), + }, + ], + }); + await withPyInstallerReader(bytes, async (reader) => { + const consume = async () => { + for await (const entry of reader.entries()) + if (entry.kind === "file") await buffer(await reader.open(entry)); + }; + await expect(consume()).rejects.toMatchObject({ + reason: "integrity", + message: expect.stringContaining("complete declared source span"), + }); + }); + }, + ); +} + +it("fills successful short reads through actual classification, TOCs and selected module hashing", async () => { + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "app", typeCode: 0, body: Buffer.from("module body") }, + ], + }), + }, + ], + }); + await withPyInstallerReader(bytes, async (_reader, handle, path) => { + const read = handle.read; + let shortReads = 0; + const spy = vi + .spyOn(Object.getPrototypeOf(handle), "read") + .mockImplementation(function (this: unknown, ...args: unknown[]) { + if (typeof args[2] === "number" && args[2] > 7) { + args[2] = 7; + shortReads += 1; + } + return Reflect.apply(read, this, args); + }); + try { + const inventory = await scanCanonicalArtifactInventory(path); + expect(inventory.manifest.root_format).toBe("pyinstaller"); + expect( + inventory.occurrences.find( + (row) => row.logical_path === "PYZ.pyz/app.pyc", + ), + ).toMatchObject({ + hash_status: "verified", + format_metadata: { source_name: "app" }, + }); + expect(shortReads).toBeGreaterThan(20); + } finally { + spy.mockRestore(); + } + }); +}); + +it("distinguishes actual EOF after a short PYZ header read", async () => { + const bytes = pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }); + await withPyInstallerReader(bytes, async (_reader, handle) => { + const read = handle.read.bind(handle); + let calls = 0; + const spy = vi + .spyOn(handle, "read") + .mockImplementation((...args: unknown[]) => { + calls += 1; + if (calls > 1) + return Promise.resolve({ bytesRead: 0, buffer: args[0] }); + args[2] = 3; + return Reflect.apply(read, handle, args); + }); + try { + await expect( + readPyzArchive(handle, 0, bytes.length), + ).resolves.toMatchObject({ + ok: false, + error: { reason: "format", message: expect.stringContaining("header") }, + }); + expect(calls).toBe(2); + } finally { + spy.mockRestore(); + } + }); +}); + +it("retains native I/O failure after a short read and simultaneous cancellation", async () => { + const bytes = pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }); + await withPyInstallerReader(bytes, async (_reader, handle) => { + const read = handle.read.bind(handle), + caller = new AbortController(); + const failure = Object.assign(new Error("late native EIO"), { + code: "EIO", + }); + let calls = 0; + const spy = vi + .spyOn(handle, "read") + .mockImplementation((...args: unknown[]) => { + calls += 1; + if (calls > 1) { + caller.abort(new Error("caller leaves")); + return Promise.reject(failure); + } + args[2] = 3; + return Reflect.apply(read, handle, args); + }); + try { + await expect( + readPyzArchive(handle, 0, bytes.length, caller.signal), + ).rejects.toMatchObject({ reason: "io", cause: failure }); + } finally { + spy.mockRestore(); + } + }); +}); + +it("rejects path-component expansion before normalization or consumer trie allocation", async () => { + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "a/".repeat(100_000) + "x", + typeCode: "x", + payload: Buffer.alloc(0), + compress: false, + }, + ], + }), + async (reader) => { + const list = async () => { + for await (const entry of reader.entries()) void entry; + }; + await expect(list()).rejects.toMatchObject({ + reason: "limit", + boundary: "memory", + message: expect.stringContaining("namespace budget"), + }); + }, + ); +}); + +it.each([true, false])( + "extracts a real file sharing a metadata query path (option first: %s)", + async (optionFirst) => { + const path = `__pyinstaller__/runtime-options/${optionFirst ? "0" : "1"}`; + const option = { + name: "pyi-option value", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }; + const file = { + name: path, + typeCode: "x", + payload: Buffer.from("real file payload"), + compress: false, + }; + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: optionFirst ? [option, file] : [file, option], + }), + async (_reader, _handle, source) => { + const inventory = await scanCanonicalArtifactInventory(source); + const output = join(dirname(source), "extracted"); + await materializeArtifactInventory( + { + inputPath: source, + inputFormat: "elf", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + source, + inventory, + ); + expect(await readFile(join(output, path))).toEqual(file.payload); + }, + ); + }, +); + +it("reserves possible case-annotation expansion along deep sibling paths", async () => { + const names = ["a/".repeat(4000) + "x", "A/".repeat(4000) + "X"]; + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: names.map((name) => ({ + name, + typeCode: "x", + payload: Buffer.alloc(0), + compress: false, + })), + }), + async (reader) => { + const list = async () => { + for await (const entry of reader.entries()) void entry; + }; + await expect(list()).rejects.toMatchObject({ + reason: "limit", + boundary: "memory", + }); + }, + ); +}); diff --git a/src/artifacts/pyinstaller/PyInstallerMemberStream.ts b/src/artifacts/pyinstaller/PyInstallerMemberStream.ts new file mode 100644 index 0000000000..480df2d002 --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerMemberStream.ts @@ -0,0 +1,147 @@ +import { Readable, Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { createInflate } from "node:zlib"; +import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { abortIfNeeded } from "../ArtifactHash.js"; + +interface MemberStreamOptions { + readonly sourceBytes: number; + readonly compressed: boolean; + readonly header: Buffer | undefined; + readonly expectedBytes: number | undefined; + readonly maxBytes: number; + readonly signal: AbortSignal; + readonly charge?: (bytes: number) => void; + readonly complete?: (bytes: number) => Promise; +} + +/** Backpressure bounds plaintext independently of the untrusted declared size. */ +export const pyInstallerMemberStream = ( + source: Readable, + options: MemberStreamOptions, +): { readonly output: Readable; readonly completion: Promise } => { + let bytes = 0; + let headerSent = false; + let completing = false; + const prependHeader = (): void => { + if (!headerSent && options.header !== undefined) + output.push(options.header); + headerSent = true; + }; + const output = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + try { + abortIfNeeded(options.signal); + bytes += chunk.length; + if (bytes > options.maxBytes) + throw new ArtifactReaderFailure( + "limit", + "PyInstaller member exceeds its plaintext budget", + ); + options.charge?.(chunk.length); + prependHeader(); + callback(null, chunk); + } catch (cause: unknown) { + callback(asError(cause)); + } + }, + flush(callback) { + completing = true; + void (async () => { + abortIfNeeded(options.signal); + if ( + inflate !== undefined && + inflate.bytesWritten !== options.sourceBytes + ) + throw new ArtifactReaderFailure( + "integrity", + "Compressed PyInstaller member did not consume its complete declared source span", + ); + if ( + options.expectedBytes !== undefined && + bytes !== options.expectedBytes + ) + throw new ArtifactReaderFailure( + "integrity", + "PyInstaller member size disagrees with its declaration or previous observation", + ); + await options.complete?.(bytes); + abortIfNeeded(options.signal); + prependHeader(); + })().then( + () => { + completing = false; + try { + abortIfNeeded(options.signal); + } catch (cause: unknown) { + callback(asError(cause)); + return; + } + callback(); + }, + (cause: unknown) => { + completing = false; + callback(asError(cause)); + }, + ); + }, + }); + // A consumer can retire its reader before attaching its stream error handler. + output.on("error", () => undefined); + const onAbort = (): void => { + // Verification and private-file cleanup own their accepted native I/O. + if (completing) return; + // The descriptor source owns cancellation until its native read settles. + // Retiring it downstream earlier would hide a native I/O rejection. + if (!source.readableEnded && !source.destroyed) return; + output.destroy( + new ArtifactReaderFailure( + "cancelled", + "PyInstaller member read cancelled", + { cause: options.signal.reason }, + ), + ); + }; + options.signal.addEventListener("abort", onAbort, { once: true }); + const onSourceFailure = (cause: unknown): void => { + output.destroy( + cause instanceof ArtifactReaderFailure + ? cause + : new ArtifactReaderFailure( + "io", + "PyInstaller member source could not be read", + { cause }, + ), + ); + }; + source.on("error", onSourceFailure); + const inflate = options.compressed ? createInflate() : undefined; + inflate?.on("error", (cause: unknown) => + output.destroy( + new ArtifactReaderFailure( + "format", + "Compressed PyInstaller member is not a complete zlib stream", + { cause }, + ), + ), + ); + const completion = ( + inflate === undefined + ? pipeline(source, output) + : pipeline(source, inflate, output) + ).finally(() => { + options.signal.removeEventListener("abort", onAbort); + source.removeListener("error", onSourceFailure); + }); + // Preserve rejection for the owning scope without an early unhandled promise. + void completion.catch(() => undefined); + if (options.signal.aborted) onAbort(); + return { output, completion }; +}; + +const asError = (cause: unknown): Error => + cause instanceof Error + ? cause + : new ArtifactReaderFailure("format", "PyInstaller member read failed", { + cause, + }); diff --git a/src/artifacts/pyinstaller/PyInstallerMetadata.ts b/src/artifacts/pyinstaller/PyInstallerMetadata.ts new file mode 100644 index 0000000000..8319721064 --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerMetadata.ts @@ -0,0 +1,179 @@ +import type { + PyInstallerMetadata, + PyInstallerEntryMetadata, +} from "../../domain/artifactFormatMetadata.js"; +import type { CArchiveEntry, PyInstallerCookie } from "./CArchiveLayout.js"; +import type { PyzArchive, PyzEntry } from "./PyzArchive.js"; +import { pythonVersionForMagic } from "./PythonBytecode.js"; + +const pythonVersion = (value: number) => + value >= 100 + ? { major: Math.floor(value / 100), minor: value % 100 } + : value >= 20 + ? { major: Math.floor(value / 10), minor: value % 10 } + : undefined; + +export const cookieMetadata = ( + cookie: PyInstallerCookie, + entries: readonly CArchiveEntry[], + encryptionMarker: boolean, +): PyInstallerMetadata => { + const version = pythonVersion(cookie.pythonVersion); + return { + kind: "pyinstaller", + layout_version: cookie.layout, + archive_start: cookie.archiveStart, + archive_size: cookie.packageLength, + cookie_offset: cookie.cookieOffset, + toc_offset: cookie.tocOffset, + toc_size: cookie.tocLength, + entry_count: entries.length, + runtime_option_count: entries.filter((entry) => entry.typeCode === "o") + .length, + external_dependency_count: entries.filter((entry) => entry.typeCode === "d") + .length, + python_version_raw: cookie.pythonVersion, + python_major: version?.major ?? null, + python_minor: version?.minor ?? null, + python_library: cookie.pythonLibrary, + encryption_marker_present: encryptionMarker, + }; +}; + +/** A sibling PYZ header is usable for scripts only when every image and the cookie agree. */ +export const scriptPythonMagic = ( + cookie: PyInstallerCookie, + archives: Iterable, +): Buffer | undefined => { + const version = pythonVersion(cookie.pythonVersion); + if (version === undefined) return undefined; + let selected: Buffer | undefined; + for (const archive of archives) { + const magic = archive.pythonMagic; + if ( + pythonVersionForMagic(magic) !== + `${String(version.major)}.${String(version.minor)}` || + (selected !== undefined && !selected.equals(magic)) + ) + return undefined; + selected = magic; + } + return selected; +}; + +const entryMetadata = ( + source: Pick< + PyInstallerEntryMetadata, + | "container" + | "container_name" + | "index" + | "source_name" + | "type_code" + | "relative_offset" + | "compression_flag" + >, + bytecode: boolean, + header: Buffer | undefined, +): PyInstallerEntryMetadata => ({ + kind: "pyinstaller-entry", + ...source, + bytecode_header: bytecode + ? header === undefined + ? "unknown" + : "derived" + : "not-bytecode", + bytecode_magic: header?.subarray(0, 4).toString("hex") ?? null, + pyz_status: "not-pyz", + directory_record_offset: null, + directory_record_size: null, + pyz_sha256: null, + pyz_python_magic: null, + pyz_python_version: null, + pyz_entry_count: null, + encryption_source: null, +}); + +export const cArchiveEntryMetadata = ( + entry: CArchiveEntry, + header?: Buffer, + archive?: PyzArchive, + encryptionMarker = false, +): PyInstallerEntryMetadata => ({ + ...entryMetadata( + { + container: "carchive", + container_name: null, + index: entry.index, + source_name: entry.name, + type_code: entry.typeCode, + relative_offset: entry.offset, + compression_flag: entry.compressionFlag === 1 ? 1 : 0, + }, + ["s", "m", "M"].includes(entry.typeCode), + header, + ), + ...(entry.typeCode === "d" ? dependencyMetadata(entry.name) : {}), + pyz_status: + archive?.kind === "entries" + ? "decoded" + : archive === undefined + ? "not-pyz" + : "opaque", + directory_record_offset: entry.tocRecordOffset, + directory_record_size: entry.tocRecordLength, + pyz_sha256: archive?.sha256 ?? null, + pyz_python_magic: archive?.pythonMagic.toString("hex") ?? null, + pyz_python_version: + archive === undefined + ? null + : (pythonVersionForMagic(archive.pythonMagic) ?? null), + pyz_entry_count: archive?.kind === "entries" ? archive.entries.length : null, + encryption_source: + archive !== undefined && encryptionMarker ? "carchive-marker" : null, +}); + +/** The bootloader splits at the first colon; neither part is a local read grant. */ +const dependencyMetadata = ( + source: string, +): Pick< + PyInstallerEntryMetadata, + "dependency_status" | "dependency_archive" | "dependency_member" +> => { + const colon = source.indexOf(":"); + const valid = colon > 0 && colon < source.length - 1; + return { + dependency_status: valid ? "unresolved" : "invalid-reference", + dependency_archive: valid ? source.slice(0, colon) : null, + dependency_member: valid ? source.slice(colon + 1) : null, + }; +}; + +/** Legacy writers encrypt nonempty namespace payloads as well as module payloads. */ +export const pyzEntryEncrypted = ( + entry: PyzEntry, + encryptionMarker: boolean, +): boolean => encryptionMarker && entry.length > 0; + +export const pyzEntryMetadata = ( + container: CArchiveEntry, + entry: PyzEntry, + header: Buffer | undefined, + encryptionMarker: boolean, +): PyInstallerEntryMetadata => ({ + ...entryMetadata( + { + container: "pyz", + container_name: container.name, + index: entry.index, + source_name: entry.name, + type_code: entry.typeCode, + relative_offset: entry.offset, + compression_flag: entry.compressionFlag, + }, + entry.typeCode === 0 || entry.typeCode === 1, + header, + ), + encryption_source: pyzEntryEncrypted(entry, encryptionMarker) + ? "carchive-marker" + : null, +}); diff --git a/src/artifacts/pyinstaller/PyInstallerPyzStore.ts b/src/artifacts/pyinstaller/PyInstallerPyzStore.ts new file mode 100644 index 0000000000..04d17b17c1 --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerPyzStore.ts @@ -0,0 +1,255 @@ +import type { FileHandle } from "node:fs/promises"; +import { lstat, mkdtemp, open, rm } from "node:fs/promises"; +import type { Stats } from "node:fs"; +import { Readable } from "node:stream"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; +import { + openRegularFile, + RegularFileAdmissionFailure, + sameRegularFileState, +} from "../../filesystem/RegularFile.js"; +import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { + ArtifactResourceScope, + type ArtifactResourceOwner, +} from "../ArtifactResourceScope.js"; +import { abortIfNeeded } from "../ArtifactHash.js"; +import { readArtifactFileChunks } from "../readArtifactFileChunks.js"; +import { pyInstallerMemberStream } from "./PyInstallerMemberStream.js"; +import type { CArchiveEntry } from "./CArchiveLayout.js"; +import type { PyInstallerMember } from "./CArchiveMembers.js"; +import { pyInstallerFileAdmissionFailure } from "./PyInstallerFileAdmission.js"; + +// Each outer-compressed image retains one inode and its identity until close. +// This is a per-reader filesystem capacity budget, independent of byte budgets. +const MAX_PRIVATE_PYZ_IMAGES = 128; + +interface StoredImage { + readonly path: string; + readonly initial: Stats; +} +export interface PyInstallerMemberSource { + readonly source: Readable; + readonly verify?: () => Promise; + readonly close?: (primary?: unknown) => Promise; +} + +/** The reader's operation scope drains work before retiring these private files. */ +export class PyInstallerPyzStore { + #root: string | undefined; + #serial = 0; + readonly #owners = new Map< + FileHandle, + Extract + >(); + readonly #cleanup = new ArtifactResourceScope(); + readonly #images = new Map(); + + reserveImages(count: number): void { + if (count > MAX_PRIVATE_PYZ_IMAGES) + throw new ArtifactReaderFailure( + "limit", + "PyInstaller compressed PYZ images exceed the 128-file private storage budget", + { boundary: "capacity" }, + ); + } + + async spool( + handle: FileHandle, + archiveStart: number, + entry: CArchiveEntry, + signal?: AbortSignal, + ): Promise<{ handle: FileHandle; start: number }> { + abortIfNeeded(signal); + if (entry.compressedSize === 0) + throw new ArtifactReaderFailure( + "format", + "Compressed PYZ image has no zlib bytes", + ); + this.#root ??= await mkdtemp(join(tmpdir(), "rea-pyz-")); + const path = join(this.#root, String(this.#serial++)); + const writer = await open(path, "wx", 0o600); + this.#retain(path, new OwnedFileHandle(writer)); + abortIfNeeded(signal); + const source = readArtifactFileChunks(handle, { + start: archiveStart + entry.offset, + end: archiveStart + entry.offset + entry.compressedSize - 1, + signal, + }); + const stream = pyInstallerMemberStream(source, { + sourceBytes: entry.compressedSize, + compressed: true, + header: undefined, + expectedBytes: entry.uncompressedSize, + maxBytes: entry.uncompressedSize, + signal: signal ?? new AbortController().signal, + }); + const copied = this.#writeImage(writer, stream.output, signal); + const outcomes = await Promise.allSettled([stream.completion, copied]); + // A failed native write can retire the upstream stream. Preserve that I/O + // failure instead of replacing it with a secondary premature-close error. + const copy = outcomes[1]; + if (copy?.status === "rejected") throw copy.reason; + const decoded = outcomes[0]; + if (decoded?.status === "rejected") throw decoded.reason; + await this.release(writer); + abortIfNeeded(signal); + const opened = await this.#admit(path, signal); + this.#images.set(opened, { path, initial: await opened.stat() }); + return { handle: opened, start: 0 }; + } + + async release(handle: FileHandle): Promise { + const owner = this.#owners.get(handle); + if (owner === undefined) return; + const outcome = await this.#cleanup.release(owner); + if (outcome.kind === "failed") throw outcome.cause; + this.#owners.delete(handle); + } + + async memberSource( + read: NonNullable, + root: FileHandle | undefined, + signal: AbortSignal, + ): Promise { + let handle = read.kind === "carchive" ? root : read.handle; + if (handle === undefined) + throw new ArtifactReaderFailure( + "unavailable", + "PyInstaller archive is closed", + ); + const image = read.kind === "pyz" ? this.#images.get(handle) : undefined; + let verify: (() => Promise) | undefined; + let close: ((primary?: unknown) => Promise) | undefined; + if (image !== undefined) { + handle = await this.#admit(image.path, signal); + const opened = handle; + verify = () => this.#verifyImage(opened, image); + close = async (primary) => { + try { + await this.release(opened); + } catch (cause: unknown) { + throw ArtifactReaderFailure.withCleanup( + primary ?? cause, + ArtifactReaderFailure.cleanupObservation(cause, image.path), + ); + } + }; + await verify(); + } + abortIfNeeded(signal); + const start = + read.kind === "carchive" + ? read.archiveStart + read.source.offset + : read.offset; + const length = + read.kind === "carchive" ? read.source.compressedSize : read.length; + const source = + length === 0 + ? Readable.from([]) + : readArtifactFileChunks(handle, { + start, + end: start + length - 1, + signal, + }); + source.on("error", () => undefined); + return { + source, + ...(verify === undefined ? {} : { verify }), + ...(close === undefined ? {} : { close }), + }; + } + + async close(): Promise { + await this.#cleanup.close(); + this.#owners.clear(); + if (this.#root !== undefined) { + try { + await rm(this.#root, { recursive: true, force: true }); + } catch (cause: unknown) { + throw ArtifactReaderFailure.withCleanup( + cause, + ArtifactReaderFailure.cleanupObservation(cause, this.#root), + ); + } + this.#root = undefined; + } + this.#images.clear(); + } + + async #admit(path: string, signal?: AbortSignal): Promise { + try { + const handle = await openRegularFile(path, { + symlinks: "reject", + signal, + }); + this.#retain(path, new OwnedFileHandle(handle)); + return handle; + } catch (cause: unknown) { + if (cause instanceof RegularFileAdmissionFailure) + this.#retain(path, cause.owner); + throw pyInstallerFileAdmissionFailure(path, cause); + } + } + + #retain(path: string, handle: OwnedFileHandle): void { + const owner = { kind: "file-handle" as const, resource: path, handle }; + this.#owners.set(handle.handle, owner); + this.#cleanup.retain(owner); + } + + async #writeImage( + handle: FileHandle, + source: Readable, + signal?: AbortSignal, + ): Promise { + let position = 0; + for await (const chunk of source as AsyncIterable) { + // Stop writing after cancellation; drain only already-produced bytes so + // the descriptor source can report its in-flight I/O outcome truthfully. + if (signal?.aborted) continue; + let offset = 0; + try { + while (offset < chunk.length) { + const { bytesWritten } = await handle.write( + chunk, + offset, + chunk.length - offset, + position, + ); + if (bytesWritten === 0) + throw new ArtifactReaderFailure( + "io", + "Private PYZ write made no progress", + ); + offset += bytesWritten; + position += bytesWritten; + if (signal?.aborted) break; + } + } catch (cause: unknown) { + const failure = + cause instanceof ArtifactReaderFailure + ? cause + : new ArtifactReaderFailure( + "io", + "Private PYZ image could not be written", + { cause }, + ); + source.destroy(failure); + throw failure; + } + } + abortIfNeeded(signal); + } + + async #verifyImage(handle: FileHandle, image: StoredImage): Promise { + const states = await Promise.all([handle.stat(), lstat(image.path)]); + if (states.some((state) => !sameRegularFileState(image.initial, state))) + throw new ArtifactReaderFailure( + "integrity", + "Private PYZ image changed after parsing", + ); + } +} diff --git a/src/artifacts/pyinstaller/PythonBytecode.ts b/src/artifacts/pyinstaller/PythonBytecode.ts new file mode 100644 index 0000000000..cefcc35174 --- /dev/null +++ b/src/artifacts/pyinstaller/PythonBytecode.ts @@ -0,0 +1,33 @@ +/** + * Final CPython bytecode magics for the minor versions whose 16-byte `.pyc` + * header this reader can reconstruct. Other magics leave the PYZ opaque. + * Each value is the four-byte little-endian magic, including the trailing CR LF. + */ +const PYTHON_BYTECODE_MAGICS: Readonly> = { + "420d0d0a": "3.7", + "550d0d0a": "3.8", + "610d0d0a": "3.9", + "6f0d0d0a": "3.10", + a70d0d0a: "3.11", + cb0d0d0a: "3.12", + f30d0d0a: "3.13", + "2b0e0d0a": "3.14", +}; + +/** Limitation recorded when a member's digest covers a derived `.pyc` header. */ +export const RECONSTRUCTED_PYTHON_HEADER_LIMITATION = + "Python bytecode header was reconstructed from the archive Python magic; this digest covers the derived file."; + +/** Dotted Python version for a supported bytecode magic, when one is known. */ +export const pythonVersionForMagic = (magic: Buffer): string | undefined => + PYTHON_BYTECODE_MAGICS[magic.toString("hex")]; + +/** + * 16-byte `.pyc` header used since Python 3.7: magic, zero flags, zero + * timestamp, and zero source size. The flags word selects the timestamp form. + */ +export const reconstructedPycHeader = (magic: Buffer): Buffer => { + const header = Buffer.alloc(16); + magic.copy(header, 0, 0, 4); + return header; +}; diff --git a/src/artifacts/pyinstaller/PyzArchive.test.ts b/src/artifacts/pyinstaller/PyzArchive.test.ts new file mode 100644 index 0000000000..6ea1fedf16 --- /dev/null +++ b/src/artifacts/pyinstaller/PyzArchive.test.ts @@ -0,0 +1,137 @@ +import { open, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; + +import { readPyzArchive, MAX_PYZ_TABLE_BYTES } from "./PyzArchive.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { PYTHON_39_MAGIC, pyzArchive } from "./CArchive.fixture.js"; + +it("lists modules from a supported PYZ and leaves an unknown magic opaque", async () => { + const body = Buffer.from("module"); + const parsed = await readImage( + pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "json.decoder", typeCode: 0, body }], + }), + ); + expect(parsed.ok).toBe(true); + if (!parsed.ok || parsed.value.kind !== "entries") return; + expect(parsed.value.pythonVersion).toBe("3.9"); + expect(parsed.value.entries).toMatchObject([ + { name: "json.decoder", typeCode: 0, package: false }, + ]); + + const opaque = await readImage( + pyzArchive({ + magic: Buffer.alloc(4), + modules: [{ name: "json.decoder", typeCode: 0, body }], + }), + ); + expect(opaque.ok).toBe(true); + if (!opaque.ok || opaque.value.kind !== "opaque") return; + expect(opaque.value.pythonMagic).toEqual(Buffer.alloc(4)); + expect(opaque.value.limitation).toContain("left opaque"); +}); + +it("leaves a PYZ opaque when the table uses an unsupported marshal type", async () => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [], + toc: Buffer.from([0x4e]), + }); + const parsed = await readImage(image); + expect(parsed.ok).toBe(true); + if (!parsed.ok || parsed.value.kind !== "opaque") return; + expect(parsed.value.limitation).toContain("left opaque"); + expect(parsed.value.pythonMagic).toEqual(PYTHON_39_MAGIC); +}); + +it("reports an entry that extends outside the PYZ as an integrity error", async () => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "missing", typeCode: 0, body: Buffer.from("x") }], + }); + image.writeInt32LE(100_000, image.length - 4); + const parsed = await readImage(image); + expect(parsed.ok).toBe(false); + if (parsed.ok) return; + expect(parsed.error.reason).toBe("integrity"); +}); + +it("rejects an oversized sparse PYZ table before allocating its bytes", async () => { + const root = await createTestTempDirectory("rea-pyz-table-budget-"); + const path = join(root, "image.pyz"); + const header = Buffer.alloc(12); + header.write("PYZ\0"); + PYTHON_39_MAGIC.copy(header, 4); + header.writeUInt32BE(12, 8); + await writeFile(path, header); + const handle = await open(path, "r+"); + try { + const size = 12 + MAX_PYZ_TABLE_BYTES + 1; + await handle.truncate(size); + expect(await readPyzArchive(handle, 0, size)).toMatchObject({ + ok: false, + error: { reason: "limit" }, + }); + } finally { + await handle.close(); + } +}); + +it("rejects a PYZ table whose back-references repeat one long row", async () => { + // One namespace row with a 16 KiB name, flagged for reference, then 999 + // back-references to it. The table is about 21 KiB but would expand to + // 16 MiB of entry names. + const name = Buffer.alloc(16 * 1024, 0x61); + const refs = 1_000; + const row = Buffer.concat([ + Buffer.from([0x28 | 0x80]), + int32(2), + Buffer.from([0x61]), + int32(name.length), + name, + Buffer.from([0x29, 3, 0x69]), + int32(3), + Buffer.from([0x69]), + int32(0), + Buffer.from([0x69]), + int32(0), + ]); + const reference = Buffer.concat([Buffer.from([0x72]), int32(0)]); + const table = Buffer.concat([ + Buffer.from([0x5b]), + int32(refs), + row, + ...Array.from({ length: refs - 1 }, () => reference), + ]); + const header = Buffer.alloc(12); + header.write("PYZ\0"); + PYTHON_39_MAGIC.copy(header, 4); + header.writeUInt32BE(12, 8); + const image = Buffer.concat([header, table]); + expect(image.length).toBeLessThan(32 * 1024); + + const parsed = await readImage(image); + expect(parsed).toMatchObject({ ok: false, error: { reason: "limit" } }); + if (parsed.ok) return; + expect(parsed.error.message).toContain("expand past"); +}); + +const int32 = (value: number): Buffer => { + const bytes = Buffer.alloc(4); + bytes.writeInt32LE(value); + return bytes; +}; + +const readImage = async (bytes: Buffer) => { + const root = await createTestTempDirectory("rea-pyz-file-"); + const path = join(root, "image.pyz"); + await writeFile(path, bytes); + const handle = await open(path, "r"); + try { + return await readPyzArchive(handle, 0, bytes.length); + } finally { + await handle.close(); + } +}; diff --git a/src/artifacts/pyinstaller/PyzArchive.ts b/src/artifacts/pyinstaller/PyzArchive.ts new file mode 100644 index 0000000000..c398cfae0f --- /dev/null +++ b/src/artifacts/pyinstaller/PyzArchive.ts @@ -0,0 +1,266 @@ +import type { FileHandle } from "node:fs/promises"; +import { hashReadable } from "../ArtifactHash.js"; +import { readArtifactFileChunks } from "../readArtifactFileChunks.js"; +import { readPyInstallerBytes } from "./PyInstallerBytes.js"; + +import { err, ok, type Result } from "../../domain/result.js"; +import { readMarshalSubset, type MarshalValue } from "./MarshalSubset.js"; +import { + pyInstallerFailure, + type PyInstallerFailure, +} from "./PyInstallerFailure.js"; +import { pythonVersionForMagic } from "./PythonBytecode.js"; + +const PYZ_MAGIC = Buffer.from("PYZ\0"); +const PYZ_HEADER_BYTES = 12; + +/** One PYZ table entry. Type codes follow the archive's module, package, data, and namespace values. */ +export interface PyzEntry { + readonly index: number; + readonly name: string; + readonly typeCode: number; + readonly offset: number; + readonly length: number; + /** PYZ compresses every nonempty payload, independently of its logical module type. */ + readonly compressionFlag: 0 | 1; + readonly package: boolean; + readonly namespace: boolean; +} + +export type PyzArchive = + | { + readonly kind: "entries"; + readonly pythonMagic: Buffer; + readonly pythonVersion: string; + readonly sha256: string; + readonly entries: readonly PyzEntry[]; + } + | { + readonly kind: "opaque"; + readonly pythonMagic: Buffer; + readonly sha256: string; + readonly limitation: string; + }; + +/** Bound the transient marshal table allocation independently of image expansion. */ +export const MAX_PYZ_TABLE_BYTES = 16 * 1024 * 1024; + +/** Parse only the header and table in memory; hash the complete file range in chunks. */ +export const readPyzArchive = async ( + handle: FileHandle, + start: number, + size: number, + signal?: AbortSignal, +): Promise> => { + if (size < PYZ_HEADER_BYTES) + return err( + pyInstallerFailure("format", "PYZ archive magic does not match"), + ); + const headerRead = await readPyInstallerBytes(handle, { + position: start, + length: PYZ_HEADER_BYTES, + label: "PYZ header", + signal, + }); + if (!headerRead.ok) return headerRead; + const header = headerRead.value; + const tocOffset = + header.length === PYZ_HEADER_BYTES ? header.readUInt32BE(8) : size; + const validOffset = tocOffset >= PYZ_HEADER_BYTES && tocOffset <= size; + const knownMagic = pythonVersionForMagic(header.subarray(4, 8)) !== undefined; + const tableSize = validOffset && knownMagic ? size - tocOffset : 0; + if (tableSize > MAX_PYZ_TABLE_BYTES) + return err( + pyInstallerFailure( + "limit", + "PYZ table exceeds the 16 MiB in-memory table budget", + ), + ); + const read = await readPyInstallerBytes(handle, { + position: start + tocOffset, + length: tableSize, + label: "PYZ table", + signal, + }); + if (!read.ok) return read; + const table = read.value; + const digest = await hashReadable( + readArtifactFileChunks(handle, { + start, + end: start + size - 1, + ...(signal === undefined ? {} : { signal }), + }), + signal, + ); + if (digest.bytes !== size) + return err( + pyInstallerFailure("format", "PYZ image extends outside the file"), + ); + return parsePyzParts(header, table, size, digest.sha256); +}; + +const parsePyzParts = ( + header: Buffer, + table: Buffer, + size: number, + sha256: string, +): Result => { + if ( + header.length < PYZ_HEADER_BYTES || + !header.subarray(0, 4).equals(PYZ_MAGIC) + ) + return err( + pyInstallerFailure("format", "PYZ archive magic does not match"), + ); + const pythonMagic = Buffer.from(header.subarray(4, 8)); + const pythonVersion = pythonVersionForMagic(pythonMagic); + if (pythonVersion === undefined) + return ok({ + kind: "opaque", + pythonMagic, + sha256, + limitation: `PYZ Python magic ${pythonMagic.toString("hex")} is not in the supported set; the member was left opaque.`, + }); + const tocOffset = header.readUInt32BE(8); + if (tocOffset < PYZ_HEADER_BYTES || tocOffset > size) + return err( + pyInstallerFailure( + "format", + "PYZ table of contents is outside the archive", + ), + ); + const marshalled = readMarshalSubset(table); + if (!marshalled.ok) { + if (marshalled.error.reason !== "format") return marshalled; + return ok({ + kind: "opaque", + pythonMagic, + sha256, + limitation: `PYZ table of contents could not be parsed (${marshalled.error.message}); the member was left opaque.`, + }); + } + const entries = pyzEntries(marshalled.value, tocOffset, table.length); + if (!entries.ok) + return entries.error.reason === "format" + ? ok({ + kind: "opaque", + pythonMagic, + sha256, + limitation: `${entries.error.message}; the member was left opaque.`, + }) + : entries; + return ok({ + kind: "entries", + pythonMagic, + pythonVersion, + sha256, + entries: entries.value, + }); +}; + +/** + * Marshal back-references let one stored row appear many times, so a small + * table can name far more bytes than it holds. Each genuine entry stores its + * own name inline, so the expanded names can never exceed the table itself. + */ +const pyzEntries = ( + value: MarshalValue, + payloadEnd: number, + tableBytes: number, +): Result => { + if (!Array.isArray(value)) + return err( + pyInstallerFailure("format", "PYZ table of contents is not a list"), + ); + const entries: PyzEntry[] = []; + let nameBytes = 0; + for (const item of value) { + const entry = onePyzEntry(item, payloadEnd, entries.length); + if (!entry.ok) return entry; + nameBytes += Buffer.byteLength(entry.value.name, "utf8"); + if (nameBytes > tableBytes) + return err( + pyInstallerFailure( + "limit", + `PYZ table entry names expand past the ${String(tableBytes)}-byte table that stores them`, + ), + ); + entries.push(entry.value); + } + return ok(entries); +}; + +const onePyzEntry = ( + item: MarshalValue, + payloadEnd: number, + index: number, +): Result => { + if (!Array.isArray(item) || item.length !== 2) + return err( + pyInstallerFailure( + "format", + "PYZ table entry is not a name and position pair", + ), + ); + const [name, position] = item; + if ( + typeof name !== "string" || + !Array.isArray(position) || + position.length !== 3 + ) + return err( + pyInstallerFailure( + "format", + "PYZ table entry is not a name, type, offset, and length", + ), + ); + const [typeCode, offset, length] = position; + if ( + !Number.isSafeInteger(typeCode) || + !Number.isSafeInteger(offset) || + !Number.isSafeInteger(length) + ) + return err( + pyInstallerFailure( + "format", + "PYZ table entry positions are not integers", + ), + ); + if (typeCode < 0 || typeCode > 3) + return err( + pyInstallerFailure( + "format", + `PYZ table entry type ${String(typeCode)} is not a known module, package, data, or namespace code`, + ), + ); + if (typeCode !== 3 && length === 0) + return err( + pyInstallerFailure( + "format", + `PYZ entry at table index ${String(index)} has no compressed bytes`, + ), + ); + if ( + length < 0 || + offset < 0 || + offset > payloadEnd || + length > payloadEnd - offset || + (length > 0 && offset < PYZ_HEADER_BYTES) + ) + return err( + pyInstallerFailure( + "integrity", + `PYZ entry at table index ${String(index)} extends outside the archive payload region`, + ), + ); + return ok({ + index, + name, + typeCode, + offset, + length, + compressionFlag: length === 0 ? 0 : 1, + package: typeCode === 1 || typeCode === 3, + namespace: typeCode === 3, + }); +}; diff --git a/src/domain/analysisSnapshot.ts b/src/domain/analysisSnapshot.ts index 92a49ea60c..eb975a835e 100644 --- a/src/domain/analysisSnapshot.ts +++ b/src/domain/analysisSnapshot.ts @@ -90,6 +90,7 @@ const subjectFormatSchema = z.enum([ "unknown", "javascript-bundle", "entitlements", + "pyinstaller", ]); // Snapshot entries retain the same raw-only architecture boundary as targets. const subjectSchema = z.discriminatedUnion("format", [ diff --git a/src/domain/analysisView/artifactInventoryView.ts b/src/domain/analysisView/artifactInventoryView.ts index 89e1c8ce95..730d200116 100644 --- a/src/domain/analysisView/artifactInventoryView.ts +++ b/src/domain/analysisView/artifactInventoryView.ts @@ -51,6 +51,9 @@ export const artifactInventoryViewRequestSchema = z.discriminatedUnion( collection: z.literal("occurrences"), occurrence_ids: z.array(prefixedDigestSchema("occ")).optional(), logical_paths: z.array(z.string()).optional(), + entry_kinds: z + .array(artifactOccurrenceSchema.shape.entry_kind) + .optional(), fields: z.array(artifactOccurrenceSchema.keyof()).optional(), }), z.strictObject({ @@ -104,6 +107,7 @@ export const artifactInventorySummarySchema = z.strictObject({ directory: z.number().int().nonnegative(), symlink: z.number().int().nonnegative(), slice: z.number().int().nonnegative(), + metadata: z.number().int().nonnegative().exactOptional(), }), source_complete: z.boolean(), inventory_evidence_ids: z.array(prefixedDigestSchema("ev")).min(1), @@ -140,6 +144,7 @@ interface InventorySource { index?: { byId: ReadonlyMap; byPath: ReadonlyMap; + byKind: ReadonlyMap; }; counts?: InventoryCounts; } @@ -241,7 +246,9 @@ export const projectArtifactInventoryView = ( ?.format_metadata ?? null; const rootMetadata = source.rootMetadata; const includeCoverage = - view.kind === "inventory" || inventory.manifest.root_format === "godot-pck"; + view.kind === "inventory" || + inventory.manifest.root_format === "godot-pck" || + inventory.manifest.root_format === "pyinstaller"; const sourceCoverage = includeCoverage ? { source_coverage: inventory.coverage } : {}; @@ -309,7 +316,15 @@ export const projectArtifactInventoryView = ( edges: inventory.edges.length, integrity_contradictions: inventory.integrityContradictions.length, }, - observed_entry_kind_counts: counts.kinds, + observed_entry_kind_counts: { + file: counts.kinds.file, + directory: counts.kinds.directory, + symlink: counts.kinds.symlink, + slice: counts.kinds.slice, + ...(counts.kinds.metadata === 0 + ? {} + : { metadata: counts.kinds.metadata }), + }, ...(includeCoverage ? { available_hash_status_counts: counts.hashes } : {}), @@ -406,7 +421,7 @@ const nodePage = ( const inventoryCounts = (source: InventorySource): InventoryCounts => { if (source.counts !== undefined) return source.counts; - const kinds = { file: 0, directory: 0, symlink: 0, slice: 0 }; + const kinds = { file: 0, directory: 0, symlink: 0, slice: 0, metadata: 0 }; const hashes = { verified: 0, mismatched: 0, @@ -435,27 +450,43 @@ const occurrenceIndices = ( source: InventorySource, view: OccurrenceView, ): number[] | undefined => { - if (view.occurrence_ids === undefined && view.logical_paths === undefined) + if ( + view.occurrence_ids === undefined && + view.logical_paths === undefined && + view.entry_kinds === undefined + ) return undefined; - if (view.occurrence_ids?.length === 0 || view.logical_paths?.length === 0) + if ( + view.occurrence_ids?.length === 0 || + view.logical_paths?.length === 0 || + view.entry_kinds?.length === 0 + ) return []; if (source.index === undefined) { const byId = new Map(); const byPath = new Map(); + const byKind = new Map(); for (const [index, row] of source.inventory.occurrences.entries()) { byId.set(row.occurrence_id, index); const existing = byPath.get(row.logical_path); if (existing === undefined) byPath.set(row.logical_path, [index]); else existing.push(index); + const kind = byKind.get(row.entry_kind); + if (kind === undefined) byKind.set(row.entry_kind, [index]); + else kind.push(index); } - source.index = { byId, byPath }; + source.index = { byId, byPath, byKind }; } const lookup = source.index; const paths = view.logical_paths === undefined ? undefined : new Set(view.logical_paths); const selected = view.occurrence_ids === undefined - ? [...(paths ?? [])].flatMap((path) => lookup.byPath.get(path) ?? []) + ? paths === undefined + ? [...new Set(view.entry_kinds)].flatMap( + (kind) => lookup.byKind.get(kind) ?? [], + ) + : [...paths].flatMap((path) => lookup.byPath.get(path) ?? []) : [...new Set(view.occurrence_ids)].flatMap((id) => { const index = lookup.byId.get(id); return index === undefined || @@ -464,7 +495,15 @@ const occurrenceIndices = ( ? [] : [index]; }); - return selected.sort((left, right) => left - right); + const kinds = + view.entry_kinds === undefined ? undefined : new Set(view.entry_kinds); + return selected + .filter( + (index) => + kinds === undefined || + kinds.has(source.inventory.occurrences[index]!.entry_kind), + ) + .sort((left, right) => left - right); }; const nodeIndices = ( diff --git a/src/domain/artifactFormatMetadata.ts b/src/domain/artifactFormatMetadata.ts index d31b936521..4ba7f6475d 100644 --- a/src/domain/artifactFormatMetadata.ts +++ b/src/domain/artifactFormatMetadata.ts @@ -29,9 +29,70 @@ export const godotPckEntryMetadataSchema = z.strictObject({ flags: uint32, md5_status: z.enum(["declared", "all-zero"]), }); + +/** Source cookie fields and derived directory counts without Python execution. */ +export const pyInstallerMetadataSchema = z.strictObject({ + kind: z.literal("pyinstaller"), + layout_version: z.enum(["2.0", "2.1"]), + archive_start: offset, + archive_size: offset, + cookie_offset: offset, + toc_offset: offset, + toc_size: offset, + entry_count: offset, + runtime_option_count: offset, + external_dependency_count: offset, + python_version_raw: uint32, + python_major: uint32.nullable(), + python_minor: uint32.nullable(), + python_library: z.string().nullable(), + encryption_marker_present: z.boolean(), +}); +export type PyInstallerMetadata = z.infer; + +/** Primitive facts preserve source spelling and keep metadata copies independent. */ +export const pyInstallerEntryMetadataSchema = z.strictObject({ + kind: z.literal("pyinstaller-entry"), + container: z.enum(["carchive", "pyz"]), + container_name: z.string().nullable(), + index: offset, + source_name: z.string(), + type_code: z.union([z.string().length(1), uint32]), + relative_offset: offset, + directory_record_offset: offset.nullable(), + directory_record_size: offset.nullable(), + compression_flag: z.union([z.literal(0), z.literal(1)]), + bytecode_header: z.enum(["not-bytecode", "unknown", "derived"]), + bytecode_magic: z + .string() + .regex(/^[0-9a-f]{8}$/u) + .nullable(), + pyz_status: z.enum(["not-pyz", "decoded", "opaque"]), + pyz_sha256: z + .string() + .regex(/^[0-9a-f]{64}$/u) + .nullable(), + pyz_python_magic: z + .string() + .regex(/^[0-9a-f]{8}$/u) + .nullable(), + pyz_python_version: z.string().nullable(), + pyz_entry_count: offset.nullable(), + encryption_source: z.literal("carchive-marker").nullable(), + dependency_status: z + .enum(["unresolved", "invalid-reference"]) + .exactOptional(), + dependency_archive: z.string().nullable().exactOptional(), + dependency_member: z.string().nullable().exactOptional(), +}); +export type PyInstallerEntryMetadata = z.infer< + typeof pyInstallerEntryMetadataSchema +>; export const artifactFormatMetadataSchema = z.discriminatedUnion("kind", [ godotPckMetadataSchema, godotPckEntryMetadataSchema, + pyInstallerMetadataSchema, + pyInstallerEntryMetadataSchema, ]); export type ArtifactFormatMetadata = z.infer< typeof artifactFormatMetadataSchema diff --git a/src/domain/artifactGraph.ts b/src/domain/artifactGraph.ts index 91c97d6930..61d0be7717 100644 --- a/src/domain/artifactGraph.ts +++ b/src/domain/artifactGraph.ts @@ -83,6 +83,7 @@ const artifactFormatSchema = z.enum([ "entitlements", "directory", "file", + "pyinstaller", "unknown", ]); @@ -138,7 +139,7 @@ export const artifactOccurrenceSchema = z.object({ artifact_id: artifactIdSchema.nullable(), parent_occurrence_id: occurrenceIdSchema.nullable(), logical_path: boundedRelativePathSchema, - entry_kind: z.enum(["file", "directory", "symlink", "slice"]), + entry_kind: z.enum(["file", "directory", "symlink", "slice", "metadata"]), artifact_kind: artifactKindSchema, artifact_format: artifactFormatSchema, declared_size: z diff --git a/src/domain/artifactInventoryEvidence.ts b/src/domain/artifactInventoryEvidence.ts index 395915c69a..c7fcc3ec13 100644 --- a/src/domain/artifactInventoryEvidence.ts +++ b/src/domain/artifactInventoryEvidence.ts @@ -348,19 +348,20 @@ const orderOccurrences = ( .flatMap(([path, group]) => { if (group.length < 2) return group; const ordinalById = new Map(); - const first = group[0]; - if (first === undefined) - throw new TypeError("Duplicate occurrence group is empty"); + const kinds = new Set(group.map(({ entry_kind }) => entry_kind)); for (let ordinal = 0; ordinal < group.length; ordinal++) { - ordinalById.set( - occurrenceIdForLocation({ - rootArtifactId, - logicalPath: path, - entryKind: first.entry_kind, - ...(ordinal === 0 ? {} : { duplicateOrdinal: ordinal }), - }), - ordinal, - ); + // Entry kinds come from a fixed schema enum, so work stays linear in + // the group size without changing or guessing producer identities. + for (const entryKind of kinds) + ordinalById.set( + occurrenceIdForLocation({ + rootArtifactId, + logicalPath: path, + entryKind, + ...(ordinal === 0 ? {} : { duplicateOrdinal: ordinal }), + }), + ordinal, + ); } return group.sort( (left, right) => diff --git a/src/domain/evidence.ts b/src/domain/evidence.ts index 92c9400b90..cf32dee6c1 100644 --- a/src/domain/evidence.ts +++ b/src/domain/evidence.ts @@ -70,6 +70,7 @@ const subjectFormatSchema = z.enum([ "mach-o-universal", "javascript-bundle", "entitlements", + "pyinstaller", ]); const subjectBaseSchema = z.strictObject({ name: z.string().min(1), diff --git a/tests/acceptance/pyinstallerArchive.test.ts b/tests/acceptance/pyinstallerArchive.test.ts new file mode 100644 index 0000000000..1039e14c1b --- /dev/null +++ b/tests/acceptance/pyinstallerArchive.test.ts @@ -0,0 +1,203 @@ +import { execFile } from "node:child_process"; +import { readFile, readdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import { expect, it } from "vitest"; + +import { + PYTHON_39_MAGIC, + cArchiveExecutable, + pyzArchive, +} from "../../src/artifacts/pyinstaller/CArchive.fixture.js"; +import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; +import { artifactExtractionResultSchema } from "../../src/domain/artifactGraph.js"; +import { parseEvidence } from "../../src/domain/evidence.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const cli = fileURLToPath(new URL("../../scripts/rea.mjs", import.meta.url)); + +const onefile = async ( + magic = PYTHON_39_MAGIC, + marker = false, + compressPyz = false, +): Promise => { + const directory = await createTestTempDirectory("rea-pyinstaller-cli-"); + const path = join(directory, "hello"); + await writeFile( + path, + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 314, + pythonLibrary: "libpython3.14.dylib", + trailer: Buffer.from("signature"), + members: [ + ...(marker + ? [ + { + name: "pyimod00_crypto_key", + typeCode: "m", + payload: Buffer.from("key"), + compress: false, + }, + ] + : []), + { + name: "pyi-contents-directory _internal", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: "../shared/app:assets/data.bin", + typeCode: "d", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: "hello", + typeCode: "s", + payload: Buffer.from("print('hi')"), + compress: true, + }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic, + modules: [ + { name: "app", typeCode: 0, body: Buffer.from("app-body") }, + ], + }), + compress: compressPyz, + }, + ], + }), + ); + return path; +}; + +it("lists CArchive members and the Python version through inspect-artifact", async () => { + const path = await onefile(); + const result = await execute(process.execPath, [ + cli, + "inspect-artifact", + path, + "--json", + ]); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(JSON.parse(result.stdout)).normalized_result, + ); + expect(inspection.subject.root_format).toBe("pyinstaller"); + expect(inspection.next_probes).toEqual([]); + const inventory = inspection.substeps[0]?.evidence.normalized_result; + expect(inventory?.manifest.root_format).toBe("pyinstaller"); + const root = inventory?.occurrences.find( + (occurrence) => occurrence.logical_path === ".", + ); + expect(root?.format_metadata).toMatchObject({ + kind: "pyinstaller", + python_version_raw: 314, + python_major: 3, + python_minor: 14, + runtime_option_count: 1, + external_dependency_count: 1, + }); + expect( + inventory?.occurrences.find( + (item) => + item.entry_kind === "metadata" && + item.format_metadata?.kind === "pyinstaller-entry" && + item.format_metadata.type_code === "o", + ), + ).toMatchObject({ + artifact_id: null, + format_metadata: { + source_name: "pyi-contents-directory _internal", + type_code: "o", + }, + }); + expect( + inventory?.occurrences.map((occurrence) => occurrence.logical_path), + ).toEqual(expect.arrayContaining(["hello", "PYZ.pyz", "PYZ.pyz/app.pyc"])); + expect( + inventory?.occurrences.find( + (row) => + row.format_metadata?.kind === "pyinstaller-entry" && + row.format_metadata.type_code === "d", + ), + ).toMatchObject({ + entry_kind: "metadata", + artifact_id: null, + format_metadata: { + dependency_archive: "../shared/app", + dependency_member: "assets/data.bin", + dependency_status: "unresolved", + }, + }); +}); + +it("extracts CArchive members through extract-artifact", async () => { + const path = await onefile(); + const result = await execute(process.execPath, [ + cli, + "extract-artifact", + path, + "--json", + ]); + const extracted = artifactExtractionResultSchema.parse( + parseEvidence(JSON.parse(result.stdout)).normalized_result, + ); + const hello = extracted.artifacts.find( + (artifact) => artifact.relative_path === "hello", + ); + expect(hello).toBeDefined(); + expect(await readFile(join(extracted.output_root, "hello"))).toHaveLength( + hello?.bytes_written ?? 0, + ); + expect( + await readFile(join(extracted.output_root, "PYZ.pyz", "app.pyc")), + ).toHaveLength(16 + Buffer.from("app-body").length); + expect(await readFile(join(extracted.output_root, "hello"))).toEqual( + Buffer.from("print('hi')"), + ); + expect(await readdir(extracted.output_root)).not.toContain("__pyinstaller__"); + expect( + extracted.artifacts.some((item) => + item.relative_path.includes("pyi-contents-directory"), + ), + ).toBe(false); +}); + +it("reports unknown traversal through the CLI for an opaque PYZ", async () => { + const path = await onefile(Buffer.alloc(4), true, true); + const result = await execute(process.execPath, [ + cli, + "inspect-artifact", + path, + "--json", + ]); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(JSON.parse(result.stdout)).normalized_result, + ); + const inventory = inspection.substeps[0]?.evidence.normalized_result; + expect(inventory?.coverage.status).toBe("unknown"); + expect(inventory?.limitations.join(" ")).toContain("opaque"); + const opaque = inventory?.occurrences.find( + (row) => row.logical_path === "PYZ.pyz", + ); + expect(opaque).toMatchObject({ + encrypted: false, + hash_status: "verified", + artifact_id: expect.any(String), + source_location: null, + }); + expect(opaque?.format_metadata).toMatchObject({ + pyz_python_magic: "00000000", + pyz_status: "opaque", + encryption_source: "carchive-marker", + compression_flag: 1, + }); +}); diff --git a/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts b/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts new file mode 100644 index 0000000000..8dfd22fcf8 --- /dev/null +++ b/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts @@ -0,0 +1,277 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { expect, it } from "vitest"; + +import { createArtifactBinarySession } from "../../../src/composition/binary.js"; +import { + PYTHON_39_MAGIC, + cArchiveExecutable, + pyzArchive, +} from "../../../src/artifacts/pyinstaller/CArchive.fixture.js"; +import { artifactInspectionResultSchema } from "../../../src/domain/artifactInspection.js"; +import { parseEvidence, type Evidence } from "../../../src/domain/evidence.js"; +import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const assertMetadataQueries = async ( + client: Client, + parent: Evidence, + expectedCoverage: "complete" | "unknown", +): Promise => { + const definition = (await client.listTools()).tools.find( + (tool) => tool.name === "inspect_analysis_view", + ); + expect(definition?.outputSchema).toBeDefined(); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const inputSchema: Record = definition!.inputSchema; + const outputSchema: Record = definition!.outputSchema!; + const input = ajv.compile(inputSchema); + const output = ajv.compile(outputSchema); + const ids: string[] = []; + const selected: unknown[] = []; + for (const offset of [0, 1, 2, 3]) { + const args = { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { + kind: "inventory", + collection: "occurrences", + entry_kinds: ["metadata"], + fields: ["format_metadata"], + offset, + limit: 1, + }, + }; + expect(input(args), JSON.stringify(input.errors)).toBe(true); + const reply = await client.callTool({ + name: "inspect_analysis_view", + arguments: args, + }); + expect(reply.isError).not.toBe(true); + expect(output(reply.structuredContent), JSON.stringify(output.errors)).toBe( + true, + ); + const evidence = parseEvidence(reply.structuredContent); + const view = analysisViewResultSchema.parse(evidence.normalized_result); + if (view.kind !== "inventory") throw new Error("Expected inventory page"); + expect(view.parent_evidence_id).toBe(parent.evidence_id); + expect(evidence.evidence_links).toContain(parent.evidence_id); + expect(view.coverage.total).toBe(3); + expect(view.source_coverage?.status).toBe(expectedCoverage); + if (offset < 3) { + expect(view.items).toHaveLength(1); + expect(view.items[0]).toMatchObject({ + artifact_id: null, + format_metadata: { kind: "pyinstaller-entry" }, + }); + expect(view.items[0]).not.toHaveProperty("logical_path"); + const row = view.items[0]; + if (row === undefined || !("occurrence_id" in row)) + throw new Error("Expected occurrence identity"); + ids.push(row.occurrence_id); + selected.push(row); + } else expect(view.items).toEqual([]); + } + expect(new Set(ids).size).toBe(3); + expect(selected).toEqual( + expect.arrayContaining([ + ...[0, 1].map((index) => + expect.objectContaining({ + format_metadata: expect.objectContaining({ + index, + type_code: "o", + source_name: "pyi-contents-directory _internal", + }), + }), + ), + expect.objectContaining({ + format_metadata: expect.objectContaining({ + type_code: "d", + source_name: "../shared/app:assets/data.bin", + dependency_status: "unresolved", + dependency_archive: "../shared/app", + dependency_member: "assets/data.bin", + }), + }), + ]), + ); +}; + +const writeExecutable = async (opaque: boolean): Promise => { + const directory = await createTestTempDirectory("rea-pyinstaller-mcp-"); + const path = join(directory, "hello"); + await writeFile( + path, + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 312, + pythonLibrary: "libpython3.12.so", + members: [ + { + name: "pyi-contents-directory _internal", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: "pyi-contents-directory _internal", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: "hello", + typeCode: "s", + payload: Buffer.from("print('hi')"), + compress: false, + }, + { + name: "../shared/app:assets/data.bin", + typeCode: "d", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: opaque ? Buffer.alloc(4) : PYTHON_39_MAGIC, + modules: [{ name: "app", typeCode: 0, body: Buffer.from("app") }], + }), + compress: opaque, + }, + ...(opaque + ? [ + { + name: "pyimod00_crypto_key", + typeCode: "m", + payload: Buffer.from("key"), + compress: false, + }, + ] + : []), + ], + }), + ); + return path; +}; + +it.each([false, true])( + "inventories a PyInstaller executable through inspect_artifact (opaque: %s)", + async (opaque) => { + const path = await writeExecutable(opaque); + const session = await createArtifactBinarySession(process.env); + const server = createServer({ kind: "session", session }); + const client = new Client({ name: "pyinstaller-mcp-test", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path }, + }); + expect(opened.isError).not.toBe(true); + const inspected = await client.callTool({ + name: "inspect_artifact", + arguments: {}, + }); + expect(inspected.isError).not.toBe(true); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(inspected.structuredContent).normalized_result, + ); + expect(inspection.subject.root_format).toBe("pyinstaller"); + if (!opaque) expect(inspection.next_probes).toEqual([]); + expect( + inspection.substeps[0]?.evidence.normalized_result.coverage.status, + ).toBe(opaque ? "unknown" : "complete"); + if (opaque) + expect( + inspection.substeps[0]?.evidence.normalized_result.occurrences.find( + (row) => row.logical_path === "PYZ.pyz", + ), + ).toMatchObject({ + encrypted: false, + hash_status: "verified", + artifact_id: expect.any(String), + format_metadata: { encryption_source: "carchive-marker" }, + }); + expect( + inspection.substeps[0]?.evidence.normalized_result.occurrences.map( + (occurrence) => occurrence.logical_path, + ), + ).toEqual( + expect.arrayContaining([ + "hello", + ...(opaque ? [] : ["PYZ.pyz/app.pyc"]), + ]), + ); + const parent = parseEvidence(inspected.structuredContent); + const compact = await client.callTool({ + name: "inspect_artifact", + arguments: { detail: "summary" }, + }); + expect(compact.isError).not.toBe(true); + const summary = parseEvidence( + compact.structuredContent, + ).normalized_result; + expect(summary).toMatchObject({ + kind: "summary", + summary: { + root_format_metadata: { + kind: "pyinstaller", + python_version_raw: 312, + external_dependency_count: 1, + }, + observed_entry_kind_counts: { metadata: 3 }, + source_coverage: { status: opaque ? "unknown" : "complete" }, + }, + }); + await assertMetadataQueries( + client, + parent, + opaque ? "unknown" : "complete", + ); + if (opaque) { + const missing = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { + kind: "retained-evidence", + evidence_id: parent.evidence_id, + }, + view: { + kind: "inventory", + collection: "occurrences", + logical_paths: ["PYZ.pyz/app.pyc"], + entry_kinds: ["file"], + fields: [], + offset: 0, + limit: 1, + }, + }, + }); + expect(missing.isError).not.toBe(true); + const page = analysisViewResultSchema.parse( + parseEvidence(missing.structuredContent).normalized_result, + ); + expect(page).toMatchObject({ + kind: "inventory", + items: [], + source_coverage: { status: "unknown" }, + }); + expect(page.unknowns).toContain( + "Producer traversal coverage is unknown; missing matches do not prove source absence.", + ); + } + } finally { + await client.close(); + await server.close(); + await session.close(); + } + }, +); diff --git a/tests/fixtures/pyinstallerReader.ts b/tests/fixtures/pyinstallerReader.ts new file mode 100644 index 0000000000..495ad9942c --- /dev/null +++ b/tests/fixtures/pyinstallerReader.ts @@ -0,0 +1,31 @@ +import { open, writeFile, type FileHandle } from "node:fs/promises"; +import { join } from "node:path"; +import { PyInstallerArtifactReader } from "../../src/artifacts/PyInstallerArtifactReader.js"; +import { createTestTempDirectory } from "./temporaryDirectory.js"; + +export const withPyInstallerReader = async ( + bytes: Buffer, + run: ( + reader: PyInstallerArtifactReader, + handle: FileHandle, + path: string, + ) => Promise, +) => { + const root = await createTestTempDirectory("rea-pyi-boundary-"); + const path = join(root, "app"); + await writeFile(path, bytes); + const handle = await open(path, "r"); + const reader = new PyInstallerArtifactReader(path, { + handle, + initial: await handle.stat(), + }); + try { + await run(reader, handle, path); + } finally { + try { + await reader.close(); + } finally { + await handle.close(); + } + } +};