diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts index 7b7f58afb0..e03dc1e620 100644 --- a/src/application/BinaryTargetResolver.ts +++ b/src/application/BinaryTargetResolver.ts @@ -41,12 +41,14 @@ import { import { hasGodotPckHeader } from "../domain/godotPckFormat.js"; import { hasZipSignature, + locateTrailingZipArchive, isZipPackageFormat, - zipPackageFormatForPath, + zipRootPackageFormat, } from "../domain/zipPackageFormat.js"; import { parseCodeViewData, + claimsExecutableHeader, parseExecutableHeader, parsePeRecord, peDebugDirectoryEntry, @@ -219,12 +221,32 @@ export const parseBinaryTarget = async ( hostArchitecture, signal, ); - if (!detected.ok) + if (!detected.ok) { + // Only bytes no other format claims may lead a located ZIP, so an + // executable with an appended archive keeps its executable route. + if ( + !(await claimsLeadingExecutableHeader( + handle, + metadata.size, + signal, + )) && + (await locateTrailingZipArchive(metadata.size, (position, length) => + readHandleRange(handle, position, length, signal), + )) !== undefined + ) + return ok({ + path, + sourcePath, + sha256: await sha256Handle(handle, path, metadata, signal), + kind: "archive", + format: zipRootPackageFormat(path), + }); return err( new BinaryTargetError(path, detected.error, { constraint: "file_format_rejected", }), ); + } return ok({ path, sourcePath, @@ -356,12 +378,8 @@ const detectArtifactFormat = async ( "WebAssembly v1 artifacts are not supported by the native opening route", { constraint: "wasm_requires_static_inspection" }, ); - if (hasZipSignature(magic.subarray(0, observed.bytesRead))) { - // The HarmonyOS App Pack suffix `.app` is a root-input concept; a nested - // ZIP member ending in `.app` keeps the generic zip classification. - if (lower.endsWith(".app")) return "app-pack"; - return zipPackageFormatForPath(lower) ?? "zip"; - } + if (hasZipSignature(magic.subarray(0, observed.bytesRead))) + return zipRootPackageFormat(lower); if (hasGodotPckHeader(magic.subarray(0, observed.bytesRead))) return "godot-pck"; // Binary plists, including keyed archives, often use other extensions. @@ -446,6 +464,31 @@ const isArchiveFormat = ( isZipPackageFormat(format) || ["asar", "godot-pck", "dmg", "pkg"].includes(format); +/** Whether the leading bytes begin any executable header family, valid or not. */ +const claimsLeadingExecutableHeader = async ( + handle: FileHandle, + size: number, + signal?: AbortSignal, +): Promise => + claimsExecutableHeader( + Buffer.from(await readHandleRange(handle, 0, Math.min(size, 4096), signal)), + size, + ); + +/** Read one bounded range, honoring cancellation around the host read. */ +const readHandleRange = async ( + handle: FileHandle, + position: number, + length: number, + signal?: AbortSignal, +): Promise => { + throwIfTargetResolutionCancelled(signal); + const bytes = Buffer.alloc(length); + const observed = await handle.read(bytes, 0, length, position); + throwIfTargetResolutionCancelled(signal); + return bytes.subarray(0, observed.bytesRead); +}; + const sha256Handle = async ( handle: FileHandle, path: string, diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.ts index c36d6e43df..0e20fe67cc 100644 --- a/src/artifacts/inventory/classify.ts +++ b/src/artifacts/inventory/classify.ts @@ -8,13 +8,14 @@ import type { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; import { containsPyInstallerArchive } from "../pyinstaller/CArchiveLayout.js"; import { classifyArtifactContent } from "./ArtifactGraphConstruction.js"; import { abortIfNeeded } from "../ArtifactHash.js"; +import { claimsExecutableHeader } from "../../domain/binaryTarget.js"; import { createArtifactClassificationProbe } from "../ArtifactClassificationProbe.js"; import type { ArtifactOccurrence } from "../../domain/artifactGraph.js"; import { hasZipSignature, + locateTrailingZipArchive, isZipPackageFormat, - zipPackageFormatForPath, - type ZipPackageFormat, + zipRootPackageFormat, } from "../../domain/zipPackageFormat.js"; import { ArtifactReaderFailure } from "../ArtifactReader.js"; import type { ArtifactResourceOwner } from "../ArtifactResourceScope.js"; @@ -188,6 +189,20 @@ const classifyRootFormat = async ( }, archiveBytes, ).format; + // Only bytes no other format claims may lead a located ZIP, so an + // executable with an appended archive keeps its executable classification. + if ( + (format === "file" || format === "unknown") && + !claimsExecutableHeader(evidence.prefix, archiveBytes) && + (await locateTrailingZipArchive(archiveBytes, async (position, length) => { + abortIfNeeded(signal); + const bytes = Buffer.alloc(length); + const read = await handle.read(bytes, 0, length, position); + abortIfNeeded(signal); + return bytes.subarray(0, read.bytesRead); + })) !== undefined + ) + return zipRootPackageFormat(lower); if (["elf", "pe", "mach-o", "mach-o-universal"].includes(format)) { const packaged = await containsPyInstallerArchive( handle, @@ -243,12 +258,6 @@ const readRootFatSliceHeaders = async ( return { header, sliceHeadersByOffset }; }; -/** App Pack is a verified ZIP root role; nested .app members retain their observed format. */ -const zipRootPackageFormat = (lower: string): ZipPackageFormat => - lower.endsWith(".app") - ? "app-pack" - : (zipPackageFormatForPath(lower) ?? "zip"); - const hasKolyTrailer = async ( handle: FileHandle, size: number, diff --git a/src/domain/binaryTarget.ts b/src/domain/binaryTarget.ts index 9b265f2786..406ff509ec 100644 --- a/src/domain/binaryTarget.ts +++ b/src/domain/binaryTarget.ts @@ -75,6 +75,22 @@ export type ExecutableMetadata = readonly codeView?: PeCodeViewRecord; }; +/** + * Whether leading bytes begin an executable header family that + * `parseExecutableHeader` interprets, whether or not that header is valid. + */ +export const claimsExecutableHeader = ( + bytes: Buffer, + fileSize = bytes.length, +): boolean => + (bytes.length >= 4 && + (bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46])) || + [0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes( + bytes.readUInt32BE(0), + ))) || + (bytes.length >= 2 && bytes[0] === 0x4d && bytes[1] === 0x5a) || + readFatContainerHeader(bytes, fileSize).state !== "not-fat"; + /** * Parse supported executable headers without I/O. The caller supplies the host * architecture used for deterministic FAT Mach-O slice selection. diff --git a/src/domain/zipPackageFormat.ts b/src/domain/zipPackageFormat.ts index 6d9fd4663b..26f8af7c16 100644 --- a/src/domain/zipPackageFormat.ts +++ b/src/domain/zipPackageFormat.ts @@ -26,6 +26,79 @@ export const hasZipSignature = (bytes: Uint8Array): boolean => (bytes[2] === 0x05 && bytes[3] === 0x06) || (bytes[2] === 0x07 && bytes[3] === 0x08)); +const END_OF_CENTRAL_DIRECTORY = 0x06054b50; +const CENTRAL_DIRECTORY_HEADER = 0x02014b50; +const LOCAL_FILE_HEADER = 0x04034b50; +const END_RECORD_BYTES = 22; +const MAXIMUM_COMMENT_BYTES = 0xffff; +const CENTRAL_HEADER_BYTES = 46; + +/** Positional read of at most `length` bytes; a short result means end of file. */ +export type PositionalRead = ( + position: number, + length: number, +) => Promise; + +/** + * Locate a single-disk ZIP whose end-of-central-directory record ends the + * file after leading non-ZIP bytes, such as a self-extractor or script stub. + * Its central directory and first local header must resolve, whether member + * offsets include the leading bytes or were written before them. Zip64, + * split and empty archives are not located. Returns the leading byte count. + */ +export const locateTrailingZipArchive = async ( + size: number, + read: PositionalRead, +): Promise<{ readonly leadingBytes: number } | undefined> => { + if (!Number.isSafeInteger(size) || size < END_RECORD_BYTES) return undefined; + const tailStart = Math.max( + 0, + size - END_RECORD_BYTES - MAXIMUM_COMMENT_BYTES, + ); + const tail = Buffer.from(await read(tailStart, size - tailStart)); + if (tail.length !== size - tailStart) return undefined; + // The record's comment must end exactly at end of file. + let record = -1; + for (let at = tail.length - END_RECORD_BYTES; at >= 0; at -= 1) + if ( + tail.readUInt32LE(at) === END_OF_CENTRAL_DIRECTORY && + at + END_RECORD_BYTES + tail.readUInt16LE(at + 20) === tail.length + ) { + record = at; + break; + } + if (record < 0) return undefined; + const entriesOnDisk = tail.readUInt16LE(record + 8); + const directoryBytes = tail.readUInt32LE(record + 12); + const declaredDirectory = tail.readUInt32LE(record + 16); + if ( + tail.readUInt16LE(record + 4) !== 0 || + tail.readUInt16LE(record + 6) !== 0 || + entriesOnDisk === 0 || + entriesOnDisk === 0xffff || + entriesOnDisk !== tail.readUInt16LE(record + 10) || + directoryBytes === 0xffffffff || + declaredDirectory === 0xffffffff + ) + return undefined; + const directory = tailStart + record - directoryBytes; + // Offsets written before leading bytes were prepended are shifted by them. + const shift = directory - declaredDirectory; + if (directory < 0 || shift < 0) return undefined; + const header = Buffer.from(await read(directory, CENTRAL_HEADER_BYTES)); + if ( + header.length !== CENTRAL_HEADER_BYTES || + header.readUInt32LE(0) !== CENTRAL_DIRECTORY_HEADER + ) + return undefined; + const firstLocal = header.readUInt32LE(42) + shift; + if (firstLocal === 0 || firstLocal >= directory) return undefined; + const local = Buffer.from(await read(firstLocal, 4)); + return local.length === 4 && local.readUInt32LE(0) === LOCAL_FILE_HEADER + ? { leadingBytes: firstLocal } + : undefined; +}; + /** * Classify a ZIP-backed package by its complete lower-cased path suffix. * @@ -52,6 +125,18 @@ export const zipPackageFormatForPath = ( return undefined; }; +/** + * Classify a verified ZIP root input by its path. The HarmonyOS App Pack + * suffix `.app` applies only to root inputs; a nested ZIP member ending in + * `.app` keeps its observed format. + */ +export const zipRootPackageFormat = (path: string): ZipPackageFormat => { + const lower = path.toLowerCase(); + return lower.endsWith(".app") + ? "app-pack" + : (zipPackageFormatForPath(lower) ?? "zip"); +}; + /** Inventory lists entries, not all bytes surrounding the central directory. */ export const ZIP_NON_ENTRY_LIMITATION = - "ZIP inventory lists central-directory entries only. Non-entry bytes, including an APK Signing Block before the central directory or a ZIP comment after it, are not occurrences."; + "ZIP inventory lists central-directory entries only. Non-entry bytes, including leading bytes before the first entry such as a self-extractor stub, an APK Signing Block before the central directory or a ZIP comment after it, are not occurrences."; diff --git a/tests/acceptance/artifactZipLeadingBytes.test.ts b/tests/acceptance/artifactZipLeadingBytes.test.ts new file mode 100644 index 0000000000..4e3add055e --- /dev/null +++ b/tests/acceptance/artifactZipLeadingBytes.test.ts @@ -0,0 +1,74 @@ +import { readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; +import { expect } from "vitest"; + +import { + artifactExtractionResultSchema, + artifactInventoryResultSchema, +} from "../../src/domain/artifactGraph.js"; +import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; +import { parseEvidence } from "../../src/domain/evidence.js"; +import { ZIP_NON_ENTRY_LIMITATION } from "../../src/domain/zipPackageFormat.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +cliTest( + "inspects and extracts a ZIP after an inert script stub", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-zip-script-stub-"); + const archivePath = join(root, "installer.sh"); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("a.txt", new TextReader("hello world\n")); + await writer.add("d/b.txt", new TextReader("x\n")); + // A self-extractor stub prepended to an unchanged archive. + await writeFile( + archivePath, + Buffer.concat([Buffer.from("#!/bin/sh\nexit 0\n"), await writer.close()]), + ); + + const inspected = await cli.run({ + arguments: ["inspect-artifact", archivePath, "--json"], + }); + expect(inspected.exitCode, inspected.stdout + inspected.stderr).toBe(0); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(inspected.json).normalized_result, + ); + expect(inspection.subject.root_format).toBe("zip"); + const inventoryEvidence = inspection.substeps[0]?.evidence; + if (inventoryEvidence === undefined) + throw new Error("CLI inspection did not retain ZIP inventory Evidence"); + const inventory = artifactInventoryResultSchema.parse( + inventoryEvidence.normalized_result, + ); + expect( + inventory.occurrences + .filter(({ entry_kind }) => entry_kind === "file") + .map(({ logical_path, hash_status }) => [logical_path, hash_status]), + ).toEqual([ + [".", "verified"], + ["a.txt", "verified"], + ["d/b.txt", "verified"], + ]); + expect(inventory.limitations).toContain(ZIP_NON_ENTRY_LIMITATION); + + const extracted = await cli.run({ + arguments: ["extract-artifact", archivePath, "--json"], + }); + expect(extracted.exitCode, extracted.stdout + extracted.stderr).toBe(0); + const extraction = artifactExtractionResultSchema.parse( + parseEvidence(extracted.json).normalized_result, + ); + try { + expect( + extraction.artifacts.map(({ relative_path }) => relative_path), + ).toEqual(["a.txt", "d/b.txt"]); + expect( + await readFile(join(extraction.output_root, "a.txt"), "utf8"), + ).toBe("hello world\n"); + } finally { + await rm(extraction.output_root, { force: true, recursive: true }); + } + }, +); diff --git a/tests/boundary/filesystem/artifactComparison.test.ts b/tests/boundary/filesystem/artifactComparison.test.ts index 1c3784efdd..b29d4aef6f 100644 --- a/tests/boundary/filesystem/artifactComparison.test.ts +++ b/tests/boundary/filesystem/artifactComparison.test.ts @@ -896,7 +896,7 @@ describe("ZIP non-entry coverage", () => { left.normalized_result, ); expect(inventory.limitations).toContain( - "ZIP inventory lists central-directory entries only. Non-entry bytes, including an APK Signing Block before the central directory or a ZIP comment after it, are not occurrences.", + "ZIP inventory lists central-directory entries only. Non-entry bytes, including leading bytes before the first entry such as a self-extractor stub, an APK Signing Block before the central directory or a ZIP comment after it, are not occurrences.", ); expect( inventory.occurrences.map(({ logical_path }) => logical_path).sort(), diff --git a/tests/boundary/filesystem/zipDetection.test.ts b/tests/boundary/filesystem/zipDetection.test.ts index f7c5d6f1bd..e5b7b09202 100644 --- a/tests/boundary/filesystem/zipDetection.test.ts +++ b/tests/boundary/filesystem/zipDetection.test.ts @@ -184,6 +184,166 @@ describe("nested container family", () => { ); }); +const SCRIPT_STUB = Buffer.from("#!/bin/sh\nexit 0\n"); + +const memberZip = async (): Promise => { + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("a.txt", new TextReader("hello world\n")); + await writer.add("d/b.txt", new TextReader("x\n")); + return Buffer.from(await writer.close()); +}; + +/** Prepend bytes; optionally shift stored offsets as `zip -A` does. */ +const withLeadingBytes = ( + zip: Buffer, + leading: Buffer, + adjustOffsets: boolean, +): Buffer => { + const bytes = Buffer.concat([leading, zip]); + if (!adjustOffsets) return bytes; + const end = bytes.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06])); + const directory = bytes.readUInt32LE(end + 16) + leading.length; + bytes.writeUInt32LE(directory, end + 16); + for (let at = directory; at < end;) { + bytes.writeUInt32LE(bytes.readUInt32LE(at + 42) + leading.length, at + 42); + at += + 46 + + bytes.readUInt16LE(at + 28) + + bytes.readUInt16LE(at + 30) + + bytes.readUInt16LE(at + 32); + } + return bytes; +}; + +/** Members by content-derived identity and verification state. */ +const members = async (path: string) => { + const inventory = await inventoryArtifact(path); + return ["a.txt", "d/b.txt"].map((member) => { + const occurrence = artifactOccurrenceAt(inventory, member); + return [member, occurrence.hash_status, occurrence.artifact_id]; + }); +}; + +describe("ZIP archives after leading bytes", () => { + it.each([ + { style: "offsets that include the leading bytes", adjust: true }, + { style: "offsets written before the bytes were prepended", adjust: false }, + ])( + "admits a ZIP after an unclaimed script stub with $style", + async ({ adjust }) => { + const root = await createTestTempDirectory("rea-zip-leading-"); + const plain = join(root, "plain.zip"); + const prefixed = join(root, "installer.bin"); + const zip = await memberZip(); + await writeFile(plain, zip); + await writeFile(prefixed, withLeadingBytes(zip, SCRIPT_STUB, adjust)); + + expect( + (await classifyAndHashRoot(prefixed, false, await lstat(prefixed))) + .format, + ).toBe("zip"); + expect(await parseBinaryTarget(prefixed)).toMatchObject({ + ok: true, + value: { kind: "archive", format: "zip" }, + }); + // Members are the same verified bytes as in the plain archive. + const expected = await members(plain); + expect(expected.map(([, status]) => status)).toEqual([ + "verified", + "verified", + ]); + expect(await members(prefixed)).toEqual(expected); + }, + ); + + it("keeps the root App Pack family for a ZIP after a script stub", async () => { + const root = await createTestTempDirectory("rea-zip-leading-app-pack-"); + const path = join(root, "Example.app"); + await writeFile( + path, + withLeadingBytes(await memberZip(), SCRIPT_STUB, false), + ); + expect( + (await classifyAndHashRoot(path, false, await lstat(path))).format, + ).toBe("app-pack"); + expect(await parseBinaryTarget(path)).toMatchObject({ + ok: true, + value: { kind: "archive", format: "app-pack" }, + }); + }); + + it.each([ + { family: "ELF", leading: Buffer.from([0x7f, 0x45, 0x4c, 0x46, 1, 1, 1]) }, + { + family: "MZ", + leading: Buffer.concat([Buffer.from("MZ"), Buffer.alloc(62)]), + }, + { + family: "Mach-O", + leading: Buffer.from([0xcf, 0xfa, 0xed, 0xfe, 0, 0, 0, 0]), + }, + ])( + "keeps a $family-led file with an appended ZIP out of the ZIP route", + async ({ leading }) => { + const root = await createTestTempDirectory("rea-zip-claimed-lead-"); + const path = join(root, "program"); + await writeFile(path, withLeadingBytes(await memberZip(), leading, true)); + expect( + (await classifyAndHashRoot(path, false, await lstat(path))).format, + ).not.toBe("zip"); + expect(await parseBinaryTarget(path)).not.toMatchObject({ + ok: true, + value: { format: "zip" }, + }); + }, + ); + + it.each([ + { + shape: "bytes after the end record", + mutate: (bytes: Buffer) => Buffer.concat([bytes, Buffer.from("junk")]), + }, + { + shape: "a corrupt central directory header", + mutate: (bytes: Buffer) => { + bytes.write("XXXX", bytes.indexOf(Buffer.from([0x50, 0x4b, 1, 2]))); + return bytes; + }, + }, + { + shape: "a Zip64 directory offset", + mutate: (bytes: Buffer) => { + const end = bytes.lastIndexOf(Buffer.from([0x50, 0x4b, 5, 6])); + bytes.writeUInt32LE(0xffffffff, end + 16); + return bytes; + }, + }, + { + shape: "an empty archive record", + mutate: (bytes: Buffer) => { + const end = bytes.lastIndexOf(Buffer.from([0x50, 0x4b, 5, 6])); + bytes.writeUInt16LE(0, end + 8); + bytes.writeUInt16LE(0, end + 10); + return bytes; + }, + }, + ])("does not locate a trailing ZIP with $shape", async ({ mutate }) => { + const root = await createTestTempDirectory("rea-zip-leading-negative-"); + const path = join(root, "input.bin"); + await writeFile( + path, + mutate(withLeadingBytes(await memberZip(), SCRIPT_STUB, true)), + ); + expect( + (await classifyAndHashRoot(path, false, await lstat(path))).format, + ).toBe("file"); + expect(await parseBinaryTarget(path)).toMatchObject({ + ok: false, + error: { _tag: "BinaryTargetError" }, + }); + }); +}); + describe("package signature byte identity", () => { it.each([ { extension: "pkg", signature: "xar!", offset: 0, size: 4 },