diff --git a/src/artifacts/ArtifactClassificationProbe.ts b/src/artifacts/ArtifactClassificationProbe.ts new file mode 100644 index 0000000000..bc661f3787 --- /dev/null +++ b/src/artifacts/ArtifactClassificationProbe.ts @@ -0,0 +1,89 @@ +import { + createAsarHeaderSignatureProbe, + type FinalAsarHeaderProbeObservation, +} from "./AsarHeader.js"; +import { + createXmlPlistSignatureProbe, + MAX_XML_PLIST_PROBE_BYTES, +} from "../domain/apple/appleArchiveFormat.js"; + +/** Initial bytes required by the other bounded format classifiers. */ +export const ARTIFACT_CLASSIFICATION_PREFIX_BYTES = 8_192; + +/** Retained bytes and independently parsed ASAR signature facts. */ +export interface ArtifactClassificationEvidence { + readonly prefix: Buffer; + readonly asarHeader: FinalAsarHeaderProbeObservation; +} + +/** Share acquisition bounds and signature observations across files and streams. */ +export const createArtifactClassificationProbe = () => { + const xml = createXmlPlistSignatureProbe(); + const asar = createAsarHeaderSignatureProbe(); + const blocks: Buffer[] = []; + let retainedBytes = 0; + let observedBytes = 0; + const retain = (bytes: Uint8Array): void => { + let offset = 0; + while (offset < bytes.length) { + const blockOffset = retainedBytes % ARTIFACT_CLASSIFICATION_PREFIX_BYTES; + if (blockOffset === 0) + blocks.push(Buffer.allocUnsafe(ARTIFACT_CLASSIFICATION_PREFIX_BYTES)); + const count = Math.min( + ARTIFACT_CLASSIFICATION_PREFIX_BYTES - blockOffset, + bytes.length - offset, + ); + blocks[blocks.length - 1]!.set( + bytes.subarray(offset, offset + count), + blockOffset, + ); + retainedBytes += count; + offset += count; + } + }; + return { + get length(): number { + return observedBytes; + }, + get limit(): number { + if (observedBytes < ARTIFACT_CLASSIFICATION_PREFIX_BYTES) + return ARTIFACT_CLASSIFICATION_PREFIX_BYTES; + return Math.max( + ARTIFACT_CLASSIFICATION_PREFIX_BYTES, + xml.state === "pending" ? MAX_XML_PLIST_PROBE_BYTES : 0, + asar.observation.state === "pending" ? asar.limit : 0, + ); + }, + write(bytes: Uint8Array): void { + asar.write(bytes); + let offset = 0; + while (offset < bytes.length) { + const limit = + retainedBytes < ARTIFACT_CLASSIFICATION_PREFIX_BYTES + ? ARTIFACT_CLASSIFICATION_PREFIX_BYTES + : xml.state === "pending" + ? MAX_XML_PLIST_PROBE_BYTES + : retainedBytes; + if (retainedBytes >= limit) break; + const selected = bytes.subarray( + offset, + offset + Math.min(limit - retainedBytes, bytes.length - offset), + ); + retain(selected); + xml.write(selected); + offset += selected.length; + } + observedBytes += bytes.length; + }, + finish(archiveBytes: number): ArtifactClassificationEvidence { + const prefix = Buffer.allocUnsafe(retainedBytes); + let copied = 0; + for (const block of blocks) { + const count = Math.min(block.length, retainedBytes - copied); + prefix.set(block.subarray(0, count), copied); + copied += count; + } + return { prefix, asarHeader: asar.finish(archiveBytes) }; + }, + }; +}; diff --git a/src/artifacts/ArtifactHash.test.ts b/src/artifacts/ArtifactHash.test.ts new file mode 100644 index 0000000000..fe428a500a --- /dev/null +++ b/src/artifacts/ArtifactHash.test.ts @@ -0,0 +1,20 @@ +import { Readable } from "node:stream"; +import { expect, it } from "vitest"; + +import { hashReadable } from "./ArtifactHash.js"; + +it("preserves cancellation arriving with the final stream result", async () => { + const controller = new AbortController(); + const reason = { stage: "source-eof" }; + const source = Readable.from( + (async function* () { + yield Buffer.from("complete bytes"); + controller.abort(reason); + })(), + ); + + await expect(hashReadable(source, controller.signal)).rejects.toMatchObject({ + reason: "cancelled", + cause: reason, + }); +}); diff --git a/src/artifacts/ArtifactHash.ts b/src/artifacts/ArtifactHash.ts index 2c06ae2c26..2512bf88c8 100644 --- a/src/artifacts/ArtifactHash.ts +++ b/src/artifacts/ArtifactHash.ts @@ -4,20 +4,16 @@ import type { Readable } from "node:stream"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; import { streamChunkToBuffer } from "./StreamBytes.js"; import { - createXmlPlistSignatureProbe, - MAX_XML_PLIST_PROBE_BYTES, -} from "../domain/apple/appleArchiveFormat.js"; + createArtifactClassificationProbe, + type ArtifactClassificationEvidence, +} from "./ArtifactClassificationProbe.js"; /** Full stream digest and byte count with bounded classification evidence. */ -export type HashResult = { +export type HashResult = ArtifactClassificationEvidence & { readonly sha256: string; readonly bytes: number; - readonly prefix: Buffer; }; -/** Bounded classification evidence; this is not an executable format size limit. */ -export const ARTIFACT_CLASSIFICATION_PREFIX_BYTES = 8_192; - /** Preserve the inventory cancellation reason at a stream boundary. */ export const abortIfNeeded = (signal?: AbortSignal): void => { if (signal?.aborted === true) @@ -34,36 +30,15 @@ export const hashReadable = async ( signal?: AbortSignal, ): Promise => { const hash = createHash("sha256"); - const prefixes: Buffer[] = []; - let prefixBytes = 0; - const xmlProbe = createXmlPlistSignatureProbe(); + const probe = createArtifactClassificationProbe(); let bytes = 0; for await (const raw of stream) { abortIfNeeded(signal); const chunk = streamChunkToBuffer(raw); bytes += chunk.length; hash.update(chunk); - let chunkOffset = 0; - while ( - chunkOffset < chunk.length && - (prefixBytes < ARTIFACT_CLASSIFICATION_PREFIX_BYTES || - (xmlProbe.state === "pending" && - prefixBytes < MAX_XML_PLIST_PROBE_BYTES)) - ) { - const limit = - prefixBytes < ARTIFACT_CLASSIFICATION_PREFIX_BYTES - ? ARTIFACT_CLASSIFICATION_PREFIX_BYTES - : MAX_XML_PLIST_PROBE_BYTES; - const selected = chunk.subarray( - chunkOffset, - Math.min(chunkOffset + limit - prefixBytes, chunk.length), - ); - prefixes.push(selected); - prefixBytes += selected.length; - chunkOffset += selected.length; - xmlProbe.write(selected); - } + probe.write(chunk); } abortIfNeeded(signal); - return { sha256: hash.digest("hex"), bytes, prefix: Buffer.concat(prefixes) }; + return { sha256: hash.digest("hex"), bytes, ...probe.finish(bytes) }; }; diff --git a/src/artifacts/AsarArtifactReader.ts b/src/artifacts/AsarArtifactReader.ts index cb54d80b86..20fe8ce049 100644 --- a/src/artifacts/AsarArtifactReader.ts +++ b/src/artifacts/AsarArtifactReader.ts @@ -63,7 +63,7 @@ export class AsarArtifactReader implements ArtifactReader { const snapshotPath = this.#snapshotPath(); this.#resetArchiveState(); this.#entries.clear(); - const admitted = await this.#admitHeader(snapshotPath); + const admitted = await this.#admitHeader(snapshotPath, signal); this.#archiveSize = admitted.archiveBytes; this.#headerSize = admitted.headerBytes; abortIfNeeded(signal); @@ -122,7 +122,7 @@ export class AsarArtifactReader implements ArtifactReader { } } - async #admitHeader(snapshotPath: string) { + async #admitHeader(snapshotPath: string, signal?: AbortSignal) { const headerHandle = await this.#openOwned( snapshotPath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, @@ -134,7 +134,10 @@ export class AsarArtifactReader implements ArtifactReader { } | { readonly ok: false; readonly cause: unknown }; try { - admission = { ok: true, value: await admitAsarHeader(headerHandle) }; + admission = { + ok: true, + value: await admitAsarHeader(headerHandle, signal), + }; } catch (cause: unknown) { admission = { ok: false, cause }; } diff --git a/src/artifacts/AsarHeader.test.ts b/src/artifacts/AsarHeader.test.ts new file mode 100644 index 0000000000..3cf4f776ee --- /dev/null +++ b/src/artifacts/AsarHeader.test.ts @@ -0,0 +1,80 @@ +import type { Stats } from "node:fs"; +import { expect, it, vi } from "vitest"; + +import { + admitAsarHeader, + createAsarHeaderSignatureProbe, +} from "./AsarHeader.js"; +import { asarHeaderBytes } from "../../tests/fixtures/asarHeader.js"; + +const makeArchive = (): Buffer => asarHeaderBytes(' \t\r\n{"files":{}} '); + +it("fills the fixed Pickle prefix across controlled short reads", async () => { + const archive = makeArchive(); + const calls: Array = []; + const handle = { + stat: vi.fn( + async () => + ({ + isFile: () => true, + size: archive.length, + }) as Stats, + ), + read: vi.fn( + async ( + target: Buffer, + offset: number, + length: number, + position: number, + ) => { + calls.push([offset, length, position]); + const bytesRead = Math.min(3, length, archive.length - position); + archive.copy(target, offset, position, position + bytesRead); + return { bytesRead, buffer: target }; + }, + ), + } as unknown as Parameters[0]; + + await expect(admitAsarHeader(handle)).resolves.toEqual({ + archiveBytes: archive.length, + headerBytes: archive.readUInt32LE(4), + jsonBytes: archive.readUInt32LE(12), + }); + expect(calls).toEqual([ + [0, 16, 0], + [3, 13, 3], + [6, 10, 6], + [9, 7, 9], + [12, 4, 12], + [15, 1, 15], + ]); +}); + +it("recognizes JSON whitespace before the object token and finalizes file extents", () => { + const archive = makeArchive(); + const probe = createAsarHeaderSignatureProbe(); + probe.write(archive.subarray(0, 7)); + expect(probe.observation.state).toBe("pending"); + probe.write(archive.subarray(7, 21)); + expect(probe.observation.state).toBe("matched"); + const result = probe.finish(archive.length); + expect(result).toMatchObject({ + state: "matched", + extents: { + archiveBytes: archive.length, + headerBytes: archive.readUInt32LE(4), + jsonBytes: archive.readUInt32LE(12), + }, + signatureOffset: 20, + }); +}); + +it("rejects a signature whose declared header exceeds the observed archive", () => { + const archive = makeArchive(); + const probe = createAsarHeaderSignatureProbe(); + probe.write(archive); + expect(probe.finish(archive.length - 1)).toMatchObject({ + state: "rejected", + reason: "archive-too-short", + }); +}); diff --git a/src/artifacts/AsarHeader.ts b/src/artifacts/AsarHeader.ts index 216b053295..f98d026e19 100644 --- a/src/artifacts/AsarHeader.ts +++ b/src/artifacts/AsarHeader.ts @@ -1,49 +1,359 @@ import type { Stats } from "node:fs"; import type { FileHandle } from "node:fs/promises"; +import { assertArtifactReadActive } from "./ArtifactFileOperation.js"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; +import { err, ok, type Result } from "../domain/result.js"; /** Shared resource budget for the eagerly decoded ASAR header, not member bytes. */ export const MAX_ASAR_HEADER_BYTES = 16 * 1024 * 1024; -/** Admit both Chromium pickle extents before an upstream decoder allocates them. */ -export const admitAsarHeader = async ( - handle: Pick & { stat(): Promise }, -): Promise<{ +/** Absolute archive prefix required to inspect an admitted header. */ +export const MAX_ASAR_HEADER_PROBE_BYTES = MAX_ASAR_HEADER_BYTES + 8; + +export interface AsarHeaderExtents { readonly archiveBytes: number; readonly headerBytes: number; readonly jsonBytes: number; -}> => { +} + +export type AsarHeaderProbeExtents = Pick< + AsarHeaderExtents, + "headerBytes" | "jsonBytes" +> & { readonly archiveBytes?: number }; + +export type AsarHeaderProbeObservation = + | { + readonly state: "pending"; + readonly extents?: AsarHeaderProbeExtents; + readonly scannedJsonBytes: number; + } + | { + /** The declared JSON begins with an object token; this does not validate JSON. */ + readonly state: "matched"; + readonly extents: AsarHeaderProbeExtents; + readonly signatureOffset: number; + readonly scannedJsonBytes: number; + } + | { + readonly state: "rejected"; + readonly reason: + | "invalid-pickle-extents" + | "archive-too-short" + | "json-not-object"; + readonly extents?: AsarHeaderProbeExtents; + readonly scannedJsonBytes: number; + } + | { + readonly state: "limit"; + readonly extents?: AsarHeaderProbeExtents; + readonly scannedJsonBytes: number; + readonly limitBytes: number; + }; + +export type FinalAsarHeaderProbeObservation = Exclude< + AsarHeaderProbeObservation, + { readonly state: "pending" } +>; + +/** + * Parse the fixed two-Pickle prefix and optionally check it against the actual + * observed archive size. This establishes byte extents only; JSON validity and + * the ASAR header schema belong to the decoder at the interpretation boundary. + */ +export const parseAsarHeaderExtents = ( + prefix: Uint8Array, + archiveBytes?: number, +): Result, ArtifactReaderFailure> => { + if (prefix.byteLength < 16) + return err( + new ArtifactReaderFailure( + "format", + "Truncated ASAR Pickle extent prefix", + ), + ); + const bytes = Buffer.from(prefix.buffer, prefix.byteOffset, 16); + const headerBytes = bytes.readUInt32LE(4); + const payloadBytes = bytes.readUInt32LE(8); + const jsonBytes = bytes.readUInt32LE(12); + if ( + bytes.readUInt32LE(0) !== 4 || + headerBytes < 8 || + payloadBytes !== headerBytes - 4 || + jsonBytes > payloadBytes - 4 + ) + return err( + new ArtifactReaderFailure( + "format", + "ASAR header has invalid Pickle extents", + ), + ); + if ( + archiveBytes !== undefined && + (!Number.isSafeInteger(archiveBytes) || + archiveBytes < 8 || + headerBytes > archiveBytes - 8) + ) + return err( + new ArtifactReaderFailure( + "format", + "ASAR header leaves its observed container", + ), + ); + return ok({ headerBytes, jsonBytes }); +}; + +/** + * Bounded ASAR signature probe. It retains only the 16-byte fixed prefix and + * scans the declared JSON bytes until the first non-JSON-whitespace byte. + */ +export interface AsarHeaderSignatureProbe { + readonly limit: number; + readonly observation: AsarHeaderProbeObservation; + write(input: Uint8Array): AsarHeaderProbeObservation; + finish(archiveBytes: number): FinalAsarHeaderProbeObservation; +} + +class AsarHeaderSignatureProbeImpl implements AsarHeaderSignatureProbe { + readonly limit = MAX_ASAR_HEADER_PROBE_BYTES; + readonly #fixed = Buffer.alloc(16); + #fixedBytes = 0; + #scannedJsonBytes = 0; + #extents: Omit | undefined; + #stop: ProbeStop = { state: "pending" }; + + get observation(): AsarHeaderProbeObservation { + return this.#observation(); + } + + /** Consume sequential archive bytes without retaining JSON or member data. */ + write(input: Uint8Array): AsarHeaderProbeObservation { + if (this.#stop.state === "finalized") return this.#stop.observation; + const bytes = Buffer.from(input.buffer, input.byteOffset, input.byteLength); + let offset = 0; + while (offset < bytes.length && !this.#isTerminal()) { + if (this.#fixedBytes < this.#fixed.length) { + const count = Math.min( + this.#fixed.length - this.#fixedBytes, + bytes.length - offset, + ); + bytes.copy(this.#fixed, this.#fixedBytes, offset, offset + count); + this.#fixedBytes += count; + offset += count; + if (this.#fixedBytes === this.#fixed.length) this.#parseFixed(); + continue; + } + const extents = this.#extents; + if (extents === undefined || this.#stop.state === "matched") break; + const jsonRemaining = extents.jsonBytes - this.#scannedJsonBytes; + if (jsonRemaining <= 0) { + this.#stop = { state: "rejected", reason: "json-not-object" }; + break; + } + const archiveBytesScanned = 16 + this.#scannedJsonBytes; + if (archiveBytesScanned >= this.limit) { + this.#stop = { state: "limit" }; + break; + } + const count = Math.min( + bytes.length - offset, + jsonRemaining, + this.limit - archiveBytesScanned, + ); + for (let index = 0; index < count; index += 1) { + const value = bytes[offset + index]!; + if (!isJsonWhitespace(value)) { + if (value === 0x7b) + this.#stop = { + state: "matched", + signatureOffset: 16 + this.#scannedJsonBytes, + extents, + }; + else this.#stop = { state: "rejected", reason: "json-not-object" }; + break; + } + this.#scannedJsonBytes += 1; + } + offset += count; + if (this.#stop.state !== "pending") break; + if (this.#scannedJsonBytes >= extents.jsonBytes) { + this.#stop = { state: "rejected", reason: "json-not-object" }; + break; + } + if (16 + this.#scannedJsonBytes >= this.limit) + this.#stop = { state: "limit" }; + } + return this.#observation(); + } + + /** Finalize against the actual byte count; a signature alone is not validity. */ + finish(archiveBytes: number): FinalAsarHeaderProbeObservation { + if (this.#stop.state === "finalized") return this.#stop.observation; + if (this.#fixedBytes < this.#fixed.length) { + this.#stop = { state: "rejected", reason: "invalid-pickle-extents" }; + } else if (this.#extents !== undefined) { + if ( + !Number.isSafeInteger(archiveBytes) || + archiveBytes < 8 || + this.#extents.headerBytes > archiveBytes - 8 + ) { + this.#stop = { state: "rejected", reason: "archive-too-short" }; + } else if ( + this.#extents.headerBytes > MAX_ASAR_HEADER_BYTES && + this.#stop.state === "matched" + ) { + this.#stop = { state: "limit" }; + } + } + const observation = this.#observation(archiveBytes); + this.#stop = { state: "finalized", observation }; + return observation; + } + + #parseFixed(): void { + const result = parseAsarHeaderExtents(this.#fixed); + if (!result.ok) { + this.#stop = { state: "rejected", reason: "invalid-pickle-extents" }; + return; + } + this.#extents = result.value; + if (this.#extents.jsonBytes === 0) + this.#stop = { state: "rejected", reason: "json-not-object" }; + } + + #isTerminal(): boolean { + return this.#stop.state !== "pending"; + } + + #observation(): AsarHeaderProbeObservation; + #observation(archiveBytes: number): FinalAsarHeaderProbeObservation; + #observation( + archiveBytes?: number, + ): AsarHeaderProbeObservation | FinalAsarHeaderProbeObservation { + if (this.#stop.state === "finalized") return this.#stop.observation; + const extents: AsarHeaderProbeExtents | undefined = + this.#extents === undefined + ? undefined + : archiveBytes === undefined + ? this.#extents + : { archiveBytes, ...this.#extents }; + switch (this.#stop.state) { + case "rejected": + return { + state: "rejected", + reason: this.#stop.reason, + ...(extents === undefined ? {} : { extents }), + scannedJsonBytes: this.#scannedJsonBytes, + }; + case "limit": + return { + state: "limit", + ...(extents === undefined ? {} : { extents }), + scannedJsonBytes: this.#scannedJsonBytes, + limitBytes: this.limit, + }; + case "matched": + return { + state: "matched", + extents: + archiveBytes === undefined + ? this.#stop.extents + : { archiveBytes, ...this.#stop.extents }, + signatureOffset: this.#stop.signatureOffset, + scannedJsonBytes: this.#scannedJsonBytes, + }; + case "pending": + if (archiveBytes !== undefined && this.#extents !== undefined) { + const finalizedExtents = { + archiveBytes, + ...this.#extents, + }; + if (this.#scannedJsonBytes >= this.#extents.jsonBytes) + return { + state: "rejected", + reason: "json-not-object", + extents: finalizedExtents, + scannedJsonBytes: this.#scannedJsonBytes, + }; + return { + state: "limit", + extents: finalizedExtents, + scannedJsonBytes: this.#scannedJsonBytes, + limitBytes: this.limit, + }; + } + return { + state: "pending", + ...(extents === undefined ? {} : { extents }), + scannedJsonBytes: this.#scannedJsonBytes, + }; + } + } +} + +type ProbeStop = + | { readonly state: "pending" } + | { + readonly state: "matched"; + readonly signatureOffset: number; + readonly extents: Omit; + } + | { + readonly state: "rejected"; + readonly reason: + | "invalid-pickle-extents" + | "archive-too-short" + | "json-not-object"; + } + | { readonly state: "limit" } + | { + readonly state: "finalized"; + readonly observation: FinalAsarHeaderProbeObservation; + }; + +const isJsonWhitespace = (value: number): boolean => + value === 0x20 || value === 0x09 || value === 0x0a || value === 0x0d; + +/** Admit Pickle extents before an upstream decoder allocates them. */ +export const admitAsarHeader = async ( + handle: Pick & { stat(): Promise }, + signal?: AbortSignal, +): Promise => { + assertArtifactReadActive(signal); const observed = await handle.stat(); + assertArtifactReadActive(signal); if (!observed.isFile()) throw new ArtifactReaderFailure( "format", "ASAR container is not a regular file", ); const prefix = Buffer.alloc(16); - const read = await handle.read(prefix, 0, prefix.length, 0); - if (read.bytesRead !== prefix.length || prefix.readUInt32LE(0) !== 4) - throw new ArtifactReaderFailure( - "format", - "Truncated or invalid ASAR size pickle", + let bytesRead = 0; + while (bytesRead < prefix.length) { + assertArtifactReadActive(signal); + const read = await handle.read( + prefix, + bytesRead, + prefix.length - bytesRead, + bytesRead, ); - const headerBytes = prefix.readUInt32LE(4); - const payloadBytes = prefix.readUInt32LE(8); - const jsonBytes = prefix.readUInt32LE(12); - if ( - headerBytes < 8 || - headerBytes > observed.size - 8 || - payloadBytes !== headerBytes - 4 || - jsonBytes > payloadBytes - 4 - ) - throw new ArtifactReaderFailure( - "format", - "ASAR header leaves its container or has invalid pickle extents", - ); - if (headerBytes > MAX_ASAR_HEADER_BYTES) + assertArtifactReadActive(signal); + if (read.bytesRead === 0) + throw new ArtifactReaderFailure("format", "Truncated ASAR size pickle"); + bytesRead += read.bytesRead; + } + const parsed = parseAsarHeaderExtents(prefix, observed.size); + if (!parsed.ok) throw parsed.error; + if (parsed.value.headerBytes > MAX_ASAR_HEADER_BYTES) throw new ArtifactReaderFailure( "limit", `ASAR header exceeds the ${String(MAX_ASAR_HEADER_BYTES)} byte decode budget`, ); - return { archiveBytes: observed.size, headerBytes, jsonBytes }; + return { + archiveBytes: observed.size, + ...parsed.value, + }; }; + +export const createAsarHeaderSignatureProbe = (): AsarHeaderSignatureProbe => + new AsarHeaderSignatureProbeImpl(); diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts index 64b928900a..95cdcd5a34 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts @@ -2,7 +2,6 @@ import { describe, expect, it } from "vitest"; import { CPU, fatImage, machoImage } from "../apple/MachoImage.fixture.js"; import { - classifyArtifactContent, createArtifactEdges, createOccurrence, indexOccurrencesByPath, @@ -11,6 +10,8 @@ import { import { artifactIdForContent } from "../../domain/artifactIdentity.js"; import type { ArtifactEntry } from "../ArtifactReader.js"; +import { classifyArtifactBytesAtPath } from "../../../tests/fixtures/artifactClassification.js"; + const entry = (path: string, kind: ArtifactEntry["kind"]): ArtifactEntry => ({ path, kind, @@ -168,7 +169,7 @@ describe("bounded FAT inventory classification", () => { const bytes = fat(); for (const length of [4, 7, 8, 27]) { expect( - classifyArtifactContent( + classifyArtifactBytesAtPath( "native", bytes.subarray(0, length), bytes.length, @@ -176,7 +177,7 @@ describe("bounded FAT inventory classification", () => { ).toEqual({ kind: "unknown", format: "unknown" }); } expect( - classifyArtifactContent( + classifyArtifactBytesAtPath( "native.class", bytes.subarray(0, 28), bytes.length, @@ -197,7 +198,7 @@ describe("bounded FAT inventory classification", () => { ] as const)("keeps %s unknown", (_label, mutate) => { const bytes = fat(); mutate(bytes); - expect(classifyArtifactContent("native", bytes)).toEqual({ + expect(classifyArtifactBytesAtPath("native", bytes)).toEqual({ kind: "unknown", format: "unknown", }); @@ -206,7 +207,7 @@ describe("bounded FAT inventory classification", () => { it("does not infer Mach-O from an out-of-bounds architecture count", () => { const bytes = fat(); bytes.writeUInt32BE(0xffffffff, 4); - expect(classifyArtifactContent("native", bytes)).toEqual({ + expect(classifyArtifactBytesAtPath("native", bytes)).toEqual({ kind: "resource", format: "file", }); @@ -215,7 +216,7 @@ describe("bounded FAT inventory classification", () => { it("does not lose unsafe FAT64 range values to number conversion", () => { const bytes = fat(true); bytes.writeBigUInt64BE(BigInt(Number.MAX_SAFE_INTEGER) + 1n, 16); - expect(classifyArtifactContent("native", bytes)).toEqual({ + expect(classifyArtifactBytesAtPath("native", bytes)).toEqual({ kind: "unknown", format: "unknown", }); @@ -229,7 +230,7 @@ describe("bounded FAT inventory classification", () => { ]), ); bytes.writeUInt32BE(bytes.length, 40); - expect(classifyArtifactContent("native", bytes)).toEqual({ + expect(classifyArtifactBytesAtPath("native", bytes)).toEqual({ kind: "unknown", format: "unknown", }); @@ -239,9 +240,13 @@ describe("bounded FAT inventory classification", () => { const cpu = { type: 0x12345678, subtype: 0 }; const bytes = Buffer.from(fatImage([{ cpu, bytes: machoImage({ cpu }) }])); expect( - classifyArtifactContent("native", bytes.subarray(0, 28), bytes.length), + classifyArtifactBytesAtPath( + "native", + bytes.subarray(0, 28), + bytes.length, + ), ).toEqual({ kind: "unknown", format: "unknown" }); - expect(classifyArtifactContent("native", bytes)).toEqual({ + expect(classifyArtifactBytesAtPath("native", bytes)).toEqual({ kind: "executable", format: "mach-o-universal", }); @@ -251,7 +256,7 @@ describe("bounded FAT inventory classification", () => { "keeps invalid or contradictory file size %s unknown", (size) => { const bytes = fat(); - expect(classifyArtifactContent("native", bytes, size)).toEqual({ + expect(classifyArtifactBytesAtPath("native", bytes, size)).toEqual({ kind: "unknown", format: "unknown", }); @@ -263,19 +268,19 @@ it("preserves path-selected inventory roles for XML content", () => { const xml = Buffer.from( '', ); - expect(classifyArtifactContent("state.archive", xml)).toEqual({ + expect(classifyArtifactBytesAtPath("state.archive", xml)).toEqual({ kind: "plist", format: "plist", }); - expect(classifyArtifactContent("state.js", xml)).toEqual({ + expect(classifyArtifactBytesAtPath("state.js", xml)).toEqual({ kind: "javascript", format: "javascript-bundle", }); - expect(classifyArtifactContent("state.map", xml)).toEqual({ + expect(classifyArtifactBytesAtPath("state.map", xml)).toEqual({ kind: "source-map", format: "source-map", }); - expect(classifyArtifactContent("state.asar", xml)).toEqual({ + expect(classifyArtifactBytesAtPath("state.asar", xml)).toEqual({ kind: "resource", format: "file", }); diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.ts b/src/artifacts/inventory/ArtifactGraphConstruction.ts index 4d80479151..90d4684727 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.ts @@ -1,3 +1,5 @@ +import type { ArtifactClassificationEvidence } from "../ArtifactClassificationProbe.js"; +import { ArtifactReaderFailure } from "../ArtifactReader.js"; import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; import { isJavaScriptSourcePath } from "../../domain/javascript/javascriptArtifactFiles.js"; import { digestCanonicalValue } from "../../domain/canonicalDigest.js"; @@ -473,12 +475,13 @@ export const classifyArtifactPath = ( /** Classify executable formats from bytes while retaining path-specific roles. */ export const classifyArtifactContent = ( path: string, - prefix: Buffer, - fileSize = prefix.length, + evidence: ArtifactClassificationEvidence, + fileSize = evidence.prefix.length, ): { readonly kind: ArtifactOccurrence["artifact_kind"]; readonly format: ArtifactOccurrence["artifact_format"]; } => { + const { prefix, asarHeader } = evidence; const format = classifyArtifactBytes(prefix, fileSize); const byPath = classifyArtifactPath(path); if ( @@ -491,27 +494,19 @@ export const classifyArtifactContent = ( if ( format === "file" && path.toLowerCase().endsWith(".asar") && - hasAsarHeader(prefix) - ) - return { kind: "container", format: "asar" }; + asarHeader.state !== "rejected" + ) { + if (asarHeader.state === "limit") + throw new ArtifactReaderFailure( + "limit", + "ASAR header exceeds the bounded classification or decode budget", + ); + if (asarHeader.state === "matched") + return { kind: "container", format: "asar" }; + } return artifactRoleForFormat(path, format); }; -/** - * Electron ASAR starts with a size pickle whose payload is one uint32, then - * a header pickle whose payload begins with a JSON object. - */ -export const hasAsarHeader = (prefix: Buffer): boolean => { - if (prefix.length < 17 || prefix.readUInt32LE(0) !== 4) return false; - const headerSize = prefix.readUInt32LE(4); - const payloadSize = prefix.readUInt32LE(8); - if (headerSize < 16 || payloadSize !== headerSize - 4) return false; - const stringLength = prefix.readUInt32LE(12); - return ( - stringLength >= 2 && stringLength <= payloadSize - 4 && prefix[16] === 0x7b - ); -}; - const artifactRoleForFormat = ( path: string, format: ArtifactOccurrence["artifact_format"], diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.ts index 0df8784ebb..9cf72e979d 100644 --- a/src/artifacts/inventory/classify.ts +++ b/src/artifacts/inventory/classify.ts @@ -5,14 +5,8 @@ import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.j import type { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; import { classifyArtifactContent } from "./ArtifactGraphConstruction.js"; -import { - abortIfNeeded, - ARTIFACT_CLASSIFICATION_PREFIX_BYTES, -} from "../ArtifactHash.js"; -import { - createXmlPlistSignatureProbe, - MAX_XML_PLIST_PROBE_BYTES, -} from "../../domain/apple/appleArchiveFormat.js"; +import { abortIfNeeded } from "../ArtifactHash.js"; +import { createArtifactClassificationProbe } from "../ArtifactClassificationProbe.js"; import type { ArtifactOccurrence } from "../../domain/artifactGraph.js"; import { hasZipSignature, @@ -143,41 +137,36 @@ const classifyRootFormat = async ( handle: FileHandle, signal?: AbortSignal, ): Promise => { - const magic = Buffer.alloc(ARTIFACT_CLASSIFICATION_PREFIX_BYTES); - const observed = await handle.read(magic, 0, magic.length, 0); - let prefix = magic.subarray(0, observed.bytesRead); - const lower = path.toLowerCase(); - if (hasZipSignature(prefix)) return zipRootPackageFormat(lower); - if ( - lower.endsWith(".pkg") && - prefix.subarray(0, 4).equals(Buffer.from("xar!")) - ) - return "pkg"; - if (lower.endsWith(".dmg") && (await hasKolyTrailer(handle))) return "dmg"; - const xmlProbe = createXmlPlistSignatureProbe(); - xmlProbe.write(prefix); - const chunks = [prefix]; - let prefixBytes = prefix.length; - while ( - xmlProbe.state === "pending" && - prefixBytes < MAX_XML_PLIST_PROBE_BYTES - ) { + const probe = createArtifactClassificationProbe(); + const chunk = Buffer.alloc(64 * 1024); + while (probe.length < probe.limit) { abortIfNeeded(signal); - const chunk = Buffer.alloc( - Math.min(64 * 1024, MAX_XML_PLIST_PROBE_BYTES - prefixBytes), + const read = await handle.read( + chunk, + 0, + Math.min(chunk.length, probe.limit - probe.length), + probe.length, ); - const read = await handle.read(chunk, 0, chunk.length, prefixBytes); abortIfNeeded(signal); if (read.bytesRead === 0) break; - const selected = chunk.subarray(0, read.bytesRead); - chunks.push(selected); - prefixBytes += selected.length; - xmlProbe.write(selected); + probe.write(chunk.subarray(0, read.bytesRead)); } - prefix = Buffer.concat(chunks); - // ASAR is chosen from its header pickle; any other suffix is a role hint. - return classifyArtifactContent(path, prefix, (await handle.stat()).size) - .format; + const archiveBytes = (await handle.stat()).size; + const evidence = probe.finish(archiveBytes); + abortIfNeeded(signal); + const lower = path.toLowerCase(); + if (hasZipSignature(evidence.prefix)) return zipRootPackageFormat(lower); + if ( + lower.endsWith(".pkg") && + evidence.prefix.subarray(0, 4).equals(Buffer.from("xar!")) + ) + return "pkg"; + if ( + lower.endsWith(".dmg") && + (await hasKolyTrailer(handle, archiveBytes, signal)) + ) + return "dmg"; + return classifyArtifactContent(path, evidence, archiveBytes).format; }; /** App Pack is a verified ZIP root role; nested .app members retain their observed format. */ @@ -186,10 +175,25 @@ const zipRootPackageFormat = (lower: string): ZipPackageFormat => ? "app-pack" : (zipPackageFormatForPath(lower) ?? "zip"); -const hasKolyTrailer = async (handle: FileHandle): Promise => { - const size = (await handle.stat()).size; +const hasKolyTrailer = async ( + handle: FileHandle, + size: number, + signal?: AbortSignal, +): Promise => { if (size < 512) return false; const trailer = Buffer.alloc(4); - const read = await handle.read(trailer, 0, trailer.length, size - 512); - return read.bytesRead === 4 && trailer.equals(Buffer.from("koly")); + let bytes = 0; + while (bytes < trailer.length) { + abortIfNeeded(signal); + const read = await handle.read( + trailer, + bytes, + trailer.length - bytes, + size - 512 + bytes, + ); + abortIfNeeded(signal); + if (read.bytesRead === 0) return false; + bytes += read.bytesRead; + } + return trailer.equals(Buffer.from("koly")); }; diff --git a/src/artifacts/inventory/scanReader.ts b/src/artifacts/inventory/scanReader.ts index 88ad14c801..378e293081 100644 --- a/src/artifacts/inventory/scanReader.ts +++ b/src/artifacts/inventory/scanReader.ts @@ -317,7 +317,7 @@ const digestArtifactEntry = async ( const classified = entry.kind === "slice" ? ({ kind: "universal-slice", format: "mach-o" } as const) - : classifyArtifactContent(logicalPath, digest.prefix, digest.bytes); + : classifyArtifactContent(logicalPath, digest, digest.bytes); return { node: createArtifactNode({ sha256: digest.sha256, diff --git a/src/browser/ElectronFileScope.ts b/src/browser/ElectronFileScope.ts index 6dd3adba9c..0ca6db7be4 100644 --- a/src/browser/ElectronFileScope.ts +++ b/src/browser/ElectronFileScope.ts @@ -236,7 +236,7 @@ const asarFiles = async ( throwIfAborted(signal); if (!opened.isFile() || archiveIdentity(opened) !== identity) throw new Error("ASAR identity changed before header read"); - const header = await admitAsarHeader(file); + const header = await admitAsarHeader(file, signal); throwIfAborted(signal); jsonBytes = header.jsonBytes; const json = Buffer.alloc(jsonBytes); diff --git a/tests/acceptance/applications/javascriptSourceSelectionCli.test.ts b/tests/acceptance/applications/javascriptSourceSelectionCli.test.ts index 31cad3a7ee..de9abcf76d 100644 --- a/tests/acceptance/applications/javascriptSourceSelectionCli.test.ts +++ b/tests/acceptance/applications/javascriptSourceSelectionCli.test.ts @@ -1,12 +1,16 @@ import { createPackageWithOptions } from "@electron/asar"; import { createHash } from "node:crypto"; -import { readFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; import { writeJavaScriptSourceSelectionFixture } from "../../fixtures/javascriptSourceSelection.js"; +import { + asarHeaderBytes, + prependAsarJsonWhitespace, +} from "../../fixtures/asarHeader.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; @@ -145,3 +149,78 @@ cliTest( }); }, ); + +cliTest( + "accepts legal ASAR JSON whitespace while preserving selected-source identity", + { timeout: 60_000 }, + async ({ cli }) => { + const root = await createTestTempDirectory("rea-js-asar-whitespace-"); + await writeJavaScriptSourceSelectionFixture(root); + const archive = join( + await createTestTempDirectory("rea-js-asar-whitespace-output-"), + "app.asar", + ); + await createPackageWithOptions(root, archive, {}); + + for (const prefix of [" \t\r\n", " ".repeat(9_001)]) { + await prependAsarJsonWhitespace(archive, prefix); + const response = await cli.run({ + arguments: [ + "analyze-javascript-application", + archive, + "--source-path", + "main.mjs", + "--format", + "json", + ], + }); + expect(response.exitCode).toBe(0); + const evidence = parseEvidence(response.json); + const result = javascriptApplicationAnalysisResultSchema.parse( + evidence.normalized_result, + ); + expect(result.statistics).toMatchObject({ + nested_asar_containers: 0, + parsed_javascript_files: 1, + parse_failures: 0, + }); + expect(result.semantic_graph.nodes).toContainEqual( + expect.objectContaining({ kind: "function", label: "selected" }), + ); + expect(result.root_artifact_sha256).toBe( + createHash("sha256") + .update(await readFile(archive)) + .digest("hex"), + ); + } + }, +); + +cliTest( + "does not parse ASAR-looking JSON beyond the declared JSON extent", + async ({ cli }) => { + const archive = join( + await createTestTempDirectory("rea-js-asar-padding-output-"), + "opaque.asar", + ); + await writeFile( + archive, + asarHeaderBytes(" ", { + padding: Buffer.from('{"files":{"main.mjs":{"size":0,"offset":"0"}}}'), + }), + ); + const response = await cli.run({ + arguments: [ + "analyze-javascript-application", + archive, + "--format", + "json", + ], + }); + expect(response.exitCode).toBe(1); + expect(response.json).toMatchObject({ + code: "artifact_operation_failed", + details: { reason: "format" }, + }); + }, +); diff --git a/tests/boundary/filesystem/artifactInventoryDescriptorCleanup.test.ts b/tests/boundary/filesystem/artifactInventoryDescriptorCleanup.test.ts index 1817d7c3ea..1826083ad3 100644 --- a/tests/boundary/filesystem/artifactInventoryDescriptorCleanup.test.ts +++ b/tests/boundary/filesystem/artifactInventoryDescriptorCleanup.test.ts @@ -1,9 +1,12 @@ +import { createHash } from "node:crypto"; import type { FileHandle } from "node:fs/promises"; -import { writeFile } from "node:fs/promises"; +import { lstat, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it, onTestFinished, vi } from "vitest"; import { ArtifactProvider } from "../../../src/artifacts/ArtifactProvider.js"; +import { classifyAndHashRoot } from "../../fixtures/artifactInventory.js"; +import { asarHeaderBytes } from "../../fixtures/asarHeader.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; const injected = vi.hoisted(() => ({ @@ -92,3 +95,64 @@ it.each([false, true])( if (invalidated) expect(closeAttempts).toBe(1); }, ); + +it("classifies root ASAR and DMG signatures across controlled short reads and closes the descriptor", async () => { + const root = await createTestTempDirectory("rea-root-short-read-signature-"); + const compactPath = join(root, "compact.asar"); + const asarPath = join(root, "whitespace.asar"); + const dmgPath = join(root, "trailer.dmg"); + const asarBytes = asarHeaderBytes( + `${" ".repeat(9_000)}\t\r\n${JSON.stringify({ files: {} })}`, + ); + const dmgBytes = Buffer.alloc(1_024); + const compactBytes = asarHeaderBytes(JSON.stringify({ files: {} })); + dmgBytes.write("koly", dmgBytes.length - 512); + await writeFile(asarPath, asarBytes); + await writeFile(compactPath, compactBytes); + await writeFile(dmgPath, dmgBytes); + let openedPath: string | undefined; + const descriptors: FileHandle[] = []; + let largestActualRead = 0; + injected.opened = (candidate, handle) => { + if (candidate !== openedPath) return; + descriptors.push(handle); + const nativeRead = handle.read.bind(handle); + const readAtMostThreeBytes = async ( + buffer: Buffer, + offset: number, + length: number, + position: number | null, + ) => { + const boundedLength = Math.min(length, 3); + largestActualRead = Math.max(largestActualRead, boundedLength); + return nativeRead(buffer, offset, boundedLength, position); + }; + vi.spyOn(handle, "read").mockImplementation( + readAtMostThreeBytes as FileHandle["read"], + ); + }; + onTestFinished(() => { + injected.opened = undefined; + vi.restoreAllMocks(); + }); + + for (const [path, bytes, format] of [ + [compactPath, compactBytes, "asar"], + [asarPath, asarBytes, "asar"], + [dmgPath, dmgBytes, "dmg"], + ] as const) { + openedPath = path; + descriptors.length = 0; + largestActualRead = 0; + const result = await classifyAndHashRoot(path, false, await lstat(path)); + expect(result.format).toBe(format); + expect(result.digest?.sha256).toBe( + createHash("sha256").update(bytes).digest("hex"), + ); + if (format === "asar") + expect(result.digest?.asarHeader).toMatchObject({ state: "matched" }); + expect(largestActualRead).toBeLessThanOrEqual(3); + expect(descriptors).toHaveLength(1); + expect(descriptors[0]?.fd).toBe(-1); + } +}); diff --git a/tests/boundary/filesystem/artifactInventoryRegularRoot.test.ts b/tests/boundary/filesystem/artifactInventoryRegularRoot.test.ts index 05e6a0e1a2..8d58f93d56 100644 --- a/tests/boundary/filesystem/artifactInventoryRegularRoot.test.ts +++ b/tests/boundary/filesystem/artifactInventoryRegularRoot.test.ts @@ -1,5 +1,12 @@ +import { createHash } from "node:crypto"; import { execFile } from "node:child_process"; -import { lstat, rename, rm, writeFile } from "node:fs/promises"; +import { + lstat, + open as openFile, + rename, + rm, + writeFile, +} from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -9,8 +16,10 @@ import { classifyAndHashRoot, inventoryArtifact, } from "../../fixtures/artifactInventory.js"; +import { asarHeaderBytes } from "../../fixtures/asarHeader.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { readWithoutFifoWriter } from "../../fixtures/fifoInput.js"; +import { MAX_ASAR_HEADER_BYTES } from "../../../src/artifacts/AsarHeader.js"; it.skipIf(process.platform === "win32")( "rejects a FIFO artifact root without waiting for a writer", @@ -71,3 +80,80 @@ it("keeps unavailable container traversal distinct from observed root bytes", as status: "unknown", }); }); + +it("recognizes a root ASAR when its exact JSON extent has legal surrounding whitespace", async () => { + const root = await createTestTempDirectory("rea-asar-root-whitespace-"); + const path = join(root, "app.asar"); + const json = ` \t\r\n${JSON.stringify({ + files: { "main.js": { size: 0, offset: "0" } }, + })}\r\n `; + const bytes = asarHeaderBytes(json); + await writeFile(path, bytes); + + const inventory = await inventoryArtifact(path); + expect(inventory.manifest.root_format).toBe("asar"); + const rootNode = inventory.nodes.find( + ({ artifact_id }) => artifact_id === inventory.manifest.root_artifact_id, + ); + expect(rootNode?.sha256).toBe( + createHash("sha256").update(bytes).digest("hex"), + ); + expect( + inventory.occurrences.map(({ logical_path }) => logical_path), + ).toContain("main.js"); +}); + +it("reports the bounded probe limit for an oversized ASAR candidate without penalizing opaque files", async () => { + const root = await createTestTempDirectory("rea-asar-probe-limit-"); + const candidatePath = join(root, "large.asar"); + const opaquePath = join(root, "opaque.bin"); + const jsonObject = Buffer.from('{"files":{}}'); + const signatureOffset = MAX_ASAR_HEADER_BYTES + 9; + const whitespaceBytes = signatureOffset - 16; + const jsonBytes = whitespaceBytes + jsonObject.length; + const headerBytes = 8 + Math.ceil(jsonBytes / 4) * 4; + const archiveBytes = 8 + headerBytes; + const fixed = Buffer.alloc(16); + fixed.writeUInt32LE(4, 0); + fixed.writeUInt32LE(headerBytes, 4); + fixed.writeUInt32LE(headerBytes - 4, 8); + fixed.writeUInt32LE(jsonBytes, 12); + + const handle = await openFile(candidatePath, "w"); + try { + const writeAt = async (bytes: Buffer, start: number): Promise => { + let written = 0; + while (written < bytes.length) { + const result = await handle.write( + bytes, + written, + bytes.length - written, + start + written, + ); + if (result.bytesWritten === 0) + throw new Error("Could not write ASAR signature fixture"); + written += result.bytesWritten; + } + }; + await writeAt(fixed, 0); + const spaces = Buffer.alloc(64 * 1024, 0x20); + let position = 16; + while (position < signatureOffset) { + const count = Math.min(spaces.length, signatureOffset - position); + await writeAt(spaces.subarray(0, count), position); + position += count; + } + await writeAt(jsonObject, signatureOffset); + await handle.truncate(archiveBytes); + } finally { + await handle.close(); + } + + await writeFile(opaquePath, "ordinary opaque bytes"); + await expect(inventoryArtifact(candidatePath)).rejects.toMatchObject({ + reason: "limit", + }); + expect((await inventoryArtifact(opaquePath)).manifest.root_format).toBe( + "file", + ); +}); diff --git a/tests/boundary/filesystem/artifactMzInventory.test.ts b/tests/boundary/filesystem/artifactMzInventory.test.ts index 613b5feb82..675069f748 100644 --- a/tests/boundary/filesystem/artifactMzInventory.test.ts +++ b/tests/boundary/filesystem/artifactMzInventory.test.ts @@ -3,15 +3,15 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { classifyArtifactContent } from "../../../src/artifacts/inventory/ArtifactGraphConstruction.js"; -import { ARTIFACT_CLASSIFICATION_PREFIX_BYTES } from "../../../src/artifacts/ArtifactHash.js"; +import { classifyArtifactBytesAtPath } from "../../fixtures/artifactClassification.js"; +import { ARTIFACT_CLASSIFICATION_PREFIX_BYTES } from "../../../src/artifacts/ArtifactClassificationProbe.js"; import { scanCanonicalArtifactInventory } from "../../fixtures/artifactInventory.js"; import { artifactInventoryResultSchema } from "../../../src/domain/artifactGraph.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; describe("MZ artifact classification", () => { it("rejects a Windows header pointer overlapping the DOS header", () => { - expect(classifyArtifactContent("invalid.exe", peFixture(32))).toEqual({ + expect(classifyArtifactBytesAtPath("invalid.exe", peFixture(32))).toEqual({ kind: "unknown", format: "unknown", }); @@ -19,31 +19,37 @@ describe("MZ artifact classification", () => { it("distinguishes DOS, observed PE, malformed MZ and unavailable prefix evidence", () => { const dos = dosFixture(); - expect(classifyArtifactContent("LEGACY.EXE", dos)).toEqual({ + expect(classifyArtifactBytesAtPath("LEGACY.EXE", dos)).toEqual({ kind: "executable", format: "dos-mz", }); expect( - classifyArtifactContent("LEGACY.EXE", dos.subarray(0, 16), dos.length), + classifyArtifactBytesAtPath( + "LEGACY.EXE", + dos.subarray(0, 16), + dos.length, + ), + ).toEqual({ kind: "unknown", format: "unknown" }); + expect( + classifyArtifactBytesAtPath("bad.exe", Buffer.from("MZbroken")), ).toEqual({ kind: "unknown", format: "unknown" }); - expect(classifyArtifactContent("bad.exe", Buffer.from("MZbroken"))).toEqual( - { kind: "unknown", format: "unknown" }, - ); const distantPe = peFixture(ARTIFACT_CLASSIFICATION_PREFIX_BYTES + 64); expect( - classifyArtifactContent( + classifyArtifactBytesAtPath( "NATIVE.EXE", distantPe.subarray(0, ARTIFACT_CLASSIFICATION_PREFIX_BYTES), distantPe.length, ), ).toEqual({ kind: "unknown", format: "unknown" }); - expect(classifyArtifactContent("NATIVE.EXE", distantPe).format).toBe("pe"); + expect(classifyArtifactBytesAtPath("NATIVE.EXE", distantPe).format).toBe( + "pe", + ); const distantRelocations = dosWithDistantRelocations(); expect( - classifyArtifactContent("LEGACY.EXE", distantRelocations).format, + classifyArtifactBytesAtPath("LEGACY.EXE", distantRelocations).format, ).toBe("dos-mz"); expect( - classifyArtifactContent( + classifyArtifactBytesAtPath( "LEGACY.EXE", distantRelocations.subarray(0, ARTIFACT_CLASSIFICATION_PREFIX_BYTES), distantRelocations.length, diff --git a/tests/boundary/filesystem/asarHeaderAdmission.test.ts b/tests/boundary/filesystem/asarHeaderAdmission.test.ts index e16e2ea2de..479d4e14b1 100644 --- a/tests/boundary/filesystem/asarHeaderAdmission.test.ts +++ b/tests/boundary/filesystem/asarHeaderAdmission.test.ts @@ -27,10 +27,26 @@ const admit = async (path: string) => { } finally { // Admission must inspect only the fixed prefix, even for forged extents. const calls = [...read.mock.calls]; + const results = await Promise.all( + read.mock.results.map( + ({ value }) => value as Promise<{ readonly bytesRead: number }>, + ), + ); read.mockRestore(); await handle.close(); expect(handle.fd).toBe(-1); - expect(calls).toEqual([[expect.any(Buffer), 0, 16, 0]]); + expect(calls.length).toBeGreaterThan(0); + let filled = 0; + for (const [index, call] of calls.entries()) { + const [buffer, offset, length, position]: readonly unknown[] = + call as unknown as readonly unknown[]; + expect(buffer).toBeInstanceOf(Buffer); + expect(offset).toBe(filled); + expect(position).toBe(filled); + expect(length).toBe(16 - filled); + filled += results[index]!.bytesRead; + } + expect(filled).toBeLessThanOrEqual(16); } }; diff --git a/tests/boundary/filesystem/asarInventoryContainment.test.ts b/tests/boundary/filesystem/asarInventoryContainment.test.ts index 42e470c54a..5d6d323eb4 100644 --- a/tests/boundary/filesystem/asarInventoryContainment.test.ts +++ b/tests/boundary/filesystem/asarInventoryContainment.test.ts @@ -12,6 +12,7 @@ import { import { describe, expect, it } from "vitest"; import { inventoryArtifact } from "../../fixtures/artifactInventory.js"; +import { asarHeaderBytes } from "../../fixtures/asarHeader.js"; import { artifactOccurrenceAt, artifactParentPaths, @@ -48,6 +49,88 @@ const expectedNestedParents = { "resources/app.asar/ordinary.asar": "resources/app.asar", }; +describe("ASAR JSON signature boundaries", () => { + it("expands a nested ASAR with a long legal JSON prefix and retains exact bytes", async () => { + const root = await createTestTempDirectory("rea-asar-json-whitespace-"); + const application = join(root, "application"); + const archivePath = join(application, "resources", "app.asar"); + const archiveBytes = asarHeaderBytes( + `${" ".repeat(8_193)}\t\r\n${JSON.stringify({ + files: { "main.js": { size: 0, offset: "0" } }, + })}\r\n `, + ); + await mkdir(join(application, "resources"), { recursive: true }); + await writeFile(archivePath, archiveBytes); + + const inventory = await inventoryArtifact(application); + expect(artifactParentPaths(inventory)).toEqual({ + ".": null, + resources: ".", + "resources/app.asar": "resources", + "resources/app.asar/main.js": "resources/app.asar", + }); + const container = artifactOccurrenceAt(inventory, "resources/app.asar"); + expect(container).toMatchObject({ + artifact_kind: "container", + artifact_format: "asar", + hash_status: "verified", + }); + const direct = await inventoryArtifact(archivePath); + expect(container.artifact_id).toBe(direct.manifest.root_artifact_id); + expect( + inventory.nodes.find( + ({ artifact_id }) => artifact_id === container.artifact_id, + )?.sha256, + ).toBe(createHash("sha256").update(archiveBytes).digest("hex")); + expect( + artifactOccurrenceAt(inventory, "resources/app.asar/main.js").artifact_id, + ).toBe(artifactOccurrenceAt(direct, "main.js").artifact_id); + expect(await inventoryArtifact(application)).toEqual(inventory); + }); + + it.each([ + { label: "vertical tab", prefix: "\v", padding: undefined }, + { label: "form feed", prefix: "\f", padding: undefined }, + { label: "nonbreaking space", prefix: "\u00a0", padding: undefined }, + { label: "byte order mark", prefix: "\ufeff", padding: undefined }, + { + label: "pickle padding outside jsonBytes", + prefix: " ", + padding: Buffer.from("{"), + }, + ])("requires an object within jsonBytes for $label", async (fixture) => { + const root = await createTestTempDirectory("rea-asar-invalid-json-space-"); + const json = + fixture.padding === undefined + ? `${fixture.prefix}${JSON.stringify({ + files: { "main.js": { size: 0, offset: "0" } }, + })}` + : fixture.prefix; + const invalid = asarHeaderBytes( + json, + fixture.padding === undefined ? {} : { padding: fixture.padding }, + ); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("resources/app.asar", new Uint8ArrayReader(invalid)); + const outerPath = join(root, "opaque.zip"); + await writeFile(outerPath, await writer.close()); + + const inventory = await inventoryArtifact(outerPath); + expect(artifactOccurrenceAt(inventory, "resources/app.asar")).toMatchObject( + { + artifact_kind: "resource", + artifact_format: "file", + hash_status: "verified", + }, + ); + expect( + inventory.occurrences.some(({ logical_path }) => + logical_path.startsWith("resources/app.asar/"), + ), + ).toBe(false); + }); +}); + describe("expanded ASAR inventory containment", () => { it("parents expanded members to their archive without changing byte identities", async () => { const { application, archive } = await createNestedAsar(); @@ -226,13 +309,10 @@ describe("unexpanded ASAR inventory controls", () => { it("keeps filesystem archives with a malformed ASAR header rejected", async () => { const root = await createTestTempDirectory("rea-malformed-asar-"); - const json = Buffer.from("{not json"); - const header = Buffer.alloc(16); - header.writeUInt32LE(4, 0); - header.writeUInt32LE(json.length + 12, 4); - header.writeUInt32LE(json.length + 8, 8); - header.writeUInt32LE(json.length, 12); - await writeFile(join(root, "broken.asar"), Buffer.concat([header, json])); + await writeFile( + join(root, "broken.asar"), + asarHeaderBytes('{"files":{not-json}}'), + ); await expect(inventoryArtifact(root)).rejects.toMatchObject({ reason: "format", }); diff --git a/tests/boundary/filesystem/asarReaderCleanup.test.ts b/tests/boundary/filesystem/asarReaderCleanup.test.ts index 170c4fde9b..15202aab31 100644 --- a/tests/boundary/filesystem/asarReaderCleanup.test.ts +++ b/tests/boundary/filesystem/asarReaderCleanup.test.ts @@ -248,7 +248,7 @@ describe("ASAR preparation cleanup", () => { try { await expect(packedMember(reader)).rejects.toMatchObject({ reason: "format", - message: "Truncated or invalid ASAR size pickle", + message: "ASAR header has invalid Pickle extents", cleanup: { reason: closeFailure.message, resources: [expect.any(String)], diff --git a/tests/boundary/filesystem/asarStreamingAllocation.test.ts b/tests/boundary/filesystem/asarStreamingAllocation.test.ts index dcde4272ff..9eee454386 100644 --- a/tests/boundary/filesystem/asarStreamingAllocation.test.ts +++ b/tests/boundary/filesystem/asarStreamingAllocation.test.ts @@ -22,6 +22,8 @@ import { readValidatedAsarEntry, } from "../../../src/artifacts/AsarEntryStream.js"; import { hashReadable } from "../../../src/artifacts/ArtifactHash.js"; +import { ARTIFACT_CLASSIFICATION_PREFIX_BYTES } from "../../../src/artifacts/ArtifactClassificationProbe.js"; +import { asarHeaderBytes } from "../../fixtures/asarHeader.js"; import { readArtifactFileChunks } from "../../../src/artifacts/readArtifactFileChunks.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -747,6 +749,29 @@ it("preserves filesystem errors raised while consuming a member stream", async ( }); describe("ASAR entry streaming", () => { + it("matches ASAR after a long JSON prefix without retaining it in the classification prefix", async () => { + const bytes = asarHeaderBytes(`${" ".repeat(9_000)}{"files":{}}`); + const observed = await hashReadable( + Readable.from( + (async function* () { + for (const byte of bytes) yield Buffer.of(byte); + })(), + ), + ); + + expect(observed.prefix).toHaveLength(ARTIFACT_CLASSIFICATION_PREFIX_BYTES); + expect(observed.asarHeader).toMatchObject({ + state: "matched", + signatureOffset: 16 + 9_000, + extents: { + jsonBytes: Buffer.byteLength(`${" ".repeat(9_000)}{"files":{}}`), + }, + }); + expect(observed.sha256).toBe( + createHash("sha256").update(bytes).digest("hex"), + ); + }); + it("reports the observed digest when an unpacked member changes size", async () => { const root = await createTestTempDirectory( "rea-asar-unpacked-size-change-", diff --git a/tests/boundary/mcp/javascriptSourceSelection.test.ts b/tests/boundary/mcp/javascriptSourceSelection.test.ts index 609cbbe607..9375b4b193 100644 --- a/tests/boundary/mcp/javascriptSourceSelection.test.ts +++ b/tests/boundary/mcp/javascriptSourceSelection.test.ts @@ -1,7 +1,8 @@ import { Client } from "@modelcontextprotocol/client"; import { createPackageWithOptions } from "@electron/asar"; import { Ajv2020 } from "ajv/dist/2020.js"; -import { rm } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { readFile, rm, writeFile } from "node:fs/promises"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { join, resolve } from "node:path"; import { expect, it } from "vitest"; @@ -10,6 +11,10 @@ import { z } from "zod"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; import { writeJavaScriptSourceSelectionFixture } from "../../fixtures/javascriptSourceSelection.js"; +import { + asarHeaderBytes, + prependAsarJsonWhitespace, +} from "../../fixtures/asarHeader.js"; import { createStrippedAsarAddon } from "../../fixtures/strippedAsarAddon.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; @@ -241,7 +246,9 @@ it("selects nested ASAR paths and distinguishes missing inventoried bytes from i const nestedRoot = await createTestTempDirectory( "rea-js-nested-selection-", ); - await createPackageWithOptions(root, join(nestedRoot, "nested.asar"), {}); + const nestedArchive = join(nestedRoot, "nested.asar"); + await createPackageWithOptions(root, nestedArchive, {}); + await prependAsarJsonWhitespace(nestedArchive, " \t\r\n".repeat(3_001)); const nested = await client.callTool({ name: "analyze_javascript_application", arguments: { @@ -298,3 +305,79 @@ it("selects nested ASAR paths and distinguishes missing inventoried bytes from i await transport.close(); } }, 60_000); + +it("accepts ASAR headers with long legal JSON whitespace through the public application tool", async () => { + const root = await createTestTempDirectory("rea-js-asar-whitespace-mcp-"); + await writeJavaScriptSourceSelectionFixture(root); + const archive = join( + await createTestTempDirectory("rea-js-asar-whitespace-mcp-output-"), + "app.asar", + ); + await createPackageWithOptions(root, archive, {}); + await prependAsarJsonWhitespace(archive, `${" ".repeat(9_001)}\r\n\t`); + const { client, transport } = createSelectionClient(); + try { + await client.connect(transport); + const output = await client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: archive, source_paths: ["main.mjs"] }, + }); + expect(output.isError, JSON.stringify(output.content)).not.toBe(true); + const result = javascriptApplicationAnalysisResultSchema.parse( + parseEvidence(output.structuredContent).normalized_result, + ); + expect(result.statistics).toMatchObject({ + nested_asar_containers: 0, + parsed_javascript_files: 1, + parse_failures: 0, + }); + expect( + result.semantic_graph.nodes.some( + ({ kind, label }) => kind === "function" && label === "selected", + ), + ).toBe(true); + expect(result.root_artifact_sha256).toBe( + createHash("sha256") + .update(await readFile(archive)) + .digest("hex"), + ); + } finally { + await client.close(); + await transport.close(); + } +}, 60_000); + +it("does not parse ASAR-looking JSON beyond the declared JSON extent through MCP", async () => { + const archive = join( + await createTestTempDirectory("rea-js-asar-padding-mcp-"), + "opaque.asar", + ); + await writeFile( + archive, + asarHeaderBytes(" ", { + padding: Buffer.from('{"files":{"main.mjs":{"size":0,"offset":"0"}}}'), + }), + ); + const { client, transport } = createSelectionClient(); + try { + await client.connect(transport); + const output = await client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: archive }, + }); + expect(output.isError).toBe(true); + const message = output.content.find((item) => item.type === "text"); + if (message?.type !== "text") + throw new Error("Missing invalid ASAR response"); + expect(JSON.parse(message.text)).toMatchObject({ + error: { + code: "artifact_operation_failed", + details: { reason: "format" }, + }, + }); + await client.ping(); + } finally { + await client.close(); + await transport.close(); + } +}, 60_000); diff --git a/tests/fixtures/artifactClassification.ts b/tests/fixtures/artifactClassification.ts new file mode 100644 index 0000000000..f5e6617c69 --- /dev/null +++ b/tests/fixtures/artifactClassification.ts @@ -0,0 +1,17 @@ +import { createAsarHeaderSignatureProbe } from "../../src/artifacts/AsarHeader.js"; +import { classifyArtifactContent } from "../../src/artifacts/inventory/ArtifactGraphConstruction.js"; + +/** Observe a supplied byte prefix before assigning an inventory path role. */ +export const classifyArtifactBytesAtPath = ( + path: string, + bytes: Buffer, + fileSize = bytes.length, +) => { + const probe = createAsarHeaderSignatureProbe(); + probe.write(bytes); + return classifyArtifactContent( + path, + { prefix: bytes, asarHeader: probe.finish(fileSize) }, + fileSize, + ); +}; diff --git a/tests/fixtures/asarHeader.ts b/tests/fixtures/asarHeader.ts index 65fc0601b7..909d06ae14 100644 --- a/tests/fixtures/asarHeader.ts +++ b/tests/fixtures/asarHeader.ts @@ -1,13 +1,21 @@ +import { readFile, writeFile } from "node:fs/promises"; + /** Serialize a header-only ASAR, including Chromium pickle alignment. */ -export const asarHeaderBytes = (json: string): Buffer => { +export const asarHeaderBytes = ( + json: string, + options: { readonly padding?: Uint8Array } = {}, +): Buffer => { const text = Buffer.from(json); - const headerSize = 8 + Math.ceil(text.length / 4) * 4; + const padding = options.padding ?? Buffer.alloc(0); + const alignedJsonBytes = Math.ceil((text.length + padding.length) / 4) * 4; + const headerSize = 8 + alignedJsonBytes; const bytes = Buffer.alloc(8 + headerSize); bytes.writeUInt32LE(4, 0); bytes.writeUInt32LE(headerSize, 4); bytes.writeUInt32LE(headerSize - 4, 8); bytes.writeUInt32LE(text.length, 12); text.copy(bytes, 16); + Buffer.from(padding).copy(bytes, 16 + text.length); return bytes; }; @@ -25,3 +33,21 @@ export const nestedAsarHeader = ( json = `{"files":{${JSON.stringify(name)}:${json}}}`; return asarHeaderBytes(json); }; + +/** Reserialize only the header of an official archive, preserving member bytes. */ +export const prependAsarJsonWhitespace = async ( + archive: string, + prefix: string, +): Promise => { + const bytes = await readFile(archive); + const jsonBytes = bytes.readUInt32LE(12); + const oldHeaderEnd = 8 + bytes.readUInt32LE(4); + const json = bytes.subarray(16, 16 + jsonBytes).toString("utf8"); + await writeFile( + archive, + Buffer.concat([ + asarHeaderBytes(`${prefix}${json}`), + bytes.subarray(oldHeaderEnd), + ]), + ); +};