From 231e458246c05c15fd2dcd1061d0dfa3d03139f9 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 07:28:09 +0800 Subject: [PATCH] fix(native): preserve parsed section and architecture facts Classify lipo parse failures once for native and artifact consumers, retain valid read-only ELF ranges independently of optional section names, and map IL2CPP reader architecture through the existing CPU identity owner before Mach-O parsing. Regression-first checks reproduced malformed lipo output, lost unnamed-section compiler paths, and the amd64/x86_64 adapter mismatch. Focused parser, artifact, Rust CLI/MCP, Go, and IL2CPP controls passed; authored fixtures do not establish real-provider behavior. Fixes #5982 Fixes #3397 --- docs/il2cpp-metadata.md | 6 +- docs/native-investigation.md | 2 + docs/rust-binaries.md | 13 +++- .../MachOSliceArtifactReader.test.ts | 43 ++++++++++++++ src/artifacts/MachOSliceArtifactReader.ts | 11 +++- src/il2cpp/Il2cppBinaryImage.ts | 7 ++- src/native/metadata/ElfImage.ts | 3 +- src/native/metadata/readBinaryContainer.ts | 2 +- src/native/parsers/lipo.test.ts | 35 ++++++++++- src/native/parsers/lipo.ts | 58 ++++++++++-------- tests/acceptance/rustUnnamedSections.test.ts | 59 +++++++++++++++++++ .../boundary/il2cpp/il2cppBinaryFile.test.ts | 31 ++++++++++ tests/boundary/rust/rustBinaryImage.test.ts | 55 +++++++++++++++++ tests/fixtures/rust/image.ts | 12 ++++ 14 files changed, 301 insertions(+), 36 deletions(-) create mode 100644 tests/acceptance/rustUnnamedSections.test.ts diff --git a/docs/il2cpp-metadata.md b/docs/il2cpp-metadata.md index 7793442282..db066bbdf9 100644 --- a/docs/il2cpp-metadata.md +++ b/docs/il2cpp-metadata.md @@ -147,8 +147,10 @@ in [CLI and Evidence](cli.md#run-a-command). Compact delivery requires preserves every decoded row for subsequent queries without the original file. `--binary PATH` links method addresses and `--architecture arm64|amd64` selects -exactly one slice of a universal Mach-O. Multiple slices with the same -architecture label are refused as `ambiguous-slice-selection`, including +exactly one slice of a universal Mach-O. The report retains the native reader's +`amd64` spelling for x86-64; loader CPU checks use the corresponding `x86_64` +identity. Multiple slices with the same architecture label are refused as +`ambiguous-slice-selection`, including distinct ARM subtypes that both map to `arm64`. On macOS, use `lipo -info` to enumerate exact subtypes, then extract the intended subtype into a separate file, for example `lipo universal -thin arm64e -output thin-arm64e`. Inspect diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 7817302b19..6e71760097 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -433,6 +433,8 @@ Available CPU type, subtype/capability bits, byte range, and alignment remain checked against the source bytes. Inventory records source ranges and hashes, and extraction copies those ranges without running the target. This does not expand any analysis provider's supported architectures. +Malformed successful `lipo` output reports a format failure with its parse +diagnostic. Apple lipo's hidden ARM64 compatibility row is retained separately from the raw `nfat_arch` count and admitted only from its extra FAT table slot when its in-range Mach-O header matches the row. `list_architectures` preserves the diff --git a/docs/rust-binaries.md b/docs/rust-binaries.md index 407f303783..b235ce0137 100644 --- a/docs/rust-binaries.md +++ b/docs/rust-binaries.md @@ -117,9 +117,16 @@ removed the field entirely, as recorded in its [changelog](https://github.com/rust-secure-code/cargo-auditable/blob/v0.5.0/CHANGELOG.md). `rustc_commit` is present only when a `/rustc/<40-hex>/` path is found in -read-only data. For ELF, read-only non-executable `PT_LOAD` file ranges are -searched alongside read-only sections, so an optional missing section-header -table does not remove this search source. Its `evidence_kind` is `inferred`. +read-only data; its `evidence_kind` is `inferred`. For ELF, read-only +non-executable `PT_LOAD` file ranges are searched alongside read-only sections, +so an optional missing section-header +table does not remove this search source. An empty or unavailable section name +does not erase a section's validated read-only range: ELF defines the +[section-name index](https://gabi.xinuos.com/elf/03-sheader.html#section-header-table-entry) +separately from its type and flags, and +[string-table index zero](https://gabi.xinuos.com/elf/04-strtab.html) means no name. +REA preserves known empty names separately from unavailable names; an unavailable +name table still leaves `.dep-v0` presence unknown. The hash is not a claim that that compiler produced the binary. Several distinct hashes leave a single commit unknown. diff --git a/src/artifacts/MachOSliceArtifactReader.test.ts b/src/artifacts/MachOSliceArtifactReader.test.ts index 7985e7c823..a3630479b3 100644 --- a/src/artifacts/MachOSliceArtifactReader.test.ts +++ b/src/artifacts/MachOSliceArtifactReader.test.ts @@ -248,6 +248,49 @@ describe("MachOSliceArtifactReader hidden architectures", () => { }); describe("MachOSliceArtifactReader", () => { + it("classifies malformed successful lipo output at the reader boundary", async () => { + const root = await createTestTempDirectory("rea-slices-malformed-lipo-"); + const binary = join(root, "fat"); + await writeFile( + binary, + fatImage([ + { cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) }, + { cpu: CPU.arm64, bytes: machoImage({ cpu: CPU.arm64 }) }, + ]), + ); + const reader = new MachOSliceArtifactReader( + binary, + {}, + { + run: async () => + ok({ + tool: "lipo", + executable: "/usr/bin/lipo", + executableSha256: "1".repeat(64), + toolVersion: null, + versionReason: "fixture", + arguments: ["-detailed_info", binary], + stdout: "Fat header in: fixture\n", + stderr: "", + stdoutBytes: 23, + stderrBytes: 0, + exitCode: 0, + signal: null, + }), + }, + ); + onTestFinished(() => reader.close()); + + await expect( + reader.entries()[Symbol.asyncIterator]().next(), + ).rejects.toMatchObject({ + reason: "format", + message: + "lipo did not produce readable architecture records: lipo output contained no architectures", + cause: new TypeError("lipo output contained no architectures"), + }); + }); + it("keeps a borrowed source usable after idle and active cancelled slice reads", async () => { const root = await createTestTempDirectory("rea-slices-cancelled-"); const binary = join(root, "fat"); diff --git a/src/artifacts/MachOSliceArtifactReader.ts b/src/artifacts/MachOSliceArtifactReader.ts index cf4b77522a..9336ee39fa 100644 --- a/src/artifacts/MachOSliceArtifactReader.ts +++ b/src/artifacts/MachOSliceArtifactReader.ts @@ -9,7 +9,7 @@ import { type NativeCommandRunner, } from "../native/CommandRunner.js"; import { - parseLipoOutput, + parseLipoOutputResult, type LipoArchitecture, } from "../native/parsers/lipo.js"; import type { MachoSlice } from "../domain/apple/dylibResolution.js"; @@ -115,7 +115,14 @@ export class MachOSliceArtifactReader implements ArtifactReader { }; await this.#verifySource(source); const fileSize = source.initial.size; - const lipo = parseLipoOutput(captured.value.stdout, this.path); + const parsed = parseLipoOutputResult(captured.value.stdout, this.path); + if (!parsed.ok) + throw new ArtifactReaderFailure( + "format", + `lipo did not produce readable architecture records: ${parsed.error.message}`, + { cause: parsed.error }, + ); + const lipo = parsed.value; const { architectures } = lipo; const structural = await readStructuralSlices(source.handle, fileSize); await this.#verifySource(source); diff --git a/src/il2cpp/Il2cppBinaryImage.ts b/src/il2cpp/Il2cppBinaryImage.ts index dc12b2ba1c..3abf781856 100644 --- a/src/il2cpp/Il2cppBinaryImage.ts +++ b/src/il2cpp/Il2cppBinaryImage.ts @@ -2,6 +2,7 @@ import { readFatContainerHeader, validateFatSlice, } from "../domain/apple/machoContainer.js"; +import { binaryMetadataArchitecture } from "../domain/binaryMetadataArchitecture.js"; import { BinaryFormatFailure, BinaryResourceFailure, @@ -144,7 +145,11 @@ export function* openImageSteps( let machoPointerFixups: PointerFixups | undefined; if (container.format === "macho") { try { - const layout = parseMachoLayout(image, container.architecture); + const layout = parseMachoLayout( + image, + binaryMetadataArchitecture(container.architecture) ?? + container.architecture, + ); machoPointerFixups = parsePointerFixups( image, layout.fixups, diff --git a/src/native/metadata/ElfImage.ts b/src/native/metadata/ElfImage.ts index 846baa8619..07377c602f 100644 --- a/src/native/metadata/ElfImage.ts +++ b/src/native/metadata/ElfImage.ts @@ -67,7 +67,6 @@ export const readElfImage = (bytes: Buffer): BinaryContainer => { zeroFills.push({ address: address + size, size: memorySize - size }); } for (const section of directory.sections) { - if (section.name === "") continue; const range = declaredFileRange(bytes.length, section.offset, section.size); const file_backed = section.type !== 8 && range.file_backed; const compressed = (section.flags & 0x800n) !== 0n; @@ -78,7 +77,7 @@ export const readElfImage = (bytes: Buffer): BinaryContainer => { (section.flags & 1n) === 0n && (section.flags & 4n) === 0n && (section.flags & 2n) !== 0n; - if (section.name === null && !read_only) continue; + if ((section.name === null || section.name === "") && !read_only) continue; sections.push({ format: "elf", name: section.name, diff --git a/src/native/metadata/readBinaryContainer.ts b/src/native/metadata/readBinaryContainer.ts index d8fa65f8a0..18bc93ceae 100644 --- a/src/native/metadata/readBinaryContainer.ts +++ b/src/native/metadata/readBinaryContainer.ts @@ -10,7 +10,7 @@ import { readElfImage } from "./ElfImage.js"; import { readPeImage } from "./PeImage.js"; import { readMachoImage } from "./MachoImage.js"; -/** Identify an ELF, PE, or thin Mach-O image and its named file sections. */ +/** Identify an ELF, PE, or thin Mach-O image and its file sections. */ export const readBinaryContainer = (bytes: Buffer): BinaryContainer => { if (bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) return readElfImage(bytes); diff --git a/src/native/parsers/lipo.test.ts b/src/native/parsers/lipo.test.ts index 7b3aa19a57..e98c39a84e 100644 --- a/src/native/parsers/lipo.test.ts +++ b/src/native/parsers/lipo.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { parseLipoOutput } from "./lipo.js"; +import { parseLipoCapture, parseLipoOutput } from "./lipo.js"; const fatRecords = `fat_magic 0xcafebabe nfat_arch 2 @@ -20,6 +20,39 @@ architecture x86_64 align 2^12 (4096) `; +it("retains native command capture when shared lipo parsing rejects records", () => { + const capture = { + tool: "lipo", + executable: "/usr/bin/lipo", + executableSha256: "1".repeat(64), + toolVersion: null, + versionReason: null, + arguments: ["-detailed_info", "/fixture/fat"], + stdout: "Fat header in: /fixture/fat\n", + stderr: "lipo emitted no slice rows", + stdoutBytes: 28, + stderrBytes: 26, + exitCode: 0, + signal: null, + }; + const parsed = parseLipoCapture(capture, "inspect_macho", "/fixture/fat"); + expect(parsed).toMatchObject({ + ok: false, + error: { + operation: "inspect_macho", + cause: new TypeError("lipo output contained no architectures"), + capturedOutput: { + stdout: capture.stdout, + stderr: capture.stderr, + stdout_bytes: capture.stdoutBytes, + stderr_bytes: capture.stderrBytes, + exit_code: 0, + signal: null, + }, + }, + }); +}); + describe("lipo architecture facts", () => { it("preserves Apple's declared and hidden ARM64 architecture scopes", () => { const output = parseLipoOutput(`fat_magic 0xcafebabe diff --git a/src/native/parsers/lipo.ts b/src/native/parsers/lipo.ts index 2880328be3..c985ddfb14 100644 --- a/src/native/parsers/lipo.ts +++ b/src/native/parsers/lipo.ts @@ -138,36 +138,46 @@ export const parseLipoOutput = ( }; }; -/** The producing parser owns malformed tool-output classification for every adapter. */ +/** Classify expected lipo parse failures once for all command adapters. */ +export const parseLipoOutputResult = ( + output: string, + operandPath?: string, +): Result => { + try { + return ok(parseLipoOutput(output, operandPath)); + } catch (cause: unknown) { + if (!(cause instanceof TypeError) && !(cause instanceof z.ZodError)) + throw cause; + return err(cause); + } +}; + +/** Translate classified parse failure into native command output failure. */ export const parseLipoCapture = ( capture: NativeCommandCapture, operation: "inspect_macho" | "list_architectures", operandPath: string, ): Result => { - try { - return ok(parseLipoOutput(capture.stdout, operandPath)); - } catch (cause) { - if (!(cause instanceof TypeError) && !(cause instanceof z.ZodError)) - throw cause; - return err( - new AnalysisOutputError( - operation, - "lipo did not produce readable architecture records", - { - cause, - capturedOutput: { - stdout: capture.stdout, - stderr: capture.stderr, - truncated: false, - stdout_bytes: capture.stdoutBytes, - stderr_bytes: capture.stderrBytes, - exit_code: capture.exitCode, - signal: capture.signal, - }, + const parsed = parseLipoOutputResult(capture.stdout, operandPath); + if (parsed.ok) return parsed; + return err( + new AnalysisOutputError( + operation, + "lipo did not produce readable architecture records", + { + cause: parsed.error, + capturedOutput: { + stdout: capture.stdout, + stderr: capture.stderr, + truncated: false, + stdout_bytes: capture.stdoutBytes, + stderr_bytes: capture.stderrBytes, + exit_code: capture.exitCode, + signal: capture.signal, }, - ), - ); - } + }, + ), + ); }; const integer = (value: string | undefined): number | null => { diff --git a/tests/acceptance/rustUnnamedSections.test.ts b/tests/acceptance/rustUnnamedSections.test.ts new file mode 100644 index 0000000000..f6be212e94 --- /dev/null +++ b/tests/acceptance/rustUnnamedSections.test.ts @@ -0,0 +1,59 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; + +import { toolContract } from "../../src/contracts/toolContracts.js"; +import { rustBinarySchema } from "../../src/domain/rust/rustBinary.js"; +import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; +import { + RUSTC_HASH, + rustElfSectionNameFixture, +} from "../fixtures/rust/image.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +cliTest( + "finds compiler paths independently of ELF section names through CLI and stdio MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-rust-unnamed-sections-"); + const { client, call } = await connectLocalToolsMcp(); + for (const name of ["named", "empty", "unavailable", "writable"] as const) { + const bytes = rustElfSectionNameFixture( + name === "writable" ? "empty" : name, + ); + if (name === "writable") bytes.writeBigUInt64LE(3n, 384 + 8); + const path = join(root, `${name}.elf`); + await writeFile(path, bytes); + const output = await cli.sharedCliRun!({ + arguments: ["inspect-rust-binary", path, "--json"], + }); + expect(output.exitCode, output.stderr).toBe(0); + const evidence = toolContract("inspect_rust_binary").outputSchema.parse( + output.json, + ); + const report = rustBinarySchema.parse(evidence.normalized_result); + expect(report.artifact.sha256).toBe( + createHash("sha256").update(bytes).digest("hex"), + ); + expect(report.auditable_metadata).toBeNull(); + expect(report.rustc_commit).toEqual( + name === "writable" + ? null + : { + evidence_kind: "inferred", + hashes: [{ hash: RUSTC_HASH, file_offset: 2048 }], + }, + ); + if (name === "unavailable") + expect(report.limitations).toContainEqual( + expect.stringContaining("no section-name string table is declared"), + ); + const response = await call("inspect_rust_binary", { path }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(response.structuredContent).toEqual(evidence); + expect(await readFile(path)).toEqual(bytes); + } + await client.ping(); + }, +); diff --git a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts index 8ac6472f18..5abc5646f6 100644 --- a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts +++ b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts @@ -187,6 +187,37 @@ it("requires an explicit slice of a universal Mach-O and reads only that slice", expect(failure(thin, "arm64").reason).toBe("slice-not-found"); }); +it.each(["thin", "universal"] as const)( + "links a %s x86-64 Mach-O while retaining the reader's amd64 spelling", + (format) => { + const thin = createIl2cppBinaryFixture(); + thin.writeUInt32LE(0x01000007, 4); + thin.writeUInt32LE(3, 8); + const bytes = + format === "thin" + ? thin + : createUniversalFixture([ + { cpu: 0x0100000c, bytes: createIl2cppBinaryFixture() }, + { cpu: 0x01000007, bytes: thin }, + ]); + // Match the selected FAT declaration to CPU_SUBTYPE_X86_64_ALL in its header. + if (format === "universal") bytes.writeUInt32BE(3, 8 + 20 + 4); + const original = Buffer.from(bytes); + const linked = link(bytes, format === "thin" ? undefined : "amd64"); + expect(linked).toMatchObject({ + format: "macho", + architecture: "amd64", + code_gen_modules_field: FIXTURE_BINARY_BASE + 0x600n, + code_gen_modules_array: FIXTURE_BINARY_BASE + 0x500n, + }); + expect(linked.method_addresses).toEqual(expectedAddresses); + expect(linked.return_types).toEqual( + link(createIl2cppBinaryFixture()).return_types, + ); + expect(bytes).toEqual(original); + }, +); + it.each< [string, Il2cppBinaryFixtureOptions, string, "unsupported" | "malformed"] >([ diff --git a/tests/boundary/rust/rustBinaryImage.test.ts b/tests/boundary/rust/rustBinaryImage.test.ts index a48bfe2e61..96e7ae99c8 100644 --- a/tests/boundary/rust/rustBinaryImage.test.ts +++ b/tests/boundary/rust/rustBinaryImage.test.ts @@ -5,6 +5,7 @@ import { createRustImageWithGoSection, createRustHashOverflowImage, compressRustElfSectionNames, + rustElfSectionNameFixture, RUSTC_HASH, rustDependencyPayload, } from "../../fixtures/rust/image.js"; @@ -34,6 +35,60 @@ const packages = [ }, ]; +it.each([ + ["named", ".rodata"], + ["empty", ""], + ["unavailable", null], +] as const)( + "retains read-only ELF section bytes with a %s name", + (name, expectedName) => { + const bytes = rustElfSectionNameFixture(name); + const container = readBinaryContainer(bytes); + expect(container.load_regions?.every(({ write }) => write)).toBe(true); + expect(container.sections).toContainEqual( + expect.objectContaining({ + format: "elf", + name: expectedName, + offset: 2048, + read_only: true, + file_backed: true, + }), + ); + expect(readRustBinaryImage(bytes).rustc_commit?.hashes).toEqual([ + { hash: RUSTC_HASH, file_offset: 2048 }, + ]); + }, +); + +it.each([ + ["writable", 1, 3n], + ["executable", 1, 6n], + ["unallocated", 1, 0n], + ["NOBITS", 8, 2n], + ["NULL", 0, 2n], +] as const)( + "does not scan unnamed ELF %s sections for compiler paths", + (_kind, type, flags) => { + const bytes = rustElfSectionNameFixture("empty"); + bytes.writeUInt32LE(type, 384 + 4); + bytes.writeBigUInt64LE(flags, 384 + 8); + expect(readBinaryContainer(bytes).sections).not.toContainEqual( + expect.objectContaining({ offset: 2048, read_only: true }), + ); + expect(readRustBinaryImage(bytes).rustc_commit).toBeNull(); + }, +); + +it("does not scan an unnamed ELF section whose declared range leaves the file", () => { + const bytes = rustElfSectionNameFixture("empty"); + // The compiler path still exists at 2048, but its declared section no longer fits. + bytes.writeBigUInt64LE(BigInt(bytes.length), 384 + 32); + expect(readBinaryContainer(bytes).sections).not.toContainEqual( + expect.objectContaining({ read_only: true }), + ); + expect(readRustBinaryImage(bytes).rustc_commit).toBeNull(); +}); + it.each(["elf", "pe"] as const)( "reads a .dep-v0 section and an inferred rustc hash from %s", (format) => { diff --git a/tests/fixtures/rust/image.ts b/tests/fixtures/rust/image.ts index b557bf8967..7afd36739e 100644 --- a/tests/fixtures/rust/image.ts +++ b/tests/fixtures/rust/image.ts @@ -81,6 +81,18 @@ export const createRustImage = (options: { return bytes; }; +/** Read-only ELF data stays independent of its section-name spelling or availability. */ +export const rustElfSectionNameFixture = ( + name: "named" | "empty" | "unavailable", +): Buffer => { + const bytes = createRustImage({ format: "elf", omitSection: true }); + // This fixture's read-only PROGBITS header is at 384. Its PT_LOAD is writable, + // so compiler-path recovery must use the section rather than segment fallback. + if (name === "empty") bytes.writeUInt32LE(0, 384); + if (name === "unavailable") bytes.writeUInt16LE(0, 62); + return bytes; +}; + interface RustImageWrite { readonly bytes: Buffer; readonly little: (offset: number, value: number, width: 16 | 32 | 64) => void;