diff --git a/docs/gpu-code-objects.md b/docs/gpu-code-objects.md index 0171b1b0c4..fb02ac7bf7 100644 --- a/docs/gpu-code-objects.md +++ b/docs/gpu-code-objects.md @@ -32,6 +32,12 @@ unexamined payloads remain `unexamined`. An exact AMDGPU target is normalized at its note owner for comparison with a Clang bundle declaration; raw values remain intact. Unresolved feature comparisons stay `unknown`. +A HIP target ID identifies a toolchain target, not its payload representation. +Recognized raw or wrapped LLVM bitcode stays `unsupported` for kernel metadata +inspection, with the bundle ID, range and selected payload digest retained. +Bitcode validity and inner targets remain unexamined; malformed recognized ELF +still reports `malformed`. Standalone bitcode is outside GPU container coverage. + HIP bundle and metadata note recognition accept both the legacy `amdgcn` triple and LLVM's `amdgpu` subarchitecture triples, including declared generic processors such as `gfx11-generic` and `gfx9-4-generic`. Notes with the empty or explicit @@ -40,9 +46,18 @@ retaining their raw triple spelling. CUDA's legacy fatbinary compression flag is retained as an unsupported codec with its encoded and decoded extents; compressed bytes are not treated as PTX text. CCOB is recognized by the exact four signature bytes `CCOB`; high-bit-corrupted bytes are not decoded as ASCII -and accepted as a bundle. CCOB v1 has no stored total size, so a following -legacy bundle is located by the next `CCOB` signature and zero-byte alignment -padding remains part of the first container extent. +and accepted as a bundle. CCOB v1 has no stored total size: the completed zlib +or Zstd stream determines its encoded extent. Embedded `CCOB` text is ordinary +compressed data; zero-byte alignment after the stream belongs to neither +payload. If decoding fails or its codec is unsupported, a later `CCOB` marker +is only an uncertain recovery candidate. The affected row retains that +limitation and discovery remains partial. + +The declared decoded length bounds decompression. An understated length can +stop decoding before codec completion, so any subsequent recovery extent stays +unknown; the decoder does not expand its allowance to prove the bad declaration. +Interrupted observations omit unfinished v1 rows. A terminal capacity row +qualifies its retained range as the containing tail and does not infer siblings. `cuda_target_header` retains the OS ABI, normalized raw `e_flags` and exact header location. CUDA OS ABI `0x33` and the observed `0x41` flags layout use diff --git a/src/gpu/ClangOffloadBundle.ts b/src/gpu/ClangOffloadBundle.ts index 0a42368f5b..fe76dfc9d4 100644 --- a/src/gpu/ClangOffloadBundle.ts +++ b/src/gpu/ClangOffloadBundle.ts @@ -118,8 +118,8 @@ export const readCcobHeader = (bytes: Buffer, remaining: number) => { return { version, method, width, total, decoded, hash }; }; -/** Legacy v1 has no size field; LLVM finds the next CCOB magic as its boundary. */ -export const readLegacyCcobTotal = async ( +/** After failed/unsupported v1 decoding only: a marker is an uncertain recovery candidate. */ +export const findLegacyCcobRecoveryTotal = async ( source: GpuSource, base: number, ): Promise => { diff --git a/src/gpu/GpuInventory.test.ts b/src/gpu/GpuInventory.test.ts index e98be43042..269b14eb22 100644 --- a/src/gpu/GpuInventory.test.ts +++ b/src/gpu/GpuInventory.test.ts @@ -1,24 +1,29 @@ import { expect, it } from "vitest"; import { GpuInventory } from "./GpuInventory.js"; import { GpuBufferSource } from "./GpuSource.js"; +import { assertArtifactReadActive } from "../artifacts/ArtifactFileOperation.js"; import { ccobFixture, cubinFixture, cudaFixture, } from "../../tests/fixtures/gpu/codeObjects.js"; -it.each(["cuda", "ccob"] as const)( +it.each(["cuda", "ccob", "ccob-v1"] as const)( "refuses oversized %s output before reading compressed input", async (kind) => { const bytes = kind === "cuda" ? cudaFixture("zstd", [{ kind: 2, target: 80, bytes: cubinFixture() }]) - : ccobFixture(3); - const payloadOffset = kind === "cuda" ? 80 : 32; - bytes.writeBigUInt64LE( - BigInt(128 * 1024 * 1024 + 1), - kind === "cuda" ? 72 : 16, - ); + : kind === "ccob" + ? ccobFixture(3) + : Buffer.concat([ccobFixture(1), ccobFixture(2)]); + const payloadOffset = kind === "cuda" ? 80 : kind === "ccob" ? 32 : 20; + if (kind === "ccob-v1") bytes.writeUInt32LE(128 * 1024 * 1024 + 1, 8); + else + bytes.writeBigUInt64LE( + BigInt(128 * 1024 * 1024 + 1), + kind === "cuda" ? 72 : 16, + ); const source = new GpuBufferSource(bytes, "fixture"), reads: number[] = [], inventory = new GpuInventory({ path: "/fixture", analysis: "kernels" }); @@ -36,6 +41,65 @@ it.each(["cuda", "ccob"] as const)( sha256: null, }); expect(inventory.kernels).toEqual([]); + if (kind === "ccob-v1") { + expect(inventory.objects).toHaveLength(1); + expect(inventory.objects[0]!.location.size).toBe(bytes.length - 20); + expect(inventory.objects[0]!.limitations.join(" ")).toContain( + "only the containing tail", + ); + expect(inventory.objects[0]!.limitations.join(" ")).toContain( + "Later siblings were not inferred", + ); + } + }, +); +it.each(["zlib", "zstd"] as const)( + "omits an unfinished v1 %s extent when cancellation interrupts its source read", + async (codec) => { + const first = ccobFixture(2, codec), + second = ccobFixture(1, codec), + source = new GpuBufferSource(Buffer.concat([first, second]), "fixture"), + controller = new AbortController(), + reason = { selected: "cancel-v1-payload-read" }, + inventory = new GpuInventory( + { path: "/fixture", analysis: "inventory" }, + controller.signal, + ); + let interrupted = false; + const result = inventory.inspect({ + size: source.size, + containerId: source.containerId, + read: async (offset, size) => { + if (offset === first.length + 20) { + interrupted = true; + controller.abort(reason); + assertArtifactReadActive(controller.signal); + } + return source.read(offset, size); + }, + }); + await expect(result).rejects.toMatchObject({ + reason: "cancelled", + cause: reason, + }); + expect(interrupted).toBe(true); + const wrappers = inventory.objects.filter( + ({ kind }) => kind === "hip-bundle", + ); + expect(wrappers).toHaveLength(1); + expect(wrappers[0]).toMatchObject({ + id: "fixture@0", + inspection: "complete", + encoded_size: first.length - 24, + location: { offset: 24, size: first.length - 24 }, + }); + expect( + inventory.objects.every( + ({ id, parent_id }) => + id !== `fixture@${first.length.toString(16)}` && + parent_id !== `fixture@${first.length.toString(16)}`, + ), + ).toBe(true); }, ); it("observes cancellation while walking flat empty bundles in a decoded buffer", async () => { @@ -55,3 +119,34 @@ it("observes cancellation while walking flat empty bundles in a decoded buffer", cause: reason, }); }); +it.each(["zlib", "zstd"] as const)( + "reuses observed %s input across multiple valid legacy CCOB siblings", + async (codec) => { + const count = 16, + bytes = Buffer.concat( + Array.from({ length: count }, () => ccobFixture(1, codec)), + ), + source = new GpuBufferSource(bytes, "fixture"), + inventory = new GpuInventory({ path: "/fixture", analysis: "inventory" }); + let observedReadBytes = 0; + await inventory.inspect({ + size: source.size, + containerId: source.containerId, + read: async (offset, size) => { + observedReadBytes += size; + return source.read(offset, size); + }, + }); + const bundles = inventory.objects.filter( + ({ kind }) => kind === "hip-bundle", + ); + expect(bundles).toHaveLength(count); + expect(bundles.every(({ inspection }) => inspection === "complete")).toBe( + true, + ); + expect(inventory.discovery).toBe("complete"); + // Includes prefix/alignment reads. Re-reading every suffix grows with the + // square of the sibling count and exceeds this physical-input allowance. + expect(observedReadBytes).toBeLessThan(2 * bytes.length); + }, +); diff --git a/src/gpu/GpuInventory.ts b/src/gpu/GpuInventory.ts index 8926f0e66e..3b1c310459 100644 --- a/src/gpu/GpuInventory.ts +++ b/src/gpu/GpuInventory.ts @@ -1,14 +1,14 @@ import { CCOB_MAGIC, CLANG_BUNDLE_MAGIC, - readLegacyCcobTotal, + findLegacyCcobRecoveryTotal, readClangBundleDirectory, readCcobHeader, } from "./ClangOffloadBundle.js"; import { gpuObject, gpuLocation, type GpuPayloadTarget } from "./GpuObject.js"; import { readCudaTarget } from "./CudaTarget.js"; import { readCudaFatbinary } from "./CudaFatbinary.js"; -import { decodeGpuPayload } from "./GpuPayloadCodec.js"; +import { decodeGpuPayload, decodeGpuPayloadStream } from "./GpuPayloadCodec.js"; import { createHash } from "node:crypto"; import { scheduler } from "node:timers/promises"; import type { @@ -99,6 +99,11 @@ export class GpuInventory { row.sha256 = sha256; row.decoded_size = size; } + #encodedExtent(row: GpuObject, size: number): void { + this.#retain({ encoded_size: size, size }); + row.encoded_size = size; + row.location.size = size; + } #objectLimitations(row: GpuObject, messages: string[]): void { this.#retain(messages); row.limitations.push(...messages); @@ -394,6 +399,8 @@ export class GpuInventory { this.kernels.push(kernel); } } else if (row.kind === "cubin" || row.kind === "amdgpu") { + if (row.kind === "amdgpu" && isLlvmBitcode(bytes)) + throw new BinaryFormatFailure("unsupported", LLVM_BITCODE_LIMITATION); coverage = await this.#elfFacts(decoded, row, row.kind, preparedAmd); } else { row.inspection = "unsupported"; @@ -464,6 +471,19 @@ export class GpuInventory { }; try { + // An AMD target declaration does not establish an ELF representation. + // Without an outer target, inventory must still avoid an ELF corruption + // claim for recognizable bitcode; its inner target remains unexamined. + if ( + directory === undefined && + isLlvmBitcode(await child.read(0, Math.min(4, child.size))) + ) { + this.discovery = "partial"; + if (this.input.analysis !== "kernels" || !this.#selected(row)) + this.#objectLimitations(row, [LLVM_BITCODE_LIMITATION]); + await this.#payload(source, row); + return; + } const native = directory ?? (await readGpuNativeDirectory( @@ -528,6 +548,7 @@ export class GpuInventory { async #hip(source: GpuSource, depth: number): Promise { let base = 0; let bundles = 0; + let legacyWindow: { offset: number; bytes: Buffer } | undefined; while (base < source.size) { if (++bundles % 128 === 0) { await scheduler.yield(); @@ -552,12 +573,10 @@ export class GpuInventory { } if (!prefix.subarray(0, CCOB_MAGIC.length).equals(CCOB_MAGIC)) rejectGpu("Unexpected bytes between Clang offload bundles"); - const legacyTotal = - prefix.length >= 6 && prefix.readUInt16LE(4) === 1 - ? await readLegacyCcobTotal(source, base) - : source.size - base; - const header = readCcobHeader(prefix, legacyTotal), - { width, total, decoded, hash, version, method } = header; + const header = readCcobHeader(prefix, source.size - base), + { width, decoded, hash, version, method } = header; + let total = header.total; + let extentKnown = version !== 1; const row = gpuObject(source, { offset: base + width, size: total - width, @@ -568,21 +587,81 @@ export class GpuInventory { method === 0 ? "zlib" : method === 1 ? "zstd" : "unsupported"; row.compression_id = `ccob:${version}:${method}`; row.decoded_size = decoded; - this.#add(row); + if (extentKnown) this.#add(row); if (row.compression === "unsupported") { row.inspection = "unsupported"; this.#objectLimitations(row, ["Unsupported CCOB compression method"]); this.discovery = "partial"; } else try { - this.#budgetWork(total - width, decoded); - const data = await decodeGpuPayload( - await source.read(base + width, total - width), - row.compression, - decoded, - this.signal, - version === 1, - ); + // v1 has no encoded size. Supply a bounded suffix, then retain only + // the codec's completed extent, never an embedded CCOB marker. + const inputSize = + version === 1 + ? Math.min(total - width, GPU_OBJECT_BYTES) + : total - width; + let data: Buffer; + if (version === 1) { + const start = base + width; + const cached = + legacyWindow !== undefined && + start >= legacyWindow.offset && + start < legacyWindow.offset + legacyWindow.bytes.length; + const readWindow = async (): Promise => { + this.#budgetWork(inputSize, decoded); + legacyWindow = undefined; + const bytes = await source.read(start, inputSize); + legacyWindow = { offset: start, bytes }; + return bytes; + }; + if (cached) this.#budgetWork(decoded); + let encoded: Buffer | undefined = cached + ? legacyWindow!.bytes.subarray(start - legacyWindow!.offset) + : await readWindow(); + const available = encoded.length; + const compression = row.compression; + const completed = await decodeGpuPayloadStream( + encoded, + compression, + decoded, + this.signal, + ).catch(async (cause: unknown) => { + // A cached window may end inside this sibling. Refill once + // before treating that incomplete view as a malformed stream. + if ( + !(cause instanceof BinaryFormatFailure) || + !cached || + available >= inputSize + ) + throw cause; + // Retire the previous window before allocating its replacement. + encoded = undefined; + return decodeGpuPayloadStream( + await readWindow(), + compression, + decoded, + this.signal, + ); + }); + total = width + completed.consumedBytes; + extentKnown = true; + this.#encodedExtent(row, completed.consumedBytes); + // Publish the completed parent extent before inspecting its children. + this.#add(row); + data = completed.buffer; + if (data.length !== decoded) + rejectGpu( + "GPU decompressed length disagrees with its declaration", + ); + } else { + this.#budgetWork(inputSize, decoded); + data = await decodeGpuPayload( + await source.read(base + width, inputSize), + row.compression, + decoded, + this.signal, + ); + } if ( createHash("md5").update(data).digest().readBigUInt64LE(0) !== hash ) @@ -594,10 +673,41 @@ export class GpuInventory { this.#retainFailure(row, cause); this.discovery = "partial"; } + if (!extentKnown) { + if (row.inspection !== "capacity") + try { + total = await findLegacyCcobRecoveryTotal(source, base); + } catch (cause) { + if (!(cause instanceof BinaryResourceFailure)) throw cause; + this.#retainFailure(row, cause); + } + // A capacity refusal cannot establish a boundary for later siblings. + if (row.inspection === "capacity") { + this.#objectLimitations(row, [ + "CCOB v1 codec extent is unknown; the retained size is only the containing tail, not a completed codec extent. Later siblings were not inferred.", + ]); + this.#add(row); + return; + } + this.#encodedExtent(row, total - width); + this.#objectLimitations(row, [ + total < source.size - base + ? "CCOB v1 codec extent is unknown; the next CCOB marker is only an uncertain recovery candidate." + : "CCOB v1 codec extent is unknown; the retained range extends to the container end.", + ]); + this.#add(row); + } base = await nextGpuContainerOffset(source, base + total); } } } +const LLVM_BITCODE_LIMITATION = + "LLVM bitcode kernel metadata is outside current coverage; bitcode validity and inner target were not inspected."; +/** LLVM raw and wrapper signatures identify representation, not bitcode validity. */ +const isLlvmBitcode = (bytes: Buffer): boolean => + bytes.length >= 4 && + (bytes.readUInt32LE(0) === 0xdec04342 || + bytes.readUInt32LE(0) === 0x0b17c0de); const isGpuSection = (name: string): boolean => [ ".nv_fatbin", diff --git a/src/gpu/GpuPayloadCodec.test.ts b/src/gpu/GpuPayloadCodec.test.ts index 97fc039ee8..465fc725c4 100644 --- a/src/gpu/GpuPayloadCodec.test.ts +++ b/src/gpu/GpuPayloadCodec.test.ts @@ -1,6 +1,6 @@ import { expect, it } from "vitest"; import { deflateSync, zstdCompressSync } from "node:zlib"; -import { decodeGpuPayload } from "./GpuPayloadCodec.js"; +import { decodeGpuPayload, decodeGpuPayloadStream } from "./GpuPayloadCodec.js"; it.each(["zlib", "zstd"] as const)( "requires all declared %s input bytes to belong to the codec stream", async (codec) => { @@ -19,3 +19,27 @@ it.each(["zlib", "zstd"] as const)( ).rejects.toMatchObject({ kind: "malformed" }); }, ); +it.each(["zlib", "zstd"] as const)( + "retains completed %s input accounting before trailing container bytes", + async (codec) => { + const source = Buffer.from("ordinary CCOB marker"), + compressed = + codec === "zstd" + ? zstdCompressSync(source) + : deflateSync(source, { level: 0 }), + input = Buffer.concat([ + compressed, + Buffer.from("CCOB following container"), + ]); + expect(await decodeGpuPayloadStream(input, codec, source.length)).toEqual({ + buffer: source, + consumedBytes: compressed.length, + }); + await expect( + decodeGpuPayloadStream(compressed.subarray(0, -1), codec, source.length), + ).rejects.toMatchObject({ kind: "malformed" }); + await expect( + decodeGpuPayload(compressed, codec, source.length + 1), + ).rejects.toMatchObject({ kind: "malformed" }); + }, +); diff --git a/src/gpu/GpuPayloadCodec.ts b/src/gpu/GpuPayloadCodec.ts index e84a94ad86..29248d5ecc 100644 --- a/src/gpu/GpuPayloadCodec.ts +++ b/src/gpu/GpuPayloadCodec.ts @@ -6,18 +6,17 @@ import { decodeLz4Block } from "./Lz4Block.js"; import { rejectGpu } from "./GpuSource.js"; const unzip = promisify(inflate); const unzstd = promisify(zstdDecompress); -export const decodeGpuPayload = async ( +/** Completed codec bytes and their encoded extent share one completion owner. */ +export const decodeGpuPayloadStream = async ( bytes: Buffer, - compression: "zlib" | "zstd" | "lz4", - expected: number, + compression: "zlib" | "zstd", + maximumDecodedSize: number, signal?: AbortSignal, - allowZeroPadding = false, -): Promise => { +): Promise<{ buffer: Buffer; consumedBytes: number }> => { assertArtifactReadActive(signal); - if (compression === "lz4") return decodeLz4Block(bytes, expected, signal); let result: unknown; try { - const maxOutputLength = Math.max(1, expected); + const maxOutputLength = Math.max(1, maximumDecodedSize); result = await (compression === "zstd" ? unzstd(bytes, { maxOutputLength, @@ -34,16 +33,32 @@ export const decodeGpuPayload = async ( assertArtifactReadActive(signal); const { buffer: decoded, consumedBytes: consumed } = nodeCodecResult(result); if ( - consumed !== bytes.length && - (!allowZeroPadding || - consumed < 0 || - consumed > bytes.length || - !bytes.subarray(consumed).every((byte) => byte === 0)) + !Number.isSafeInteger(consumed) || + consumed <= 0 || + consumed > bytes.length ) + rejectGpu("GPU decoder returned an invalid complete codec extent"); + return { buffer: decoded, consumedBytes: consumed }; +}; +export const decodeGpuPayload = async ( + bytes: Buffer, + compression: "zlib" | "zstd" | "lz4", + expected: number, + signal?: AbortSignal, +): Promise => { + assertArtifactReadActive(signal); + if (compression === "lz4") return decodeLz4Block(bytes, expected, signal); + const decoded = await decodeGpuPayloadStream( + bytes, + compression, + expected, + signal, + ); + if (decoded.consumedBytes !== bytes.length) rejectGpu( "GPU compressed payload contains bytes outside its complete codec stream", ); - if (decoded.length !== expected) + if (decoded.buffer.length !== expected) rejectGpu("GPU decompressed length disagrees with its declaration"); - return decoded; + return decoded.buffer; }; diff --git a/tests/acceptance/gpuCodeObjectsCli.test.ts b/tests/acceptance/gpuCodeObjectsCli.test.ts index b9f5ea8165..7d9c3e73ad 100644 --- a/tests/acceptance/gpuCodeObjectsCli.test.ts +++ b/tests/acceptance/gpuCodeObjectsCli.test.ts @@ -2,6 +2,7 @@ import { expect, it } from "vitest"; import { writeFile, unlink } from "node:fs/promises"; import { join } from "node:path"; import { createHash } from "node:crypto"; +import { deflateSync } from "node:zlib"; import { toolContract } from "../../src/contracts/toolContracts.js"; import { gpuCodeObjectsSchema } from "../../src/domain/gpu/gpuCodeObjects.js"; import { @@ -21,6 +22,7 @@ import { cubinFixture, hipFixture, elfFixture, + llvmBitcodeFixture, } from "../fixtures/gpu/codeObjects.js"; cliTest( @@ -65,6 +67,148 @@ cliTest( }, ); +cliTest( + "inventories a complete CCOB stream containing ordinary CCOB payload text", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-ccob-embedded-marker-"); + const bundle = hipFixture( + Buffer.from("ordinary CCOB marker"), + "host-other", + ), + compressed = deflateSync(bundle, { level: 0 }); + expect(compressed.indexOf("CCOB")).toBeGreaterThan(0); + for (const version of [1, 2] as const) { + const width = version === 1 ? 20 : 24, + header = Buffer.from( + ccobFixture(version, "zlib", bundle).subarray(0, width), + ); + if (version === 2) header.writeUInt32LE(width + compressed.length, 8); + const bytes = Buffer.concat([header, compressed]), + path = join(root, `v${version}`); + await writeFile(path, bytes); + const response = await cli.run({ + arguments: [ + "inspect-gpu-code-objects", + path, + "--analysis", + "inventory", + "--detail", + "complete", + "--json", + ], + }); + expect(response.exitCode, response.stderr).toBe(0); + const evidence = parseEvidence( + toolContract("inspect_gpu_code_objects").outputSchema.parse( + response.json, + ), + ); + const report = gpuCodeObjectsSchema.parse(evidence.normalized_result); + expect(report.coverage).toMatchObject({ + discovery: "complete", + payloads: "not-requested", + }); + expect(report.objects).toHaveLength(3); + expect(report.objects[0]).toMatchObject({ + kind: "hip-bundle", + inspection: "complete", + encoded_size: compressed.length, + decoded_size: bundle.length, + sha256: createHash("sha256").update(bundle).digest("hex"), + location: { offset: width, size: compressed.length }, + }); + expect( + report.objects + .slice(1) + .map(({ bundle_id, inspection }) => [bundle_id, inspection]), + ).toEqual([ + ["host-x86_64-unknown-linux", "complete"], + ["host-other", "complete"], + ]); + } + }, +); + +cliTest( + "distinguishes unsupported HIP bitcode from corrupted ELF through the public CLI", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-hip-bitcode-cli-"), + bitcode = llvmBitcodeFixture(), + corruptedElf = amdFixture(), + bundleId = "hip-amdgcn-amd-amdhsa--gfx900"; + corruptedElf.writeBigUInt64LE(BigInt(corruptedElf.length + 1), 40); + for (const [name, payload, analysis, inspection] of [ + ["inventory", bitcode, "inventory", "inventory"], + ["bitcode", bitcode, "kernels", "unsupported"], + ["corrupt-elf", corruptedElf, "kernels", "malformed"], + ] as const) { + const bytes = hipFixture(payload, bundleId), + path = join(root, name); + await writeFile(path, bytes); + const response = await cli.run({ + arguments: [ + "inspect-gpu-code-objects", + path, + "--analysis", + analysis, + "--targets", + '["gfx900"]', + "--detail", + "complete", + "--json", + ], + }); + expect(response.exitCode, response.stderr).toBe(0); + const evidence = parseEvidence( + toolContract("inspect_gpu_code_objects").outputSchema.parse( + response.json, + ), + ); + const report = gpuCodeObjectsSchema.parse(evidence.normalized_result); + const row = report.objects.find(({ kind }) => kind === "amdgpu")!; + expect(row).toMatchObject({ + bundle_id: bundleId, + target: "gfx900", + payload_target: null, + inspection, + encoded_size: payload.length, + location: { + container_id: null, + offset: bytes.length - payload.length, + size: payload.length, + }, + sha256: + analysis === "inventory" + ? null + : createHash("sha256").update(payload).digest("hex"), + }); + expect(report.coverage.payloads).toBe( + analysis === "inventory" ? "not-requested" : "partial", + ); + expect(report.kernels).toEqual([]); + if (name === "bitcode") { + expect(row.limitations.join(" ")).toContain("LLVM bitcode"); + expect(row.limitations.join(" ")).toContain( + "validity and inner target were not inspected", + ); + } + } + const path = join(root, "raw-bitcode"); + await writeFile(path, bitcode); + const raw = await cli.run({ + arguments: [ + "inspect-gpu-code-objects", + path, + "--analysis", + "kernels", + "--json", + ], + }); + expect(raw.exitCode).toBe(1); + expect(raw.json).toMatchObject({ code: "unsupported_target" }); + }, +); + cliTest( "keeps embedded Clang bundle reads inside their native section", async ({ cli }) => { diff --git a/tests/boundary/gpu/codeObjects.test.ts b/tests/boundary/gpu/codeObjects.test.ts index 029cdb14b8..f172ff2786 100644 --- a/tests/boundary/gpu/codeObjects.test.ts +++ b/tests/boundary/gpu/codeObjects.test.ts @@ -2,6 +2,7 @@ import { expect, it, onTestFinished } from "vitest"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { deflateSync } from "node:zlib"; +import { createHash } from "node:crypto"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { gpuCodeObjectsPartialSchema } from "../../../src/domain/gpu/gpuCodeObjects.js"; import { GpuCodeObjectsProvider } from "../../../src/gpu/GpuCodeObjectsProvider.js"; @@ -15,6 +16,7 @@ import { elfFixture, ptxFixture, amdFixture, + llvmBitcodeFixture, } from "../../fixtures/gpu/codeObjects.js"; const inspect = async ( bytes: Buffer, @@ -278,6 +280,37 @@ it("keeps AMDGPU processor identity unknown when ELF flags are unassigned", asyn inspection: "inventory", }); }); +it.each([false, true])( + "keeps HIP bitcode representation unsupported with wrapped=%s and no declared processor", + async (wrapped) => { + const payload = llvmBitcodeFixture(wrapped), + bytes = hipFixture(payload, "hip-amdgcn-amd-amdhsa--"); + for (const analysis of ["inventory", "kernels"] as const) { + const result = await inspect(bytes, { analysis }); + if (!result.ok) throw result.error; + const row = result.value.objects.find(({ kind }) => kind === "amdgpu")!; + expect(row).toMatchObject({ + bundle_id: "hip-amdgcn-amd-amdhsa--", + target: null, + payload_target: null, + inspection: analysis === "inventory" ? "inventory" : "unsupported", + encoded_size: payload.length, + location: { size: payload.length }, + sha256: + analysis === "inventory" + ? null + : createHash("sha256").update(payload).digest("hex"), + }); + expect(row.limitations.join(" ")).toContain("LLVM bitcode"); + expect(row.limitations.join(" ")).not.toContain("not an ELF image"); + expect(result.value.coverage).toMatchObject({ + discovery: "partial", + payloads: analysis === "inventory" ? "not-requested" : "partial", + }); + expect(result.value.kernels).toEqual([]); + } + }, +); it.each([ ["amdgcn", "gfx900:xnack+"], ["amdgcn", "gfx900+xnack"], @@ -930,26 +963,34 @@ it("does not dispatch a high-bit-corrupted DOS signature to the PE parser", asyn error: { _tag: "AnalysisUnsupportedTargetError" }, }); }); -it("uses each legacy CCOB v1 stream boundary across alignment padding", async () => { - const first = ccobFixture(1, "zlib"), - second = ccobFixture(2, "zlib"), - padding = Buffer.alloc((8 - (first.length % 8)) % 8); - const result = await inspect(Buffer.concat([first, padding, second])); - if (!result.ok) throw result.error; - const bundles = result.value.objects.filter( - ({ kind }) => kind === "hip-bundle", - ); - expect(bundles).toHaveLength(2); - expect( - bundles.map(({ inspection, decoded_size }) => [inspection, decoded_size]), - ).toEqual([ - ["complete", expect.any(Number)], - ["complete", expect.any(Number)], - ]); - expect(bundles[0]!.sha256).toBe(bundles[1]!.sha256); - expect(result.value.kernels).toHaveLength(2); - expect(result.value.coverage.discovery).toBe("complete"); -}); +it.each(["zlib", "zstd"] as const)( + "uses each legacy CCOB v1 %s stream boundary across alignment padding", + async (codec) => { + const first = ccobFixture(1, codec), + second = ccobFixture(2, "zlib"), + padding = Buffer.alloc((8 - (first.length % 8)) % 8); + const result = await inspect(Buffer.concat([first, padding, second])); + if (!result.ok) throw result.error; + const bundles = result.value.objects.filter( + ({ kind }) => kind === "hip-bundle", + ); + expect(bundles).toHaveLength(2); + expect( + bundles.map(({ inspection, decoded_size }) => [inspection, decoded_size]), + ).toEqual([ + ["complete", expect.any(Number)], + ["complete", expect.any(Number)], + ]); + expect(bundles[0]!.sha256).toBe(bundles[1]!.sha256); + expect(bundles[0]!.encoded_size).toBe(first.length - 20); + expect(bundles[0]!.location.size).toBe(first.length - 20); + expect(bundles[1]!.metadata_location.offset).toBe( + first.length + padding.length, + ); + expect(result.value.kernels).toHaveLength(2); + expect(result.value.coverage.discovery).toBe("complete"); + }, +); it("keeps a later CCOB bundle after a truncated legacy v1 stream", async () => { const truncated = ccobFixture(1, "zlib").subarray(0, -1), later = ccobFixture(2, "zlib"); @@ -965,7 +1006,42 @@ it("keeps a later CCOB bundle after a truncated legacy v1 stream", async () => { ]); expect(result.value.kernels).toHaveLength(1); expect(result.value.coverage.discovery).toBe("partial"); + expect(bundles[0]!.limitations.join(" ")).toContain( + "uncertain recovery candidate", + ); }); +it.each(["checksum", "decoded-size"] as const)( + "preserves a completed v1 codec extent after a bad %s declaration", + async (declaration) => { + const bundle = hipFixture( + Buffer.from("ordinary CCOB marker"), + "host-other", + ), + compressed = deflateSync(bundle, { level: 0 }), + header = Buffer.from(ccobFixture(1, "zlib", bundle).subarray(0, 20)); + if (declaration === "checksum") header[12] = header[12]! ^ 1; + else header.writeUInt32LE(bundle.length + 1, 8); + const first = Buffer.concat([header, compressed]); + const result = await inspect( + Buffer.concat([first, ccobFixture(2, "zlib")]), + ); + if (!result.ok) throw result.error; + const wrappers = result.value.objects.filter( + ({ kind }) => kind === "hip-bundle", + ); + expect(wrappers.map(({ inspection }) => inspection)).toEqual([ + "malformed", + "complete", + ]); + expect(wrappers[0]).toMatchObject({ + encoded_size: compressed.length, + location: { size: compressed.length }, + }); + expect(wrappers[0]!.limitations.join(" ")).not.toContain("recovery"); + expect(wrappers[1]!.metadata_location.offset).toBe(first.length); + expect(result.value.coverage.discovery).toBe("partial"); + }, +); it("continues at the declared CCOB boundary after an unsupported codec", async () => { const unknown = ccobFixture(); unknown.writeUInt16LE(99, 6); diff --git a/tests/fixtures/gpu/codeObjects.ts b/tests/fixtures/gpu/codeObjects.ts index 7a39a4a562..49a3ca55b7 100644 --- a/tests/fixtures/gpu/codeObjects.ts +++ b/tests/fixtures/gpu/codeObjects.ts @@ -321,6 +321,16 @@ export const hipFixture = ( head.writeBigUInt64LE(2n, 24); return Buffer.concat([head, ...headers, payload]); }; +/** Representation recognition only; these bytes do not establish valid LLVM IR. */ +export const llvmBitcodeFixture = (wrapped = false): Buffer => { + const raw = Buffer.from([0x42, 0x43, 0xc0, 0xde, 0, 0, 0, 0]); + if (!wrapped) return raw; + const header = Buffer.alloc(20); + header.writeUInt32LE(0x0b17c0de, 0); + header.writeUInt32LE(20, 8); + header.writeUInt32LE(raw.length, 12); + return Buffer.concat([header, raw]); +}; export const ccobFixture = ( version: 1 | 2 | 3 = 2, method: "zlib" | "zstd" = "zstd",