diff --git a/docs/model-artifacts.md b/docs/model-artifacts.md index 5ebb5e2c17..293970856d 100644 --- a/docs/model-artifacts.md +++ b/docs/model-artifacts.md @@ -30,9 +30,11 @@ result records the exact selected path, file size, header length, header SHA-256 bytes read and read scope. GGUF alignment padding is outside this prefix. `artifact.sha256` and the Evidence subject are null: a header digest is not a commitment to the weights. Different files can have identical headers and sizes. -The reported GGUF tensor-data offset is aligned after the tensor directory, -including when the directory contains no tensors; `header_length` still ends at -the directory, before that padding. +The reported GGUF tensor-data offset follows the aligned tensor directory when +padding is present. For zero-tensor metadata-only files that end at the +directory, it is the directory end; no tensor payload is implied. A partial +alignment gap is rejected. `header_length` still ends at the directory, before +any padding. `hash: "full"` additionally streams the remaining file bytes through SHA-256. It returns a full-file digest and Evidence subject, while retaining the same @@ -55,9 +57,20 @@ name, original type ID, dimensions, relative data offset, file offset and derive byte length. Alignment accepts every positive UINT32 multiple of eight, including non-powers of two. Partial Evidence retains explicit alignment once validated; the default alignment becomes known after metadata completes. Cancellation -during tensor parsing or hashing retains that fact without claiming finalized -digests or file identity. Metadata keys use ASCII lower-snake-case hierarchy components; +digests or file identity. Metadata keys preserve their spelling and use lowercase +ASCII alphanumeric components separated by dots, underscores or hyphens; standard numeric components such as `general.base_model.0.name` are preserved. +Hyphenated components are retained for compatibility with the published +llama.cpp v0.7.0 writer, which emits keys such as +`gpt-oss.context_length` and `wavtokenizer-dec.context_length`; the current +[GGUF specification](https://github.com/ggml-org/ggml/blob/master/docs/gguf.md#standardized-key-value-pairs) +restricts `general.architecture` values to letters and digits. +This documents producer output that conflicts with that written restriction, +not a claim that the specification permits hyphens. See the pinned +[WAVTOKENIZER_DEC](https://github.com/ggml-org/llama.cpp/blob/620d2225cc436ab0b001e3200e590e3b286ef9ab/gguf-py/gguf/constants.py#L1447) +and [GPT_OSS](https://github.com/ggml-org/llama.cpp/blob/620d2225cc436ab0b001e3200e590e3b286ef9ab/gguf-py/gguf/constants.py#L1466) +architecture names, [context-length key template](https://github.com/ggml-org/llama.cpp/blob/620d2225cc436ab0b001e3200e590e3b286ef9ab/gguf-py/gguf/constants.py#L110), +and [writer method](https://github.com/ggml-org/llama.cpp/blob/620d2225cc436ab0b001e3200e590e3b286ef9ab/gguf-py/gguf/gguf_writer.py#L698-L699). Scalar metadata is complete. Arrays have declared element types, counts and at most eight preview elements per array, with an explicit `preview_complete` flag. This includes tokenizer vocabularies: omitted values diff --git a/src/model/GgufHeader.ts b/src/model/GgufHeader.ts index 1c5fa8c618..49268fe3b4 100644 --- a/src/model/GgufHeader.ts +++ b/src/model/GgufHeader.ts @@ -287,10 +287,12 @@ export class GgufHeader { const key = await this.#text(true, 65535); // Numeric hierarchy components occur in standardized base_model.{id} keys. if ( - !/^[a-z0-9]+(?:_[a-z0-9]+)*(?:\.[a-z0-9]+(?:_[a-z0-9]+)*)*$/u.test(key) + !/^[a-z0-9]+(?:[_-][a-z0-9]+)*(?:\.[a-z0-9]+(?:[_-][a-z0-9]+)*)*$/u.test( + key, + ) ) rejectModel( - "GGUF metadata keys require ASCII lower-snake-case hierarchy components", + "GGUF metadata keys require lowercase ASCII hierarchy components", ); if (keys.has(key)) rejectModel(`Duplicate GGUF metadata key ${key}`); keys.add(key); @@ -383,8 +385,12 @@ export class GgufHeader { tensors.push(tensor); progress?.onTensorDeclaration(tensor); } + const directoryEnd = this.reader.position; + const alignedDataOffset = Math.ceil(directoryEnd / alignment) * alignment; const tensor_data_offset = - Math.ceil(this.reader.position / alignment) * alignment; + tensorCount === 0 && this.reader.size === directoryEnd + ? directoryEnd + : alignedDataOffset; if (tensor_data_offset > this.reader.size) rejectModel("GGUF tensor data section begins outside the file"); for (const tensor of tensors) { diff --git a/tests/acceptance/modelArtifactCli.test.ts b/tests/acceptance/modelArtifactCli.test.ts index feb84649b7..c855ea4be1 100644 --- a/tests/acceptance/modelArtifactCli.test.ts +++ b/tests/acceptance/modelArtifactCli.test.ts @@ -19,6 +19,56 @@ import { ggufFixture, safetensorsFixture } from "../fixtures/model/model.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { cliTest } from "../support/cli/cliFixture.js"; +cliTest( + "preserves lowercase hyphenated GGUF architecture metadata from the published writer", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-model-hyphen-architecture-", + ); + const base = ggufFixture(); + for (const architecture of ["gpt-oss", "wavtokenizer-dec"] as const) { + const path = join(root, `${architecture}.gguf`); + const bytes = ggufFixture({ + tensors: [], + metadata: [ + { + key: "general.architecture", + type: 8, + value: base.text(architecture), + }, + { + key: `${architecture}.context_length`, + type: 4, + value: base.u32(4096), + }, + ], + }).bytes; + await writeFile(path, bytes); + const response = await cli.run({ + arguments: [ + "inspect-model-artifact", + path, + "--detail", + "complete", + "--json", + ], + }); + expect(response.exitCode, response.stderr).toBe(0); + const report = modelArtifactSchema.parse( + toolContract("inspect_model_artifact").outputSchema.parse(response.json) + .normalized_result, + ); + expect(report.metadata.map(({ key, value }) => ({ key, value }))).toEqual( + [ + { key: "general.architecture", value: architecture }, + { key: `${architecture}.context_length`, value: 4096 }, + ], + ); + expect(report.artifact.sha256).toBeNull(); + } + }, +); + cliTest( "preserves exact large safetensors dimensions while bounding actual offsets", async ({ cli }) => { @@ -564,7 +614,6 @@ cliTest( emptyGgufHeader.header, Buffer.alloc(alignedOffset - emptyGgufHeader.header.length), ]); - for (const [name, bytes] of [ ["optional.safetensors", safetensors.bytes], ["empty.gguf", emptyGguf], @@ -677,6 +726,116 @@ cliTest( }, ); +cliTest( + "inspects unpadded zero-tensor vocab GGUF through CLI and MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-model-vocab-only-"); + const { call } = await connectLocalToolsMcp(); + const base = ggufFixture(); + const vocab = ggufFixture({ + tensors: [], + metadata: [ + { key: "general.type", type: 8, value: base.text("vocab") }, + { + key: "tokenizer.ggml.model", + type: 8, + value: base.text("gpt2"), + }, + ], + }); + expect(vocab.header.length).toBe(105); + expect(vocab.bytes.length).toBe(128); + const padded = Buffer.concat([ + vocab.header, + Buffer.alloc(vocab.bytes.length - vocab.header.length), + ]); + const files = [ + ["vocab-unpadded.gguf", vocab.header, vocab.header.length], + ["vocab-padded.gguf", padded, vocab.bytes.length], + ] as const; + + for (const [name, bytes, expectedOffset] of files) { + const path = join(root, name); + await writeFile(path, bytes); + const cliResult = await cli.run({ + arguments: [ + "inspect-model-artifact", + path, + "--detail", + "complete", + "--hash", + "full", + "--json", + ], + }); + expect(cliResult.exitCode, cliResult.stderr).toBe(0); + const report = modelArtifactSchema.parse( + toolContract("inspect_model_artifact").outputSchema.parse( + cliResult.json, + ).normalized_result, + ); + expect(report.metadata.map(({ key, value }) => ({ key, value }))).toEqual( + [ + { key: "general.type", value: "vocab" }, + { key: "tokenizer.ggml.model", value: "gpt2" }, + ], + ); + expect(report.tensors).toEqual([]); + expect(report.tensor_data_offset).toBe(expectedOffset); + expect(report.artifact.header_length).toBe(vocab.header.length); + expect(report.artifact.header_sha256).toBe( + createHash("sha256").update(vocab.header).digest("hex"), + ); + expect(report.artifact.sha256).toBe( + createHash("sha256").update(bytes).digest("hex"), + ); + const mcpResult = await call("inspect_model_artifact", { + path, + detail: "complete", + hash: "full", + }); + expect(mcpResult.isError, JSON.stringify(mcpResult)).not.toBe(true); + expect( + parseEvidence(mcpResult.structuredContent).normalized_result, + ).toEqual(report); + } + + const malformed = [ + ["partial-padding.gguf", Buffer.concat([vocab.header, Buffer.alloc(1)])], + [ + "truncated-tensor.gguf", + ggufFixture({ + metadata: [], + tensors: [ + { name: "weight", type: 0, shape: [4n], offset: 0n, bytes: 16 }, + ], + }).header, + ], + ] as const; + for (const [name, bytes] of malformed) { + const path = join(root, name); + await writeFile(path, bytes); + const cliResult = await cli.run({ + arguments: ["inspect-model-artifact", path, "--json"], + }); + expect(cliResult.exitCode).toBe(1); + expect(cliResult.json).toMatchObject({ + code: "target_unavailable", + category: "unsupported_target", + details: { constraint: "file_format_rejected" }, + }); + const mcpResult = parseMcpToolError( + await call("inspect_model_artifact", { path }), + ); + expect(mcpResult.error).toMatchObject({ + code: "target_unavailable", + category: "unsupported_target", + details: { constraint: "file_format_rejected" }, + }); + } + }, +); + cliTest( "exports complete model Evidence for portable paging and returns typed malformed errors", async ({ cli }) => { diff --git a/tests/boundary/model/modelArtifact.test.ts b/tests/boundary/model/modelArtifact.test.ts index 7c9a29ec98..f3b6404113 100644 --- a/tests/boundary/model/modelArtifact.test.ts +++ b/tests/boundary/model/modelArtifact.test.ts @@ -244,13 +244,6 @@ it("aligns the GGUF data section after an empty tensor directory", async () => { expect(result.value.artifact.header_length).toBe(base.header.length); }); -it("rejects an empty GGUF whose required directory padding is truncated", async () => { - const fixture = ggufFixture({ tensors: [], metadata: [] }); - const { path, provider } = await providerAt(fixture.header); - const result = await provider.inspect({ path, hash: "header" }); - expect(result.ok).toBe(false); -}); - it.each([0, 1, 2, 4, 12])( "rejects GGUF alignment %s outside positive multiples of eight", async (alignment) => { @@ -276,6 +269,8 @@ it.each([ "general..name", "general/name", "general.-name", + "general.name-", + "general.double--name", "general.name_", "general.__name", ])(