diff --git a/src/model/GgufHeader.ts b/src/model/GgufHeader.ts index 060e322340..d2fae04284 100644 --- a/src/model/GgufHeader.ts +++ b/src/model/GgufHeader.ts @@ -429,6 +429,15 @@ export class GgufHeader { storage_validation: tensor.byte_length === null ? "unknown" : "consistent", })); + const intervals = completeTensors + .filter((row) => row.byte_length !== null) + .sort((a, b) => a.file_offset - b.file_offset); + let end = tensor_data_offset; + for (const row of intervals) { + if (row.file_offset < end && row.byte_length !== 0) + rejectModel(`Overlapping tensor byte ranges at ${row.name}`); + end = Math.max(end, row.file_offset + (row.byte_length ?? 0)); + } return { version, byte_order: this.#little ? "little" : "big", diff --git a/src/model/ModelArtifactProvider.ts b/src/model/ModelArtifactProvider.ts index 9f021a3bfa..49b0735c46 100644 --- a/src/model/ModelArtifactProvider.ts +++ b/src/model/ModelArtifactProvider.ts @@ -40,7 +40,6 @@ import { GgufHeader } from "./GgufHeader.js"; import { readSafetensorsHeader } from "./SafetensorsHeader.js"; import { ModelFormatFailure, - rejectModel, rejectPickle, MODEL_REPORT_BYTES, modelCapacityRemediation, @@ -311,15 +310,6 @@ export class ModelArtifactProvider implements ModelArtifactPort { onLimitation: (message) => partial.limitations.push(message), }) : await readSafetensorsHeader(reader, first); - const intervals = parsed.tensors - .filter((row) => row.byte_length !== null) - .sort((a, b) => a.file_offset - b.file_offset); - let end = parsed.tensor_data_offset; - for (const row of intervals) { - if (row.file_offset < end && row.byte_length !== 0) - rejectModel(`Overlapping tensor byte ranges at ${row.name}`); - end = Math.max(end, row.file_offset + (row.byte_length ?? 0)); - } partial.version = parsed.version; partial.byte_order = parsed.byte_order; partial.alignment = parsed.alignment; diff --git a/tests/boundary/model/modelArtifact.test.ts b/tests/boundary/model/modelArtifact.test.ts index ba4aa7e2cc..374f1c895c 100644 --- a/tests/boundary/model/modelArtifact.test.ts +++ b/tests/boundary/model/modelArtifact.test.ts @@ -15,6 +15,7 @@ import { import { readSafetensorsHeader } from "../../../src/model/SafetensorsHeader.js"; import { GgufHeader } from "../../../src/model/GgufHeader.js"; import { createModelArtifactService } from "../../../src/composition/model.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; import { modelArtifactSchema } from "../../../src/domain/model/modelArtifact.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { @@ -1063,6 +1064,48 @@ it.each(malformed)( }, ); +it("retains GGUF declarations and metadata when storage overlap is refused", async () => { + const seed = ggufFixture(); + const fixture = ggufFixture({ + metadata: [{ key: "general.name", type: 8, value: seed.text("Overlap") }], + tensors: [ + { name: "a", type: 0, shape: [1n], offset: 0n, bytes: 4 }, + { name: "b", type: 0, shape: [1n], offset: 0n, bytes: 4 }, + ], + }); + const { provider, path } = await providerAt(fixture.bytes); + const result = await provider.inspect({ path, hash: "header" }); + if (result.ok) throw new Error("Overlapping tensors admitted"); + expect(result.error).toMatchObject({ + _tag: "BinaryTargetError", + constraint: "file_format_rejected", + }); + const evidence = parseEvidence(result.error.partialObservation); + expect(evidence.subject).toBeNull(); + expect(evidence.normalized_result).toMatchObject({ + artifact: { + header_sha256: null, + sha256: null, + file_identity: "unverified", + }, + coverage: { header: "partial", metadata: "complete", tensors: "partial" }, + metadata: [{ key: "general.name", value: "Overlap" }], + tensors: [], + tensor_declarations: [ + { + name: "a", + file_offset: fixture.dataOffset, + storage_validation: "not-established", + }, + { + name: "b", + file_offset: fixture.dataOffset, + storage_validation: "not-established", + }, + ], + }); +}); + it("accepts null safetensors metadata and ignores tensor extension fields", async () => { const fixture = safetensorsFixture( '{"__metadata__":null,"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":{"source":"local","flags":[null,false,-1,1.25]}}}',