diff --git a/src/reference/ReferenceSourceReaderEntries.ts b/src/reference/ReferenceSourceReaderEntries.ts index c7f634168e..39d2b520c8 100644 --- a/src/reference/ReferenceSourceReaderEntries.ts +++ b/src/reference/ReferenceSourceReaderEntries.ts @@ -66,9 +66,16 @@ export const traverseDirectory = async ( } const directories: PendingDirectory[] = []; let readFailure: string | undefined; + let unrepresentableNameCount = 0; + let firstUnrepresentableNameHex: string | undefined; let handle: Dir | undefined; try { - handle = await opendir(current.path); + // Preserve directory-entry identity until UTF-8 validity is established. + // Node's opendir accepts this byte-preserving encoding; @types/node's + // OpenDirOptions currently narrows it to text encodings. + handle = await opendir(current.path, { + encoding: "buffer" as BufferEncoding, + }); } catch (cause: unknown) { readFailure = directoryReadFailure(cause); } @@ -86,7 +93,17 @@ export const traverseDirectory = async ( break; } if (child === null) break; - result = await processEntry(state, current, child.name, directories); + if (!Buffer.isBuffer(child.name)) { + readFailure = "Directory entry name was not returned as raw bytes"; + break; + } + const name = child.name.toString("utf8"); + if (!Buffer.from(name, "utf8").equals(child.name)) { + unrepresentableNameCount += 1; + firstUnrepresentableNameHex ??= child.name.toString("hex"); + continue; + } + result = await processEntry(state, current, name, directories); if (!result.ok) break; } } finally { @@ -152,7 +169,14 @@ export const traverseDirectory = async ( scheduleObservedDirectories(state, directories); return { ok: true, value: undefined }; } - if (current.path !== state.root) + if (unrepresentableNameCount > 0) { + recordDirectoryFailure( + state, + current.path, + "unsupported", + `Directory contains ${String(unrepresentableNameCount)} entry name(s) that are not valid UTF-8; no filesystem operations were performed for them. First raw basename bytes: ${firstUnrepresentableNameHex ?? "unknown"}.`, + ); + } else if (current.path !== state.root) state.entries.push({ status: "read", kind: "directory", diff --git a/tests/boundary/cli/referenceSourceUtf8.test.ts b/tests/boundary/cli/referenceSourceUtf8.test.ts index 0a1243a913..33af3c0209 100644 --- a/tests/boundary/cli/referenceSourceUtf8.test.ts +++ b/tests/boundary/cli/referenceSourceUtf8.test.ts @@ -84,6 +84,55 @@ cliTest( }, ); +cliTest.skipIf(process.platform !== "linux")( + "reports unsupported raw POSIX filename bytes without substituting a Unicode sibling", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-reference-raw-name-"); + const config = await createTestTempDirectory( + "rea-reference-raw-name-config-", + ); + const rawName = Buffer.from([0x62, 0x61, 0x64, 0xff, 0x2e, 0x6a, 0x73]); + const rawPath = Buffer.concat([Buffer.from(`${root}/`), rawName]); + await Promise.all([ + writeFile(rawPath, "RAW_FILE"), + writeFile(join(root, "bad\ufffd.js"), "UNICODE_FILE"), + writeFile(join(root, "good.js"), "GOOD_FILE"), + ]); + + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + environment: { + HOME: config, + XDG_CONFIG_HOME: config, + XDG_CACHE_HOME: config, + }, + }); + expect(result.exitCode, result.stderr + JSON.stringify(result.json)).toBe( + 0, + ); + const graph = historicalSourceGraphSchema.parse(result.json); + expect(graph.inventory_state).toBe("partial"); + expect(graph.entries.map(({ path }) => path)).toEqual([ + "bad\ufffd.js", + "good.js", + ]); + expect(graph.entries).toContainEqual( + expect.objectContaining({ + path: "bad\ufffd.js", + sha256: createHash("sha256").update("UNICODE_FILE").digest("hex"), + size: 12, + }), + ); + expect(graph.limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining( + `First raw basename bytes: ${rawName.toString("hex")}`, + ), + ]), + ); + }, +); + cliTest.skipIf(process.platform === "win32")( "preserves historical literal-backslash entries and their exact owning import base", async ({ cli }) => { diff --git a/tests/boundary/filesystem/referenceSourceFilenameBytes.test.ts b/tests/boundary/filesystem/referenceSourceFilenameBytes.test.ts new file mode 100644 index 0000000000..48c2f3f919 --- /dev/null +++ b/tests/boundary/filesystem/referenceSourceFilenameBytes.test.ts @@ -0,0 +1,63 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; +import { readReferenceSource } from "../../../src/reference/ReferenceSourceReader.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it.skipIf(process.platform !== "linux")( + "does not read an invalid UTF-8 directory name through its replacement-character sibling", + async () => { + const root = await createTestTempDirectory("rea-reference-filename-bytes-"); + const rawName = Buffer.from([0x62, 0x61, 0x64, 0xff, 0x2e, 0x6a, 0x73]); + const rawPath = Buffer.concat([Buffer.from(`${root}/`), rawName]); + const replacementPath = join(root, "bad\ufffd.js"); + await Promise.all([ + writeFile(rawPath, "RAW_FILE"), + writeFile(replacementPath, "UNICODE_FILE"), + writeFile(join(root, "good.js"), "GOOD_FILE"), + ]); + + const resources = new ArtifactResourceScope(); + try { + const result = await readReferenceSource(root, resources); + if (!result.ok) throw result.error; + expect(result.value.rootFailure).toMatchObject({ + code: "unsupported", + message: expect.stringContaining( + `First raw basename bytes: ${rawName.toString("hex")}`, + ), + }); + expect(result.value.entries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + status: "read", + kind: "file", + path: "bad\ufffd.js", + size: 12, + bytes: Buffer.from("UNICODE_FILE"), + }), + expect.objectContaining({ + status: "read", + kind: "file", + path: "good.js", + size: 9, + bytes: Buffer.from("GOOD_FILE"), + }), + ]), + ); + expect( + result.value.entries.filter( + (entry) => entry.kind === "file" && entry.path === "bad\ufffd.js", + ), + ).toHaveLength(1); + expect(result.value.bytesRead).toBe( + "UNICODE_FILE".length + "GOOD_FILE".length, + ); + } finally { + await resources.close(); + } + }, +);