fix(reference-source): preserve raw POSIX filename identity

Mark directories with non-UTF-8 names unsupported without following replacement-character paths, while retaining readable sibling observations.

Fixes #6653

src/reference/ReferenceSourceReaderEntries.ts

@@ -66,9 +66,16 @@ export const traverseDirectory = async (
   }
   const directories: PendingDirectory[] = [];
   let readFailure: string | undefined;
+  let unrepresentableNameCount = 0;
+  let firstUnrepresentableNameHex: string | undefined;
   let handle: Dir | undefined;
   try {
-    handle = await opendir(current.path);
+    // Preserve directory-entry identity until UTF-8 validity is established.
+    // Node's opendir accepts this byte-preserving encoding; @types/node's
+    // OpenDirOptions currently narrows it to text encodings.
+    handle = await opendir(current.path, {
+      encoding: "buffer" as BufferEncoding,
+    });
   } catch (cause: unknown) {
     readFailure = directoryReadFailure(cause);
   }
@@ -86,7 +93,17 @@ export const traverseDirectory = async (
           break;
         }
         if (child === null) break;
-        result = await processEntry(state, current, child.name, directories);
+        if (!Buffer.isBuffer(child.name)) {
+          readFailure = "Directory entry name was not returned as raw bytes";
+          break;
+        }
+        const name = child.name.toString("utf8");
+        if (!Buffer.from(name, "utf8").equals(child.name)) {
+          unrepresentableNameCount += 1;
+          firstUnrepresentableNameHex ??= child.name.toString("hex");
+          continue;
+        }
+        result = await processEntry(state, current, name, directories);
         if (!result.ok) break;
       }
     } finally {
@@ -152,7 +169,14 @@ export const traverseDirectory = async (
     scheduleObservedDirectories(state, directories);
     return { ok: true, value: undefined };
   }
-  if (current.path !== state.root)
+  if (unrepresentableNameCount > 0) {
+    recordDirectoryFailure(
+      state,
+      current.path,
+      "unsupported",
+      `Directory contains ${String(unrepresentableNameCount)} entry name(s) that are not valid UTF-8; no filesystem operations were performed for them. First raw basename bytes: ${firstUnrepresentableNameHex ?? "unknown"}.`,
+    );
+  } else if (current.path !== state.root)
     state.entries.push({
       status: "read",
       kind: "directory",

tests/boundary/cli/referenceSourceUtf8.test.ts

@@ -84,6 +84,55 @@ cliTest(
   },
 );
 
+cliTest.skipIf(process.platform !== "linux")(
+  "reports unsupported raw POSIX filename bytes without substituting a Unicode sibling",
+  async ({ cli }) => {
+    const root = await createTestTempDirectory("rea-reference-raw-name-");
+    const config = await createTestTempDirectory(
+      "rea-reference-raw-name-config-",
+    );
+    const rawName = Buffer.from([0x62, 0x61, 0x64, 0xff, 0x2e, 0x6a, 0x73]);
+    const rawPath = Buffer.concat([Buffer.from(`${root}/`), rawName]);
+    await Promise.all([
+      writeFile(rawPath, "RAW_FILE"),
+      writeFile(join(root, "bad\ufffd.js"), "UNICODE_FILE"),
+      writeFile(join(root, "good.js"), "GOOD_FILE"),
+    ]);
+
+    const result = await cli.run({
+      arguments: ["import-reference-source", root, "--json"],
+      environment: {
+        HOME: config,
+        XDG_CONFIG_HOME: config,
+        XDG_CACHE_HOME: config,
+      },
+    });
+    expect(result.exitCode, result.stderr + JSON.stringify(result.json)).toBe(
+      0,
+    );
+    const graph = historicalSourceGraphSchema.parse(result.json);
+    expect(graph.inventory_state).toBe("partial");
+    expect(graph.entries.map(({ path }) => path)).toEqual([
+      "bad\ufffd.js",
+      "good.js",
+    ]);
+    expect(graph.entries).toContainEqual(
+      expect.objectContaining({
+        path: "bad\ufffd.js",
+        sha256: createHash("sha256").update("UNICODE_FILE").digest("hex"),
+        size: 12,
+      }),
+    );
+    expect(graph.limitations).toEqual(
+      expect.arrayContaining([
+        expect.stringContaining(
+          `First raw basename bytes: ${rawName.toString("hex")}`,
+        ),
+      ]),
+    );
+  },
+);
+
 cliTest.skipIf(process.platform === "win32")(
   "preserves historical literal-backslash entries and their exact owning import base",
   async ({ cli }) => {

tests/boundary/filesystem/referenceSourceFilenameBytes.test.ts

@@ -0,0 +1,63 @@
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { expect, it } from "vitest";
+
+import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js";
+import { readReferenceSource } from "../../../src/reference/ReferenceSourceReader.js";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+
+it.skipIf(process.platform !== "linux")(
+  "does not read an invalid UTF-8 directory name through its replacement-character sibling",
+  async () => {
+    const root = await createTestTempDirectory("rea-reference-filename-bytes-");
+    const rawName = Buffer.from([0x62, 0x61, 0x64, 0xff, 0x2e, 0x6a, 0x73]);
+    const rawPath = Buffer.concat([Buffer.from(`${root}/`), rawName]);
+    const replacementPath = join(root, "bad\ufffd.js");
+    await Promise.all([
+      writeFile(rawPath, "RAW_FILE"),
+      writeFile(replacementPath, "UNICODE_FILE"),
+      writeFile(join(root, "good.js"), "GOOD_FILE"),
+    ]);
+
+    const resources = new ArtifactResourceScope();
+    try {
+      const result = await readReferenceSource(root, resources);
+      if (!result.ok) throw result.error;
+      expect(result.value.rootFailure).toMatchObject({
+        code: "unsupported",
+        message: expect.stringContaining(
+          `First raw basename bytes: ${rawName.toString("hex")}`,
+        ),
+      });
+      expect(result.value.entries).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            status: "read",
+            kind: "file",
+            path: "bad\ufffd.js",
+            size: 12,
+            bytes: Buffer.from("UNICODE_FILE"),
+          }),
+          expect.objectContaining({
+            status: "read",
+            kind: "file",
+            path: "good.js",
+            size: 9,
+            bytes: Buffer.from("GOOD_FILE"),
+          }),
+        ]),
+      );
+      expect(
+        result.value.entries.filter(
+          (entry) => entry.kind === "file" && entry.path === "bad\ufffd.js",
+        ),
+      ).toHaveLength(1);
+      expect(result.value.bytesRead).toBe(
+        "UNICODE_FILE".length + "GOOD_FILE".length,
+      );
+    } finally {
+      await resources.close();
+    }
+  },
+);