From b9e5a9f4635e339e90b948d4f94f75be2eaa57cb Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:31:47 +0800 Subject: [PATCH] fix(process): retain unrepresentable filename evidence --- docs/process-capture.md | 5 + src/domain/process/processCaptureCoverage.ts | 19 ++- src/process/capture/FilesystemSnapshot.ts | 41 ++++- .../processCaptureFilenameBytes.test.ts | 159 ++++++++++++++++++ 4 files changed, 210 insertions(+), 14 deletions(-) create mode 100644 tests/process-global/processCaptureFilenameBytes.test.ts diff --git a/docs/process-capture.md b/docs/process-capture.md index 5a7c33533c..89c6ce8dd9 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -500,6 +500,11 @@ Every capture requires `truncation_details`, with separate accounting for: An unreadable directory is recorded in `enumeration_omissions` with its root-aliased path and operating-system error code. Entries observed before that directory remain available, while descendants beneath it stay unknown. + A basename that is not valid UTF-8 is never looked up through replacement + characters. It consumes entry capacity and records an `unrepresentable_name` + omission at its containing directory, with exact basename bytes in + `raw_name_hex` and a null `system_code`. That root's enumeration remains + incomplete, so missing paths beneath it remain unknown. Each retained regular file without a digest has an aliased path, size, remaining budget and reason: `file_bytes_budget`, `file_changed_or_short_read`, or `file_unavailable`. Its `system_code` is diff --git a/src/domain/process/processCaptureCoverage.ts b/src/domain/process/processCaptureCoverage.ts index 50a020c66f..e51054528b 100644 --- a/src/domain/process/processCaptureCoverage.ts +++ b/src/domain/process/processCaptureCoverage.ts @@ -28,15 +28,24 @@ export const filesystemCoverageSchema = z.strictObject({ z.enum([ "files_limit", "depth_limit", + "unrepresentable_name", ...enumerationOmissionReasonSchema.options, ]), ), enumeration_omissions: z.array( - z.strictObject({ - path: z.string(), - reason: enumerationOmissionReasonSchema, - system_code: z.string().nullable(), - }), + z.union([ + z.strictObject({ + path: z.string(), + reason: enumerationOmissionReasonSchema, + system_code: z.string().nullable(), + }), + z.strictObject({ + path: z.string(), + reason: z.literal("unrepresentable_name"), + system_code: z.null(), + raw_name_hex: z.string().regex(/^(?:[0-9a-f]{2})+$/u), + }), + ]), ), unobserved_subtrees: z .array( diff --git a/src/process/capture/FilesystemSnapshot.ts b/src/process/capture/FilesystemSnapshot.ts index 8ca0bf0087..1bc48beb84 100644 --- a/src/process/capture/FilesystemSnapshot.ts +++ b/src/process/capture/FilesystemSnapshot.ts @@ -5,6 +5,7 @@ import { lstat, open, opendir, readlink, realpath } from "node:fs/promises"; import { basename, dirname, join, relative } from "node:path"; import type { FileHandle } from "node:fs/promises"; import { appendCallerPath } from "../../filesystem/callerPath.js"; +import { decodeDirectoryEntryName } from "../../filesystem/decodeDirectoryEntryName.js"; import { sameRegularFileState } from "../../filesystem/RegularFile.js"; import type { ProcessScenario } from "../../domain/process/processScenario.js"; import type { FilesystemCoverage } from "../../domain/process/processCaptureCoverage.js"; @@ -387,6 +388,7 @@ export const snapshotRoots = async ( const completeRoots: string[] = []; let remainingBytes = scenario.limits.file_bytes; let pendingNames = 0; + let unrepresentableNames = 0; let truncated = false; const enumerationReasons = new Set< FilesystemCoverage["enumeration_reasons"][number] @@ -472,11 +474,11 @@ export const snapshotRoots = async ( signal?.throwIfAborted(); if (await omitObserverPath(root, rootAlias, path)) return true; if ( - entries.length >= scenario.limits.files || + entries.length + unrepresentableNames >= scenario.limits.files || depth > scenario.limits.filesystem_depth ) { truncated = true; - if (entries.length >= scenario.limits.files) + if (entries.length + unrepresentableNames >= scenario.limits.files) enumerationReasons.add("files_limit"); if (depth > scenario.limits.filesystem_depth) enumerationReasons.add("depth_limit"); @@ -579,11 +581,16 @@ export const snapshotRoots = async ( if (type !== "directory") return true; const children: string[] = []; let complete = true; - const remainingEntries = - scenario.limits.files - entries.length - pendingNames; + let remainingEntries = + scenario.limits.files - + entries.length - + pendingNames - + unrepresentableNames; let directory; try { - directory = await opendir(path); + directory = await opendir(path, { + encoding: "buffer" as BufferEncoding, + }); } catch (cause: unknown) { const systemCode = fileUnavailableSystemCode(cause); if (systemCode === undefined) throw cause; @@ -609,12 +616,14 @@ export const snapshotRoots = async ( signal?.throwIfAborted(); const child = await ownedDirectory.read(); if (child === null) break; + const decoded = decodeDirectoryEntryName(child.name); if ( - await omitObserverPath( + decoded.kind === "decoded" && + (await omitObserverPath( root, rootAlias, - appendCallerPath(path, child.name), - ) + appendCallerPath(path, decoded.name), + )) ) continue; if ( @@ -629,7 +638,21 @@ export const snapshotRoots = async ( enumerationReasons.add("depth_limit"); break; } - children.push(child.name); + if (decoded.kind === "unrepresentable") { + unrepresentableNames += 1; + remainingEntries -= 1; + truncated = true; + complete = false; + enumerationReasons.add("unrepresentable_name"); + enumerationOmissions.push({ + path: `${rootAlias}:${relativePath}`, + reason: "unrepresentable_name", + system_code: null, + raw_name_hex: decoded.rawHex, + }); + continue; + } + children.push(decoded.name); pendingNames += 1; } }, diff --git a/tests/process-global/processCaptureFilenameBytes.test.ts b/tests/process-global/processCaptureFilenameBytes.test.ts new file mode 100644 index 0000000000..7b268fab83 --- /dev/null +++ b/tests/process-global/processCaptureFilenameBytes.test.ts @@ -0,0 +1,159 @@ +import { createHash } from "node:crypto"; +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import type { Dirent } from "node:fs"; +import * as fs from "node:fs/promises"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { processFilesystemSnapshotSchema } from "../../src/domain/process/processCapture.js"; +import { processScenarioSchema } from "../../src/domain/process/processScenario.js"; +import { snapshotRoots } from "../../src/process/capture/FilesystemSnapshot.js"; +import { createServer } from "../../src/server/createServer.js"; +import { createTestBinarySession } from "../fixtures/binarySession.js"; +import { advertisedSchemaErrors } from "../fixtures/mcpAdvertisedSchemaValidity.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, opendir: vi.fn(actual.opendir) }; +}); + +const actualFs = await vi.importActual("node:fs/promises"); +afterEach(() => { + vi.restoreAllMocks(); + vi.mocked(fs.opendir).mockImplementation(actualFs.opendir); +}); + +// macOS rejects these names at creation. Simulate only directory-entry bytes; +// subsequent lookup, hashing, resource cleanup and coverage use production code. +const injectNames = (names: Buffer[]): void => { + vi.mocked(fs.opendir).mockImplementation(async (...args) => { + expect(args[1]).toMatchObject({ encoding: "buffer" }); + const directory = await actualFs.opendir(...args); + const read = directory.read.bind(directory); + const pending = [...names]; + vi.spyOn(directory, "read").mockImplementation(async () => { + const name = pending.shift(); + return name === undefined ? read() : ({ name } as unknown as Dirent); + }); + return directory; + }); +}; + +it.each([false, true])( + "retains invalid basename bytes without looking up a replacement alias (sibling=%s)", + async (sibling) => { + const root = await createTestTempDirectory("rea-capture-name-bytes-"); + const content = "replacement sibling"; + if (sibling) await fs.writeFile(join(root, "�"), content); + injectNames([Buffer.from([0xff])]); + const snapshot = await snapshotRoots( + processScenarioSchema.parse({ + executable: process.execPath, + working_directory: root, + filesystem_observation_paths: [root], + }), + ); + expect(snapshot.truncated).toBe(true); + expect(snapshot.completeRoots).toEqual([]); + expect(snapshot.coverage.enumeration_omissions).toEqual([ + { + path: "root_0:.", + reason: "unrepresentable_name", + system_code: null, + raw_name_hex: "ff", + }, + ]); + expect(snapshot.coverage.enumeration_truncated).toBe(true); + expect(snapshot.files.filter(({ type }) => type === "file")).toEqual( + sibling + ? [ + expect.objectContaining({ + path: "root_0:�", + sha256: createHash("sha256").update(content).digest("hex"), + }), + ] + : [], + ); + expect(snapshot.coverage.hashed_bytes).toBe(sibling ? content.length : 0); + expect(processFilesystemSnapshotSchema.parse(snapshot)).toEqual(snapshot); + }, +); + +it("bounds invalid-name enumeration with the selected files limit", async () => { + const root = await createTestTempDirectory("rea-capture-name-budget-"); + injectNames(Array.from({ length: 10 }, () => Buffer.from([0xff]))); + const snapshot = await snapshotRoots( + processScenarioSchema.parse({ + executable: process.execPath, + working_directory: root, + filesystem_observation_paths: [root], + limits: { files: 3 }, + }), + ); + expect(snapshot.files).toHaveLength(1); + expect(snapshot.coverage.enumeration_omissions).toHaveLength(2); + expect(snapshot.coverage.enumeration_reasons).toEqual([ + "unrepresentable_name", + "files_limit", + ]); + expect(snapshot.completeRoots).toEqual([]); +}); + +it.skipIf(process.platform !== "linux")( + "keeps actual Linux non-UTF-8 filenames distinct from a replacement sibling", + async () => { + const root = await createTestTempDirectory("rea-capture-linux-name-"); + await fs.writeFile( + Buffer.concat([Buffer.from(`${root}/`), Buffer.from([0xff])]), + "invalid basename", + ); + await fs.writeFile(join(root, "�"), "valid sibling"); + const snapshot = await snapshotRoots( + processScenarioSchema.parse({ + executable: process.execPath, + working_directory: root, + filesystem_observation_paths: [root], + }), + ); + expect(snapshot.files.map(({ path }) => path)).toEqual([ + "root_0:.", + "root_0:�", + ]); + expect(snapshot.coverage.enumeration_omissions).toEqual([ + { + path: "root_0:.", + reason: "unrepresentable_name", + system_code: null, + raw_name_hex: "ff", + }, + ]); + expect(snapshot.coverage.hashed_bytes).toBe(13); + expect(snapshot.completeRoots).toEqual([]); + }, +); + +it("advertises byte-preserving omissions in valid SDK-converted MCP schemas", async () => { + const session = createTestBinarySession(() => { + throw new Error("Schema discovery must not launch a provider"); + }); + const server = createServer({ kind: "session", session }); + const client = new Client({ name: "capture-name-bytes", version: "1" }); + try { + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + const tool = (await client.listTools()).tools.find( + ({ name }) => name === "capture_process_scenario", + ); + expect(tool).toBeDefined(); + if (tool === undefined) throw new Error("Missing process capture contract"); + expect(advertisedSchemaErrors([tool])).toEqual([]); + expect(JSON.stringify(tool.outputSchema)).toContain("unrepresentable_name"); + expect(JSON.stringify(tool.outputSchema)).toContain("raw_name_hex"); + } finally { + await client.close(); + await server.close(); + await session.close(); + } +});