From 54646e8dc4aaa647d1b66db26d2cddffebf91753 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 08:58:21 +0800 Subject: [PATCH] fix(environment): preserve raw UTF-16 token boundaries Pair a preceding low surrogate only with its adjacent high surrogate. Unpaired code units must not make an earlier letter become the token's neighbor. Keep UTF-16 offsets and source strings unchanged at the shared delimiter owner used by rule discovery, copy checks, and replacement. Cover valid astral letters, digits, emoji, and lone-surrogate controls, plus create/read behavior through a CLI-started environment MCP server. Validation: regression-first run had 4 expected failures; after the fix, 437 selected domain, application, and server-boundary tests passed. --- .../environmentStateWriteValues.test.ts | 16 ++++ .../environmentStateWriteValues.ts | 8 +- .../mcp/environmentServeWritesMcp.test.ts | 81 ++++++++++++++++++- 3 files changed, 101 insertions(+), 4 deletions(-) diff --git a/src/domain/analysisView/environmentStateWriteValues.test.ts b/src/domain/analysisView/environmentStateWriteValues.test.ts index a825cb5f8b..fc8d3103e4 100644 --- a/src/domain/analysisView/environmentStateWriteValues.test.ts +++ b/src/domain/analysisView/environmentStateWriteValues.test.ts @@ -120,6 +120,22 @@ it("preserves UTF-16 offsets while accepting emoji as a delimiter", () => { expect(replaceDelimited("😀42😀", "42", "43")).toBe("😀43😀"); }); +it.each<[string, number[], string]>([ + ["A\uDC0042", [2], "A\uDC0043"], + ["é\uDFFF42", [2], "é\uDFFF43"], + ["\uDC0042", [1], "\uDC0043"], + ["A\uDC0042\uD800B", [2], "A\uDC0043\uD800B"], + ["A\uD80042", [2], "A\uD80043"], + ["42\uDC00A", [0], "43\uDC00A"], + ["\uD800\uDC0042", [], "\uD800\uDC0042"], +])( + "classifies only the immediate code point beside raw UTF-16 text %j", + (text, offsets, replaced) => { + expect(delimitedOffsets(text, "42")).toEqual(offsets); + expect(replaceDelimited(text, "42", "43")).toBe(replaced); + }, +); + it.each(["0000-01-01T00:00:00+01:00", "9999-12-31T23:59:59-01:00"])( "keeps %s when conversion cannot fit the four-digit year format", (value) => { diff --git a/src/domain/analysisView/environmentStateWriteValues.ts b/src/domain/analysisView/environmentStateWriteValues.ts index 6327515d02..c9eb0eb3e2 100644 --- a/src/domain/analysisView/environmentStateWriteValues.ts +++ b/src/domain/analysisView/environmentStateWriteValues.ts @@ -266,9 +266,11 @@ const isAlphanumeric = (character: string | undefined): boolean => const previousCharacter = (text: string, at: number): string | undefined => { if (at === 0) return undefined; const last = text.charCodeAt(at - 1); - return last >= 0xdc00 && last <= 0xdfff && at > 1 - ? text.slice(at - 2, at) - : text[at - 1]; + if (last >= 0xdc00 && last <= 0xdfff && at > 1) { + const first = text.charCodeAt(at - 2); + if (first >= 0xd800 && first <= 0xdbff) return text.slice(at - 2, at); + } + return text[at - 1]; }; const nextCharacter = (text: string, at: number): string | undefined => { diff --git a/tests/boundary/mcp/environmentServeWritesMcp.test.ts b/tests/boundary/mcp/environmentServeWritesMcp.test.ts index 046b0cc85c..8ce3c75031 100644 --- a/tests/boundary/mcp/environmentServeWritesMcp.test.ts +++ b/tests/boundary/mcp/environmentServeWritesMcp.test.ts @@ -4,7 +4,10 @@ import { } from "@modelcontextprotocol/client"; import { describe, expect } from "vitest"; -import { timedCaptureEvidence } from "../../fixtures/analysisView.js"; +import { + timedCaptureEvidence, + type TimedCaptureRow, +} from "../../fixtures/analysisView.js"; import { environmentStateServeFixture, writeEnvironmentServeFiles, @@ -94,4 +97,80 @@ describe("rea environment-serve write tools", () => { }, HTTP_BOUNDARY_TIMEOUT_MS, ); + + processTest( + "rebuilds ids beside unpaired surrogates while preserving raw UTF-16 through creates and reads", + async ({ processes }) => { + const captured = (id: number, name: string) => ({ + ...book(id, name, bookClock(id)), + low: `A\uDC00${String(id)}`, + high: `A\uD800${String(id)}`, + right: `${String(id)}\uDC00A`, + word: "𐐀3𐐁", + emoji: `😀${String(id)}😀`, + }); + const first = captured(1, "First Book"); + const second = captured(2, "Second Book"); + const recorded = captured(3, "Third Book"); + const rows: readonly TimedCaptureRow[] = [ + ["GET", "/api/books", [first, second]], + ["GET", "/api/books/1", first], + ["GET", "/api/books/2", second], + ["POST", "/api/books", recorded, 201, { name: "Third Book" }], + ["GET", "/api/books/3", recorded], + ]; + const root = await createTestTempDirectory("rea-raw-utf16-writes-mcp-"); + const files = await writeEnvironmentServeFiles( + root, + environmentStateServeFixture(timedCaptureEvidence(rows)), + ); + const started = await startListeningProcess(processes, [ + "environment-serve", + files.manifestPath, + "--evidence", + files.capturePath, + "--port", + "0", + ]); + const client = new Client({ name: "raw-utf16-writes", version: "1" }); + await client.connect(new StreamableHTTPClientTransport(started.url)); + try { + for (const id of [3, 4]) { + const name = `Created Book ${String(id)}`; + const expected = { + id, + name, + low: `A\uDC00${String(id)}`, + high: `A\uD800${String(id)}`, + right: `${String(id)}\uDC00A`, + word: "𐐀3𐐁", + emoji: `😀${String(id)}😀`, + }; + const created = await client.callTool({ + name: "post_books", + arguments: { body: { name } }, + }); + expect(created.isError).toBeFalsy(); + expect(created.structuredContent).toMatchObject({ + status: 201, + body: expected, + state: { source: "state", view: "create", id: String(id) }, + }); + const read = await client.callTool({ + name: "get_books_by_id", + arguments: { id: String(id) }, + }); + expect(read.isError).toBeFalsy(); + expect(read.structuredContent).toMatchObject({ + status: 200, + body: expected, + state: { source: "state", view: "item", id: String(id) }, + }); + } + } finally { + await client.close(); + } + }, + HTTP_BOUNDARY_TIMEOUT_MS, + ); });