fix(environment): preserve raw UTF-16 token boundaries

Pair a preceding low surrogate only with its adjacent high surrogate.
Unpaired code units must not make an earlier letter become the token's
neighbor. Keep UTF-16 offsets and source strings unchanged at the shared
delimiter owner used by rule discovery, copy checks, and replacement.

Cover valid astral letters, digits, emoji, and lone-surrogate controls,
plus create/read behavior through a CLI-started environment MCP server.
Validation: regression-first run had 4 expected failures; after the fix,
437 selected domain, application, and server-boundary tests passed.
This commit is contained in:
morluto
2026-10-24 08:58:21 +08:00
parent 8681752c7b
commit 54646e8dc4
3 changed files with 101 additions and 4 deletions
@@ -120,6 +120,22 @@ it("preserves UTF-16 offsets while accepting emoji as a delimiter", () => {
expect(replaceDelimited("😀42😀", "42", "43")).toBe("😀43😀");
});
it.each<[string, number[], string]>([
["A\uDC0042", [2], "A\uDC0043"],
["é\uDFFF42", [2], "é\uDFFF43"],
["\uDC0042", [1], "\uDC0043"],
["A\uDC0042\uD800B", [2], "A\uDC0043\uD800B"],
["A\uD80042", [2], "A\uD80043"],
["42\uDC00A", [0], "43\uDC00A"],
["\uD800\uDC0042", [], "\uD800\uDC0042"],
])(
"classifies only the immediate code point beside raw UTF-16 text %j",
(text, offsets, replaced) => {
expect(delimitedOffsets(text, "42")).toEqual(offsets);
expect(replaceDelimited(text, "42", "43")).toBe(replaced);
},
);
it.each(["0000-01-01T00:00:00+01:00", "9999-12-31T23:59:59-01:00"])(
"keeps %s when conversion cannot fit the four-digit year format",
(value) => {
@@ -266,9 +266,11 @@ const isAlphanumeric = (character: string | undefined): boolean =>
const previousCharacter = (text: string, at: number): string | undefined => {
if (at === 0) return undefined;
const last = text.charCodeAt(at - 1);
return last >= 0xdc00 && last <= 0xdfff && at > 1
? text.slice(at - 2, at)
: text[at - 1];
if (last >= 0xdc00 && last <= 0xdfff && at > 1) {
const first = text.charCodeAt(at - 2);
if (first >= 0xd800 && first <= 0xdbff) return text.slice(at - 2, at);
}
return text[at - 1];
};
const nextCharacter = (text: string, at: number): string | undefined => {
@@ -4,7 +4,10 @@ import {
} from "@modelcontextprotocol/client";
import { describe, expect } from "vitest";
import { timedCaptureEvidence } from "../../fixtures/analysisView.js";
import {
timedCaptureEvidence,
type TimedCaptureRow,
} from "../../fixtures/analysisView.js";
import {
environmentStateServeFixture,
writeEnvironmentServeFiles,
@@ -94,4 +97,80 @@ describe("rea environment-serve write tools", () => {
},
HTTP_BOUNDARY_TIMEOUT_MS,
);
processTest(
"rebuilds ids beside unpaired surrogates while preserving raw UTF-16 through creates and reads",
async ({ processes }) => {
const captured = (id: number, name: string) => ({
...book(id, name, bookClock(id)),
low: `A\uDC00${String(id)}`,
high: `A\uD800${String(id)}`,
right: `${String(id)}\uDC00A`,
word: "𐐀3𐐁",
emoji: `😀${String(id)}😀`,
});
const first = captured(1, "First Book");
const second = captured(2, "Second Book");
const recorded = captured(3, "Third Book");
const rows: readonly TimedCaptureRow[] = [
["GET", "/api/books", [first, second]],
["GET", "/api/books/1", first],
["GET", "/api/books/2", second],
["POST", "/api/books", recorded, 201, { name: "Third Book" }],
["GET", "/api/books/3", recorded],
];
const root = await createTestTempDirectory("rea-raw-utf16-writes-mcp-");
const files = await writeEnvironmentServeFiles(
root,
environmentStateServeFixture(timedCaptureEvidence(rows)),
);
const started = await startListeningProcess(processes, [
"environment-serve",
files.manifestPath,
"--evidence",
files.capturePath,
"--port",
"0",
]);
const client = new Client({ name: "raw-utf16-writes", version: "1" });
await client.connect(new StreamableHTTPClientTransport(started.url));
try {
for (const id of [3, 4]) {
const name = `Created Book ${String(id)}`;
const expected = {
id,
name,
low: `A\uDC00${String(id)}`,
high: `A\uD800${String(id)}`,
right: `${String(id)}\uDC00A`,
word: "𐐀3𐐁",
emoji: `😀${String(id)}😀`,
};
const created = await client.callTool({
name: "post_books",
arguments: { body: { name } },
});
expect(created.isError).toBeFalsy();
expect(created.structuredContent).toMatchObject({
status: 201,
body: expected,
state: { source: "state", view: "create", id: String(id) },
});
const read = await client.callTool({
name: "get_books_by_id",
arguments: { id: String(id) },
});
expect(read.isError).toBeFalsy();
expect(read.structuredContent).toMatchObject({
status: 200,
body: expected,
state: { source: "state", view: "item", id: String(id) },
});
}
} finally {
await client.close();
}
},
HTTP_BOUNDARY_TIMEOUT_MS,
);
});