mirror of
https://github.com/morluto/rea
synced 2026-10-26 06:16:17 -04:00
fix(environment): preserve raw UTF-16 token boundaries
Pair a preceding low surrogate only with its adjacent high surrogate. Unpaired code units must not make an earlier letter become the token's neighbor. Keep UTF-16 offsets and source strings unchanged at the shared delimiter owner used by rule discovery, copy checks, and replacement. Cover valid astral letters, digits, emoji, and lone-surrogate controls, plus create/read behavior through a CLI-started environment MCP server. Validation: regression-first run had 4 expected failures; after the fix, 437 selected domain, application, and server-boundary tests passed.
This commit is contained in:
@@ -120,6 +120,22 @@ it("preserves UTF-16 offsets while accepting emoji as a delimiter", () => {
|
||||
expect(replaceDelimited("😀42😀", "42", "43")).toBe("😀43😀");
|
||||
});
|
||||
|
||||
it.each<[string, number[], string]>([
|
||||
["A\uDC0042", [2], "A\uDC0043"],
|
||||
["é\uDFFF42", [2], "é\uDFFF43"],
|
||||
["\uDC0042", [1], "\uDC0043"],
|
||||
["A\uDC0042\uD800B", [2], "A\uDC0043\uD800B"],
|
||||
["A\uD80042", [2], "A\uD80043"],
|
||||
["42\uDC00A", [0], "43\uDC00A"],
|
||||
["\uD800\uDC0042", [], "\uD800\uDC0042"],
|
||||
])(
|
||||
"classifies only the immediate code point beside raw UTF-16 text %j",
|
||||
(text, offsets, replaced) => {
|
||||
expect(delimitedOffsets(text, "42")).toEqual(offsets);
|
||||
expect(replaceDelimited(text, "42", "43")).toBe(replaced);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["0000-01-01T00:00:00+01:00", "9999-12-31T23:59:59-01:00"])(
|
||||
"keeps %s when conversion cannot fit the four-digit year format",
|
||||
(value) => {
|
||||
|
||||
@@ -266,9 +266,11 @@ const isAlphanumeric = (character: string | undefined): boolean =>
|
||||
const previousCharacter = (text: string, at: number): string | undefined => {
|
||||
if (at === 0) return undefined;
|
||||
const last = text.charCodeAt(at - 1);
|
||||
return last >= 0xdc00 && last <= 0xdfff && at > 1
|
||||
? text.slice(at - 2, at)
|
||||
: text[at - 1];
|
||||
if (last >= 0xdc00 && last <= 0xdfff && at > 1) {
|
||||
const first = text.charCodeAt(at - 2);
|
||||
if (first >= 0xd800 && first <= 0xdbff) return text.slice(at - 2, at);
|
||||
}
|
||||
return text[at - 1];
|
||||
};
|
||||
|
||||
const nextCharacter = (text: string, at: number): string | undefined => {
|
||||
|
||||
@@ -4,7 +4,10 @@ import {
|
||||
} from "@modelcontextprotocol/client";
|
||||
import { describe, expect } from "vitest";
|
||||
|
||||
import { timedCaptureEvidence } from "../../fixtures/analysisView.js";
|
||||
import {
|
||||
timedCaptureEvidence,
|
||||
type TimedCaptureRow,
|
||||
} from "../../fixtures/analysisView.js";
|
||||
import {
|
||||
environmentStateServeFixture,
|
||||
writeEnvironmentServeFiles,
|
||||
@@ -94,4 +97,80 @@ describe("rea environment-serve write tools", () => {
|
||||
},
|
||||
HTTP_BOUNDARY_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
processTest(
|
||||
"rebuilds ids beside unpaired surrogates while preserving raw UTF-16 through creates and reads",
|
||||
async ({ processes }) => {
|
||||
const captured = (id: number, name: string) => ({
|
||||
...book(id, name, bookClock(id)),
|
||||
low: `A\uDC00${String(id)}`,
|
||||
high: `A\uD800${String(id)}`,
|
||||
right: `${String(id)}\uDC00A`,
|
||||
word: "𐐀3𐐁",
|
||||
emoji: `😀${String(id)}😀`,
|
||||
});
|
||||
const first = captured(1, "First Book");
|
||||
const second = captured(2, "Second Book");
|
||||
const recorded = captured(3, "Third Book");
|
||||
const rows: readonly TimedCaptureRow[] = [
|
||||
["GET", "/api/books", [first, second]],
|
||||
["GET", "/api/books/1", first],
|
||||
["GET", "/api/books/2", second],
|
||||
["POST", "/api/books", recorded, 201, { name: "Third Book" }],
|
||||
["GET", "/api/books/3", recorded],
|
||||
];
|
||||
const root = await createTestTempDirectory("rea-raw-utf16-writes-mcp-");
|
||||
const files = await writeEnvironmentServeFiles(
|
||||
root,
|
||||
environmentStateServeFixture(timedCaptureEvidence(rows)),
|
||||
);
|
||||
const started = await startListeningProcess(processes, [
|
||||
"environment-serve",
|
||||
files.manifestPath,
|
||||
"--evidence",
|
||||
files.capturePath,
|
||||
"--port",
|
||||
"0",
|
||||
]);
|
||||
const client = new Client({ name: "raw-utf16-writes", version: "1" });
|
||||
await client.connect(new StreamableHTTPClientTransport(started.url));
|
||||
try {
|
||||
for (const id of [3, 4]) {
|
||||
const name = `Created Book ${String(id)}`;
|
||||
const expected = {
|
||||
id,
|
||||
name,
|
||||
low: `A\uDC00${String(id)}`,
|
||||
high: `A\uD800${String(id)}`,
|
||||
right: `${String(id)}\uDC00A`,
|
||||
word: "𐐀3𐐁",
|
||||
emoji: `😀${String(id)}😀`,
|
||||
};
|
||||
const created = await client.callTool({
|
||||
name: "post_books",
|
||||
arguments: { body: { name } },
|
||||
});
|
||||
expect(created.isError).toBeFalsy();
|
||||
expect(created.structuredContent).toMatchObject({
|
||||
status: 201,
|
||||
body: expected,
|
||||
state: { source: "state", view: "create", id: String(id) },
|
||||
});
|
||||
const read = await client.callTool({
|
||||
name: "get_books_by_id",
|
||||
arguments: { id: String(id) },
|
||||
});
|
||||
expect(read.isError).toBeFalsy();
|
||||
expect(read.structuredContent).toMatchObject({
|
||||
status: 200,
|
||||
body: expected,
|
||||
state: { source: "state", view: "item", id: String(id) },
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
await client.close();
|
||||
}
|
||||
},
|
||||
HTTP_BOUNDARY_TIMEOUT_MS,
|
||||
);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user