From 23ab2c4069a335d87bdc4a3bce6bfaebecce05e4 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:27:07 +0800 Subject: [PATCH] fix(environment): preserve literal values and recorded comparison chronology --- docs/agent-environments.md | 12 ++-- .../fidelity-compare/spec.md | 7 ++- .../environment/environmentCompare.test.ts | 56 +++++++++++++++++++ .../environment/environmentCompare.ts | 23 +++++++- .../environmentCompareDiff.test.ts | 34 +++++++++-- .../environment/environmentCompareDiff.ts | 39 ++++--------- .../environmentCompareIdsIntegration.test.ts | 35 ++++++++++++ .../environment/environmentCompareProbe.ts | 21 +++++-- .../EnvironmentStateAccumulator.ts | 39 ++++++++++--- .../environmentStateWriteValues.test.ts | 27 +++++++++ .../environmentStateWriteValues.ts | 38 ++++++++++--- 11 files changed, 266 insertions(+), 65 deletions(-) diff --git a/docs/agent-environments.md b/docs/agent-environments.md index e1d0689554..8b5bbcdaf7 100644 --- a/docs/agent-environments.md +++ b/docs/agent-environments.md @@ -771,8 +771,12 @@ recording replayed in its place and never from a recording whose body the probe two collections can reuse a live id, and an equal pair stays in that collection. A live id in a later request's path, query or body is translated before the environment answers, a path parameter by the collection the live path names (a nested collection keeps its parent values, each decoded once, and a literal route segment stays), except a query value that pages a list, such as `limit` or `page`. Each call is checked against the served tool's input schema first, as the MCP server checks it. After mapping, two clock values compare by format only (ISO-8601 strings, and epoch numbers under -`at`, `time`, `date`, `timestamp`, `created`, `updated` or `modified` field names, including snake-case and camel-case forms), and two different values each side generated on its own compare equal: both UUIDs, or one length of -eight or more with the same character classes and a digit. The report counts each kind of normalised value. +`at`, `time`, `date`, `timestamp`, `created`, `updated` or `modified` field names, including snake-case and camel-case forms). +Numeric array elements have no time-named field and remain literal measurements. +Different string values compare as independently generated only at entity fields +whose executed served write plan declares a unique token or UUID/token identity +generator. UUID-like or token-like spelling alone does not establish generation; +other business strings compare literally. The report counts each kind of normalised value. ISO clock offsets are accepted only within RFC 3339's `00`–`23` hour and `00`–`59` minute ranges. Timestamp ordering retains submillisecond precision when the source records it. @@ -809,8 +813,8 @@ body the probe did not record compares by status only and is counted as `unobser carries one (204, 205, 304, `HEAD`). A browser probe captured without `capture.network.response_body` is refused, as is one that makes POST, PUT or PATCH calls without `capture.network.request_body`. -Agreement covers only the calls the probes make, after the mapping the answers allow. Values compared by form, clocks -and generated values, can still differ in content, and an equal answer after a write shows the state matched for that +Agreement covers only the calls the probes make, after the mapping the answers allow. Clocks compared by format +and fields normalized under explicit generation rules can still differ in content, and an equal answer after a write shows the state matched for that read, not that the write had no other effect. ## Compatibility verification diff --git a/docs/specs/agent-environments/fidelity-compare/spec.md b/docs/specs/agent-environments/fidelity-compare/spec.md index 58ccc279b5..9fe7091413 100644 --- a/docs/specs/agent-environments/fidelity-compare/spec.md +++ b/docs/specs/agent-environments/fidelity-compare/spec.md @@ -74,8 +74,11 @@ credentials and resets it between sequences. another environment value (a reference keeps raw equality, as the probe's mapping cannot tell which collection it names); an object's members compare under the environment's mapped keys only when it is keyed by learned ids as learning reads one, and any other object compares field by field; two clock values (ISO-8601 strings, or epoch numbers under - time-named fields) compare by format only; two different generated values (both UUIDs, or both of one length and the - same character classes with a digit) that neither side maps compare equal and are counted. A body over the replay + time-named fields) compare by format only; numeric array elements remain literal. + Different strings normalize as generated only at direct entity fields where + the executed served write plan proves a unique token or UUID/token identity + generator, and neither side participates in an ID mapping. A UUID or token + shape alone is insufficient evidence. A body over the replay limit, text or binary, compares by status only, as does a live body the probe did not record, unless the answer has none by definition (204, 205, 304, or an answer to `HEAD`). Both are counted. - A step agrees when the statuses are equal and the normalised bodies are. Differences are categorised: `status`, diff --git a/src/application/environment/environmentCompare.test.ts b/src/application/environment/environmentCompare.test.ts index fd8853fe44..0d449081b0 100644 --- a/src/application/environment/environmentCompare.test.ts +++ b/src/application/environment/environmentCompare.test.ts @@ -472,6 +472,27 @@ const compareRows = ( }; describe("ids the comparison used to misread", () => { + it.each([ + ["", "", true], + ['""', "", false], + ["null", "", true], + ] as const)( + "compares a recorded missing-item body representation %j against %j", + (recorded, live, equal) => { + const probe = compareRows( + [ + ["GET", "/api/books", [FIRST_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/9", recorded, 404], + ], + [ + ["GET", "/api/books", [{ ...FIRST_BOOK, id: 101 }]], + ["GET", "/api/books/999", live, 404], + ], + ); + expect(probe.results[1]?.equal).toBe(equal); + }, + ); it("reads a twice-encoded path id once, so book%2531 is not book1", () => { const probe = compareRows( [ @@ -767,6 +788,41 @@ describe("what a probe did not observe", () => { expect(report.value.summary).toMatchObject({ steps: 2, equal: 2 }); }); + it.each([ + ["0002", "0001"], + ["002", "001"], + ])( + "orders probe requests by complete fractional timestamps: %s after %s", + (readFraction, writeFraction) => { + const report = run( + analysisViewCaptureEvidence( + environmentManifestCaptureFixture([ + { + ...exchange(0, "GET", "http://app.example/api/books/2", 404), + startedDateTime: `2026-10-20T09:00:00.${readFraction}Z`, + response: { + status: 404, + content: { + mimeType: "application/json", + text: JSON.stringify(BOOK_NOT_FOUND), + }, + }, + }, + { + ...exchange(0, "DELETE", "http://app.example/api/books/2", 204), + startedDateTime: `2026-10-20T09:00:00.${writeFraction}Z`, + }, + ]), + ), + ); + if (!report.ok) throw report.error; + expect(report.value.summary).toMatchObject({ steps: 2, equal: 2 }); + expect( + report.value.probes[0]?.results.map(({ ordinal }) => ordinal), + ).toEqual([1, 0]); + }, + ); + it("refuses a browser probe that makes writes without recorded request bodies", () => { const report = run( scenarioCaptureEvidence( diff --git a/src/application/environment/environmentCompare.ts b/src/application/environment/environmentCompare.ts index 2c0e470599..cd6cf9a02f 100644 --- a/src/application/environment/environmentCompare.ts +++ b/src/application/environment/environmentCompare.ts @@ -157,8 +157,11 @@ const entityOf = ( return { scope, at, identity }; }; -const fromState = (answer: ReplayResult): boolean => - (answer as Partial).state?.source === "state"; +const fromState = (answer: ReplayResult): boolean => { + const state = (answer as Partial).state; + // Missing-item selection comes from state, but its response is a recording. + return state?.source === "state" && state.view !== "missing"; +}; /** Identity facts come from decoded JSON or text scalars, never binary presentations or truncated bodies. */ const identityValue = (body: ReplayedBody): JsonValue | undefined => { @@ -359,7 +362,21 @@ const compareStep = ( entityAt: entity.at, identity: entity.identity, }); - const compared = compareAnswers(live, env, run.map, entity); + const plan = run.store?.state.writes.get(call.tool.name); + const generatedFields = new Set( + fromState(answered.value) && plan?.served === true + ? plan.rules.flatMap((rule) => + rule.rule === "unique" || + (rule.rule === "identity" && rule.generator !== "sequence") + ? [rule.field] + : [], + ) + : [], + ); + const compared = compareAnswers(live, env, run.map, { + ...entity, + generatedFields, + }); for (const key of NORMALISED_KEYS) run.normalised[key] += compared.normalised[key]; const differences = [...compared.differences]; diff --git a/src/application/environment/environmentCompareDiff.test.ts b/src/application/environment/environmentCompareDiff.test.ts index 315a0d4c90..02aba394d8 100644 --- a/src/application/environment/environmentCompareDiff.test.ts +++ b/src/application/environment/environmentCompareDiff.test.ts @@ -17,7 +17,7 @@ const compare = ( ); describe("compareAnswers", () => { - it("agrees on mapped ids, clocks of one format and generated values, and counts each", () => { + it("normalizes mapped ids and clocks while comparing unproven generated-looking values literally", () => { const map = new IdMap(); map.learn(101, 1); const result = compare( @@ -35,16 +35,40 @@ describe("compareAnswers", () => { }, map, ); - expect(result.differences).toEqual([]); + expect( + result.differences.map(({ category, pointer }) => [category, pointer]), + ).toEqual([ + ["value", "/token"], + ["value", "/uuid"], + ]); expect(result.normalised).toEqual({ ids: 1, clocks: 1, - generated: 2, + generated: 0, large_bodies: 0, unobserved_bodies: 0, }); }); + it("normalizes string generation only at fields identified by an executed write plan", () => { + const result = compareAnswers( + { + status: 201, + body: { kind: "json", value: { token: "live-token", sku: "product2" } }, + }, + { + status: 201, + body: { kind: "json", value: { token: "env-token", sku: "product1" } }, + }, + new IdMap(), + { generatedFields: new Set(["token"]) }, + ); + expect(result.differences).toMatchObject([ + { category: "value", pointer: "/sku" }, + ]); + expect(result.normalised.generated).toBe(1); + }); + it("categorises status, missing and extra fields, values and clock formats", () => { const result = compare( { name: "A", gone: 1, at: "2026-10-20T10:00:00.123+00:00" }, @@ -152,7 +176,7 @@ describe("compareAnswers", () => { ).toEqual([]); }); - it("reports a reordered set of scalars as order alone and reads epoch array items as clocks", () => { + it("reports scalar reordering and keeps unnamed epoch-shaped array measurements literal", () => { expect( compare( { access: ["a", "b", "c"] }, @@ -162,7 +186,7 @@ describe("compareAnswers", () => { expect( compare({ stats: [[1792450800, 3]] }, { stats: [[1792285200, 3]] }) .differences, - ).toEqual([]); + ).toMatchObject([{ category: "value", pointer: "/stats/0/0" }]); }); }); diff --git a/src/application/environment/environmentCompareDiff.ts b/src/application/environment/environmentCompareDiff.ts index 350363d842..1552799673 100644 --- a/src/application/environment/environmentCompareDiff.ts +++ b/src/application/environment/environmentCompareDiff.ts @@ -1,6 +1,5 @@ import type { JsonObject } from "../../domain/analysisView/environmentStateRules.js"; import { - isUuid, readClock, type ClockFormat, } from "../../domain/analysisView/environmentStateWriteValues.js"; @@ -74,43 +73,21 @@ const formatKey = (format: ClockFormat): string => ? `epoch-${format.unit}` : `iso${format.separator}${String(format.fraction)}${format.zone === "" ? "" : format.zone === "Z" ? "Z" : "offset"}`; -/** The format of a clock value: an ISO-8601 string, or an epoch number under a time-named field or as an array item. */ +/** ISO text has its own clock syntax; numeric epochs require a time-named field. */ const clockFormat = ( value: JsonValue, key: string | undefined, ): string | undefined => { if ( typeof value === "number" && - key !== undefined && - !TIME_FIELD.test(key) && - !TIME_FIELD_CAMEL.test(key) + (key === undefined || + (!TIME_FIELD.test(key) && !TIME_FIELD_CAMEL.test(key))) ) return undefined; const clock = readClock(value); return clock === undefined ? undefined : formatKey(clock.format); }; -const characterClasses = (text: string): string => - [ - ...new Set( - text - .replaceAll(/\d/gu, "0") - .replaceAll(/[a-z]/gu, "a") - .replaceAll(/[A-Z]/gu, "A"), - ), - ] - .sort() - .join(""); - -/** Two values each side generated on its own: both UUIDs, or one length of eight or more with the same character classes and a digit. */ -const generatedPair = (live: string, env: string): boolean => - (isUuid(live) && isUuid(env)) || - (live.length === env.length && - live.length >= 8 && - /\d/u.test(live) && - /\d/u.test(env) && - characterClasses(live) === characterClasses(env)); - class Differ { readonly differences: Difference[] = []; readonly normalised: Normalised = { @@ -128,6 +105,7 @@ class Differ { readonly scope: string | undefined; readonly at: string; readonly identity: boolean; + readonly generatedFields: ReadonlySet; }, ) {} @@ -211,9 +189,11 @@ class Differ { if ( typeof live === "string" && typeof env === "string" && + at.own === true && + key !== undefined && + this.entity.generatedFields.has(key) && !this.map.hasLive(live) && - !this.map.hasEnv(env) && - generatedPair(live, env) + !this.map.hasEnv(env) ) { this.normalised.generated += 1; return; @@ -375,12 +355,15 @@ export const compareAnswers = ( readonly scope?: string; readonly at?: string; readonly identity?: boolean; + /** Direct entity fields whose executed write plan proves independent generation. */ + readonly generatedFields?: ReadonlySet; } = {}, ): { readonly differences: Difference[]; readonly normalised: Normalised } => { const differ = new Differ(map, { scope: entity.scope, at: entity.at ?? "", identity: entity.identity === true, + generatedFields: entity.generatedFields ?? new Set(), }); if (live.status !== env.status) differ.add({ diff --git a/src/application/environment/environmentCompareIdsIntegration.test.ts b/src/application/environment/environmentCompareIdsIntegration.test.ts index 521853d51f..b96785734a 100644 --- a/src/application/environment/environmentCompareIdsIntegration.test.ts +++ b/src/application/environment/environmentCompareIdsIntegration.test.ts @@ -27,6 +27,41 @@ const compareRows = ( }; describe("environment comparison identifier semantics", () => { + it("reports changed business tokens and numeric arrays through the application comparison boundary", () => { + const probe = compareRows( + [ + [ + "GET", + "/api/measurements", + { sku: "product1", samples: [1700000000] }, + ], + ], + [ + [ + "GET", + "/api/measurements", + { sku: "product2", samples: [1800000000] }, + ], + ], + ); + expect(probe.results[0]?.equal).toBe(false); + expect( + probe.results[0]?.differences.map(({ category, pointer }) => [ + category, + pointer, + ]), + ).toEqual([ + ["value", "/sku"], + ["value", "/samples/0"], + ]); + expect(probe.normalised.generated).toBe(0); + expect(probe.normalised.clocks).toBe(0); + const same = compareRows( + [["GET", "/api/measurements", { samples: [1700000000] }]], + [["GET", "/api/measurements", { samples: [1700000000] }]], + ); + expect(same.results[0]?.equal).toBe(true); + }); it("compares numeric format identifiers literally while normalizing actual clock fields", () => { const fourcc = (value: string) => Buffer.from(value).readUInt32LE(0); const probe = compareRows( diff --git a/src/application/environment/environmentCompareProbe.ts b/src/application/environment/environmentCompareProbe.ts index e35d9af933..024979b909 100644 --- a/src/application/environment/environmentCompareProbe.ts +++ b/src/application/environment/environmentCompareProbe.ts @@ -1,4 +1,8 @@ import { z } from "zod"; +import { + compareClockInstants, + readClock, +} from "../../domain/analysisView/environmentStateWriteValues.js"; import { isJsonLike } from "../../domain/analysisView/EnvironmentManifestAccumulator.js"; import { primaryOrigin, @@ -46,10 +50,11 @@ const postDataSchema = z.object({ const methodSchema = z.object({ request: z.object({ method: z.string() }) }); -const startedMs = (record: WebNetworkCaptureRecord): number | undefined => { +const startedClock = ( + record: WebNetworkCaptureRecord, +): ReturnType => { const started = startedSchema.safeParse(record.reported); - const ms = started.success ? Date.parse(started.data.startedDateTime) : NaN; - return Number.isFinite(ms) ? ms : undefined; + return started.success ? readClock(started.data.startedDateTime) : undefined; }; /** Whether a probe makes calls that carry a request body: a POST, PUT or PATCH. */ @@ -118,13 +123,17 @@ export const probeRequests = ( const unobserved: UnobservedRecord[] = []; // Without a start time for every record the probe keeps its recorded order, as state extraction does. const starts = new Map( - records.map((record) => [record.ordinal, startedMs(record)]), + records.map((record) => [record.ordinal, startedClock(record)]), ); const timed = [...starts.values()].every((ms) => ms !== undefined); const ordered = [...records].sort( (left, right) => - (timed ? starts.get(left.ordinal)! - starts.get(right.ordinal)! : 0) || - left.ordinal - right.ordinal, + (timed + ? compareClockInstants( + starts.get(left.ordinal)!, + starts.get(right.ordinal)!, + ) + : 0) || left.ordinal - right.ordinal, ); const lines = new Map( records.map((record) => [ diff --git a/src/domain/analysisView/EnvironmentStateAccumulator.ts b/src/domain/analysisView/EnvironmentStateAccumulator.ts index fdb9d001f4..2763655e28 100644 --- a/src/domain/analysisView/EnvironmentStateAccumulator.ts +++ b/src/domain/analysisView/EnvironmentStateAccumulator.ts @@ -1,4 +1,8 @@ import { z } from "zod"; +import { + compareClockInstants, + readClock, +} from "./environmentStateWriteValues.js"; import type { AnalysisError } from "../analysisErrorBase.js"; import { parseExactJson } from "../exactJsonNumber.js"; import { MAX_JSON_DEPTH, type JsonValue } from "../jsonValue.js"; @@ -111,6 +115,7 @@ interface RecordEntry { readonly origin: string | null; readonly contentType: string; readonly timed: boolean; + readonly subMillisecond: number; readonly record: WebNetworkCaptureRecord; } @@ -155,16 +160,21 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator const line = readRecordedRequestLine(record.reported, true, this.scratch); if (line === null) return this.overBudget(); const timing = timingSchema.safeParse(record.reported); - const start = timing.success - ? Date.parse(timing.data.startedDateTime) - : Number.NaN; - const timed = timing.success && Number.isFinite(start); + const clock = timing.success + ? readClock(timing.data.startedDateTime) + : undefined; + const start = clock?.ms ?? record.ordinal; + const timed = timing.success && clock !== undefined; if (line.kind === "excluded") return line.reason === "url-redacted" ? this.#redactedWrite( record, timing.success && timed - ? { start, end: start + timing.data.time } + ? { + start, + end: start + timing.data.time, + subMillisecond: clock.subMillisecond, + } : undefined, ) : ok(undefined); @@ -195,6 +205,7 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator origin: line.origin, contentType: line.rawMime, timed, + subMillisecond: clock?.subMillisecond ?? 0, record, }); return ok(undefined); @@ -206,7 +217,13 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator /** Keep a successful write whose URL was redacted, with an empty path whose root relates to every type. */ #redactedWrite( record: WebNetworkCaptureRecord, - timing: { readonly start: number; readonly end: number } | undefined, + timing: + | { + readonly start: number; + readonly end: number; + readonly subMillisecond: number; + } + | undefined, ): Result { const parsed = redactedWriteSchema.safeParse(record.reported); if (!parsed.success) return ok(undefined); @@ -228,6 +245,7 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator origin: null, contentType: "", timed: timing !== undefined, + subMillisecond: timing?.subMillisecond ?? 0, record, }); return ok(undefined); @@ -307,12 +325,15 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator ? "recorded-start-time" : "record-order"; const records = entries - .map((entry) => entry.facts) .sort((left, right) => timeline === "record-order" - ? left.ordinal - right.ordinal - : left.start - right.start || left.ordinal - right.ordinal, + ? left.facts.ordinal - right.facts.ordinal + : compareClockInstants( + { ms: left.facts.start, subMillisecond: left.subMillisecond }, + { ms: right.facts.start, subMillisecond: right.subMillisecond }, + ) || left.facts.ordinal - right.facts.ordinal, ) + .map((entry) => entry.facts) .map((facts) => timeline === "record-order" ? { ...facts, start: facts.ordinal, end: facts.ordinal } diff --git a/src/domain/analysisView/environmentStateWriteValues.test.ts b/src/domain/analysisView/environmentStateWriteValues.test.ts index dc4ef3c564..905a188773 100644 --- a/src/domain/analysisView/environmentStateWriteValues.test.ts +++ b/src/domain/analysisView/environmentStateWriteValues.test.ts @@ -53,6 +53,9 @@ it.each<[string, number, ClockFormat, number?]>([ ); it.each([ + "0001-02-29T00:00:00Z", + "0099-02-29T00:00:00Z", + "0100-02-29T00:00:00Z", "2026-02-30T00:00:00Z", "2026-02-29T00:00:00Z", "2026-13-01T00:00:00Z", @@ -72,6 +75,30 @@ it.each([ expect(readClock(text)).toBeUndefined(); }); +it.each(["0000", "0001", "0099", "0100"])( + "retains first-century ISO year %s without Date.UTC's adjustment", + (year) => { + const value = `${year}-01-01T00:00:00Z`; + expect(readClock(value)).toBeDefined(); + expect(respellClock(value, iso(" ", 3, "Z"))).toBe( + `${year}-01-01 00:00:00.000Z`, + ); + }, +); + +it.each(["𐐀42𐐁", "𝟙42𝟚", "é42é", "A42B"])( + "keeps token42 embedded in complete Unicode letters or digits: %s", + (text) => { + expect(delimitedOffsets(text, "42")).toEqual([]); + expect(replaceDelimited(text, "42", "43")).toBe(text); + }, +); + +it("preserves UTF-16 offsets while accepting emoji as a delimiter", () => { + expect(delimitedOffsets("😀42😀", "42")).toEqual([2]); + expect(replaceDelimited("😀42😀", "42", "43")).toBe("😀43😀"); +}); + it.each<[JsonValue | undefined, ReturnType]>([ [ WHOLE_SECOND / 1000, diff --git a/src/domain/analysisView/environmentStateWriteValues.ts b/src/domain/analysisView/environmentStateWriteValues.ts index 6048e64917..46ac3965c7 100644 --- a/src/domain/analysisView/environmentStateWriteValues.ts +++ b/src/domain/analysisView/environmentStateWriteValues.ts @@ -18,6 +18,12 @@ interface ClockReading { readonly format: ClockFormat; } +/** Compare recorded instants without rounding away their submillisecond digits. */ +export const compareClockInstants = ( + left: Pick, + right: Pick, +): number => left.ms - right.ms || left.subMillisecond - right.subMillisecond; + const ISO_INSTANT = /^(\d{4})-(\d{2})-(\d{2})([T ])(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:?\d{2})?$/u; const MINUTE_MS = 60_000; @@ -56,23 +62,24 @@ const readIsoClock = (value: string): ClockReading | undefined => { const zone = match[9] ?? ""; const offsetMinutes = zoneMinutes(zone); if (offsetMinutes === undefined) return undefined; - const local = Date.UTC( - year as number, - (month as number) - 1, - day as number, + // setUTCFullYear avoids Date.UTC's special 1900 adjustment for years 0–99. + const rolled = new Date(0); + rolled.setUTCFullYear(year as number, (month as number) - 1, day as number); + rolled.setUTCHours( hour as number, minute as number, second as number, Number(`${fraction}000`.slice(0, 3)), ); - // Date.UTC rolls invalid fields over; the round trip refuses them. - const rolled = new Date(local); + const local = rolled.getTime(); + // Date setters roll invalid fields over; the round trip refuses them. if ( rolled.getUTCFullYear() !== year || rolled.getUTCMonth() !== (month as number) - 1 || rolled.getUTCDate() !== day || rolled.getUTCHours() !== hour || - rolled.getUTCMinutes() !== minute + rolled.getUTCMinutes() !== minute || + rolled.getUTCSeconds() !== second ) return undefined; return { @@ -256,6 +263,20 @@ export const slugOf = (text: string): string => const isAlphanumeric = (character: string | undefined): boolean => character !== undefined && /[\p{L}\p{N}]/u.test(character); +/** Source offsets stay UTF-16, while delimiter classification reads complete code points. */ +const previousCharacter = (text: string, at: number): string | undefined => { + if (at === 0) return undefined; + const last = text.charCodeAt(at - 1); + return last >= 0xdc00 && last <= 0xdfff && at > 1 + ? text.slice(at - 2, at) + : text[at - 1]; +}; + +const nextCharacter = (text: string, at: number): string | undefined => { + const point = text.codePointAt(at); + return point === undefined ? undefined : String.fromCodePoint(point); +}; + /** Start offsets of `token` in `text` where neither neighbour is a letter or digit. */ export const delimitedOffsets = (text: string, token: string): number[] => { const offsets: number[] = []; @@ -263,7 +284,8 @@ export const delimitedOffsets = (text: string, token: string): number[] => { let at = text.indexOf(token); while (at >= 0) { const delimited = - !isAlphanumeric(text[at - 1]) && !isAlphanumeric(text[at + token.length]); + !isAlphanumeric(previousCharacter(text, at)) && + !isAlphanumeric(nextCharacter(text, at + token.length)); if (delimited) offsets.push(at); // A rejected match may overlap a delimited one, so the search moves on by one character. at = text.indexOf(token, at + (delimited ? token.length : 1));