From 49b4edfe7788495ba548e423c9c3b82dc14cca03 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:46:23 +0800 Subject: [PATCH 1/2] fix(browser): validate large base64 without regexp stack overflow Share padded decoding across CDP response shapes and cache fingerprints, and count binary WebSocket bytes without allocating a decoded buffer. Preserve HAR canonical pad bits and validate before allocation. Replace repeated alphabet quartets in canonical schemas with separate alphabet, padding and fixed-width length assertions. Cover near-32MiB encodings under ordinary, Unicode and Unicode-set modes, plus complete 4MiB CLI and MCP retained-evidence export workflows. Fixes #5253 --- src/browser/CdpCaptureEventBodyShapes.ts | 7 +- src/browser/CdpCaptureEventHandlers.ts | 9 +- src/browser/CdpCaptureEventHelpers.ts | 7 - src/browser/CdpCaptureStorageFingerprints.ts | 18 +- src/browser/CdpLargeBase64.test.ts | 184 ++++++++++++++++++ src/browser/history/HarCapture.ts | 11 +- src/domain/paddedBase64.test.ts | 44 +++++ src/domain/paddedBase64.ts | 14 ++ src/domain/stringPatterns.test.ts | 23 +++ src/domain/stringPatterns.ts | 15 +- .../browser/largeBase64Capture.test.ts | 138 +++++++++++++ 11 files changed, 433 insertions(+), 37 deletions(-) create mode 100644 src/browser/CdpLargeBase64.test.ts create mode 100644 src/domain/paddedBase64.test.ts create mode 100644 src/domain/paddedBase64.ts create mode 100644 tests/boundary/browser/largeBase64Capture.test.ts diff --git a/src/browser/CdpCaptureEventBodyShapes.ts b/src/browser/CdpCaptureEventBodyShapes.ts index f3b22bfe7f..d12f21c2f1 100644 --- a/src/browser/CdpCaptureEventBodyShapes.ts +++ b/src/browser/CdpCaptureEventBodyShapes.ts @@ -1,10 +1,11 @@ import { inferJsonShape, type JsonShape } from "../domain/jsonShape.js"; +import { decodePaddedBase64 } from "../domain/paddedBase64.js"; import { recordValue, cdpStringValue, type UnknownRecord, } from "./CdpCaptureValues.js"; -import { decodeBase64, isJsonContentType } from "./CdpCaptureEventHelpers.js"; +import { isJsonContentType } from "./CdpCaptureEventHelpers.js"; import type { CdpCaptureEventsState } from "./CdpCaptureEventState.js"; import type { NetworkState } from "./CdpCaptureEventTypes.js"; @@ -20,7 +21,9 @@ export const ingestResponseBodyShape = ( invalidResponseBodyShape(state, requestId); return; } - const decoded = base64Encoded ? decodeBase64(body)?.toString("utf8") : body; + const decoded = base64Encoded + ? decodePaddedBase64(body)?.toString("utf8") + : body; if (decoded === undefined) { invalidResponseBodyShape(state, requestId); return; diff --git a/src/browser/CdpCaptureEventHandlers.ts b/src/browser/CdpCaptureEventHandlers.ts index 66f784a6e0..0f16bd0a71 100644 --- a/src/browser/CdpCaptureEventHandlers.ts +++ b/src/browser/CdpCaptureEventHandlers.ts @@ -1,6 +1,7 @@ import type { WebPageInspection } from "../domain/browserObservationSchemas.js"; import { sanitizeBrowserUrl } from "../domain/browserObservation.js"; import { inferJsonShape } from "../domain/jsonShape.js"; +import { paddedBase64ByteLength } from "../domain/paddedBase64.js"; import { safeResponseMetadata } from "./CdpSafeMetadata.js"; import { allowedSanitizedUrl, @@ -18,7 +19,6 @@ import { } from "./CdpCaptureEventBodyShapes.js"; import { consolePrimitive, - decodeBase64, exclusionReasonForUrl, executionContextKey, firstCallFrame, @@ -508,14 +508,15 @@ export const handleWebSocketFrame = ( const response = recordValue(params.response); const payload = cdpStringValue(response?.payloadData) ?? ""; const opcode = Math.max(0, Math.trunc(numberValue(response?.opcode) ?? 0)); - const decoded = opcode === 1 ? undefined : decodeBase64(payload); - if (opcode !== 1 && decoded === undefined) + const decodedBytes = + opcode === 1 ? undefined : paddedBase64ByteLength(payload); + if (opcode !== 1 && decodedBytes === undefined) state.completeness.exclude("websocket_frames", "invalid_protocol_value"); connection.events.push({ direction, opcode, payload_bytes: - opcode === 1 ? Buffer.byteLength(payload) : (decoded?.byteLength ?? null), + opcode === 1 ? Buffer.byteLength(payload) : (decodedBytes ?? null), payload_shape: captureWebSocketShape(state, payload, opcode), }); }; diff --git a/src/browser/CdpCaptureEventHelpers.ts b/src/browser/CdpCaptureEventHelpers.ts index 9b5358e8a8..6f718f0d6c 100644 --- a/src/browser/CdpCaptureEventHelpers.ts +++ b/src/browser/CdpCaptureEventHelpers.ts @@ -111,10 +111,3 @@ export const consolePrimitive = ( return undefined; } }; - -const BASE64 = /^(?:[A-Za-z\d+/]{4})*(?:[A-Za-z\d+/]{2}==|[A-Za-z\d+/]{3}=)?$/u; - -export const decodeBase64 = (value: string): Buffer | undefined => - value.length % 4 === 0 && BASE64.test(value) - ? Buffer.from(value, "base64") - : undefined; diff --git a/src/browser/CdpCaptureStorageFingerprints.ts b/src/browser/CdpCaptureStorageFingerprints.ts index 9f8f80c525..538db86c23 100644 --- a/src/browser/CdpCaptureStorageFingerprints.ts +++ b/src/browser/CdpCaptureStorageFingerprints.ts @@ -4,6 +4,7 @@ import canonicalize from "canonicalize"; import type { InspectWebPageInput } from "../domain/browserObservation.js"; import type { WebPageInspection } from "../domain/browserObservationSchemas.js"; +import { decodePaddedBase64 } from "../domain/paddedBase64.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; import { CdpConnection } from "./CdpConnection.js"; import { optionalCdpCommand } from "./CdpOptionalCommand.js"; @@ -413,13 +414,13 @@ const addCacheEntry = async ( run.limitations, ); const body = cdpStringValue(recordValue(recordValue(raw)?.response)?.body); - const decoded = body === undefined ? null : decodeBase64(body); - const complete = decoded !== null; + const decoded = body === undefined ? undefined : decodePaddedBase64(body); + const complete = decoded !== undefined; if (!complete) run.state.complete = false; addFingerprint(run.state, { scope: "cache_entry", identity: { cache: cache.name, requestURL, requestMethod, requestHeaders }, - value: decoded ?? undefined, + value: decoded, complete, }); }; @@ -541,14 +542,3 @@ const remoteObjectComplete = (value: unknown): boolean => { return false; } }; - -const decodeBase64 = (value: string): Buffer | null => { - if ( - value.length % 4 !== 0 || - !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test( - value, - ) - ) - return null; - return Buffer.from(value, "base64"); -}; diff --git a/src/browser/CdpLargeBase64.test.ts b/src/browser/CdpLargeBase64.test.ts new file mode 100644 index 0000000000..ffe7e05e7e --- /dev/null +++ b/src/browser/CdpLargeBase64.test.ts @@ -0,0 +1,184 @@ +import { createHash } from "node:crypto"; + +import { expect, it, vi } from "vitest"; + +import { inspectWebPageInputSchema } from "../domain/browserObservation.js"; +import { CdpCaptureEvents } from "./CdpCaptureEvents.js"; +import { captureStorageFingerprints } from "./CdpCaptureStorageFingerprints.js"; +import type { CdpConnection } from "./CdpConnection.js"; + +const ORIGIN = "https://example.test"; +const CDP_ENDPOINT = "http://127.0.0.1:9222"; +const BODY_BYTES = 4 * 1024 * 1024; + +const captureCacheFingerprint = async (encodedBody: string) => { + const send = vi.fn(async (method: string) => { + if (method === "Network.getCookies") return { cookies: [] }; + if (method === "CacheStorage.requestEntries") + return { + cacheDataEntries: [ + { + requestURL: `${ORIGIN}/cached`, + requestMethod: "GET", + requestHeaders: [], + }, + ], + returnCount: 1, + }; + if (method === "CacheStorage.requestCachedResponse") + return { response: { body: encodedBody } }; + throw new Error(`Unexpected CDP command ${method}`); + }); + const input = inspectWebPageInputSchema.parse({ + cdp_endpoint: CDP_ENDPOINT, + allowed_origins: [ORIGIN], + target_id: "page", + }); + return captureStorageFingerprints({ + context: { + connection: { send } as unknown as CdpConnection, + sessionId: "page-session", + input, + }, + origin: ORIGIN, + cookieUrl: `${ORIGIN}/`, + local: { items: [], complete: true }, + session: { items: [], complete: true }, + indexedDbNames: [], + caches: [{ name: "cache", id: "cache-id" }], + limitations: [], + }); +}; + +const createEvents = (includeJsonBodyShapes = true): CdpCaptureEvents => + new CdpCaptureEvents( + inspectWebPageInputSchema.parse({ + cdp_endpoint: CDP_ENDPOINT, + allowed_origins: [ORIGIN], + target_id: "page", + include_json_body_shapes: includeJsonBodyShapes, + }), + new Set([ORIGIN]), + ); + +const startJsonResponse = ( + events: CdpCaptureEvents, + requestId: string, + path: string, +): void => { + events.ingest({ + method: "Network.requestWillBeSent", + params: { + requestId, + request: { url: `${ORIGIN}${path}`, method: "GET" }, + initiator: { type: "other" }, + timestamp: 1, + }, + }); + events.ingest({ + method: "Network.responseReceived", + params: { + requestId, + response: { + url: `${ORIGIN}${path}`, + status: 200, + mimeType: "application/json", + }, + }, + }); +}; + +it("fingerprints large cached bodies and retains CDP padding-bit acceptance", async () => { + const body = Buffer.alloc(BODY_BYTES, 0x81); + const large = await captureCacheFingerprint(body.toString("base64")); + expect(large.complete).toBe(true); + expect(large.items[0]?.value_sha256).toBe( + createHash("sha256").update(body).digest("hex"), + ); + + const padBits = await captureCacheFingerprint("e31="); + expect(padBits.items[0]?.value_sha256).toBe( + createHash("sha256").update(Buffer.from("e31=", "base64")).digest("hex"), + ); + + const malformed = await captureCacheFingerprint("%%%not-base64%%%"); + expect(malformed.complete).toBe(false); + expect(malformed.items[0]?.value_sha256).toBeNull(); +}); + +it("decodes large JSON response bodies and preserves malformed and pad-bit behavior", () => { + const events = createEvents(); + startJsonResponse(events, "large-json", "/large.json"); + const encoded = Buffer.from( + JSON.stringify({ value: "x".repeat(BODY_BYTES) }), + ).toString("base64"); + const candidate = events.responseBodyRequests()[0]; + expect(candidate).toBeDefined(); + events.ingestResponseBody(candidate!, { + body: encoded, + base64Encoded: true, + }); + expect(events.network.get("large-json")?.body_shapes.response).toMatchObject({ + root_type: "object", + node_count: 2, + }); + + startJsonResponse(events, "pad-bits", "/pad-bits.json"); + const padBits = events + .responseBodyRequests() + .find(({ requestId }) => requestId === "pad-bits"); + expect(padBits).toBeDefined(); + events.ingestResponseBody(padBits!, { + body: "e31=", + base64Encoded: true, + }); + expect(events.network.get("pad-bits")?.body_shapes.response?.root_type).toBe( + "object", + ); + + startJsonResponse(events, "malformed", "/malformed.json"); + const malformed = events + .responseBodyRequests() + .find(({ requestId }) => requestId === "malformed"); + expect(malformed).toBeDefined(); + events.ingestResponseBody(malformed!, { + body: "%%%not-base64%%%", + base64Encoded: true, + }); + expect(events.network.get("malformed")?.body_shapes.response).toBeNull(); +}); + +it("counts large binary WebSocket frames without accepting malformed base64", () => { + const events = createEvents(false); + const encoded = Buffer.alloc(BODY_BYTES, 0x81).toString("base64"); + events.ingest({ + method: "Network.webSocketCreated", + params: { requestId: "socket", url: "wss://example.test/socket" }, + }); + events.ingest({ + method: "Network.webSocketFrameReceived", + params: { + requestId: "socket", + response: { opcode: 2, payloadData: encoded }, + }, + }); + expect( + events.webSocketConnectionsById.get("socket")?.events[0]?.payload_bytes, + ).toBe(BODY_BYTES); + + events.ingest({ + method: "Network.webSocketFrameReceived", + params: { + requestId: "socket", + response: { opcode: 2, payloadData: "%%%not-base64%%%" }, + }, + }); + expect( + events.webSocketConnectionsById.get("socket")?.events[1]?.payload_bytes, + ).toBeNull(); + expect(events.completeness.snapshot().excluded).toContainEqual({ + section: "websocket_frames", + reason: "invalid_protocol_value", + count: 1, + }); +}); diff --git a/src/browser/history/HarCapture.ts b/src/browser/history/HarCapture.ts index 3c0cc86bc0..9e164e92d9 100644 --- a/src/browser/history/HarCapture.ts +++ b/src/browser/history/HarCapture.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import { LosslessNumber } from "lossless-json"; import { parseCaptureJson } from "./CaptureJson.js"; import type { JsonValue } from "../../domain/jsonValue.js"; +import { CANONICAL_BASE64_PATTERN } from "../../domain/stringPatterns.js"; import { WEB_NETWORK_CAPTURE_LIMITS, type WebNetworkCapture, @@ -264,17 +265,11 @@ const integerLexeme = (literal: string): boolean => { const decodeHarBase64 = (value: string, pointer: string): Buffer => { const compact = value.replace(/[\t\n\r ]/g, ""); - const bytes = Buffer.from(compact, "base64"); - if ( - !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test( - compact, - ) || - bytes.toString("base64") !== compact - ) + if (compact.length % 4 !== 0 || !CANONICAL_BASE64_PATTERN.test(compact)) throw new CaptureFormatError( "format", "HAR content declares base64 but its text is not valid canonical base64 (ASCII whitespace is allowed).", pointer, ); - return bytes; + return Buffer.from(compact, "base64"); }; diff --git a/src/domain/paddedBase64.test.ts b/src/domain/paddedBase64.test.ts new file mode 100644 index 0000000000..092cec9adc --- /dev/null +++ b/src/domain/paddedBase64.test.ts @@ -0,0 +1,44 @@ +import { expect, it } from "vitest"; + +import { decodePaddedBase64, paddedBase64ByteLength } from "./paddedBase64.js"; + +it("preserves padded syntax, including empty bodies and nonzero pad bits", () => { + for (const [encoded, decoded] of [ + ["", ""], + ["AA==", "00"], + ["AB==", "00"], + ["AAA=", "0000"], + ["AAB=", "0000"], + ["////", "ffffff"], + ] as const) { + expect(decodePaddedBase64(encoded)?.toString("hex")).toBe(decoded); + expect(paddedBase64ByteLength(encoded)).toBe(decoded.length / 2); + } + for (const malformed of [ + "A", + "AA", + "AAA", + "=AAA", + "A===", + "====", + "AA==A", + "AA-_", + "AA==\n", + "AA==\r", + "AA== ", + "A A=", + "ＡＡ==", + ]) { + expect(paddedBase64ByteLength(malformed), malformed).toBeUndefined(); + expect(decodePaddedBase64(malformed), malformed).toBeUndefined(); + } +}); + +it("validates large bodies and malformed suffixes without a regexp stack overflow", () => { + const body = Buffer.alloc(24 * 1024 * 1024 - 3, 0x81); + const encoded = body.toString("base64"); + expect(paddedBase64ByteLength(encoded)).toBe(body.length); + expect(decodePaddedBase64(encoded)?.equals(body)).toBe(true); + const malformed = `${encoded.slice(0, -1)}!`; + expect(paddedBase64ByteLength(malformed)).toBeUndefined(); +}); diff --git a/src/domain/paddedBase64.ts b/src/domain/paddedBase64.ts new file mode 100644 index 0000000000..395bdf5b8e --- /dev/null +++ b/src/domain/paddedBase64.ts @@ -0,0 +1,14 @@ +import { PADDED_BASE64_PATTERN } from "./stringPatterns.js"; + +/** Validate padded syntax without imposing canonical zero pad bits. */ +export const paddedBase64ByteLength = (value: string): number | undefined => { + if (value.length % 4 !== 0 || !PADDED_BASE64_PATTERN.test(value)) + return undefined; + const padding = value.endsWith("==") ? 2 : value.endsWith("=") ? 1 : 0; + return (value.length / 4) * 3 - padding; +}; + +export const decodePaddedBase64 = (value: string): Buffer | undefined => + paddedBase64ByteLength(value) === undefined + ? undefined + : Buffer.from(value, "base64"); diff --git a/src/domain/stringPatterns.test.ts b/src/domain/stringPatterns.test.ts index 5e35d2ccd9..7537742bd3 100644 --- a/src/domain/stringPatterns.test.ts +++ b/src/domain/stringPatterns.test.ts @@ -3,6 +3,7 @@ import { expect, it } from "vitest"; import { CANONICAL_BASE64_PATTERN, IDENTIFIER_PATTERN, + PADDED_BASE64_PATTERN, STABLE_IDENTIFIER_PATTERN, } from "./stringPatterns.js"; @@ -57,6 +58,11 @@ it("preserves canonical base64 padding and pad-bit validation", () => { "AAA", "====", "AA==\n", + "AA==\r\n", + "AA==\u2028", + "AA==\u2029", + "AA\nAA", + "AA=\n=", "AA-_", "a b c", ]; @@ -73,3 +79,20 @@ it("preserves canonical base64 padding and pad-bit validation", () => { ); } }); + +it("validates large canonical base64 under JSON Schema Unicode modes", () => { + // Near the 32 MiB capture input budget: a quartet expansion alone still + // overflows at this size, especially in Unicode-set client validators. + const body = Buffer.alloc(24 * 1024 * 1024 - 3, 0x81); + const encoded = body.toString("base64"); + for (const flag of ["", "u", "v"]) { + const pattern = new RegExp(CANONICAL_BASE64_PATTERN.source, flag); + expect(pattern.test(encoded), flag).toBe(true); + expect(pattern.test(`${encoded.slice(0, -1)}!`), flag).toBe(false); + expect(pattern.test(`${encoded.slice(0, -3)}R==`), flag).toBe(false); + expect(pattern.test(`${encoded}A`), flag).toBe(false); + const padded = new RegExp(PADDED_BASE64_PATTERN.source, flag); + expect(padded.test(encoded), flag).toBe(true); + expect(padded.test(`${encoded.slice(0, -1)}!`), flag).toBe(false); + } +}); diff --git a/src/domain/stringPatterns.ts b/src/domain/stringPatterns.ts index 8aa65cb75c..ab8994f17f 100644 --- a/src/domain/stringPatterns.ts +++ b/src/domain/stringPatterns.ts @@ -4,6 +4,17 @@ export const IDENTIFIER_PATTERN = /^[A-Za-z][A-Za-z0-9._\x2d]*$/u; /** Stable evidence identifier, also admitting namespace and path separators. */ export const STABLE_IDENTIFIER_PATTERN = /^[A-Za-z][A-Za-z0-9._:\x2f\x2d]*$/u; +// Repeated alphabet quartets exhaust V8's regexp stack on large bodies, +// including under JSON Schema's Unicode-set mode. Check alphabet and terminal +// padding separately, leaving an unconstrained fixed-width loop for length. +const PADDED_BASE64_PREFIX = String.raw`^(?!.*[^A-Za-z0-9+\x2f=])(?!.*=[^=])(?!.*===)`; + +/** Padded base64 syntax, allowing nonzero pad bits and the empty encoding. */ +export const PADDED_BASE64_PATTERN = new RegExp( + `${PADDED_BASE64_PREFIX}(?:....)*$`, +); + /** Canonical padded base64, including zero pad bits and the empty encoding. */ -export const CANONICAL_BASE64_PATTERN = - /^(?:[A-Za-z0-9+\x2f]{4})*(?:[A-Za-z0-9+\x2f][AQgw]==|[A-Za-z0-9+\x2f]{2}[AEIMQUYcgkosw048]=)?$/; +export const CANONICAL_BASE64_PATTERN = new RegExp( + `${PADDED_BASE64_PREFIX}(?!.*[^AQgw]==$)(?!.*[^AEIMQUYcgkosw048=]=$)(?:....)*$`, +); diff --git a/tests/boundary/browser/largeBase64Capture.test.ts b/tests/boundary/browser/largeBase64Capture.test.ts new file mode 100644 index 0000000000..06ebc313d5 --- /dev/null +++ b/tests/boundary/browser/largeBase64Capture.test.ts @@ -0,0 +1,138 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; +import { z } from "zod"; + +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { evidenceBundleSchema } from "../../../src/domain/evidenceBundle.js"; +import { webNetworkCaptureSchema } from "../../../src/domain/webNetworkCapture.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { historicalHar } from "../../fixtures/historicalHar.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const sha256 = (value: Uint8Array | string): string => + createHash("sha256").update(value).digest("hex"); + +cliTest( + "retains a 4 MiB canonical HAR body through CLI and MCP evidence export", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-large-base64-har-"); + const path = join(root, "capture.har"); + const body = Buffer.alloc(4 * 1024 * 1024, 0x5a); + const contentBase64 = body.toString("base64"); + const fixture = historicalHar(); + const entry = fixture.log.entries[0]; + if (entry === undefined) throw new Error("HAR fixture has no entry"); + const original = Buffer.from( + JSON.stringify({ + log: { + ...fixture.log, + entries: [ + { + ...entry, + response: { + ...entry.response, + content: { + size: body.length, + mimeType: "application/octet-stream", + encoding: "base64", + text: contentBase64, + }, + }, + }, + ], + }, + }), + ); + await writeFile(path, original); + + const summarize = (document: unknown) => { + const evidence = parseEvidence(document); + const capture = webNetworkCaptureSchema.parse(evidence.normalized_result); + const record = capture.records[0]; + const binary = record?.binary_fields[0]; + if (record === undefined || binary === undefined) + throw new Error("HAR evidence omitted its binary sidecar"); + if (binary.state !== "retained") + throw new Error(`HAR binary sidecar was ${binary.state}`); + const reported = z + .object({ + response: z.object({ content: z.object({ text: z.string() }) }), + }) + .parse(record.reported); + const retainedBytes = Buffer.from(binary.content_base64, "base64"); + return { + state: binary.state, + bytes: binary.bytes, + sha256: binary.sha256, + contentBase64Length: binary.content_base64.length, + contentBase64Sha256: sha256(binary.content_base64), + decodedLength: retainedBytes.length, + decodedSha256: sha256(retainedBytes), + reportedTextLength: reported.response.content.text.length, + reportedTextSha256: sha256(reported.response.content.text), + }; + }; + const expected = { + state: "retained", + bytes: body.length, + sha256: sha256(body), + contentBase64Length: contentBase64.length, + contentBase64Sha256: sha256(contentBase64), + decodedLength: body.length, + decodedSha256: sha256(body), + reportedTextLength: contentBase64.length, + reportedTextSha256: sha256(contentBase64), + }; + + const fromCli = await cli.run({ + arguments: ["inspect-web-network-capture", path, "har", "--json"], + timeoutMs: 120_000, + }); + expect(fromCli.exitCode).toBe(0); + const cliSummary = summarize(fromCli.json); + + const { call } = await connectLocalToolsMcp(); + const response = await call("inspect_web_network_capture", { + capture_path: path, + format: "har", + }); + const { evidence_reference: reference } = z + .object({ + error: z.object({ + code: z.literal("resource_constraint"), + details: z.object({ + reported_limits: z.object({ + boundary: z.literal("mcp-response"), + constraint: z.literal("receive-buffer"), + evidence_reference: z.object({ + kind: z.literal("retained-evidence"), + evidence_id: z.string(), + }), + }), + }), + }), + }) + .parse(parseMcpToolError(response)).error.details.reported_limits; + const bundlePath = join(root, "evidence.json"); + const exported = await call("export_evidence_bundle", { path: bundlePath }); + expect(exported.isError).not.toBe(true); + const bundle = evidenceBundleSchema.parse( + JSON.parse(await readFile(bundlePath, "utf8")), + ); + const retained = bundle.records.find( + (record) => record.evidence_id === reference.evidence_id, + ); + expect(retained).toBeDefined(); + const mcpSummary = summarize(retained); + + expect(cliSummary).toEqual(expected); + expect(mcpSummary).toEqual(expected); + const unchanged = await readFile(path); + expect(unchanged.byteLength).toBe(original.byteLength); + expect(sha256(unchanged)).toBe(sha256(original)); + }, +); From 255d906e15a141b0f4acee60dc73976f03adc94c Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:09:42 +0800 Subject: [PATCH 2/2] fix(native): scan byte-pattern inputs without regexp stack growth Keep exact wildcard, whitespace and aligned-mask validation in the shared domain refinement. Advertise descriptive string inputs instead of repeating regexes, and remove discarded byte/mask array materialization. Preserve caller spelling and error precedence; cover large public stdio queries, provider-boundary parsing and connection recovery. Fixes #5257 --- docs/mcp-contracts.md | 6 ++ src/domain/native/bytePattern.test.ts | 76 ++++++++++++++--- src/domain/native/bytePattern.ts | 95 ++++++++++++++++----- src/ghidra/GhidraInventoryValues.test.ts | 24 ++++++ tests/boundary/mcp/largeBytePattern.test.ts | 63 ++++++++++++++ 5 files changed, 229 insertions(+), 35 deletions(-) create mode 100644 tests/boundary/mcp/largeBytePattern.test.ts diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 840e7f241e..3a4daed24f 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -776,6 +776,12 @@ after SDK conversion and check advertised validation and actual calls. Individual model APIs can impose additional nesting limits; complete producer captures can exceed ten structural levels. +`search_bytes` advertises its pattern and mask as descriptive strings. Its +shared canonical validator scans byte pairs, whitespace, wildcards and aligned +masks without a repeating regular expression or temporary byte arrays. The +server rejects malformed or unconstrained patterns before provider dispatch; +the advertised strings alone do not express those rules. + Boolean literal constraints in input schemas are relaxed only in this advertisement. For example, a boolean `const` is presented as `type: boolean`; an affected `not` is omitted, conditional predicates and their branches are diff --git a/src/domain/native/bytePattern.test.ts b/src/domain/native/bytePattern.test.ts index 9d2cc92c11..296b425149 100644 --- a/src/domain/native/bytePattern.test.ts +++ b/src/domain/native/bytePattern.test.ts @@ -1,5 +1,14 @@ import { describe, expect, it } from "vitest"; -import { parseBytePattern, byteSearchResultSchema } from "./bytePattern.js"; +import { z } from "zod"; +import { + bytePatternFields, + validateBytePattern, + byteSearchResultSchema, +} from "./bytePattern.js"; + +const patternSchema = z + .strictObject(bytePatternFields) + .superRefine(validateBytePattern); it.each(["0x20", "overlay:0x10", "invalid"])( "rejects a match outside reported searched coverage: %s", @@ -53,15 +62,13 @@ it.each(["", "overlay:"])( describe("masked byte patterns", () => { it("preserves whole-byte wildcards and combines explicit bit masks", () => { - const parsed = parseBytePattern("48 8b\t??89", "ff f0 ff 0f"); - if (!parsed.ok) throw parsed.error; - expect(parsed.value).toEqual({ - bytes: [0x48, 0x8b, 0, 0x89], - masks: [255, 240, 0, 15], - }); - const exact = parseBytePattern("Aa\u00a000"); - if (!exact.ok) throw exact.error; - expect(exact.value).toEqual({ bytes: [170, 0], masks: [255, 255] }); + for (const input of [ + { pattern: "48 8b\t??89", mask: "ff f0 ff 0f" }, + { pattern: "Aa\u00a000" }, + { pattern: "00" }, + { pattern: "????aa", mask: "000001" }, + ]) + expect(patternSchema.parse(input)).toEqual(input); }); it.each([ @@ -73,11 +80,54 @@ describe("masked byte patterns", () => { ["ff", "00", "pattern"], ["ffff", "ff", "mask"], ["ff", "??", "mask"], + ["aa ??", "00 ff", "pattern"], + ["??", "0000", "mask"], + ["aa", "00gg", "mask"], + ["??gg", "00gg", "pattern"], ])("rejects invalid pattern %s with typed issues", (pattern, mask, field) => { - const parsed = parseBytePattern(pattern, mask); - if (parsed.ok) throw new Error("Expected a rejected pattern"); + const parsed = patternSchema.safeParse({ pattern, mask }); + expect(parsed.success).toBe(false); + if (parsed.success) throw new Error("Expected a rejected pattern"); expect(parsed.error.issues).toMatchObject([ - { path: [field], reason: "invalid_value" }, + { path: [field], code: "custom" }, ]); }); + + it("accepts ECMAScript whitespace only between complete byte pairs", () => { + const whitespace = [ + "\t", + "\n", + "\v", + "\f", + "\r", + " ", + "\u00a0", + "\u1680", + ...Array.from({ length: 11 }, (_, i) => String.fromCharCode(0x2000 + i)), + "\u2028", + "\u2029", + "\u202f", + "\u205f", + "\u3000", + "\ufeff", + ]; + for (const separator of whitespace) { + expect( + patternSchema.safeParse({ + pattern: `aa${separator}??`, + mask: `ff${separator}ff`, + }).success, + ).toBe(true); + for (const pattern of [ + `${separator}aa`, + `aa${separator}`, + `a${separator}a`, + ]) + expect(patternSchema.safeParse({ pattern }).success).toBe(false); + } + for (const separator of ["\u0085", "\u180e", "\u200b", "\u200c", "\u200d"]) + expect( + patternSchema.safeParse({ pattern: `aa${separator}bb` }).success, + ).toBe(false); + }); }); diff --git a/src/domain/native/bytePattern.ts b/src/domain/native/bytePattern.ts index 118102dcf5..d0ae144585 100644 --- a/src/domain/native/bytePattern.ts +++ b/src/domain/native/bytePattern.ts @@ -3,45 +3,96 @@ import { addressCoordinate } from "./addressCoordinate.js"; import { AnalysisInputError } from "../analysisErrorCore.js"; import { err, ok, type Result } from "../result.js"; -/** Byte pairs, with whole-byte wildcards; explicit masks can select individual bits. */ +/** Exact grammar belongs to the object refinement, not a repeating advertised regex. */ export const bytePatternFields = { pattern: z .string() - .regex(/^([0-9a-fA-F]{2}|\?\?)(\s*([0-9a-fA-F]{2}|\?\?))*$/u), + .describe( + "Hexadecimal byte pairs or ?? whole-byte wildcards, with optional whitespace between complete pairs; no leading or trailing whitespace. At least one bit must be constrained after applying the mask.", + ), mask: z .string() - .regex(/^[0-9a-fA-F]{2}(\s*[0-9a-fA-F]{2})*$/u) - .optional(), + .optional() + .describe( + "Hexadecimal mask byte pairs, with optional whitespace between complete pairs; no leading or trailing whitespace. Must contain the same number of bytes as pattern.", + ), }; -/** Parse once at the input boundary, retaining typed correction issues. */ -export const parseBytePattern = ( +const hexDigit = (code: number): number | undefined => { + if (code >= 48 && code <= 57) return code - 48; + if (code >= 65 && code <= 70) return code - 55; + if (code >= 97 && code <= 102) return code - 87; + return undefined; +}; + +const readBytePair = ( + value: string, + offset: number, +): number | "wildcard" | undefined => { + if (value[offset] === "?" && value[offset + 1] === "?") return "wildcard"; + const high = hexDigit(value.charCodeAt(offset)); + const low = hexDigit(value.charCodeAt(offset + 1)); + return high === undefined || low === undefined ? undefined : high * 16 + low; +}; + +const nextPairOffset = (value: string, offset: number): number => { + const end = offset + 2; + let next = end; + while (next < value.length && /\s/u.test(value[next]!)) next += 1; + return next === value.length && next !== end ? -1 : next; +}; + +const countBytePairs = ( + value: string, + field: "pattern" | "mask", +): number | undefined => { + let count = 0; + for (let offset = 0; offset < value.length;) { + const byte = readBytePair(value, offset); + if (byte === undefined || (field === "mask" && byte === "wildcard")) + return undefined; + count += 1; + offset = nextPairOffset(value, offset); + if (offset < 0) return undefined; + } + return count === 0 ? undefined : count; +}; + +/** Validate without materializing byte arrays or using size-dependent regexes. */ +const validateBytePatternValue = ( pattern: string, mask?: string, -): Result<{ bytes: number[]; masks: number[] }, AnalysisInputError> => { +): Result => { const invalid = (field: string, message: string) => err( new AnalysisInputError("search_bytes", undefined, [ { path: [field], reason: "invalid_value", message }, ]), ); - if (!bytePatternFields.pattern.safeParse(pattern).success) + const patternCount = countBytePairs(pattern, "pattern"); + if (patternCount === undefined) return invalid("pattern", "Use hexadecimal byte pairs or ?? wildcards"); - if (mask !== undefined && !bytePatternFields.mask.safeParse(mask).success) + const maskCount = + mask === undefined ? undefined : countBytePairs(mask, "mask"); + if (mask !== undefined && maskCount === undefined) return invalid("mask", "Use hexadecimal mask byte pairs"); - const pairs = pattern.replace(/\s/gu, "").match(/../gu) ?? []; - const maskPairs = mask?.replace(/\s/gu, "").match(/../gu); - if (maskPairs && maskPairs.length !== pairs.length) + if (maskCount !== undefined && maskCount !== patternCount) return invalid("mask", "Mask length must equal pattern length"); - const bytes = pairs.map((pair) => - pair === "??" ? 0 : Number.parseInt(pair, 16), - ); - const masks = pairs.map((pair, index) => - pair === "??" ? 0 : Number.parseInt(maskPairs?.[index] ?? "ff", 16), - ); - if (!masks.some((value) => value !== 0)) - return invalid("pattern", "Pattern must constrain at least one bit"); - return ok({ bytes, masks }); + let patternOffset = 0; + let maskOffset = 0; + for (let index = 0; index < patternCount; index += 1) { + const byte = readBytePair(pattern, patternOffset); + const selected = mask === undefined ? 255 : readBytePair(mask, maskOffset); + if ( + typeof byte === "number" && + typeof selected === "number" && + selected !== 0 + ) + return ok(null); + patternOffset = nextPairOffset(pattern, patternOffset); + if (mask !== undefined) maskOffset = nextPairOffset(mask, maskOffset); + } + return invalid("pattern", "Pattern must constrain at least one bit"); }; /** Shared semantic validation for public and provider inputs. */ @@ -49,7 +100,7 @@ export const validateBytePattern = ( value: { pattern: string; mask?: string | undefined }, context: z.RefinementCtx, ): void => { - const parsed = parseBytePattern(value.pattern, value.mask); + const parsed = validateBytePatternValue(value.pattern, value.mask); if (!parsed.ok) for (const issue of parsed.error.issues) context.addIssue({ diff --git a/src/ghidra/GhidraInventoryValues.test.ts b/src/ghidra/GhidraInventoryValues.test.ts index 42d0c262de..b66fed3ca5 100644 --- a/src/ghidra/GhidraInventoryValues.test.ts +++ b/src/ghidra/GhidraInventoryValues.test.ts @@ -5,6 +5,30 @@ import { parseGhidraInventoryResult, } from "./GhidraInventoryValues.js"; +it("validates large masked byte queries at the provider boundary without changing caller spelling", () => { + const input = { + pattern: "aA??".repeat(1024 * 1024), + mask: "ff00".repeat(1024 * 1024), + }; + const parsed = parseGhidraInventoryInput("search_bytes", input); + expect(parsed.ok).toBe(true); + if (!parsed.ok) throw parsed.error; + expect(parsed.value.pattern === input.pattern).toBe(true); + expect(parsed.value.mask === input.mask).toBe(true); + expect( + parseGhidraInventoryInput("search_bytes", { + ...input, + mask: `${input.mask.slice(0, -2)}g0`, + }).ok, + ).toBe(false); + expect( + parseGhidraInventoryInput("search_bytes", { + ...input, + mask: "00ff".repeat(1024 * 1024), + }).ok, + ).toBe(false); +}); + describe("provider-owned overview boundary", () => { const overview = { document: "fixture", diff --git a/tests/boundary/mcp/largeBytePattern.test.ts b/tests/boundary/mcp/largeBytePattern.test.ts new file mode 100644 index 0000000000..21d18778a2 --- /dev/null +++ b/tests/boundary/mcp/largeBytePattern.test.ts @@ -0,0 +1,63 @@ +import { expect, it } from "vitest"; +import type { CallToolResult } from "@modelcontextprotocol/server"; + +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; + +const LARGE_PATTERN_BYTES = 4 * 1024 * 1024; + +const inputValidationText = (result: CallToolResult): string => { + expect(result.isError).toBe(true); + const text = result.content.find((item) => item.type === "text"); + if (text?.type !== "text") throw new Error("Input error omitted text"); + expect(text.text).toMatch( + /^Input validation error: Invalid arguments for tool search_bytes:/u, + ); + expect(text.text).not.toMatch(/RangeError|Maximum call stack/i); + return text.text; +}; + +it("validates large search_bytes inputs without breaking stdio MCP", async () => { + const { client, call } = await connectLocalToolsMcp(); + + const definition = (await client.listTools()).tools.find( + ({ name }) => name === "search_bytes", + ); + expect(definition?.inputSchema.properties?.pattern).toMatchObject({ + type: "string", + }); + expect(definition?.inputSchema.properties?.pattern).not.toHaveProperty( + "pattern", + ); + expect(definition?.inputSchema.properties?.mask).toMatchObject({ + type: "string", + }); + expect(definition?.inputSchema.properties?.mask).not.toHaveProperty( + "pattern", + ); + + const initial = await call("search_bytes", { pattern: "aa" }); + expect(parseMcpToolError(initial).error.code).toBe("target_unavailable"); + + const largePattern = "aa".repeat(LARGE_PATTERN_BYTES); + const largeSearch = await call("search_bytes", { pattern: largePattern }); + expect(parseMcpToolError(largeSearch).error.code).toBe("target_unavailable"); + + const malformedPattern = `${largePattern.slice(0, -2)}a?`; + const malformed = inputValidationText( + await call("search_bytes", { pattern: malformedPattern }), + ); + expect(malformed).toMatch(/pattern/i); + + const oversizedMask = inputValidationText( + await call("search_bytes", { + pattern: "aa", + mask: "ff".repeat(LARGE_PATTERN_BYTES), + }), + ); + expect(oversizedMask).toContain("Mask length must equal pattern length"); + + const final = await call("search_bytes", { pattern: "aa" }); + expect(parseMcpToolError(final).error.code).toBe("target_unavailable"); + await client.ping(); +});