diff --git a/docs/browser-observation.md b/docs/browser-observation.md index c73caa90e3..01a940e149 100644 --- a/docs/browser-observation.md +++ b/docs/browser-observation.md @@ -434,6 +434,17 @@ complete Structured Field Item, including parameters such as `report-to`. These are declarations, not an assessment of the document's effective policy; absent, malformed or non-token items project as `null`. +Console entries, WebSocket frame summaries, and selected request/response JSON +shape outputs share one 8 MiB retained-observation allowance per capture. Shape +inference admits expanded path bytes before allocating those paths, including +escaped and Unicode property names. Replacing or removing a request's shapes +refunds their retained charges. A refused body shape preserves network metadata +and any captured peer shape with `resource_budget_exhausted` coverage; a refused +WebSocket frame is counted as dropped rather than reclassified as non-JSON text. +The allowance has no runtime override. Protocol messages and JSON input parsing +remain subject to their separate transport admission; this output allowance does +not bound the whole process heap. + REA retains local observation data and removes only URL userinfo credentials, explicitly declared scenario secrets, and values from structured credential/storage fields: - URL observations preserve the complete local URL, including query values, fragments, duplicate parameters, and their order. Only username/password userinfo is removed; malformed original URL strings remain available with `origin: null`. diff --git a/src/browser/CdpCaptureEventBodyShapes.ts b/src/browser/CdpCaptureEventBodyShapes.ts index 9c6fefb023..263b240ed6 100644 --- a/src/browser/CdpCaptureEventBodyShapes.ts +++ b/src/browser/CdpCaptureEventBodyShapes.ts @@ -1,4 +1,5 @@ -import { inferJsonShape, type JsonShape } from "../domain/jsonShape.js"; +import { inferJsonShape, type AdmittedJsonShape } from "../domain/jsonShape.js"; +import { WEB_RUNTIME_LIMITS } from "../domain/webRuntime.js"; import { decodePaddedBase64 } from "../domain/paddedBase64.js"; import { recordValue, @@ -14,6 +15,7 @@ export const ingestResponseBodyShape = ( requestId: string, value: unknown, ): void => { + if (!state.network.has(requestId)) return; const result = recordValue(value); const body = cdpStringValue(result?.body); const base64Encoded = result?.base64Encoded; @@ -26,9 +28,23 @@ export const ingestResponseBodyShape = ( invalidResponseBodyShape(state, requestId); return; } - const shape = inferJsonShape(decoded); - if (shape === null) state.completeness.unavailable("json_body_shapes"); - updateResponseBodyShape(state, requestId, shape); + const previousBytes = + state.bodyShapeBytesByRequestId.get(requestId)?.response ?? 0; + const parsed = inferJsonShape( + decoded, + WEB_RUNTIME_LIMITS.retainedEventBytes - + state.retainedEventBytes + + previousBytes, + ); + if (!parsed.ok) { + if (parsed.error.kind === "capacity") + state.completeness.exclude( + "json_body_shapes", + "resource_budget_exhausted", + ); + else state.completeness.unavailable("json_body_shapes"); + } + updateResponseBodyShape(state, requestId, parsed.ok ? parsed.value : null); }; export const requestBodyShape = ( @@ -36,18 +52,37 @@ export const requestBodyShape = ( requestId: string, request: UnknownRecord, ): NetworkState["body_shapes"] => { - state.missingRequestBodyShapeIds.delete(requestId); + releaseBodyShapes(state, requestId); if (!state.input.include_json_body_shapes) return { status: "not_approved", request: null, response: null }; const headers = recordValue(request.headers); const json = isJsonContentType(headers); const body = cdpStringValue(request.postData); // CDP may omit a long body from postData while reporting hasPostData. - const shape = json && body !== undefined ? inferJsonShape(body) : null; + const parsed = + json && body !== undefined + ? inferJsonShape( + body, + WEB_RUNTIME_LIMITS.retainedEventBytes - state.retainedEventBytes, + ) + : undefined; + const shape = parsed?.ok ? parsed.value.shape : null; const expected = json && (body !== undefined || request.hasPostData === true); if (expected && shape === null) { state.missingRequestBodyShapeIds.add(requestId); - state.completeness.unavailable("json_body_shapes"); + if (parsed !== undefined && !parsed.ok && parsed.error.kind === "capacity") + state.completeness.exclude( + "json_body_shapes", + "resource_budget_exhausted", + ); + else state.completeness.unavailable("json_body_shapes"); + } + if (parsed?.ok) { + state.bodyShapeBytesByRequestId.set(requestId, { + request: parsed.value.serializedBytes, + response: 0, + }); + state.retainedEventBytes += parsed.value.serializedBytes; } return { status: shape === null ? "unavailable" : "included", @@ -59,10 +94,21 @@ export const requestBodyShape = ( export const updateResponseBodyShape = ( state: CdpCaptureEventsState, requestId: string, - response: JsonShape | null, + response: AdmittedJsonShape | null, ): void => { const current = state.network.get(requestId); if (current === undefined) return; + const previous = state.bodyShapeBytesByRequestId.get(requestId); + const responseBytes = response?.serializedBytes ?? 0; + const requestBytes = previous?.request ?? 0; + state.retainedEventBytes += responseBytes - (previous?.response ?? 0); + if (requestBytes + responseBytes === 0) + state.bodyShapeBytesByRequestId.delete(requestId); + else + state.bodyShapeBytesByRequestId.set(requestId, { + request: requestBytes, + response: responseBytes, + }); const request = current.body_shapes.request; const status = response !== null @@ -74,10 +120,22 @@ export const updateResponseBodyShape = ( : "unavailable"; state.network.set(requestId, { ...current, - body_shapes: { status, request, response }, + body_shapes: { status, request, response: response?.shape ?? null }, }); }; +/** Replacing or excluding a network record retires exactly the shapes it retained. */ +export const releaseBodyShapes = ( + state: CdpCaptureEventsState, + requestId: string, +): void => { + state.missingRequestBodyShapeIds.delete(requestId); + const previous = state.bodyShapeBytesByRequestId.get(requestId); + if (previous === undefined) return; + state.retainedEventBytes -= previous.request + previous.response; + state.bodyShapeBytesByRequestId.delete(requestId); +}; + const invalidResponseBodyShape = ( state: CdpCaptureEventsState, requestId: string, diff --git a/src/browser/CdpCaptureEventHandlers.ts b/src/browser/CdpCaptureEventHandlers.ts index a3d6bd7b84..8587393fcc 100644 --- a/src/browser/CdpCaptureEventHandlers.ts +++ b/src/browser/CdpCaptureEventHandlers.ts @@ -1,6 +1,11 @@ import type { WebPageInspection } from "../domain/browserObservationSchemas.js"; import { sanitizeBrowserUrl } from "../domain/browserObservation.js"; -import { inferJsonShape } from "../domain/jsonShape.js"; +import { + inferJsonShape, + type JsonShapeCapacityFailure, +} from "../domain/jsonShape.js"; +import { jsonValueByteLength } from "../domain/jsonValueByteLength.js"; +import { err, ok, type Result } from "../domain/result.js"; import { paddedBase64ByteLength } from "../domain/paddedBase64.js"; import { WEB_RUNTIME_LIMITS } from "../domain/webRuntime.js"; import { safeResponseMetadata } from "./CdpSafeMetadata.js"; @@ -16,6 +21,7 @@ import { } from "./CdpCaptureValues.js"; import { requestBodyShape, + releaseBodyShapes, updateResponseBodyShape, } from "./CdpCaptureEventBodyShapes.js"; import { @@ -252,6 +258,7 @@ export const handleRequestWillBeSent = ( "network_requests", exclusionReasonForUrl(cdpStringValue(request.url)), ); + releaseBodyShapes(state, requestId); state.network.delete(requestId); state.networkRequestTimestamps.delete(requestId); return; @@ -283,6 +290,7 @@ export const handleRequestWillBeSent = ( // redirectResponse belongs to the same request ID and can contain an // excluded origin. Drop the entire chain before retaining any details. state.completeness.exclude("network_requests", responseUrlReason); + releaseBodyShapes(state, requestId); state.network.delete(requestId); state.networkRequestTimestamps.delete(requestId); return; @@ -405,6 +413,7 @@ export const handleResponseReceived = ( ? "invalid_protocol_value" : exclusionReasonForUrl(cdpStringValue(response.url)), ); + releaseBodyShapes(state, requestId); state.network.delete(requestId); state.networkRequestTimestamps.delete(requestId); return; @@ -582,20 +591,27 @@ export const handleWebSocketFrame = ( : (paddedBase64ByteLength(payload) ?? null); if (opcode === null || payloadBytes === null) state.completeness.exclude("websocket_frames", "invalid_protocol_value"); - const payloadShape = captureWebSocketShape(state, payload, opcode); - // The frame retains its shape summary, not the payload, and the shape grows - // with the payload's own keys; the shared retained-event budget bounds the - // retained frames and a refused frame stays counted as a dropped event. - const entryBytes = - RETAINED_EVENT_ENTRY_OVERHEAD_BYTES + - stringByteLength(direction) + - (payloadShape === null - ? 0 - : Buffer.byteLength(JSON.stringify(payloadShape), "utf8")); - if ( - entryBytes > - WEB_RUNTIME_LIMITS.retainedEventBytes - state.retainedEventBytes - ) { + const availableBytes = + WEB_RUNTIME_LIMITS.retainedEventBytes - state.retainedEventBytes; + const baseBytes = + RETAINED_EVENT_ENTRY_OVERHEAD_BYTES + stringByteLength(direction); + if (baseBytes > availableBytes) { + state.completeness.drop("websocket_frames"); + return; + } + const captured = captureWebSocketShape( + state, + payload, + opcode, + availableBytes - baseBytes, + ); + if (!captured.ok) { + state.completeness.drop("websocket_frames"); + state.completeness.exclude("websocket_shapes", "resource_budget_exhausted"); + return; + } + const entryBytes = baseBytes + captured.value.serializedBytes; + if (entryBytes > availableBytes) { state.completeness.drop("websocket_frames"); return; } @@ -604,24 +620,56 @@ export const handleWebSocketFrame = ( direction, opcode, payload_bytes: payloadBytes, - payload_shape: payloadShape, + payload_shape: captured.value.payloadShape, }); }; +type WebSocketShape = + WebPageInspection["network"]["websocket_connections"][number]["events"][number]["payload_shape"]; +const jsonShapeEnvelopeBytes = + jsonValueByteLength({ format: "json", json_shape: null }) - + jsonValueByteLength(null); + const captureWebSocketShape = ( state: CdpCaptureEventsState, payload: string | undefined, opcode: number | null, -): WebPageInspection["network"]["websocket_connections"][number]["events"][number]["payload_shape"] => { - if (!state.input.include_websocket_shapes) return null; - if (opcode === null) return null; - if (opcode !== 1) return { format: "binary", json_shape: null }; - if (payload === undefined) return null; - const shape = inferJsonShape(payload); - return { - format: shape === null ? "text" : "json", - json_shape: shape, + maximumBytes: number, +): Result< + { readonly payloadShape: WebSocketShape; readonly serializedBytes: number }, + JsonShapeCapacityFailure +> => { + if (!state.input.include_websocket_shapes || opcode === null) + return ok({ payloadShape: null, serializedBytes: 0 }); + if (opcode !== 1) { + const payloadShape: WebSocketShape = { format: "binary", json_shape: null }; + return ok({ + payloadShape, + serializedBytes: jsonValueByteLength(payloadShape), + }); + } + if (payload === undefined) + return ok({ payloadShape: null, serializedBytes: 0 }); + const parsed = inferJsonShape( + payload, + Math.max(0, maximumBytes - jsonShapeEnvelopeBytes), + ); + if (!parsed.ok) { + if (parsed.error.kind === "capacity") return err(parsed.error); + const payloadShape: WebSocketShape = { format: "text", json_shape: null }; + return ok({ + payloadShape, + serializedBytes: jsonValueByteLength(payloadShape), + }); + } + const payloadShape: WebSocketShape = { + format: "json", + json_shape: parsed.value.shape, }; + return ok({ + payloadShape, + serializedBytes: jsonShapeEnvelopeBytes + parsed.value.serializedBytes, + }); }; export const handleWebSocketCreated = ( diff --git a/src/browser/CdpCaptureEventState.ts b/src/browser/CdpCaptureEventState.ts index 331fc2807e..b6f3d50999 100644 --- a/src/browser/CdpCaptureEventState.ts +++ b/src/browser/CdpCaptureEventState.ts @@ -12,8 +12,12 @@ export interface CdpCaptureEventsState { rejectedSourceMapMetadataBudgetCount: number; readonly scriptMetadataBytesById: Map; retainedScriptMetadataBytes: number; - /** Console-entry and websocket-frame bytes admitted against the shared retained-event budget. */ + /** Console, websocket-frame and selected request/response shape outputs share one allowance. */ retainedEventBytes: number; + readonly bodyShapeBytesByRequestId: Map< + string, + { readonly request: number; readonly response: number } + >; readonly executionContextFrames: Map; readonly network: Map; readonly networkRequestTimestamps: Map; diff --git a/src/browser/CdpCaptureEvents.eventBudget.test.ts b/src/browser/CdpCaptureEvents.eventBudget.test.ts index 496aec5537..166c366efb 100644 --- a/src/browser/CdpCaptureEvents.eventBudget.test.ts +++ b/src/browser/CdpCaptureEvents.eventBudget.test.ts @@ -80,7 +80,7 @@ it("drops a websocket frame whose retained shape alone exceeds the shared budget }, }); const keys = Array.from( - { length: 900_000 }, + { length: 120_000 }, (_value, index) => `key${String(index)}`, ); events.ingest({ @@ -125,7 +125,7 @@ it("spends one shared allowance across console entries and websocket frames", () params: { requestId: "socket", url: `ws://${origin.slice(7)}/live` }, }); const keys = Array.from( - { length: 200_000 }, + { length: 20_000 }, (_value, index) => `key${String(index)}`, ); events.ingest({ diff --git a/src/browser/CdpCaptureEvents.shapeBudget.test.ts b/src/browser/CdpCaptureEvents.shapeBudget.test.ts new file mode 100644 index 0000000000..cb6ebb5db9 --- /dev/null +++ b/src/browser/CdpCaptureEvents.shapeBudget.test.ts @@ -0,0 +1,224 @@ +import { expect, it } from "vitest"; + +import { inspectWebPageInputSchema } from "../domain/browserObservation.js"; +import { WEB_RUNTIME_LIMITS } from "../domain/webRuntime.js"; +import { CdpCaptureEvents } from "./CdpCaptureEvents.js"; + +const origin = "http://127.0.0.1:43127"; +const deep = (depth: number) => '{"a":'.repeat(depth) + "0" + "}".repeat(depth); +const createEvents = (selected = true) => + new CdpCaptureEvents( + inspectWebPageInputSchema.parse({ + cdp_endpoint: origin, + allowed_origins: [origin], + target_id: "page", + include_console_text: true, + include_json_body_shapes: selected, + include_websocket_shapes: selected, + }), + new Set([origin]), + ); +const request = ( + events: CdpCaptureEvents, + id: string, + body: string, + redirect = false, +) => { + events.ingest({ + method: "Network.requestWillBeSent", + params: { + requestId: id, + timestamp: redirect ? 2 : 1, + initiator: { type: "other" }, + request: { + url: `${origin}/${redirect ? "next" : "first"}`, + method: "POST", + headers: { "Content-Type": "application/json" }, + postData: body, + }, + ...(redirect + ? { + redirectResponse: { + url: `${origin}/first`, + status: 302, + encodedDataLength: 0, + }, + } + : {}), + }, + }); +}; +const response = (events: CdpCaptureEvents, id: string, body: string) => { + events.ingest({ + method: "Network.responseReceived", + params: { + requestId: id, + response: { + url: `${origin}/first`, + status: 200, + mimeType: "application/json", + }, + }, + }); + const candidate = events + .responseBodyRequests() + .find(({ requestId }) => requestId === id); + if (candidate === undefined) + throw new Error("Expected selected response body"); + events.ingestResponseBody(candidate, { body, base64Encoded: false }); + return candidate; +}; +const openSocket = (events: CdpCaptureEvents) => + events.ingest({ + method: "Network.webSocketCreated", + params: { requestId: "socket", url: `ws://${origin.slice(7)}/live` }, + }); +const frame = (events: CdpCaptureEvents, payloadData: string) => + events.ingest({ + method: "Network.webSocketFrameReceived", + params: { requestId: "socket", response: { opcode: 1, payloadData } }, + }); + +it("distinguishes valid over-budget websocket JSON from non-JSON text", () => { + const events = createEvents(); + openSocket(events); + frame(events, deep(760)); + frame(events, "ordinary non-JSON text"); + expect(events.webSocketConnectionsById.get("socket")?.events).toMatchObject([ + { payload_shape: { format: "text", json_shape: null } }, + ]); + const coverage = events.completeness.snapshot(); + expect(coverage.dropped_events.websocket_frames).toBe(1); + expect(coverage.excluded).toContainEqual({ + section: "websocket_shapes", + reason: "resource_budget_exhausted", + count: 1, + }); +}); + +it("retains the peer body facts when a selected request or response shape exceeds the allowance", () => { + const events = createEvents(); + request(events, "request-large", deep(760)); + response(events, "request-large", '{"response":true}'); + expect(events.network.get("request-large")?.body_shapes).toMatchObject({ + status: "partial", + request: null, + response: { root_type: "object" }, + }); + request(events, "response-large", '{"request":true}'); + response(events, "response-large", deep(760)); + expect(events.network.get("response-large")?.body_shapes).toMatchObject({ + status: "partial", + request: { root_type: "object" }, + response: null, + }); + const coverage = events.completeness.snapshot(); + expect(coverage.truncated_sections).toContain("json_body_shapes"); + expect(coverage.excluded).toContainEqual({ + section: "json_body_shapes", + reason: "resource_budget_exhausted", + count: 2, + }); +}); + +it("replaces response receipts without charging the same retained result a hundred times", () => { + const events = createEvents(); + request(events, "repeat", '{"request":true}'); + const candidate = response(events, "repeat", deep(100)); + const retained = events.retainedEventBytes; + for (let index = 0; index < 100; index++) + events.ingestResponseBody(candidate, { + body: deep(100), + base64Encoded: false, + }); + expect(events.retainedEventBytes).toBe(retained); + response(events, "repeat", '{"smaller":true}'); + const shapes = events.network.get("repeat")?.body_shapes; + if (shapes === undefined) throw new Error("Expected retained request"); + expect(shapes.status).toBe("included"); + expect(events.retainedEventBytes).toBe( + Buffer.byteLength(JSON.stringify(shapes.request)) + + Buffer.byteLength(JSON.stringify(shapes.response)), + ); + expect(events.completeness.snapshot().truncated_sections).not.toContain( + "json_body_shapes", + ); +}); + +it("refunds replaced and removed request records while ignoring stale response occurrences", () => { + const events = createEvents(); + request(events, "redirect", deep(100)); + const stale = response(events, "redirect", deep(100)); + request(events, "redirect", '{"next":true}', true); + const retained = events.retainedEventBytes; + events.ingestResponseBody(stale, { body: deep(760), base64Encoded: false }); + expect(events.retainedEventBytes).toBe(retained); + expect(events.network.get("redirect")?.body_shapes.response).toBeNull(); + events.ingest({ + method: "Network.responseReceived", + params: { + requestId: "redirect", + response: { url: "https://excluded.invalid/next", status: 200 }, + }, + }); + expect(events.network.has("redirect")).toBe(false); + expect(events.retainedEventBytes).toBe(0); + request(events, "fresh", deep(600)); + expect(events.network.get("fresh")?.body_shapes.status).toBe("included"); + events.beginAuthorizedFrame("frame"); + expect(events.retainedEventBytes).toBe(0); + request(events, "after-reset", deep(600)); + expect(events.network.get("after-reset")?.body_shapes.status).toBe( + "included", + ); +}); + +it("shares the remaining allowance across console entries, body shapes and websocket shapes", () => { + const events = createEvents(); + events.ingest({ + method: "Runtime.consoleAPICalled", + params: { + type: "log", + timestamp: 1, + args: [{ type: "string", value: "x".repeat(1024 * 1024) }], + stackTrace: { + callFrames: [ + { url: `${origin}/app.js`, lineNumber: 0, columnNumber: 0 }, + ], + }, + }, + }); + request(events, "body", deep(600)); + expect(events.console).toHaveLength(1); + expect(events.network.get("body")?.body_shapes.status).toBe("included"); + openSocket(events); + frame(events, deep(400)); + expect(events.webSocketConnectionsById.get("socket")?.events).toHaveLength(0); + expect(events.completeness.snapshot().dropped_events.websocket_frames).toBe( + 1, + ); + expect(events.retainedEventBytes).toBeLessThanOrEqual( + WEB_RUNTIME_LIMITS.retainedEventBytes, + ); + const empty = createEvents(); + openSocket(empty); + frame(empty, deep(400)); + expect(empty.webSocketConnectionsById.get("socket")?.events).toHaveLength(1); +}); + +it("does not infer unselected body or websocket shapes", () => { + const events = createEvents(false); + request(events, "unselected", deep(760)); + expect(events.network.get("unselected")?.body_shapes).toEqual({ + status: "not_approved", + request: null, + response: null, + }); + expect(events.responseBodyRequests()).toEqual([]); + openSocket(events); + frame(events, deep(760)); + expect(events.webSocketConnectionsById.get("socket")?.events).toMatchObject([ + { payload_shape: null }, + ]); + expect(events.completeness.snapshot().dropped_events.total).toBe(0); +}); diff --git a/src/browser/CdpCaptureEvents.ts b/src/browser/CdpCaptureEvents.ts index b97e114334..ff51df5921 100644 --- a/src/browser/CdpCaptureEvents.ts +++ b/src/browser/CdpCaptureEvents.ts @@ -27,6 +27,10 @@ export class CdpCaptureEvents implements CdpCaptureEventsState { readonly scriptMetadataBytesById = new Map(); retainedScriptMetadataBytes = 0; retainedEventBytes = 0; + readonly bodyShapeBytesByRequestId = new Map< + string, + { readonly request: number; readonly response: number } + >(); readonly executionContextFrames = new Map(); readonly network = new Map(); readonly networkRequestTimestamps = new Map(); @@ -74,6 +78,7 @@ export class CdpCaptureEvents implements CdpCaptureEventsState { this.scriptMetadataBytesById.clear(); this.retainedScriptMetadataBytes = 0; this.retainedEventBytes = 0; + this.bodyShapeBytesByRequestId.clear(); this.#scriptMetadataBudgetExclusionsRecorded = false; this.executionContextFrames.clear(); this.network.clear(); diff --git a/src/domain/jsonShape.admission.test.ts b/src/domain/jsonShape.admission.test.ts new file mode 100644 index 0000000000..1b45316e04 --- /dev/null +++ b/src/domain/jsonShape.admission.test.ts @@ -0,0 +1,108 @@ +import { expect, it } from "vitest"; + +import { inferJsonShape } from "./jsonShape.js"; +import { WEB_RUNTIME_LIMITS } from "./webRuntime.js"; + +it("refuses a small deeply nested source whose expanded shape exceeds the real retained allowance", () => { + const depth = 760; + const source = '{"a":'.repeat(depth) + "0" + "}".repeat(depth); + expect(Buffer.byteLength(source)).toBe(4_561); + const result = inferJsonShape(source, WEB_RUNTIME_LIMITS.retainedEventBytes); + if (result.ok) throw new Error("Expected expanded shape capacity refusal"); + expect(result.error).toMatchObject({ + kind: "capacity", + maximumBytes: WEB_RUNTIME_LIMITS.retainedEventBytes, + }); + if (result.error.kind === "capacity") + expect(result.error.requiredBytesAtLeast).toBeGreaterThan( + WEB_RUNTIME_LIMITS.retainedEventBytes, + ); +}); + +it("admits exact escaped and Unicode output bytes, independently of input whitespace", () => { + const source = JSON.stringify({ + 'quote"\\\n': [ + { "é/~": 1, "\ud800": true }, + { "é/~": "other", "\ud800": null }, + ], + "\u{10000}": "value", + }); + const reference = inferJsonShape( + source, + WEB_RUNTIME_LIMITS.retainedEventBytes, + ); + if (!reference.ok) throw reference.error; + const exact = Buffer.byteLength(JSON.stringify(reference.value.shape)); + expect(reference.value.serializedBytes).toBe(exact); + const accepted = inferJsonShape(` ${source}\n`, exact); + if (!accepted.ok) throw accepted.error; + expect(accepted.value).toEqual(reference.value); + const refused = inferJsonShape(source, exact - 1); + if (refused.ok) throw new Error("Expected one-byte-over-budget refusal"); + expect(refused.error.kind).toBe("capacity"); +}); + +it("shares repeated array paths while preserving all observation and node counts", () => { + const source = JSON.stringify( + Array.from({ length: 10_000 }, () => ({ retained: true })), + ); + const reference = inferJsonShape( + source, + WEB_RUNTIME_LIMITS.retainedEventBytes, + ); + if (!reference.ok) throw reference.error; + const result = inferJsonShape(source, reference.value.serializedBytes); + if (!result.ok) throw result.error; + expect(result.value.shape).toMatchObject({ + node_count: 20_001, + max_depth_observed: 2, + properties: [ + { path: [{ kind: "array-element" }], observations: 10_000 }, + { + path: [ + { kind: "array-element" }, + { kind: "property", name: "retained" }, + ], + observations: 10_000, + }, + ], + }); + expect(result.value.serializedBytes).toBe( + Buffer.byteLength(JSON.stringify(result.value.shape)), + ); +}); + +it("refuses wide expanded paths before admitting all source keys", () => { + const source = JSON.stringify( + Object.fromEntries( + Array.from({ length: 2_000 }, (_, index) => [ + `field_${index}_é\\"`, + index, + ]), + ), + ); + const refused = inferJsonShape(source, 4_096); + if (refused.ok) throw new Error("Expected wide shape capacity refusal"); + expect(refused.error.kind).toBe("capacity"); + const admitted = inferJsonShape( + source, + WEB_RUNTIME_LIMITS.retainedEventBytes, + ); + if (!admitted.ok) throw admitted.error; + expect(admitted.value.shape.properties).toHaveLength(2_000); + expect(admitted.value.serializedBytes).toBe( + Buffer.byteLength(JSON.stringify(admitted.value.shape)), + ); +}); + +it.each(["not-json", Uint8Array.from([0xff])])( + "distinguishes invalid source from valid JSON capacity refusal", + (source) => { + const result = inferJsonShape(source, 0); + if (result.ok) throw new Error("Expected invalid JSON refusal"); + expect(result.error.kind).toBe("invalid-json"); + const valid = inferJsonShape("null", 0); + if (valid.ok) throw new Error("Expected capacity refusal for valid JSON"); + expect(valid.error.kind).toBe("capacity"); + }, +); diff --git a/src/domain/jsonShape.test.ts b/src/domain/jsonShape.test.ts index ba0165ffa3..260b37d033 100644 --- a/src/domain/jsonShape.test.ts +++ b/src/domain/jsonShape.test.ts @@ -1,13 +1,20 @@ import { describe, expect, it } from "vitest"; import { inferJsonShape, jsonShapeSchema } from "./jsonShape.js"; +import { WEB_RUNTIME_LIMITS } from "./webRuntime.js"; + +const completeShape = (source: string | Uint8Array) => { + const result = inferJsonShape(source, WEB_RUNTIME_LIMITS.retainedEventBytes); + if (!result.ok) throw result.error; + return result.value.shape; +}; const property = (name: string) => ({ kind: "property" as const, name }); const element = { kind: "array-element" as const }; describe("inferJsonShape", () => { it("retains paths and types without retaining JSON values", () => { - const shape = inferJsonShape( + const shape = completeShape( JSON.stringify({ token: "super-secret", users: [ @@ -43,8 +50,8 @@ describe("inferJsonShape", () => { }); it("distinguishes array elements from literal star properties and counts primitive types", () => { - const shape = inferJsonShape('{"items":[{"*":1},[true,"x",null],2]}'); - expect(shape?.properties).toEqual( + const shape = completeShape('{"items":[{"*":1},[true,"x",null],2]}'); + expect(shape.properties).toEqual( expect.arrayContaining([ { path: [property("items"), element], @@ -66,20 +73,22 @@ describe("inferJsonShape", () => { }); it("rejects malformed JSON", () => { - expect(inferJsonShape("not-json")).toBeNull(); + expect( + inferJsonShape("not-json", WEB_RUNTIME_LIMITS.retainedEventBytes), + ).toEqual({ ok: false, error: { kind: "invalid-json" } }); }); it("orders mixed array and property descendants with parents first", () => { - const shape = inferJsonShape('[{"z":1,"a":{"x":true}},[null],{"a":2}]'); + const shape = completeShape('[{"z":1,"a":{"x":true}},[null],{"a":2}]'); - expect(shape?.properties.map(({ path }) => path)).toEqual([ + expect(shape.properties.map(({ path }) => path)).toEqual([ [element], [element, element], [element, property("a")], [element, property("a"), property("x")], [element, property("z")], ]); - expect(shape?.properties[2]).toMatchObject({ + expect(shape.properties[2]).toMatchObject({ types: ["number", "object"], observations: 2, }); @@ -87,10 +96,7 @@ describe("inferJsonShape", () => { it("retains deep coverage and independently owned path segments", () => { const depth = 512; - const shape = inferJsonShape("[".repeat(depth) + "1" + "]".repeat(depth)); - expect(shape).not.toBeNull(); - if (shape === null) return; - + const shape = completeShape("[".repeat(depth) + "1" + "]".repeat(depth)); expect(shape.node_count).toBe(depth + 1); expect(shape.max_depth_observed).toBe(depth); expect(shape.properties).toHaveLength(depth); @@ -121,7 +127,7 @@ describe("JSON shape completeness and ordering", () => { ]; return Object.fromEntries(reverse ? entries.reverse() : entries); }; - const shape = inferJsonShape( + const shape = completeShape( JSON.stringify({ nested: [object(1, null), object(2, [null])] }), ); @@ -199,11 +205,11 @@ describe("JSON shape completeness and ordering", () => { const content = Object.fromEntries( Array.from({ length: 5_001 }, (_, index) => [`field_${index}`, index]), ); - const shape = inferJsonShape(JSON.stringify(content)); + const shape = completeShape(JSON.stringify(content)); - expect(shape?.properties).toHaveLength(5_001); - expect(shape?.node_count).toBe(5_002); - expect(shape?.properties).toContainEqual({ + expect(shape.properties).toHaveLength(5_001); + expect(shape.node_count).toBe(5_002); + expect(shape.properties).toContainEqual({ path: [property("field_5000")], types: ["number"], observations: 1, diff --git a/src/domain/jsonShape.ts b/src/domain/jsonShape.ts index 67a5306b5d..9f6145b420 100644 --- a/src/domain/jsonShape.ts +++ b/src/domain/jsonShape.ts @@ -2,6 +2,9 @@ import { z } from "zod"; import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js"; import { safeParseJson } from "./safeJson.js"; +import { jsonStringByteLength } from "./jsonStringByteLength.js"; +import { jsonValueByteLength } from "./jsonValueByteLength.js"; +import { err, ok, type Result } from "./result.js"; const jsonValueTypeSchema = z.enum([ "object", @@ -37,107 +40,221 @@ export type JsonShape = z.infer; const arrayElementKey = Symbol("array-element"); interface ShapeNode { - readonly path: JsonShapePathSegment[]; + readonly parent?: ShapeNode; + readonly segment?: JsonShapePathSegment; + readonly depth: number; + readonly pathBytes: number; readonly types: Set; observations: number; children?: Map; } -/** Parse approved JSON text or exact UTF-8 bytes, then discard values after shape inference. */ +export interface JsonShapeCapacityFailure { + readonly kind: "capacity"; + readonly maximumBytes: number; + readonly requiredBytesAtLeast: number; +} +export type JsonShapeFailure = + | { readonly kind: "invalid-json" } + | JsonShapeCapacityFailure; + +export interface AdmittedJsonShape { + readonly shape: JsonShape; + /** Exact UTF-8 JSON bytes of the admitted shape, excluding any caller envelope. */ + readonly serializedBytes: number; +} + +const propertyBaseBytes = jsonValueByteLength({ + path: [], + types: [], + observations: 0, +}); +const digits = (value: number): number => String(value).length; + +/** Infer complete value-free facts only within the caller's remaining output allowance. */ export const inferJsonShape = ( source: string | Uint8Array, -): JsonShape | null => { - let root: unknown; + maximumBytes: number, +): Result => { + if (!Number.isSafeInteger(maximumBytes) || maximumBytes < 0) + throw new RangeError( + "JSON shape byte allowance must be a non-negative safe integer", + ); + let text: string; try { - const text = + text = typeof source === "string" ? source : new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( source, ); - root = JSON.parse(text); - } catch (cause: unknown) { - // Invalid JSON or non-UTF-8 bytes cannot establish a shape. - void cause; - return null; + } catch { + return err({ kind: "invalid-json" }); } + const parsed = safeParseJson(text); + if (!parsed.ok) return err({ kind: "invalid-json" }); + const root = parsed.value; + const rootType = jsonValueType(root); + let serializedBytes = jsonValueByteLength({ + root_type: rootType, + node_count: 0, + max_depth_observed: 0, + properties: [], + }); + let requiredBytesAtLeast = serializedBytes; + const capacity = (): Result => + err({ + kind: "capacity", + maximumBytes, + requiredBytesAtLeast, + }); + const reserve = (bytes: number): boolean => { + requiredBytesAtLeast = serializedBytes + bytes; + if (requiredBytesAtLeast > maximumBytes) return false; + serializedBytes = requiredBytesAtLeast; + return true; + }; + if (serializedBytes > maximumBytes) return capacity(); const rootShape: ShapeNode = { - path: [], + depth: 0, + pathBytes: 2, types: new Set(), observations: 0, }; - // Each structural path owns one accumulator, shared by all matching array rows. + let propertyCount = 0; const childShape = ( parent: ShapeNode, key: string | typeof arrayElementKey, - ): ShapeNode => { - const children = (parent.children ??= new Map()); - const existing = children.get(key); + ): ShapeNode | undefined => { + const existing = parent.children?.get(key); if (existing !== undefined) return existing; + const segment: JsonShapePathSegment = + typeof key === "string" + ? { kind: "property", name: key } + : { kind: "array-element" }; + const segmentBytes = jsonValueByteLength( + segment, + maximumBytes - serializedBytes, + ); + const pathBytes = + parent.pathBytes + segmentBytes + (parent.depth === 0 ? 0 : 1); + // Admit the expanded serialized path before allocating its accumulator. + // Parent links retain one segment rather than every temporary full path. + if ( + !reserve( + propertyBaseBytes - 2 + pathBytes + (propertyCount === 0 ? 0 : 1), + ) + ) + return undefined; const node: ShapeNode = { - path: [ - ...parent.path, - typeof key === "string" - ? { kind: "property", name: key } - : { kind: "array-element" }, - ], + parent, + segment, + depth: parent.depth + 1, + pathBytes, types: new Set(), observations: 0, }; - children.set(key, node); + (parent.children ??= new Map()).set(key, node); + propertyCount++; return node; }; const pending: Array<{ readonly value: unknown; readonly shape: ShapeNode; + children?: Generator; }> = [{ value: root, shape: rootShape }]; let nodeCount = 0; let maxDepthObserved = 0; while (pending.length > 0) { - const current = pending.pop(); - if (current === undefined) break; - nodeCount += 1; - maxDepthObserved = Math.max(maxDepthObserved, current.shape.path.length); - current.shape.types.add(jsonValueType(current.value)); - current.shape.observations += 1; - if (Array.isArray(current.value)) { - if (current.value.length === 0) continue; - const shape = childShape(current.shape, arrayElementKey); - for (let index = current.value.length - 1; index >= 0; index -= 1) - pending.push({ value: current.value[index], shape }); + const current = pending.at(-1)!; + if (current.children === undefined) { + const nextCount = nodeCount + 1; + const nextDepth = Math.max(maxDepthObserved, current.shape.depth); + if ( + !reserve( + digits(nextCount) - + digits(nodeCount) + + digits(nextDepth) - + digits(maxDepthObserved), + ) + ) + return capacity(); + nodeCount = nextCount; + maxDepthObserved = nextDepth; + if (current.shape.parent !== undefined) { + const type = jsonValueType(current.value); + const newType = !current.shape.types.has(type); + const nextObservations = current.shape.observations + 1; + const typeBytes = newType + ? jsonStringByteLength(type) + + (current.shape.types.size === 0 ? 0 : 1) + : 0; + if ( + !reserve( + typeBytes + + digits(nextObservations) - + digits(current.shape.observations), + ) + ) + return capacity(); + current.shape.types.add(type); + current.shape.observations = nextObservations; + } + current.children = shapeChildren(current.value); + } + const next = current.children.next(); + if (next.done) { + pending.pop(); continue; } - if (!isRecord(current.value)) continue; - const allEntries = Object.entries(current.value); - for (let index = allEntries.length - 1; index >= 0; index -= 1) { - const entry = allEntries[index]; - if (entry === undefined) continue; - const [name, value] = entry; - pending.push({ value, shape: childShape(current.shape, name) }); - } + const [key, value] = next.value; + const shape = childShape(current.shape, key); + if (shape === undefined) return capacity(); + pending.push({ value, shape }); } - // Preorder over sorted siblings gives canonical paths without repeatedly - // comparing their shared prefixes. Each returned path owns its segments. + // Every expanded output path was admitted above. Reconstruct each path once, + // preserving independently owned segments and canonical sibling ordering. const properties: JsonShape["properties"] = []; const shapes = orderedChildren(rootShape); while (shapes.length > 0) { - const shape = shapes.pop(); - if (shape === undefined) break; + const shape = shapes.pop()!; + const path: JsonShapePathSegment[] = []; + let ancestor: ShapeNode = shape; + while (ancestor.parent !== undefined && ancestor.segment !== undefined) { + path.push({ ...ancestor.segment }); + ancestor = ancestor.parent; + } + path.reverse(); properties.push({ - path: shape.path.map((segment) => ({ ...segment })), + path, types: [...shape.types].sort(), observations: shape.observations, }); for (const child of orderedChildren(shape)) shapes.push(child); } - return { - root_type: jsonValueType(root), - node_count: nodeCount, - max_depth_observed: maxDepthObserved, - properties, - }; + return ok({ + shape: { + root_type: rootType, + node_count: nodeCount, + max_depth_observed: maxDepthObserved, + properties, + }, + serializedBytes, + }); }; +/** Keep traversal scratch proportional to nesting rather than a wide input array. */ +function* shapeChildren( + value: unknown, +): Generator { + if (Array.isArray(value)) { + for (const item of value) yield [arrayElementKey, item]; + } else if (isRecord(value)) { + for (const key in value) + if (Object.hasOwn(value, key)) yield [key, value[key]]; + } +} + /** Parse text as a JSON object; anything else, including invalid JSON, is null. */ export const parseJsonObject = ( text: string,