fix(browser): preserve captures when response bodies are unavailable

scripts/fixtures/browser-verifier-site.mjs

@@ -7,6 +7,13 @@ export async function startBrowserVerifierSite() {
   let port = 0;
   let sessionGeneration = 0;
   const server = createServer((request, response) => {
+    if (request.url === "/slow-json") {
+      response.setHeader("content-type", "application/json");
+      response.write('{"pending":');
+      const timer = setTimeout(() => response.end("true}"), 350);
+      response.once("close", () => clearTimeout(timer));
+      return;
+    }
     if (request.url === "/screenshot-noise") {
       response.setHeader("content-type", "text/html");
       response.end(`<!doctype html><html><body style="margin:0"><canvas width="2048" height="1536"></canvas><script>
@@ -174,6 +181,11 @@ const browserScript = (port) => `
       },
       body: JSON.stringify({ token: "request-body-secret-value", active: true })
     });
+    fetch("/slow-json", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ pending: true })
+    });
     const socket = new WebSocket("ws://127.0.0.1:${String(port)}/live?token=websocket-url-secret");
     socket.addEventListener("open", () => socket.send(JSON.stringify({ token: "websocket-secret-value" })));
     socket.addEventListener("message", () => socket.close());

scripts/lib/browser-verifier-assertions.mjs

@@ -217,6 +217,19 @@ export function assertSensitiveShapes(result) {
   );
   if (request === undefined)
     throw new Error("Real Chrome JSON request/response shapes were missing");
+  if (
+    !result.network.requests.some(
+      (item) =>
+        item.url.endsWith("/slow-json") &&
+        item.body_shapes.status === "partial" &&
+        item.body_shapes.request !== null &&
+        item.body_shapes.response === null,
+    ) ||
+    !result.completeness.unavailable_sections.includes("json_body_shapes")
+  )
+    throw new Error(
+      "Real Chrome unfinished JSON bodies were not reported as unavailable",
+    );
   const consoleText = result.console.events.flatMap(
     (event) => event.text_capture.values,
   );

src/browser/CdpPageCapture.ts

@@ -12,6 +12,7 @@ import { CdpCaptureEvents } from "./CdpCaptureEvents.js";
 import { authorizedMainFrame } from "./CdpAuthorizedMainFrame.js";
 import { captureStorage } from "./CdpCaptureStorage.js";
 import { optionalCdpCommand } from "./CdpOptionalCommand.js";
+import { CdpCommandRejection } from "./CdpCommandRejection.js";
 import {
   captureAccessibility,
   captureDom,
@@ -185,12 +186,34 @@ const captureJsonResponseBodies = async (
   for (let index = 0; index < requestIds.length; index += 1) {
     const requestId = requestIds[index];
     if (requestId === undefined) continue;
-    const result = await optionalCdpCommand(
-      state.context,
-      "Network.getResponseBody",
-      { requestId },
-      state.limitations,
-    );
+    const request = state.events.network.get(requestId);
+    if (request === undefined) continue;
+    if (request.encoded_data_length === null) {
+      state.events.responseBodyUnavailable(requestId);
+      continue;
+    }
+    let result: unknown;
+    try {
+      result = await optionalCdpCommand(
+        state.context,
+        "Network.getResponseBody",
+        { requestId },
+        state.limitations,
+      );
+    } catch (cause: unknown) {
+      if (
+        !(cause instanceof CdpCommandRejection) ||
+        cause.command !== "Network.getResponseBody" ||
+        cause.code !== -32_000 ||
+        cause.reportedMessage === null
+      )
+        throw cause;
+      state.events.responseBodyUnavailable(requestId);
+      state.limitations.push(
+        `Response body unavailable for request ${requestId}: ${cause.userMessage}`,
+      );
+      continue;
+    }
     if (result !== undefined) {
       state.events.ingestResponseBody(requestId, result);
       continue;

tests/boundary/browser/cdpBrowserProvider.network.test.ts

@@ -5,6 +5,170 @@ import { inspectWebPageInputSchema } from "../../../src/domain/browserObservatio
 import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js";
 import { trackBrowser } from "./cdpBrowserProvider.support.js";
 
+it("preserves unavailable response observations and captures completed peers", async () => {
+  const browser = await startFakeCdpBrowser({
+    commandEvents(command, origin) {
+      if (command.method !== "Network.enable") return undefined;
+      return ["unfinished", "missing-body", "available"].flatMap(
+        (requestId) => {
+          const session =
+            command.sessionId === undefined
+              ? {}
+              : { sessionId: command.sessionId };
+          const url = `${origin}/${requestId}`;
+          return [
+            {
+              ...session,
+              method: "Network.requestWillBeSent",
+              params: {
+                requestId,
+                type: "Fetch",
+                request: {
+                  url,
+                  method: "POST",
+                  headers: { "Content-Type": "application/json" },
+                  postData: '{"selected":true}',
+                },
+              },
+            },
+            {
+              ...session,
+              method: "Network.responseReceived",
+              params: {
+                requestId,
+                response: {
+                  url,
+                  status: 200,
+                  mimeType: "application/json",
+                  headers: {},
+                },
+              },
+            },
+            ...(requestId === "unfinished"
+              ? []
+              : [
+                  {
+                    ...session,
+                    method: "Network.loadingFinished",
+                    params: { requestId, encodedDataLength: 0 },
+                  },
+                ]),
+          ];
+        },
+      );
+    },
+    commandError(command) {
+      if (
+        command.method === "Network.getResponseBody" &&
+        command.params.requestId === "missing-body"
+      )
+        return {
+          code: -32_000,
+          message: "No data found for resource with given identifier",
+        };
+      return undefined;
+    },
+    commandResult(command) {
+      return command.method === "Network.getResponseBody"
+        ? { body: '{"retained":true}', base64Encoded: false }
+        : undefined;
+    },
+  });
+  trackBrowser(browser);
+  const result = await new CdpBrowserProvider().inspectPage(
+    inspectWebPageInputSchema.parse({
+      cdp_endpoint: browser.endpoint,
+      allowed_origins: [browser.allowedOrigin],
+      target_id: "allowed-page",
+      observation_ms: 20,
+      include_json_body_shapes: true,
+    }),
+  );
+  if (!result.ok) throw result.error;
+  expect(result.value.network.requests).toEqual([
+    expect.objectContaining({
+      request_id: "unfinished",
+      status: 200,
+      encoded_data_length: null,
+      body_shapes: {
+        status: "partial",
+        request: expect.any(Object),
+        response: null,
+      },
+    }),
+    expect.objectContaining({
+      request_id: "missing-body",
+      status: 200,
+      encoded_data_length: 0,
+      body_shapes: {
+        status: "partial",
+        request: expect.any(Object),
+        response: null,
+      },
+    }),
+    expect.objectContaining({
+      request_id: "available",
+      body_shapes: {
+        status: "included",
+        request: expect.any(Object),
+        response: expect.any(Object),
+      },
+    }),
+  ]);
+  expect(result.value.completeness.unavailable_sections).toContain(
+    "json_body_shapes",
+  );
+  expect(result.value.limitations).toContainEqual(
+    expect.stringContaining(
+      "missing-body: CDP Network.getResponseBody failed (-32000): No data found",
+    ),
+  );
+  expect(
+    browser.commands
+      .filter(({ method }) => method === "Network.getResponseBody")
+      .map(({ params }) => params.requestId),
+  ).toEqual(["missing-body", "available"]);
+});
+
+it.each([
+  {
+    options: {
+      commandError: (command: { method: string }) =>
+        command.method === "Network.getResponseBody"
+          ? { code: -32_602, message: "Invalid parameters" }
+          : undefined,
+    },
+    reason: "protocol_error",
+  },
+  {
+    options: { malformedMessageOnMethod: "Network.getResponseBody" },
+    reason: "protocol_error",
+  },
+  {
+    options: { closeOnMethod: "Network.getResponseBody" },
+    reason: "disconnected",
+  },
+])(
+  "preserves fatal response capture errors ($reason)",
+  async ({ options, reason }) => {
+    const browser = await startFakeCdpBrowser(options);
+    trackBrowser(browser);
+    const result = await new CdpBrowserProvider().inspectPage(
+      inspectWebPageInputSchema.parse({
+        cdp_endpoint: browser.endpoint,
+        allowed_origins: [browser.allowedOrigin],
+        target_id: "allowed-page",
+        observation_ms: 20,
+        include_json_body_shapes: true,
+      }),
+    );
+    expect(result).toMatchObject({
+      ok: false,
+      error: { _tag: "BrowserObservationError", reason },
+    });
+  },
+);
+
 describe("CdpBrowserProvider: network 1", () => {
   it("retains same-origin redirect hops on the one final request", async () => {
     const browser = await startFakeCdpBrowser({ redirectWithinOrigin: true });