fix(cli): count emitted documents and normalize token windows

docs/cli.md

@@ -909,3 +909,13 @@ REA's failure status:
 set -o pipefail
 rea inspect-artifact ./app.asar --json | jq . > inspection.json
 ```
+
+`--token-count` reports a tokenx estimate for the selected emitted document,
+including a requested full-output envelope and metadata. JSON uses the same
+terminal indentation or compact piped representation as normal output; JSONL
+uses its emitted line representation. The final newline is included (tokenx
+assigns whitespace no tokens). This is an estimate, not a model tokenizer count.
+Counting retains the existing bounded formatted-string allocation after capacity
+admission; omit the flag to stream large JSON documents without materializing
+the complete formatted text. Exhausted token windows display an empty range at
+the total token count and have no next offset.

src/cli/cliResultOutput.ts

@@ -20,6 +20,7 @@ import { jsonLinesErrorRecord } from "./jsonLines.js";
 import { writeCliJsonLines } from "./jsonLinesOutput.js";
 import {
   jsonOutputParts,
+  isTerminalOutput,
   writeJsonOutput,
   writeOutputText,
 } from "./jsonOutput.js";
@@ -346,55 +347,6 @@ export const createCliResultOutput = (
       }
       return true;
     }
-    if (options.tokenCount) {
-      // Incur/tokenx materialize JSON. Measure its UTF-16 expansion before
-      // allowing that allocation, independently of UTF-8 bytes. Other formats
-      // retain Incur's materializing behavior.
-      const tokenValue = failure ?? filtered;
-      let characters = 0;
-      if (
-        tokenValue !== undefined &&
-        tokenValue !== null &&
-        typeof tokenValue !== "function" &&
-        typeof tokenValue !== "symbol"
-      ) {
-        if (format === "json") {
-          for (const part of jsonOutputParts(
-            tokenValue,
-            format,
-            format === "json",
-          ))
-            characters += part.length;
-          characters = Math.max(0, characters - 1);
-        }
-      }
-      if (characters <= constants.MAX_STRING_LENGTH) {
-        const count = estimateTokenCount(formatCliOutput(tokenValue, format));
-        handled = true;
-        try {
-          await writeOutputText(destination, `${count}\n`);
-          return true;
-        } catch (cause: unknown) {
-          failed = true;
-          throw cause;
-        }
-      }
-      error = projectAnalysisError(
-        new AnalysisResourceConstraintError(
-          metadata.command,
-          "memory",
-          "Token counting requires a formatted string larger than the runtime permits.",
-          {
-            formatted_characters: characters,
-            max_string_characters: constants.MAX_STRING_LENGTH,
-          },
-          {
-            remediationAction:
-              "Remove --token-count to stream the complete JSON result, or narrow it with --filter-output.",
-          },
-        ),
-      );
-    }
     const meta = {
       command: metadata.command,
       duration: metadata.duration,
@@ -422,6 +374,66 @@ export const createCliResultOutput = (
           typeof filtered === "symbol"
         ? {}
         : filtered;
+    if (options.tokenCount) {
+      // Incur/tokenx materialize JSON. Measure its UTF-16 expansion before
+      // allowing that allocation, independently of UTF-8 bytes. Other formats
+      // retain Incur's materializing behavior.
+      const tokenValue = document;
+      let characters = 0;
+      if (
+        tokenValue !== undefined &&
+        tokenValue !== null &&
+        typeof tokenValue !== "function" &&
+        typeof tokenValue !== "symbol"
+      ) {
+        if (format === "json" || format === "jsonl") {
+          for (const part of jsonOutputParts(
+            tokenValue,
+            format,
+            isTerminalOutput(destination),
+          ))
+            characters += part.length;
+        }
+      }
+      if (characters <= constants.MAX_STRING_LENGTH) {
+        let formatted: string;
+        if (format === "json" || format === "jsonl") {
+          // Reuse the writer's exact representation after its capacity admission.
+          // tokenx requires one string; retain only this bounded formatted value.
+          formatted = "";
+          for (const part of jsonOutputParts(
+            tokenValue,
+            format,
+            isTerminalOutput(destination),
+          ))
+            formatted += part;
+        } else formatted = formatCliOutput(tokenValue, format);
+        const count = estimateTokenCount(formatted);
+        handled = true;
+        try {
+          await writeOutputText(destination, `${count}\n`);
+          return true;
+        } catch (cause: unknown) {
+          failed = true;
+          throw cause;
+        }
+      }
+      error = projectAnalysisError(
+        new AnalysisResourceConstraintError(
+          metadata.command,
+          "memory",
+          "Token counting requires a formatted string larger than the runtime permits.",
+          {
+            formatted_characters: characters,
+            max_string_characters: constants.MAX_STRING_LENGTH,
+          },
+          {
+            remediationAction:
+              "Remove --token-count to stream the complete JSON result, or narrow it with --filter-output.",
+          },
+        ),
+      );
+    }
     if (ownsSuccessfulTokenWindow) {
       const formatted = formatCliOutput(filtered, format);
       const window = sliceTextByTokenWindow(
@@ -435,7 +447,7 @@ export const createCliResultOutput = (
         return false;
 
       const sliced = window.truncated
-        ? `${window.text}\n[truncated: showing tokens ${options.tokenOffset ?? 0}–${window.end} of ${window.total}]`
+        ? `${window.text}\n[truncated: showing tokens ${window.start}–${window.end} of ${window.total}]`
         : formatted;
       const outputDocument = options.fullOutput
         ? formatCliOutput(
@@ -526,7 +538,7 @@ const fullFailureEnvelope = (
   const nextOffset = window.end < window.total ? window.end : undefined;
   return {
     ok: false,
-    error: `${window.text}\n[truncated: showing tokens ${offset}–${window.end} of ${window.total}]`,
+    error: `${window.text}\n[truncated: showing tokens ${window.start}–${window.end} of ${window.total}]`,
     meta: { ...meta, ...(nextOffset === undefined ? {} : { nextOffset }) },
   };
 };

src/cli/jsonOutput.ts

@@ -18,7 +18,7 @@ export const writeJsonOutput = async (
 ): Promise<void> => {
   await writeOutputParts(
     destination,
-    jsonOutputParts(value, format, isTerminal(destination)),
+    jsonOutputParts(value, format, isTerminalOutput(destination)),
   );
 };
 
@@ -31,7 +31,7 @@ export const writeOutputParts = async (
     await writeOutputText(destination, part);
 };
 
-const isTerminal = (destination: Writable): boolean =>
+export const isTerminalOutput = (destination: Writable): boolean =>
   "isTTY" in destination && destination.isTTY === true;
 
 export const writeOutputText = (

src/cli/tokenWindow.ts

@@ -3,6 +3,7 @@ import { estimateTokenCount } from "tokenx";
 export interface TokenWindow {
   readonly text: string;
   readonly total: number;
+  readonly start: number;
   readonly end: number;
   readonly truncated: boolean;
 }
@@ -41,11 +42,17 @@ export const sliceTextByTokenWindow = (
   const total = estimateTokenCount(text);
   const end = limit === undefined ? total : offset + limit;
   if (offset === 0 && end >= total)
-    return { text, total, end: total, truncated: false };
+    return { text, total, start: 0, end: total, truncated: false };
 
   const actualEnd = Math.min(end, total);
   if (offset >= actualEnd)
-    return { text: "", total, end: actualEnd, truncated: true };
+    return {
+      text: "",
+      total,
+      start: actualEnd,
+      end: actualEnd,
+      truncated: true,
+    };
 
   const parts: string[] = [];
   let tokenPosition = 0;
@@ -75,6 +82,7 @@ export const sliceTextByTokenWindow = (
   return {
     text: parts.join(""),
     total,
+    start: offset,
     end: actualEnd,
     truncated: true,
   };

tests/boundary/cli/jsonOutput.test.ts

@@ -2,7 +2,7 @@ import { constants } from "node:buffer";
 import { createHash } from "node:crypto";
 import { Writable } from "node:stream";
 
-import { Cli, Formatter } from "incur";
+import { Cli } from "incur";
 import { estimateTokenCount } from "tokenx";
 import { parse as parseYaml } from "yaml";
 import { describe, expect, it } from "vitest";
@@ -473,21 +473,89 @@ describe("full-output token counts", () => {
       destination,
     );
     if (output === undefined) throw new Error("Missing JSON output surface");
-    const cli = Cli.create("rea-output-test", { sync: false });
-    registerManagedCommands(cli, silentLogger, async () => failure);
-    streamCliCommandResults(cli, output);
-    const previousExitCode = process.exitCode;
-    try {
-      await cli.serve(argv, { stdout: () => undefined });
-      expect(process.exitCode).toBe(1);
-    } finally {
-      process.exitCode = previousExitCode;
-    }
+    await output.write(failure, {
+      command: "inspect-managed-artifact",
+      duration: "0ms",
+      format: "json",
+    });
     expect(output.handled).toBe(true);
     expect(Buffer.concat(chunks).toString("utf8")).toBe(
-      `${estimateTokenCount(Formatter.format(failure, "json"))}\n`,
+      `${estimateTokenCount(JSON.stringify({ ok: false, error: failure, meta: { command: "inspect-managed-artifact", duration: "0ms" } }) + "\n")}\n`,
     );
   });
+  it("counts the exact emitted JSON representation and selected document", async () => {
+    const metadata = { command: "fixture", duration: "0ms", format: "json" };
+    for (const terminal of [false, true]) {
+      for (const flags of [
+        [],
+        ["--full-output"],
+        ["--filter-output", "missing"],
+        ["--format", "jsonl"],
+      ]) {
+        const argv = ["fixture", "--json", ...flags];
+        const render = async (count: boolean) => {
+          const chunks: Buffer[] = [];
+          const destination = new Writable({
+            write(chunk: Buffer, _encoding, callback) {
+              chunks.push(chunk);
+              callback();
+            },
+          });
+          Object.defineProperty(destination, "isTTY", { value: terminal });
+          const output = createCliResultOutput(
+            parseCliOutputArguments([
+              ...argv,
+              ...(count ? ["--token-count"] : []),
+            ]),
+            destination,
+          );
+          if (output === undefined) throw new Error("Missing output surface");
+          await output.write({ value: ["hello", "雪😀"] }, metadata);
+          return Buffer.concat(chunks).toString("utf8");
+        };
+        expect(Number((await render(true)).trim())).toBe(
+          estimateTokenCount(await render(false)),
+        );
+      }
+    }
+  });
+
+  it("normalizes exhausted failure window display coordinates", async () => {
+    const chunks: Buffer[] = [];
+    const destination = new Writable({
+      write(chunk: Buffer, _encoding, callback) {
+        chunks.push(chunk);
+        callback();
+      },
+    });
+    const output = createCliResultOutput(
+      parseCliOutputArguments([
+        "fixture",
+        "--format",
+        "toon",
+        "--full-output",
+        "--token-offset",
+        "100000",
+        "--token-limit",
+        "3",
+      ]),
+      destination,
+    );
+    if (output === undefined) throw new Error("Missing output surface");
+    await output.write(
+      {
+        code: "target_unavailable",
+        category: "unavailable",
+        message: "Missing target",
+        retryable: false,
+      },
+      { command: "fixture", duration: "0ms", format: "toon" },
+    );
+    const text = Buffer.concat(chunks).toString("utf8");
+    expect(text).toMatch(/showing tokens (\d+)–\1 of \1/);
+    expect(text).not.toContain("nextOffset");
+    expect(text).toContain("ok: false");
+  });
 });
 
 describe("CLI result output surface", () => {
@@ -612,7 +680,8 @@ describe("CLI result output surface", () => {
     });
     expect(exitCode).toBe(1);
     expect(fallbackBytes).toBe(0);
-    expect(JSON.parse(Buffer.concat(chunks).toString("utf8"))).toMatchObject({
+    const report = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+    expect(report).toMatchObject({
       ok: false,
       error: {
         code: "resource_constraint",
@@ -621,7 +690,11 @@ describe("CLI result output surface", () => {
         },
         details: {
           reported_limits: {
-            formatted_characters: expectedCharacters - 1,
+            formatted_characters:
+              expectedCharacters +
+              JSON.stringify({ ok: true, data: null, meta: report.meta })
+                .length -
+              4,
             max_string_characters: constants.MAX_STRING_LENGTH,
           },
         },