fix(cli): preserve Unicode and text across token windows (#5814)

* fix(cli): preserve Unicode and text across token windows

* fix(cli): avoid redundant token window formatting

src/cli/cliResultOutput.ts

@@ -2,7 +2,7 @@ import { constants } from "node:buffer";
 import type { Writable } from "node:stream";
 
 import { Filter, Formatter } from "incur";
-import { estimateTokenCount, sliceByTokens } from "tokenx";
+import { estimateTokenCount } from "tokenx";
 
 import {
   parseCliOutputArguments,
@@ -13,6 +13,7 @@ import {
   cliOperationFailurePayload,
 } from "../cliLogging.js";
 import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";
+import { sliceTextByTokenWindow } from "./tokenWindow.js";
 import {
   projectAnalysisError,
   type AnalysisErrorProjection,
@@ -33,6 +34,7 @@ export interface CliResultOutputMetadata {
   readonly command: string;
   readonly duration: string;
   readonly format: string;
+  readonly renderOutput?: boolean;
 }
 
 /** Optional executable-owned result surface for one CLI invocation. */
@@ -50,8 +52,8 @@ export const streamCliCommandResults = (
   cli: CliInstance,
   output: CliResultOutput,
 ): void => {
-  visitRegisteredCommands(cli, (command, path) =>
-    wrapCommand(command, path, output),
+  visitRegisteredCommands(cli, (command, path, inheritedPolicy) =>
+    wrapCommand(command, path, output, inheritedPolicy),
   );
 };
 
@@ -59,6 +61,7 @@ const wrapCommand = (
   command: Pick<RegisteredCommand, "run">,
   name: string,
   output: CliResultOutput,
+  inheritedPolicy?: "all" | "agent-only",
 ): void => {
   const run = command.run;
   command.run = (context) => {
@@ -82,6 +85,7 @@ const wrapCommand = (
         !(await output.write(value, {
           command: name,
           format: context.format,
+          renderOutput: context.agent || inheritedPolicy !== "agent-only",
           duration: `${Math.round(performance.now() - started)}ms`,
         }))
       )
@@ -213,9 +217,24 @@ export const createCliResultOutput = (
         ? cliOperationFailurePayload(value)
         : undefined;
     const failure = options.fullOutput ? operationFailure : undefined;
+    // Incur owns suppressed human presentation, but a filter must never erase
+    // the typed failure that this boundary already owns.
+    if (
+      operationFailure === undefined &&
+      metadata.renderOutput === false &&
+      !options.fullOutput &&
+      !options.tokenCount
+    )
+      return false;
     const filtered: unknown =
       operationFailure ??
       (filterPaths === undefined ? value : Filter.apply(value, filterPaths));
+    const ownsSuccessfulTokenWindow =
+      operationFailure === undefined &&
+      options.tokenWindow &&
+      !options.tokenCount &&
+      format !== "json" &&
+      format !== "jsonl";
     if (
       operationFailure === undefined &&
       filterPaths !== undefined &&
@@ -230,7 +249,8 @@ export const createCliResultOutput = (
     if (
       format !== "json" &&
       format !== "jsonl" &&
-      operationFailure === undefined
+      operationFailure === undefined &&
+      !ownsSuccessfulTokenWindow
     )
       return false;
     if (options.tokenCount) {
@@ -309,6 +329,53 @@ export const createCliResultOutput = (
           typeof filtered === "symbol"
         ? {}
         : filtered;
+    if (ownsSuccessfulTokenWindow) {
+      const formatted = Formatter.format(filtered, format);
+      const window = sliceTextByTokenWindow(
+        formatted,
+        options.tokenOffset ?? 0,
+        options.tokenLimit,
+      );
+      // Keep Incur's empty human/agent presentation semantics. Nonempty text
+      // already belongs to this formatter, even when the whole result fits.
+      if (!window.truncated && !options.fullOutput && formatted === "")
+        return false;
+
+      const sliced = window.truncated
+        ? `${window.text}\n[truncated: showing tokens ${options.tokenOffset ?? 0}–${window.end} of ${window.total}]`
+        : formatted;
+      const outputDocument = options.fullOutput
+        ? Formatter.format(
+            window.truncated
+              ? {
+                  ok: true,
+                  data: sliced,
+                  meta: {
+                    command: metadata.command,
+                    duration: metadata.duration,
+                    ...(window.end < window.total
+                      ? { nextOffset: window.end }
+                      : {}),
+                  },
+                }
+              : { ok: true, data: filtered, meta },
+            format,
+          )
+        : sliced;
+      handled = true;
+      try {
+        await writeOutputText(
+          destination,
+          outputDocument.endsWith("\n")
+            ? outputDocument
+            : `${outputDocument}\n`,
+        );
+      } catch (cause: unknown) {
+        failed = true;
+        throw cause;
+      }
+      return true;
+    }
     // Once writing starts, a failed destination must not receive a second
     // document from Incur's error formatter.
     const failureText =
@@ -350,15 +417,13 @@ const fullFailureEnvelope = (
   if (tokenLimit === undefined && tokenOffset === undefined)
     return { ok: false, error: failure, meta };
   const formatted = Formatter.format(failure, format);
-  const total = estimateTokenCount(formatted);
   const offset = tokenOffset ?? 0;
-  const end = tokenLimit === undefined ? total : offset + tokenLimit;
-  if (offset === 0 && end >= total) return { ok: false, error: failure, meta };
-  const actualEnd = Math.min(end, total);
-  const nextOffset = actualEnd < total ? actualEnd : undefined;
+  const window = sliceTextByTokenWindow(formatted, offset, tokenLimit);
+  if (!window.truncated) return { ok: false, error: failure, meta };
+  const nextOffset = window.end < window.total ? window.end : undefined;
   return {
     ok: false,
-    error: `${sliceByTokens(formatted, offset, end)}\n[truncated: showing tokens ${offset}–${actualEnd} of ${total}]`,
+    error: `${window.text}\n[truncated: showing tokens ${offset}–${window.end} of ${window.total}]`,
     meta: { ...meta, ...(nextOffset === undefined ? {} : { nextOffset }) },
   };
 };

src/cli/tokenWindow.ts

@@ -0,0 +1,81 @@
+import { estimateTokenCount } from "tokenx";
+
+export interface TokenWindow {
+  readonly text: string;
+  readonly total: number;
+  readonly end: number;
+  readonly truncated: boolean;
+}
+
+// tokenx 1.3.0 uses this split expression before estimating and slicing.
+// Its public API does not expose the segments, so retain the same boundaries
+// here while mapping both ends of each segment to a disjoint text range.
+const TOKEN_SPLIT_PATTERN = /(\s+|[.,!?;(){}[\]<>:/\\|@#$%^&*+=`~_-]+)/;
+
+const codePointSafeBoundary = (text: string, offset: number): number =>
+  offset > 0 &&
+  offset < text.length &&
+  text.charCodeAt(offset - 1)! >= 0xd800 &&
+  text.charCodeAt(offset - 1)! <= 0xdbff &&
+  text.charCodeAt(offset)! >= 0xdc00 &&
+  text.charCodeAt(offset)! <= 0xdfff
+    ? offset - 1
+    : offset;
+
+const segmentBoundary = (
+  segment: string,
+  tokenOffset: number,
+  tokenCount: number,
+): number =>
+  codePointSafeBoundary(
+    segment,
+    Math.floor((tokenOffset / tokenCount) * segment.length),
+  );
+
+/** Slice an approximate token range without overlapping pages or splitting a surrogate pair. */
+export const sliceTextByTokenWindow = (
+  text: string,
+  offset: number,
+  limit: number | undefined,
+): TokenWindow => {
+  const total = estimateTokenCount(text);
+  const end = limit === undefined ? total : offset + limit;
+  if (offset === 0 && end >= total)
+    return { text, total, end: total, truncated: false };
+
+  const actualEnd = Math.min(end, total);
+  if (offset >= actualEnd)
+    return { text: "", total, end: actualEnd, truncated: true };
+
+  const parts: string[] = [];
+  let tokenPosition = 0;
+  for (const segment of text.split(TOKEN_SPLIT_PATTERN).filter(Boolean)) {
+    if (tokenPosition >= actualEnd && actualEnd < total) break;
+    const tokenCount = estimateTokenCount(segment);
+    if (tokenCount === 0) {
+      if (
+        tokenPosition >= offset &&
+        (tokenPosition < actualEnd || actualEnd === total)
+      )
+        parts.push(segment);
+      continue;
+    }
+
+    const segmentEnd = tokenPosition + tokenCount;
+    if (tokenPosition < actualEnd && segmentEnd > offset) {
+      const localStart = Math.max(0, offset - tokenPosition);
+      const localEnd = Math.min(tokenCount, actualEnd - tokenPosition);
+      const charStart = segmentBoundary(segment, localStart, tokenCount);
+      const charEnd = segmentBoundary(segment, localEnd, tokenCount);
+      parts.push(segment.slice(charStart, charEnd));
+    }
+    tokenPosition = segmentEnd;
+  }
+
+  return {
+    text: parts.join(""),
+    total,
+    end: actualEnd,
+    truncated: true,
+  };
+};

src/cli/visitRegisteredCommands.ts

@@ -5,6 +5,7 @@ import type { CliInstance } from "./types.js";
 type CommandMap = NonNullable<ReturnType<typeof Cli.toCommands.get>>;
 type CommandEntry = CommandMap extends Map<string, infer Entry> ? Entry : never;
 export type RegisteredCommand = Extract<CommandEntry, { run: unknown }>;
+type OutputPolicy = "all" | "agent-only";
 
 // Incur types its router key without vars, env, or globals; accept any CLI.
 const registeredCommands: WeakMap<object, CommandMap> = Cli.toCommands;
@@ -12,24 +13,35 @@ const registeredCommands: WeakMap<object, CommandMap> = Cli.toCommands;
 /** Visit ordinary commands in map order, children before each group's root. */
 export const visitRegisteredCommands = (
   cli: CliInstance,
-  visit: (command: RegisteredCommand, path: string) => void,
+  visit: (
+    command: RegisteredCommand,
+    path: string,
+    outputPolicy: OutputPolicy | undefined,
+  ) => void,
 ): void => {
   const commands = registeredCommands.get(cli);
-  if (commands !== undefined) visitCommands(commands, [], visit);
+  if (commands !== undefined) visitCommands(commands, [], undefined, visit);
 };
 
 const visitCommands = (
   commands: CommandMap,
   parents: readonly string[],
-  visit: (command: RegisteredCommand, path: string) => void,
+  inheritedPolicy: OutputPolicy | undefined,
+  visit: (
+    command: RegisteredCommand,
+    path: string,
+    outputPolicy: OutputPolicy | undefined,
+  ) => void,
 ): void => {
   for (const [name, entry] of commands) {
     const path = [...parents, name];
     if ("_group" in entry) {
-      visitCommands(entry.commands, path, visit);
-      if (entry.root !== undefined) visit(entry.root, path.join(" "));
+      const policy = entry.outputPolicy ?? inheritedPolicy;
+      visitCommands(entry.commands, path, policy, visit);
+      if (entry.root !== undefined)
+        visit(entry.root, path.join(" "), entry.root.outputPolicy ?? policy);
     } else if (!("_alias" in entry) && !("_fetch" in entry)) {
-      visit(entry, path.join(" "));
+      visit(entry, path.join(" "), entry.outputPolicy ?? inheritedPolicy);
     }
   }
 };

tests/acceptance/cliTokenWindowUnicode.test.ts

@@ -0,0 +1,107 @@
+import { writeFile } from "node:fs/promises";
+import { resolve } from "node:path";
+
+import { expect } from "vitest";
+
+import { workspaceCliTest } from "../support/cli/workspaceCliFixture.js";
+
+workspaceCliTest(
+  "preserves successful TOON text across CLI token pages",
+  async ({ processes, workspace }) => {
+    const fileName = `${"😀é".repeat(20)}.plist`;
+    const plist = workspace.path(fileName);
+    await writeFile(
+      plist,
+      `<?xml version="1.0" encoding="UTF-8"?>\n` +
+        `<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" ` +
+        `"http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n` +
+        `<plist version="1.0"><dict>` +
+        `<key>Long</key><string>${"abcdefghij".repeat(300)}</string>` +
+        `<key>Emoji</key><string>${"😀é".repeat(300)}</string>` +
+        `</dict></plist>`,
+    );
+    const run = async (filter: string, ...window: string[]) =>
+      processes.run(
+        process.execPath,
+        [
+          resolve("scripts/rea.mjs"),
+          "inspect-artifact",
+          plist,
+          "--format",
+          "toon",
+          "--filter-output",
+          filter,
+          ...window,
+        ],
+        {
+          cwd: workspace.root,
+          env: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" },
+        },
+      );
+
+    const fullDigest = await run("subject.digest.sha256");
+    expect(
+      fullDigest.exitCode,
+      `${fullDigest.stderr}\n${fullDigest.stdout}`,
+    ).toBe(0);
+    const firstPage = await run(
+      "subject.digest.sha256",
+      "--token-limit",
+      "10",
+      "--token-offset",
+      "0",
+    );
+    expect(firstPage.exitCode, firstPage.stderr).toBe(0);
+    const firstMarker = firstPage.stdout.match(
+      /\n\[truncated: showing tokens 0–10 of (\d+)\]\n?$/,
+    );
+    expect(firstMarker).not.toBeNull();
+    const total = Number(firstMarker?.[1]);
+    let joinedDigest = "";
+    for (let offset = 0; offset < total; offset += 10) {
+      const page =
+        offset === 0
+          ? firstPage
+          : await run(
+              "subject.digest.sha256",
+              "--token-limit",
+              "10",
+              "--token-offset",
+              String(offset),
+            );
+      expect(page.exitCode, page.stderr).toBe(0);
+      joinedDigest += page.stdout.replace(
+        /\n\[truncated: showing tokens \d+–\d+ of \d+\]\n?$/,
+        "",
+      );
+    }
+    expect(joinedDigest).toBe(fullDigest.stdout.trimEnd());
+
+    // Artifact inspection exposes the actual Unicode filename on every platform.
+    const fullName = await run("subject.name");
+    expect(fullName.exitCode, fullName.stderr).toBe(0);
+    expect(fullName.stdout).toContain(fileName);
+    let joinedName = "";
+    let nameTotal = Number.POSITIVE_INFINITY;
+    for (let offset = 0; offset < nameTotal; offset += 7) {
+      const page = await run(
+        "subject.name",
+        "--token-limit",
+        "7",
+        "--token-offset",
+        String(offset),
+      );
+      expect(page.exitCode, page.stderr).toBe(0);
+      const marker = page.stdout.match(
+        /\n\[truncated: showing tokens \d+–\d+ of (\d+)\]\n?$/,
+      );
+      expect(marker).not.toBeNull();
+      nameTotal = Number(marker?.[1]);
+      const body = page.stdout.slice(0, marker?.index);
+      expect(body.length).toBeGreaterThan(0);
+      expect(body).not.toContain("\uFFFD");
+      joinedName += body;
+    }
+    expect(joinedName).toBe(fullName.stdout.trimEnd());
+  },
+);

tests/boundary/cli/cliTokenWindowOutput.test.ts

@@ -0,0 +1,288 @@
+import { Writable } from "node:stream";
+
+import { Cli, Formatter } from "incur";
+import { estimateTokenCount } from "tokenx";
+import { describe, expect, it } from "vitest";
+
+import {
+  createCliResultOutput,
+  streamCliCommandResults,
+} from "../../../src/cli/cliResultOutput.js";
+
+const renderWindow = async (
+  value: unknown,
+  offset: number,
+  limit: number,
+  fullOutput = false,
+  terminal = false,
+  formatExplicit = true,
+): Promise<{ readonly handled: boolean; readonly text: string }> => {
+  const chunks: Buffer[] = [];
+  const destination = new Writable({
+    write(chunk: Buffer, _encoding, callback) {
+      chunks.push(chunk);
+      callback();
+    },
+  });
+  if (terminal) Object.defineProperty(destination, "isTTY", { value: true });
+  const argv = [
+    "fixture",
+    ...(formatExplicit ? ["--format", "toon"] : []),
+    ...(fullOutput ? ["--full-output"] : []),
+    "--token-limit",
+    String(limit),
+    "--token-offset",
+    String(offset),
+  ];
+  const output = createCliResultOutput(argv, destination);
+  if (output === undefined) throw new Error("Missing TOON output surface");
+  const handled = await output.write(value, {
+    command: "fixture",
+    duration: "0ms",
+    format: "toon",
+  });
+  return { handled, text: Buffer.concat(chunks).toString("utf8") };
+};
+
+describe("successful CLI token windows", () => {
+  it.each([undefined, null, ""])(
+    "preserves the full-output data field for a fitting empty value (%j)",
+    async (value) => {
+      const page = await renderWindow(value, 0, 1000, true);
+      expect(page.handled).toBe(true);
+      expect(page.text).toBe(
+        `${Formatter.format(
+          {
+            ok: true,
+            data: value,
+            meta: { command: "fixture", duration: "0ms" },
+          },
+          "toon",
+        )}\n`,
+      );
+    },
+  );
+
+  it("preserves an existing final newline when the whole result fits", async () => {
+    const page = await renderWindow("abcdefghij\n", 0, 1000);
+    expect(page.handled).toBe(true);
+    expect(page.text).toBe("abcdefghij\n");
+  });
+
+  it.each([false, true])(
+    "owns fitting windows and preserves full-output structure (%s)",
+    async (fullOutput) => {
+      const value = { text: "😀é", nested: { value: 42 } };
+      const page = await renderWindow(value, 0, 1000, fullOutput);
+      expect(page.handled).toBe(true);
+      expect(page.text).toBe(
+        `${Formatter.format(
+          fullOutput
+            ? {
+                ok: true,
+                data: value,
+                meta: { command: "fixture", duration: "0ms" },
+              }
+            : value,
+          "toon",
+        )}\n`,
+      );
+      expect(page.text).not.toContain("nextOffset");
+      expect(page.text).not.toContain("[truncated:");
+    },
+  );
+
+  it("does not reformat a fitting ordinary result", async () => {
+    let reads = 0;
+    const page = await renderWindow(
+      {
+        get text() {
+          reads++;
+          return "abcdefghij";
+        },
+      },
+      0,
+      1000,
+    );
+    expect(page.handled).toBe(true);
+    // TOON reads this property once per formatting pass.
+    expect(reads).toBe(1);
+    expect(page.text).toBe("text: abcdefghij\n");
+  });
+
+  it.each([
+    { terminal: false, formatExplicit: false },
+    { terminal: true, formatExplicit: false },
+    { terminal: true, formatExplicit: true },
+  ])(
+    "preserves digest and Unicode pages ($terminal, $formatExplicit)",
+    async ({ terminal, formatExplicit }) => {
+      const value = {
+        sha256:
+          "6764c190e641f1c59c638f9bf32fee05f66992811c5605ebd25056156f21e221",
+        text: "😀é".repeat(300),
+      };
+      const full = Formatter.format(value, "toon");
+      const total = estimateTokenCount(full);
+      const pages: string[] = [];
+      for (let offset = 0; offset < total; offset += 10) {
+        const page = await renderWindow(
+          value,
+          offset,
+          10,
+          false,
+          terminal,
+          formatExplicit,
+        );
+        expect(page.handled).toBe(true);
+        const marker = page.text.lastIndexOf("\n[truncated: showing tokens ");
+        expect(marker).toBeGreaterThanOrEqual(0);
+        pages.push(page.text.slice(0, marker));
+      }
+      const joined = pages.join("");
+      expect(joined).toBe(full);
+      expect(joined).toContain(value.sha256);
+      expect(joined).not.toContain("\uFFFD");
+    },
+  );
+
+  it("retains full-output metadata while returning a truncated page", async () => {
+    const page = await renderWindow(
+      { value: "abcdefghij" },
+      0,
+      2,
+      true,
+      true,
+      false,
+    );
+    expect(page.handled).toBe(true);
+    expect(page.text).toContain("ok: true");
+    expect(page.text).toContain("nextOffset: 2");
+    expect(page.text).toContain("showing tokens 0–2 of");
+  });
+
+  it.each(["abcdefghij abcdefghij abcdefghij \n", "  😀é abcdefghij\t\r\n"])(
+    "preserves all whitespace across text pages: %j",
+    async (value) => {
+      let joined = "";
+      const total = estimateTokenCount(value);
+      for (let offset = 0; offset < total; offset += 1) {
+        const page = await renderWindow(value, offset, 1);
+        expect(page.handled).toBe(true);
+        joined += page.text.slice(
+          0,
+          page.text.lastIndexOf("\n[truncated: showing tokens "),
+        );
+      }
+      expect(joined).toBe(value);
+    },
+  );
+});
+
+describe("token windows through Incur output policies", () => {
+  it.each([{ windowFlags: [] }, { windowFlags: ["--token-limit", "2"] }])(
+    "retains filtered typed failures despite suppressed human results ($windowFlags)",
+    async ({ windowFlags }) => {
+      let text = "";
+      const destination = new Writable({
+        write(chunk, _encoding, callback) {
+          text += chunk.toString();
+          callback();
+        },
+      });
+      const output = createCliResultOutput(
+        ["fixture", "--filter-output", "missing", ...windowFlags],
+        destination,
+      )!;
+      const handled = await output.write(
+        {
+          error: "Analysis failed",
+          code: "target_unavailable",
+          category: "unavailable",
+          message: "Artifact missing",
+          retryable: false,
+          remediation: { action: "Check the path." },
+          details: {},
+        },
+        {
+          command: "fixture",
+          duration: "0ms",
+          format: "toon",
+          renderOutput: false,
+        },
+      );
+      expect(handled).toBe(true);
+      expect(text).toContain("target_unavailable");
+      expect(text).toContain("Artifact missing");
+    },
+  );
+  it.each([
+    { terminal: true, flags: [], visible: false },
+    { terminal: false, flags: [], visible: true },
+    { terminal: true, flags: ["--format", "toon"], visible: true },
+    { terminal: true, flags: ["--full-output"], visible: true },
+    { terminal: true, flags: ["--token-count"], visible: true },
+  ])(
+    "honors agent-only presentation ($terminal, $flags)",
+    async ({ terminal, flags, visible }) => {
+      const descriptor = Object.getOwnPropertyDescriptor(
+        process.stdout,
+        "isTTY",
+      );
+      Object.defineProperty(process.stdout, "isTTY", {
+        value: terminal,
+        configurable: true,
+      });
+      try {
+        for (const grouped of [false, true]) {
+          const argv = [
+            ...(grouped ? ["group"] : []),
+            "fixture",
+            "--token-limit",
+            "2",
+            ...flags,
+          ];
+          let text = "";
+          const destination = new Writable({
+            write(chunk, _encoding, callback) {
+              text += chunk.toString();
+              callback();
+            },
+          });
+          const output = createCliResultOutput(argv, destination)!;
+          const cli = Cli.create("policy-test", { sync: false });
+          if (grouped) {
+            const group = Cli.create("group", {
+              sync: false,
+              outputPolicy: "agent-only",
+            });
+            group.command("fixture", {
+              run: () => ({ value: "😀é".repeat(100) }),
+            });
+            cli.command(group);
+          } else {
+            cli.command("fixture", {
+              outputPolicy: "agent-only",
+              run: () => ({ value: "😀é".repeat(100) }),
+            });
+          }
+          streamCliCommandResults(cli, output);
+          await cli.serve(argv, {
+            stdout: (chunk) => {
+              if (!output.handled) text += chunk;
+            },
+            exit: () => {},
+          });
+          expect(text.length > 0).toBe(visible);
+          expect(text).not.toContain("\uFFFD");
+          if (visible && !flags.includes("--token-count"))
+            expect(text).toContain("showing tokens 0–2 of");
+        }
+      } finally {
+        if (descriptor)
+          Object.defineProperty(process.stdout, "isTTY", descriptor);
+        else Reflect.deleteProperty(process.stdout, "isTTY");
+      }
+    },
+  );
+});