From fc404f008d7a6f1dfc8337288a28691b76a977ce Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:28:56 +0800 Subject: [PATCH] fix(cli): count emitted documents and normalize token windows --- docs/cli.md | 10 +++ src/cli/cliResultOutput.ts | 114 ++++++++++++++------------ src/cli/jsonOutput.ts | 4 +- src/cli/tokenWindow.ts | 12 ++- tests/boundary/cli/jsonOutput.test.ts | 101 +++++++++++++++++++---- 5 files changed, 172 insertions(+), 69 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index c51ebef579..33e96b6167 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -909,3 +909,13 @@ REA's failure status: set -o pipefail rea inspect-artifact ./app.asar --json | jq . > inspection.json ``` + +`--token-count` reports a tokenx estimate for the selected emitted document, +including a requested full-output envelope and metadata. JSON uses the same +terminal indentation or compact piped representation as normal output; JSONL +uses its emitted line representation. The final newline is included (tokenx +assigns whitespace no tokens). This is an estimate, not a model tokenizer count. +Counting retains the existing bounded formatted-string allocation after capacity +admission; omit the flag to stream large JSON documents without materializing +the complete formatted text. Exhausted token windows display an empty range at +the total token count and have no next offset. diff --git a/src/cli/cliResultOutput.ts b/src/cli/cliResultOutput.ts index ff399ed97c..34474a63bd 100644 --- a/src/cli/cliResultOutput.ts +++ b/src/cli/cliResultOutput.ts @@ -20,6 +20,7 @@ import { jsonLinesErrorRecord } from "./jsonLines.js"; import { writeCliJsonLines } from "./jsonLinesOutput.js"; import { jsonOutputParts, + isTerminalOutput, writeJsonOutput, writeOutputText, } from "./jsonOutput.js"; @@ -346,55 +347,6 @@ export const createCliResultOutput = ( } return true; } - if (options.tokenCount) { - // Incur/tokenx materialize JSON. Measure its UTF-16 expansion before - // allowing that allocation, independently of UTF-8 bytes. Other formats - // retain Incur's materializing behavior. - const tokenValue = failure ?? filtered; - let characters = 0; - if ( - tokenValue !== undefined && - tokenValue !== null && - typeof tokenValue !== "function" && - typeof tokenValue !== "symbol" - ) { - if (format === "json") { - for (const part of jsonOutputParts( - tokenValue, - format, - format === "json", - )) - characters += part.length; - characters = Math.max(0, characters - 1); - } - } - if (characters <= constants.MAX_STRING_LENGTH) { - const count = estimateTokenCount(formatCliOutput(tokenValue, format)); - handled = true; - try { - await writeOutputText(destination, `${count}\n`); - return true; - } catch (cause: unknown) { - failed = true; - throw cause; - } - } - error = projectAnalysisError( - new AnalysisResourceConstraintError( - metadata.command, - "memory", - "Token counting requires a formatted string larger than the runtime permits.", - { - formatted_characters: characters, - max_string_characters: constants.MAX_STRING_LENGTH, - }, - { - remediationAction: - "Remove --token-count to stream the complete JSON result, or narrow it with --filter-output.", - }, - ), - ); - } const meta = { command: metadata.command, duration: metadata.duration, @@ -422,6 +374,66 @@ export const createCliResultOutput = ( typeof filtered === "symbol" ? {} : filtered; + if (options.tokenCount) { + // Incur/tokenx materialize JSON. Measure its UTF-16 expansion before + // allowing that allocation, independently of UTF-8 bytes. Other formats + // retain Incur's materializing behavior. + const tokenValue = document; + let characters = 0; + if ( + tokenValue !== undefined && + tokenValue !== null && + typeof tokenValue !== "function" && + typeof tokenValue !== "symbol" + ) { + if (format === "json" || format === "jsonl") { + for (const part of jsonOutputParts( + tokenValue, + format, + isTerminalOutput(destination), + )) + characters += part.length; + } + } + if (characters <= constants.MAX_STRING_LENGTH) { + let formatted: string; + if (format === "json" || format === "jsonl") { + // Reuse the writer's exact representation after its capacity admission. + // tokenx requires one string; retain only this bounded formatted value. + formatted = ""; + for (const part of jsonOutputParts( + tokenValue, + format, + isTerminalOutput(destination), + )) + formatted += part; + } else formatted = formatCliOutput(tokenValue, format); + const count = estimateTokenCount(formatted); + handled = true; + try { + await writeOutputText(destination, `${count}\n`); + return true; + } catch (cause: unknown) { + failed = true; + throw cause; + } + } + error = projectAnalysisError( + new AnalysisResourceConstraintError( + metadata.command, + "memory", + "Token counting requires a formatted string larger than the runtime permits.", + { + formatted_characters: characters, + max_string_characters: constants.MAX_STRING_LENGTH, + }, + { + remediationAction: + "Remove --token-count to stream the complete JSON result, or narrow it with --filter-output.", + }, + ), + ); + } if (ownsSuccessfulTokenWindow) { const formatted = formatCliOutput(filtered, format); const window = sliceTextByTokenWindow( @@ -435,7 +447,7 @@ export const createCliResultOutput = ( return false; const sliced = window.truncated - ? `${window.text}\n[truncated: showing tokens ${options.tokenOffset ?? 0}–${window.end} of ${window.total}]` + ? `${window.text}\n[truncated: showing tokens ${window.start}–${window.end} of ${window.total}]` : formatted; const outputDocument = options.fullOutput ? formatCliOutput( @@ -526,7 +538,7 @@ const fullFailureEnvelope = ( const nextOffset = window.end < window.total ? window.end : undefined; return { ok: false, - error: `${window.text}\n[truncated: showing tokens ${offset}–${window.end} of ${window.total}]`, + error: `${window.text}\n[truncated: showing tokens ${window.start}–${window.end} of ${window.total}]`, meta: { ...meta, ...(nextOffset === undefined ? {} : { nextOffset }) }, }; }; diff --git a/src/cli/jsonOutput.ts b/src/cli/jsonOutput.ts index 5c28b9d6ea..dbfa33cc63 100644 --- a/src/cli/jsonOutput.ts +++ b/src/cli/jsonOutput.ts @@ -18,7 +18,7 @@ export const writeJsonOutput = async ( ): Promise => { await writeOutputParts( destination, - jsonOutputParts(value, format, isTerminal(destination)), + jsonOutputParts(value, format, isTerminalOutput(destination)), ); }; @@ -31,7 +31,7 @@ export const writeOutputParts = async ( await writeOutputText(destination, part); }; -const isTerminal = (destination: Writable): boolean => +export const isTerminalOutput = (destination: Writable): boolean => "isTTY" in destination && destination.isTTY === true; export const writeOutputText = ( diff --git a/src/cli/tokenWindow.ts b/src/cli/tokenWindow.ts index 0b85e6a969..38353cada9 100644 --- a/src/cli/tokenWindow.ts +++ b/src/cli/tokenWindow.ts @@ -3,6 +3,7 @@ import { estimateTokenCount } from "tokenx"; export interface TokenWindow { readonly text: string; readonly total: number; + readonly start: number; readonly end: number; readonly truncated: boolean; } @@ -41,11 +42,17 @@ export const sliceTextByTokenWindow = ( const total = estimateTokenCount(text); const end = limit === undefined ? total : offset + limit; if (offset === 0 && end >= total) - return { text, total, end: total, truncated: false }; + return { text, total, start: 0, end: total, truncated: false }; const actualEnd = Math.min(end, total); if (offset >= actualEnd) - return { text: "", total, end: actualEnd, truncated: true }; + return { + text: "", + total, + start: actualEnd, + end: actualEnd, + truncated: true, + }; const parts: string[] = []; let tokenPosition = 0; @@ -75,6 +82,7 @@ export const sliceTextByTokenWindow = ( return { text: parts.join(""), total, + start: offset, end: actualEnd, truncated: true, }; diff --git a/tests/boundary/cli/jsonOutput.test.ts b/tests/boundary/cli/jsonOutput.test.ts index a013d21235..b7d1b7a372 100644 --- a/tests/boundary/cli/jsonOutput.test.ts +++ b/tests/boundary/cli/jsonOutput.test.ts @@ -2,7 +2,7 @@ import { constants } from "node:buffer"; import { createHash } from "node:crypto"; import { Writable } from "node:stream"; -import { Cli, Formatter } from "incur"; +import { Cli } from "incur"; import { estimateTokenCount } from "tokenx"; import { parse as parseYaml } from "yaml"; import { describe, expect, it } from "vitest"; @@ -473,21 +473,89 @@ describe("full-output token counts", () => { destination, ); if (output === undefined) throw new Error("Missing JSON output surface"); - const cli = Cli.create("rea-output-test", { sync: false }); - registerManagedCommands(cli, silentLogger, async () => failure); - streamCliCommandResults(cli, output); - const previousExitCode = process.exitCode; - try { - await cli.serve(argv, { stdout: () => undefined }); - expect(process.exitCode).toBe(1); - } finally { - process.exitCode = previousExitCode; - } + await output.write(failure, { + command: "inspect-managed-artifact", + duration: "0ms", + format: "json", + }); expect(output.handled).toBe(true); expect(Buffer.concat(chunks).toString("utf8")).toBe( - `${estimateTokenCount(Formatter.format(failure, "json"))}\n`, + `${estimateTokenCount(JSON.stringify({ ok: false, error: failure, meta: { command: "inspect-managed-artifact", duration: "0ms" } }) + "\n")}\n`, ); }); + it("counts the exact emitted JSON representation and selected document", async () => { + const metadata = { command: "fixture", duration: "0ms", format: "json" }; + for (const terminal of [false, true]) { + for (const flags of [ + [], + ["--full-output"], + ["--filter-output", "missing"], + ["--format", "jsonl"], + ]) { + const argv = ["fixture", "--json", ...flags]; + const render = async (count: boolean) => { + const chunks: Buffer[] = []; + const destination = new Writable({ + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }); + Object.defineProperty(destination, "isTTY", { value: terminal }); + const output = createCliResultOutput( + parseCliOutputArguments([ + ...argv, + ...(count ? ["--token-count"] : []), + ]), + destination, + ); + if (output === undefined) throw new Error("Missing output surface"); + await output.write({ value: ["hello", "雪😀"] }, metadata); + return Buffer.concat(chunks).toString("utf8"); + }; + expect(Number((await render(true)).trim())).toBe( + estimateTokenCount(await render(false)), + ); + } + } + }); + + it("normalizes exhausted failure window display coordinates", async () => { + const chunks: Buffer[] = []; + const destination = new Writable({ + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }); + const output = createCliResultOutput( + parseCliOutputArguments([ + "fixture", + "--format", + "toon", + "--full-output", + "--token-offset", + "100000", + "--token-limit", + "3", + ]), + destination, + ); + if (output === undefined) throw new Error("Missing output surface"); + await output.write( + { + code: "target_unavailable", + category: "unavailable", + message: "Missing target", + retryable: false, + }, + { command: "fixture", duration: "0ms", format: "toon" }, + ); + const text = Buffer.concat(chunks).toString("utf8"); + expect(text).toMatch(/showing tokens (\d+)–\1 of \1/); + expect(text).not.toContain("nextOffset"); + expect(text).toContain("ok: false"); + }); }); describe("CLI result output surface", () => { @@ -612,7 +680,8 @@ describe("CLI result output surface", () => { }); expect(exitCode).toBe(1); expect(fallbackBytes).toBe(0); - expect(JSON.parse(Buffer.concat(chunks).toString("utf8"))).toMatchObject({ + const report = JSON.parse(Buffer.concat(chunks).toString("utf8")); + expect(report).toMatchObject({ ok: false, error: { code: "resource_constraint", @@ -621,7 +690,11 @@ describe("CLI result output surface", () => { }, details: { reported_limits: { - formatted_characters: expectedCharacters - 1, + formatted_characters: + expectedCharacters + + JSON.stringify({ ok: true, data: null, meta: report.meta }) + .length - + 4, max_string_characters: constants.MAX_STRING_LENGTH, }, },