diff --git a/src/cli/cliResultOutput.ts b/src/cli/cliResultOutput.ts index 65eb80cb3a..a7a7bf43e1 100644 --- a/src/cli/cliResultOutput.ts +++ b/src/cli/cliResultOutput.ts @@ -2,7 +2,7 @@ import { constants } from "node:buffer"; import type { Writable } from "node:stream"; import { Filter, Formatter } from "incur"; -import { estimateTokenCount, sliceByTokens } from "tokenx"; +import { estimateTokenCount } from "tokenx"; import { parseCliOutputArguments, @@ -13,6 +13,7 @@ import { cliOperationFailurePayload, } from "../cliLogging.js"; import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; +import { sliceTextByTokenWindow } from "./tokenWindow.js"; import { projectAnalysisError, type AnalysisErrorProjection, @@ -33,6 +34,7 @@ export interface CliResultOutputMetadata { readonly command: string; readonly duration: string; readonly format: string; + readonly renderOutput?: boolean; } /** Optional executable-owned result surface for one CLI invocation. */ @@ -50,8 +52,8 @@ export const streamCliCommandResults = ( cli: CliInstance, output: CliResultOutput, ): void => { - visitRegisteredCommands(cli, (command, path) => - wrapCommand(command, path, output), + visitRegisteredCommands(cli, (command, path, inheritedPolicy) => + wrapCommand(command, path, output, inheritedPolicy), ); }; @@ -59,6 +61,7 @@ const wrapCommand = ( command: Pick, name: string, output: CliResultOutput, + inheritedPolicy?: "all" | "agent-only", ): void => { const run = command.run; command.run = (context) => { @@ -82,6 +85,7 @@ const wrapCommand = ( !(await output.write(value, { command: name, format: context.format, + renderOutput: context.agent || inheritedPolicy !== "agent-only", duration: `${Math.round(performance.now() - started)}ms`, })) ) @@ -213,9 +217,24 @@ export const createCliResultOutput = ( ? cliOperationFailurePayload(value) : undefined; const failure = options.fullOutput ? operationFailure : undefined; + // Incur owns suppressed human presentation, but a filter must never erase + // the typed failure that this boundary already owns. + if ( + operationFailure === undefined && + metadata.renderOutput === false && + !options.fullOutput && + !options.tokenCount + ) + return false; const filtered: unknown = operationFailure ?? (filterPaths === undefined ? value : Filter.apply(value, filterPaths)); + const ownsSuccessfulTokenWindow = + operationFailure === undefined && + options.tokenWindow && + !options.tokenCount && + format !== "json" && + format !== "jsonl"; if ( operationFailure === undefined && filterPaths !== undefined && @@ -230,7 +249,8 @@ export const createCliResultOutput = ( if ( format !== "json" && format !== "jsonl" && - operationFailure === undefined + operationFailure === undefined && + !ownsSuccessfulTokenWindow ) return false; if (options.tokenCount) { @@ -309,6 +329,53 @@ export const createCliResultOutput = ( typeof filtered === "symbol" ? {} : filtered; + if (ownsSuccessfulTokenWindow) { + const formatted = Formatter.format(filtered, format); + const window = sliceTextByTokenWindow( + formatted, + options.tokenOffset ?? 0, + options.tokenLimit, + ); + // Keep Incur's empty human/agent presentation semantics. Nonempty text + // already belongs to this formatter, even when the whole result fits. + if (!window.truncated && !options.fullOutput && formatted === "") + return false; + + const sliced = window.truncated + ? `${window.text}\n[truncated: showing tokens ${options.tokenOffset ?? 0}–${window.end} of ${window.total}]` + : formatted; + const outputDocument = options.fullOutput + ? Formatter.format( + window.truncated + ? { + ok: true, + data: sliced, + meta: { + command: metadata.command, + duration: metadata.duration, + ...(window.end < window.total + ? { nextOffset: window.end } + : {}), + }, + } + : { ok: true, data: filtered, meta }, + format, + ) + : sliced; + handled = true; + try { + await writeOutputText( + destination, + outputDocument.endsWith("\n") + ? outputDocument + : `${outputDocument}\n`, + ); + } catch (cause: unknown) { + failed = true; + throw cause; + } + return true; + } // Once writing starts, a failed destination must not receive a second // document from Incur's error formatter. const failureText = @@ -350,15 +417,13 @@ const fullFailureEnvelope = ( if (tokenLimit === undefined && tokenOffset === undefined) return { ok: false, error: failure, meta }; const formatted = Formatter.format(failure, format); - const total = estimateTokenCount(formatted); const offset = tokenOffset ?? 0; - const end = tokenLimit === undefined ? total : offset + tokenLimit; - if (offset === 0 && end >= total) return { ok: false, error: failure, meta }; - const actualEnd = Math.min(end, total); - const nextOffset = actualEnd < total ? actualEnd : undefined; + const window = sliceTextByTokenWindow(formatted, offset, tokenLimit); + if (!window.truncated) return { ok: false, error: failure, meta }; + const nextOffset = window.end < window.total ? window.end : undefined; return { ok: false, - error: `${sliceByTokens(formatted, offset, end)}\n[truncated: showing tokens ${offset}–${actualEnd} of ${total}]`, + error: `${window.text}\n[truncated: showing tokens ${offset}–${window.end} of ${window.total}]`, meta: { ...meta, ...(nextOffset === undefined ? {} : { nextOffset }) }, }; }; diff --git a/src/cli/tokenWindow.ts b/src/cli/tokenWindow.ts new file mode 100644 index 0000000000..0b85e6a969 --- /dev/null +++ b/src/cli/tokenWindow.ts @@ -0,0 +1,81 @@ +import { estimateTokenCount } from "tokenx"; + +export interface TokenWindow { + readonly text: string; + readonly total: number; + readonly end: number; + readonly truncated: boolean; +} + +// tokenx 1.3.0 uses this split expression before estimating and slicing. +// Its public API does not expose the segments, so retain the same boundaries +// here while mapping both ends of each segment to a disjoint text range. +const TOKEN_SPLIT_PATTERN = /(\s+|[.,!?;(){}[\]<>:/\\|@#$%^&*+=`~_-]+)/; + +const codePointSafeBoundary = (text: string, offset: number): number => + offset > 0 && + offset < text.length && + text.charCodeAt(offset - 1)! >= 0xd800 && + text.charCodeAt(offset - 1)! <= 0xdbff && + text.charCodeAt(offset)! >= 0xdc00 && + text.charCodeAt(offset)! <= 0xdfff + ? offset - 1 + : offset; + +const segmentBoundary = ( + segment: string, + tokenOffset: number, + tokenCount: number, +): number => + codePointSafeBoundary( + segment, + Math.floor((tokenOffset / tokenCount) * segment.length), + ); + +/** Slice an approximate token range without overlapping pages or splitting a surrogate pair. */ +export const sliceTextByTokenWindow = ( + text: string, + offset: number, + limit: number | undefined, +): TokenWindow => { + const total = estimateTokenCount(text); + const end = limit === undefined ? total : offset + limit; + if (offset === 0 && end >= total) + return { text, total, end: total, truncated: false }; + + const actualEnd = Math.min(end, total); + if (offset >= actualEnd) + return { text: "", total, end: actualEnd, truncated: true }; + + const parts: string[] = []; + let tokenPosition = 0; + for (const segment of text.split(TOKEN_SPLIT_PATTERN).filter(Boolean)) { + if (tokenPosition >= actualEnd && actualEnd < total) break; + const tokenCount = estimateTokenCount(segment); + if (tokenCount === 0) { + if ( + tokenPosition >= offset && + (tokenPosition < actualEnd || actualEnd === total) + ) + parts.push(segment); + continue; + } + + const segmentEnd = tokenPosition + tokenCount; + if (tokenPosition < actualEnd && segmentEnd > offset) { + const localStart = Math.max(0, offset - tokenPosition); + const localEnd = Math.min(tokenCount, actualEnd - tokenPosition); + const charStart = segmentBoundary(segment, localStart, tokenCount); + const charEnd = segmentBoundary(segment, localEnd, tokenCount); + parts.push(segment.slice(charStart, charEnd)); + } + tokenPosition = segmentEnd; + } + + return { + text: parts.join(""), + total, + end: actualEnd, + truncated: true, + }; +}; diff --git a/src/cli/visitRegisteredCommands.ts b/src/cli/visitRegisteredCommands.ts index 895f9e16a9..efe7989e8f 100644 --- a/src/cli/visitRegisteredCommands.ts +++ b/src/cli/visitRegisteredCommands.ts @@ -5,6 +5,7 @@ import type { CliInstance } from "./types.js"; type CommandMap = NonNullable>; type CommandEntry = CommandMap extends Map ? Entry : never; export type RegisteredCommand = Extract; +type OutputPolicy = "all" | "agent-only"; // Incur types its router key without vars, env, or globals; accept any CLI. const registeredCommands: WeakMap = Cli.toCommands; @@ -12,24 +13,35 @@ const registeredCommands: WeakMap = Cli.toCommands; /** Visit ordinary commands in map order, children before each group's root. */ export const visitRegisteredCommands = ( cli: CliInstance, - visit: (command: RegisteredCommand, path: string) => void, + visit: ( + command: RegisteredCommand, + path: string, + outputPolicy: OutputPolicy | undefined, + ) => void, ): void => { const commands = registeredCommands.get(cli); - if (commands !== undefined) visitCommands(commands, [], visit); + if (commands !== undefined) visitCommands(commands, [], undefined, visit); }; const visitCommands = ( commands: CommandMap, parents: readonly string[], - visit: (command: RegisteredCommand, path: string) => void, + inheritedPolicy: OutputPolicy | undefined, + visit: ( + command: RegisteredCommand, + path: string, + outputPolicy: OutputPolicy | undefined, + ) => void, ): void => { for (const [name, entry] of commands) { const path = [...parents, name]; if ("_group" in entry) { - visitCommands(entry.commands, path, visit); - if (entry.root !== undefined) visit(entry.root, path.join(" ")); + const policy = entry.outputPolicy ?? inheritedPolicy; + visitCommands(entry.commands, path, policy, visit); + if (entry.root !== undefined) + visit(entry.root, path.join(" "), entry.root.outputPolicy ?? policy); } else if (!("_alias" in entry) && !("_fetch" in entry)) { - visit(entry, path.join(" ")); + visit(entry, path.join(" "), entry.outputPolicy ?? inheritedPolicy); } } }; diff --git a/tests/acceptance/cliTokenWindowUnicode.test.ts b/tests/acceptance/cliTokenWindowUnicode.test.ts new file mode 100644 index 0000000000..246365745b --- /dev/null +++ b/tests/acceptance/cliTokenWindowUnicode.test.ts @@ -0,0 +1,107 @@ +import { writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { expect } from "vitest"; + +import { workspaceCliTest } from "../support/cli/workspaceCliFixture.js"; + +workspaceCliTest( + "preserves successful TOON text across CLI token pages", + async ({ processes, workspace }) => { + const fileName = `${"😀é".repeat(20)}.plist`; + const plist = workspace.path(fileName); + await writeFile( + plist, + `\n` + + `\n` + + `` + + `Long${"abcdefghij".repeat(300)}` + + `Emoji${"😀é".repeat(300)}` + + ``, + ); + const run = async (filter: string, ...window: string[]) => + processes.run( + process.execPath, + [ + resolve("scripts/rea.mjs"), + "inspect-artifact", + plist, + "--format", + "toon", + "--filter-output", + filter, + ...window, + ], + { + cwd: workspace.root, + env: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" }, + }, + ); + + const fullDigest = await run("subject.digest.sha256"); + expect( + fullDigest.exitCode, + `${fullDigest.stderr}\n${fullDigest.stdout}`, + ).toBe(0); + const firstPage = await run( + "subject.digest.sha256", + "--token-limit", + "10", + "--token-offset", + "0", + ); + expect(firstPage.exitCode, firstPage.stderr).toBe(0); + const firstMarker = firstPage.stdout.match( + /\n\[truncated: showing tokens 0–10 of (\d+)\]\n?$/, + ); + expect(firstMarker).not.toBeNull(); + const total = Number(firstMarker?.[1]); + let joinedDigest = ""; + for (let offset = 0; offset < total; offset += 10) { + const page = + offset === 0 + ? firstPage + : await run( + "subject.digest.sha256", + "--token-limit", + "10", + "--token-offset", + String(offset), + ); + expect(page.exitCode, page.stderr).toBe(0); + joinedDigest += page.stdout.replace( + /\n\[truncated: showing tokens \d+–\d+ of \d+\]\n?$/, + "", + ); + } + expect(joinedDigest).toBe(fullDigest.stdout.trimEnd()); + + // Artifact inspection exposes the actual Unicode filename on every platform. + const fullName = await run("subject.name"); + expect(fullName.exitCode, fullName.stderr).toBe(0); + expect(fullName.stdout).toContain(fileName); + let joinedName = ""; + let nameTotal = Number.POSITIVE_INFINITY; + for (let offset = 0; offset < nameTotal; offset += 7) { + const page = await run( + "subject.name", + "--token-limit", + "7", + "--token-offset", + String(offset), + ); + expect(page.exitCode, page.stderr).toBe(0); + const marker = page.stdout.match( + /\n\[truncated: showing tokens \d+–\d+ of (\d+)\]\n?$/, + ); + expect(marker).not.toBeNull(); + nameTotal = Number(marker?.[1]); + const body = page.stdout.slice(0, marker?.index); + expect(body.length).toBeGreaterThan(0); + expect(body).not.toContain("\uFFFD"); + joinedName += body; + } + expect(joinedName).toBe(fullName.stdout.trimEnd()); + }, +); diff --git a/tests/boundary/cli/cliTokenWindowOutput.test.ts b/tests/boundary/cli/cliTokenWindowOutput.test.ts new file mode 100644 index 0000000000..48e16bcafe --- /dev/null +++ b/tests/boundary/cli/cliTokenWindowOutput.test.ts @@ -0,0 +1,288 @@ +import { Writable } from "node:stream"; + +import { Cli, Formatter } from "incur"; +import { estimateTokenCount } from "tokenx"; +import { describe, expect, it } from "vitest"; + +import { + createCliResultOutput, + streamCliCommandResults, +} from "../../../src/cli/cliResultOutput.js"; + +const renderWindow = async ( + value: unknown, + offset: number, + limit: number, + fullOutput = false, + terminal = false, + formatExplicit = true, +): Promise<{ readonly handled: boolean; readonly text: string }> => { + const chunks: Buffer[] = []; + const destination = new Writable({ + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }); + if (terminal) Object.defineProperty(destination, "isTTY", { value: true }); + const argv = [ + "fixture", + ...(formatExplicit ? ["--format", "toon"] : []), + ...(fullOutput ? ["--full-output"] : []), + "--token-limit", + String(limit), + "--token-offset", + String(offset), + ]; + const output = createCliResultOutput(argv, destination); + if (output === undefined) throw new Error("Missing TOON output surface"); + const handled = await output.write(value, { + command: "fixture", + duration: "0ms", + format: "toon", + }); + return { handled, text: Buffer.concat(chunks).toString("utf8") }; +}; + +describe("successful CLI token windows", () => { + it.each([undefined, null, ""])( + "preserves the full-output data field for a fitting empty value (%j)", + async (value) => { + const page = await renderWindow(value, 0, 1000, true); + expect(page.handled).toBe(true); + expect(page.text).toBe( + `${Formatter.format( + { + ok: true, + data: value, + meta: { command: "fixture", duration: "0ms" }, + }, + "toon", + )}\n`, + ); + }, + ); + + it("preserves an existing final newline when the whole result fits", async () => { + const page = await renderWindow("abcdefghij\n", 0, 1000); + expect(page.handled).toBe(true); + expect(page.text).toBe("abcdefghij\n"); + }); + + it.each([false, true])( + "owns fitting windows and preserves full-output structure (%s)", + async (fullOutput) => { + const value = { text: "😀é", nested: { value: 42 } }; + const page = await renderWindow(value, 0, 1000, fullOutput); + expect(page.handled).toBe(true); + expect(page.text).toBe( + `${Formatter.format( + fullOutput + ? { + ok: true, + data: value, + meta: { command: "fixture", duration: "0ms" }, + } + : value, + "toon", + )}\n`, + ); + expect(page.text).not.toContain("nextOffset"); + expect(page.text).not.toContain("[truncated:"); + }, + ); + + it("does not reformat a fitting ordinary result", async () => { + let reads = 0; + const page = await renderWindow( + { + get text() { + reads++; + return "abcdefghij"; + }, + }, + 0, + 1000, + ); + expect(page.handled).toBe(true); + // TOON reads this property once per formatting pass. + expect(reads).toBe(1); + expect(page.text).toBe("text: abcdefghij\n"); + }); + + it.each([ + { terminal: false, formatExplicit: false }, + { terminal: true, formatExplicit: false }, + { terminal: true, formatExplicit: true }, + ])( + "preserves digest and Unicode pages ($terminal, $formatExplicit)", + async ({ terminal, formatExplicit }) => { + const value = { + sha256: + "6764c190e641f1c59c638f9bf32fee05f66992811c5605ebd25056156f21e221", + text: "😀é".repeat(300), + }; + const full = Formatter.format(value, "toon"); + const total = estimateTokenCount(full); + const pages: string[] = []; + for (let offset = 0; offset < total; offset += 10) { + const page = await renderWindow( + value, + offset, + 10, + false, + terminal, + formatExplicit, + ); + expect(page.handled).toBe(true); + const marker = page.text.lastIndexOf("\n[truncated: showing tokens "); + expect(marker).toBeGreaterThanOrEqual(0); + pages.push(page.text.slice(0, marker)); + } + const joined = pages.join(""); + expect(joined).toBe(full); + expect(joined).toContain(value.sha256); + expect(joined).not.toContain("\uFFFD"); + }, + ); + + it("retains full-output metadata while returning a truncated page", async () => { + const page = await renderWindow( + { value: "abcdefghij" }, + 0, + 2, + true, + true, + false, + ); + expect(page.handled).toBe(true); + expect(page.text).toContain("ok: true"); + expect(page.text).toContain("nextOffset: 2"); + expect(page.text).toContain("showing tokens 0–2 of"); + }); + + it.each(["abcdefghij abcdefghij abcdefghij \n", " 😀é abcdefghij\t\r\n"])( + "preserves all whitespace across text pages: %j", + async (value) => { + let joined = ""; + const total = estimateTokenCount(value); + for (let offset = 0; offset < total; offset += 1) { + const page = await renderWindow(value, offset, 1); + expect(page.handled).toBe(true); + joined += page.text.slice( + 0, + page.text.lastIndexOf("\n[truncated: showing tokens "), + ); + } + expect(joined).toBe(value); + }, + ); +}); + +describe("token windows through Incur output policies", () => { + it.each([{ windowFlags: [] }, { windowFlags: ["--token-limit", "2"] }])( + "retains filtered typed failures despite suppressed human results ($windowFlags)", + async ({ windowFlags }) => { + let text = ""; + const destination = new Writable({ + write(chunk, _encoding, callback) { + text += chunk.toString(); + callback(); + }, + }); + const output = createCliResultOutput( + ["fixture", "--filter-output", "missing", ...windowFlags], + destination, + )!; + const handled = await output.write( + { + error: "Analysis failed", + code: "target_unavailable", + category: "unavailable", + message: "Artifact missing", + retryable: false, + remediation: { action: "Check the path." }, + details: {}, + }, + { + command: "fixture", + duration: "0ms", + format: "toon", + renderOutput: false, + }, + ); + expect(handled).toBe(true); + expect(text).toContain("target_unavailable"); + expect(text).toContain("Artifact missing"); + }, + ); + it.each([ + { terminal: true, flags: [], visible: false }, + { terminal: false, flags: [], visible: true }, + { terminal: true, flags: ["--format", "toon"], visible: true }, + { terminal: true, flags: ["--full-output"], visible: true }, + { terminal: true, flags: ["--token-count"], visible: true }, + ])( + "honors agent-only presentation ($terminal, $flags)", + async ({ terminal, flags, visible }) => { + const descriptor = Object.getOwnPropertyDescriptor( + process.stdout, + "isTTY", + ); + Object.defineProperty(process.stdout, "isTTY", { + value: terminal, + configurable: true, + }); + try { + for (const grouped of [false, true]) { + const argv = [ + ...(grouped ? ["group"] : []), + "fixture", + "--token-limit", + "2", + ...flags, + ]; + let text = ""; + const destination = new Writable({ + write(chunk, _encoding, callback) { + text += chunk.toString(); + callback(); + }, + }); + const output = createCliResultOutput(argv, destination)!; + const cli = Cli.create("policy-test", { sync: false }); + if (grouped) { + const group = Cli.create("group", { + sync: false, + outputPolicy: "agent-only", + }); + group.command("fixture", { + run: () => ({ value: "😀é".repeat(100) }), + }); + cli.command(group); + } else { + cli.command("fixture", { + outputPolicy: "agent-only", + run: () => ({ value: "😀é".repeat(100) }), + }); + } + streamCliCommandResults(cli, output); + await cli.serve(argv, { + stdout: (chunk) => { + if (!output.handled) text += chunk; + }, + exit: () => {}, + }); + expect(text.length > 0).toBe(visible); + expect(text).not.toContain("\uFFFD"); + if (visible && !flags.includes("--token-count")) + expect(text).toContain("showing tokens 0–2 of"); + } + } finally { + if (descriptor) + Object.defineProperty(process.stdout, "isTTY", descriptor); + else Reflect.deleteProperty(process.stdout, "isTTY"); + } + }, + ); +});