From e5965adb29b0acb1de915b81fcaffe7eae7844ca Mon Sep 17 00:00:00 2001 From: KAI-CHIEH YANG <99654472+KJLavender@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:32:41 +0800 Subject: [PATCH] feat(cli): accept JavaScript source selections from a JSON array or file (#5029) * feat(cli): accept JavaScript source selections from a JSON array or file Add --source-paths-json to analyze and analyze-javascript-application. It takes an inline JSON array or a JSON file path of exact inventoried paths and appends them after any --source-path values, so a long selection no longer has to fit on one command line (cmd.exe through npx on Windows stops at 8,191 characters). Entries stay artifact-relative and go through the existing source_paths validation; the native lane rejects the option like --source-path. Refs #4987 * fix(cli): cancel --source-paths-json file reads with the command Pass the command's abort signal through parseCliJsonInput to readJsonInputFile, so SIGINT/SIGTERM stop a selection-file read instead of waiting for it to finish. A cancelled read now propagates its abort rather than being reported as a file read failure. * test(cli): preserve ASAR identity for JSON file selections * docs(cli): record source-selection task costs and identity verification --------- Co-authored-by: KJLavender Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> --- .../references/javascript-applications.md | 7 +- docs/javascript-artifact-reconstruction.md | 16 ++- ...javascript-source-paths-json-2026-10-24.md | 86 ++++++++++++ src/cli/coreBinaryCommands.ts | 29 ++-- src/cli/electronCommands.ts | 15 +- src/cli/javascriptApplicationAnalysis.ts | 40 ++++++ src/cliObservationOptions.ts | 7 + .../javascriptSourceSelectionCli.test.ts | 129 ++++++++++++++++++ .../boundary/filesystem/cliJsonInput.test.ts | 39 ++++++ 9 files changed, 352 insertions(+), 16 deletions(-) create mode 100644 docs/verification/javascript-source-paths-json-2026-10-24.md diff --git a/.agents/skills/reverse-engineer-anything/references/javascript-applications.md b/.agents/skills/reverse-engineer-anything/references/javascript-applications.md index e1894c1b63..9c3508571d 100644 --- a/.agents/skills/reverse-engineer-anything/references/javascript-applications.md +++ b/.agents/skills/reverse-engineer-anything/references/javascript-applications.md @@ -17,9 +17,10 @@ the complete artifact inventory and identity but reads source text and analyzes only selected files. The result and summary record `source_selection`; files outside scope and their relationships remain unknown, so partial application coverage does not mean the selected source failed. Use repeated `--source-path` -on either static application CLI route. Omit the selection when the question -requires complete application analysis; `source_paths: []` is invalid, not an -unfiltered request. `detail: "summary"` reduces delivery +on either static application CLI route, or `--source-paths-json` with a JSON +file for selections too long for one command line. Omit the selection when the +question requires complete application analysis; `source_paths: []` is invalid, +not an unfiltered request. `detail: "summary"` reduces delivery and does not by itself reduce analysis work. ASAR header/index admission defaults to 16 MiB / 16777216 bytes; diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md index 129e5b1091..5727eef481 100644 --- a/docs/javascript-artifact-reconstruction.md +++ b/docs/javascript-artifact-reconstruction.md @@ -101,6 +101,18 @@ caller's request and first-occurrence priority, analyzing the file once. Omit the selection for the existing complete application workflow. +For a long selection, pass the paths as a JSON array with +`--source-paths-json`, either inline or as a JSON file path. A file keeps the +command below host command-line limits, such as the 8,191-character limit of +the `cmd.exe` shim that `npx` uses on Windows. Its entries follow any +`--source-path` values and stay artifact-relative; they are not resolved +against the working directory: + +```bash +rea analyze-javascript-application /absolute/path/to/app.asar \ + --source-paths-json /absolute/path/to/paths.json --json +``` + The result and summary retain `source_selection.requested_paths` and `source_selection.unselected_files`. Selecting every relevant file leaves this count at zero and adds no unselected-source caveat. Every relevant file keeps its @@ -109,8 +121,8 @@ original inventory identity; unselected files report `text_status: coverage is partial while any file remains outside scope. Selection does not claim whole-application completeness or change integrity policy. Summary detail only reduces delivery; source selection reduces text acquisition and analysis. -Explicit native provider or snapshot routing rejects `--source-path` rather -than ignoring it. +Explicit native provider or snapshot routing rejects `--source-path` and +`--source-paths-json` rather than ignoring them. For directory input, a permission-denied child file remains an unavailable inventory occurrence, and an unreadable child directory records an unknown diff --git a/docs/verification/javascript-source-paths-json-2026-10-24.md b/docs/verification/javascript-source-paths-json-2026-10-24.md new file mode 100644 index 0000000000..4f6e61f077 --- /dev/null +++ b/docs/verification/javascript-source-paths-json-2026-10-24.md @@ -0,0 +1,86 @@ +# JavaScript source-selection JSON verification — 2026-10-24 + +## Scope and integration + +PR #5029 adds `--source-paths-json` to the two static application CLI routes. +One inline JSON array or selected local JSON file appends exact inventoried +paths after repeated `--source-path` values. The application `source_paths` +contract, ordering, duplicate requests, artifact identity and MCP contract are +preserved. This does not add folders, globs or implicit dependencies; #5048 is +separate. + +The CLI adapter reuses the existing JSON file reader and domain array schema. +Entries remain artifact-relative. It preserves cancellation and shared file +ownership rather than introducing another reader/parser. The native lane +refuses source-selection options before starting an engine. Main integration +retains existing delivery validation, the caller's summary/view settings, +environment selection and process-owned JavaScript worker cleanup. + +The initial main `7f70797b` integration passes **44** selected CLI, filesystem +JSON-input and actual SDK source-selection checks, plus full `npm run check`. +The JSON-file acceptance case now analyzes an ASAR and verifies the original +package's path and SHA-256; the inline case covers an extracted directory. +Both combine repeated and JSON entries and preserve their requested order. +Invalid shapes, empty entries, native-lane refusal and cancellation are covered. +This is static source analysis; the selected application is not executed. + +## Completed-task comparison + +Compare main `7f70797b98c226009163358f91a9cfcedf03b390` with feature integration +`e886393ae698d4ed411179430c8dbb10ce6c5647`. Three alternating pairs analyze the +same ASAR, selecting `package.json` followed by 210 named JavaScript files. +The baseline repeats 211 path options; the candidate uses one repeated option +plus an existing JSON file with the other 210 paths. All **six** tasks finish. + +Every complete Evidence object is exactly equal across all runs, including its +ID, selected paths, provenance, coverage, uncertainty and limitations. Each +result reports 210 parsed JavaScript files, no unselected files, and retains all +210 expected named functions. The ASAR remains 75,620 bytes with SHA-256 +`f1089a49e40f4477e0850787654411c0ec208934262837a19e65c2aa4dc5f035`. + +| Completed CLI task cost | Repeated options | Existing JSON file | +| ----------------------------------------- | ---------------: | -----------------: | +| Displayed invocation characters | 17,977 | 201 | +| Estimated complete Evidence stdout tokens | 1,861,409 | 1,861,409 | +| Estimated invocation tokens | 5,078 | 62 | +| Estimated stdout + invocation tokens | 1,866,487 | 1,861,471 | +| Elapsed median s | 19.149 | 18.389 | +| Waited-child CPU median s | 18.250 | 17.340 | +| Sampled family RSS median MiB | 601.4 | 594.1 | + +This solves command-line capacity without reducing the complete analysis +result. Microsoft documents an +[8,191-character Cmd.exe command-line limit](https://learn.microsoft.com/en-us/troubleshoot/windows-client/shell-experience/command-line-string-limitation). +The ASCII fixture's displayed invocation crosses that limit before the change +and stays below it with a file. This is an inference from the documented limit; +the completed tasks ran on Linux, not through Windows Cmd.exe or `npx`. + +The caller-provided selection file already exists in this timed task; its +reading/validation is included. If an agent must create it first, the file +contents add an estimated **4,411 tokens**, plus an extra write call's framing +and execution costs, which are unmeasured. Do not treat the shorter invocation +as a whole-workflow token saving in that scenario. File creation, discovery, +prompts, rendering and client JSON parsing are outside the measured scope. + +No speedup is claimed: elapsed ranges are 19.142–19.841 s before and +18.277–21.384 s after; CPU ranges are 18.140–18.821 s and 17.278–21.026 s. +The ranges overlap and include a slower candidate run. Tokenx 1.3.0 estimates +stdout and invocation text; these are not billed/model token counts. + +Each task starts a fresh Node 24.22.1 CLI and JavaScript worker with CPU affinity +0,1. OS caches were not flushed and the host is shared. These are fresh-process +runs with uncontrolled warm filesystem caches; cold-disk and warm persistent +worker costs are unmeasured. CPU is waited-child user plus system time. RSS is +a 50 ms recursive process-family sample, which can double-count shared pages +and miss short-lived or unobserved descendants. Token estimation and equality +checks run outside timing. Later main changes are outside these pinned costs. + +## Final integration checks + +Integration with main `cd10ee1f8a8e124be4864e77750a16e76c561809` passes **58** +checks, extending the initial selection with active JSON-read cancellation, +process-level error status and existing route/delivery coverage. This takes +167.935 s, 182.796 s waited-child CPU and 2,056.1 MiB sampled family RSS; these +are suite costs rather than analyst-task costs. Full `npm run check`, +`npm run docs:check` and test discovery pass. These selections overlap the +earlier checks. Later upstream integrations remain outside the cost comparison. diff --git a/src/cli/coreBinaryCommands.ts b/src/cli/coreBinaryCommands.ts index 9268ecb436..9146bd59f6 100644 --- a/src/cli/coreBinaryCommands.ts +++ b/src/cli/coreBinaryCommands.ts @@ -18,7 +18,10 @@ import { snapshotSelectionOption, } from "./options.js"; import type { CliInstance } from "./types.js"; -import { runCliJavaScriptApplicationAnalysis } from "./javascriptApplicationAnalysis.js"; +import { + cliJavaScriptSourcePaths, + runCliJavaScriptApplicationAnalysis, +} from "./javascriptApplicationAnalysis.js"; import { withCommandCancellation } from "./commandCancellation.js"; import { analysisEvidenceDeliveryOptions, @@ -98,6 +101,7 @@ const registerOverviewCommands = ( options: overviewOptions.extend({ integrityPolicy: laneIntegrityPolicy, sourcePath: javascriptApplicationOptions.shape.sourcePath, + sourcePathsJson: javascriptApplicationOptions.shape.sourcePathsJson, }), run: async ({ args, options }) => { // Route JavaScript targets exactly like analyze-javascript-application, @@ -105,23 +109,32 @@ const registerOverviewCommands = ( // code is not replaced by the logged failure status. if (await routesToJavaScriptAnalysis(args.path, options)) return withCommandCancellation((signal) => - logCliCommand(logger, "analyze", () => - runCliJavaScriptApplicationAnalysis( + logCliCommand(logger, "analyze", async () => { + const sourcePaths = await cliJavaScriptSourcePaths( + options.sourcePath, + options.sourcePathsJson, + signal, + ); + if (!sourcePaths.ok) return sourcePaths.error; + return runCliJavaScriptApplicationAnalysis( { input_path: absoluteCallerPath(args.path), integrity_policy: options.integrityPolicy ?? defaultIntegrityPolicy, - ...(options.sourcePath === undefined + ...(sourcePaths.value === undefined ? {} - : { source_paths: options.sourcePath }), + : { source_paths: sourcePaths.value }), }, environment, signal, - ), - ), + ); + }), ); return logCliCommand(logger, "analyze", () => { - if (options.sourcePath !== undefined) + if ( + options.sourcePath !== undefined || + options.sourcePathsJson !== undefined + ) throw new AnalysisInputError( "analyze_javascript_application", undefined, diff --git a/src/cli/electronCommands.ts b/src/cli/electronCommands.ts index d395fcb864..4dd452e852 100644 --- a/src/cli/electronCommands.ts +++ b/src/cli/electronCommands.ts @@ -22,7 +22,10 @@ import { javascriptApplicationOptions, selectedCommandTimeout, } from "../cliObservationOptions.js"; -import { runCliJavaScriptApplicationAnalysis } from "./javascriptApplicationAnalysis.js"; +import { + cliJavaScriptSourcePaths, + runCliJavaScriptApplicationAnalysis, +} from "./javascriptApplicationAnalysis.js"; import { withCommandCancellation } from "./commandCancellation.js"; import { absoluteCallerPath } from "../filesystem/callerPath.js"; import { finishAnalysisCommand } from "./finishAnalysisCommand.js"; @@ -322,6 +325,12 @@ const registerJavaScriptApplicationCommand = ( options, ); if (!delivery.ok) return projectAnalysisError(delivery.error); + const sourcePaths = await cliJavaScriptSourcePaths( + options.sourcePath, + options.sourcePathsJson, + signal, + ); + if (!sourcePaths.ok) return sourcePaths.error; return runCliJavaScriptApplicationAnalysis( { // An empty path names no file; leave it for input validation. @@ -347,9 +356,9 @@ const registerJavaScriptApplicationCommand = ( max_semantic_unknowns_per_file: options.maxSemanticUnknownsPerFile, }), - ...(options.sourcePath === undefined + ...(sourcePaths.value === undefined ? {} - : { source_paths: options.sourcePath }), + : { source_paths: sourcePaths.value }), ...(options.maxHeapMb === undefined ? {} : { max_heap_mb: options.maxHeapMb }), diff --git a/src/cli/javascriptApplicationAnalysis.ts b/src/cli/javascriptApplicationAnalysis.ts index c0f717f0bf..df2b80e813 100644 --- a/src/cli/javascriptApplicationAnalysis.ts +++ b/src/cli/javascriptApplicationAnalysis.ts @@ -8,6 +8,46 @@ import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup. import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; +import { javaScriptSourcePathsSchema } from "../domain/javascript/javascriptSourceSelection.js"; +import { parseCliJsonInput } from "../cliJsonInput.js"; + +const OPERATION = "analyze_javascript_application"; + +/** + * Combine repeated `--source-path` values with `--source-paths-json`, an inline + * JSON array or a JSON file. A file keeps large selections below the host + * command-line limit; entries remain exact application-relative paths and are + * not resolved against the working directory. + */ +export const cliJavaScriptSourcePaths = async ( + sourcePath: readonly string[] | undefined, + sourcePathsJson: string | undefined, + signal?: AbortSignal, +): Promise< + | { readonly ok: true; readonly value: readonly string[] | undefined } + | { readonly ok: false; readonly error: JsonValue } +> => { + if (sourcePathsJson === undefined) return { ok: true, value: sourcePath }; + const input = await parseCliJsonInput(sourcePathsJson, OPERATION, signal); + if (!input.ok) return input; + const parsed = javaScriptSourcePathsSchema.safeParse(input.value); + if (!parsed.success) + return { + ok: false, + error: cliError( + analysisInputErrorFromIssues( + OPERATION, + parsed.error.issues.map((issue) => ({ + ...issue, + path: ["source_paths_json", ...issue.path], + })), + { source_paths_json: input.value }, + ), + ), + }; + return { ok: true, value: [...(sourcePath ?? []), ...parsed.data] }; +}; /** * One scope for every JavaScript analysis in this process. A healthy worker diff --git a/src/cliObservationOptions.ts b/src/cliObservationOptions.ts index e78df7403e..ad15af9146 100644 --- a/src/cliObservationOptions.ts +++ b/src/cliObservationOptions.ts @@ -88,6 +88,13 @@ export const javascriptApplicationOptions = z.object({ .describe( "Exact inventoried file path relative to the application root; repeat to select several. No implicit dependencies.", ), + sourcePathsJson: z + .string() + .min(1) + .optional() + .describe( + "Inline JSON array or JSON file path of exact inventoried file paths relative to the application root, selected after any --source-path values. Use a file for selections that exceed the host command-line limit.", + ), maxHeapMb: javaScriptAnalysisResourceControlsSchema.shape.max_heap_mb, analysisTimeoutMs: javaScriptAnalysisResourceControlsSchema.shape.analysis_timeout_ms, diff --git a/tests/acceptance/applications/javascriptSourceSelectionCli.test.ts b/tests/acceptance/applications/javascriptSourceSelectionCli.test.ts index 1e26196c37..9fc6f49f6f 100644 --- a/tests/acceptance/applications/javascriptSourceSelectionCli.test.ts +++ b/tests/acceptance/applications/javascriptSourceSelectionCli.test.ts @@ -182,6 +182,135 @@ cliTest( }, ); +cliTest.for([ + { command: "analyze", source: "file", format: "asar" }, + { + command: "analyze-javascript-application", + source: "inline", + format: "directory", + }, +])( + "appends $source --source-paths-json selections for $format through $command", + { timeout: 60_000 }, + async ({ command, source, format }, { cli }) => { + const directory = await createTestTempDirectory( + "rea-js-source-paths-json-", + ); + await writeJavaScriptSourceSelectionFixture(directory); + let target = directory; + if (format === "asar") { + target = join( + await createTestTempDirectory("rea-js-json-selected-asar-"), + "app.asar", + ); + await createPackageWithOptions(directory, target, {}); + } + const listed = ["main.mjs", "package.json"]; + let sourcePathsJson = JSON.stringify(listed); + if (source === "file") { + sourcePathsJson = join( + await createTestTempDirectory("rea-js-source-paths-list-"), + "paths.json", + ); + await writeFile(sourcePathsJson, JSON.stringify(listed)); + } + const response = await cli.run({ + arguments: [ + command, + target, + "--source-path", + "package.json", + "--source-paths-json", + sourcePathsJson, + "--format", + "json", + ], + }); + expect(response.exitCode).toBe(0); + const evidence = parseEvidence(response.json); + const result = javascriptApplicationAnalysisResultSchema.parse( + evidence.normalized_result, + ); + const requested = ["package.json", ...listed]; + expect(result.source_selection).toEqual({ + requested_paths: requested, + unselected_files: 4, + }); + expect(evidence.parameters.source_paths).toEqual(requested); + expect(result.statistics.parsed_javascript_files).toBe(1); + if (format === "asar") { + expect(evidence.subject).toMatchObject({ + local_path: target, + digest: { + sha256: createHash("sha256") + .update(await readFile(target)) + .digest("hex"), + }, + }); + } + }, +); + +cliTest.for([ + { label: "an object", value: '{"paths":["main.mjs"]}', path: [] }, + { label: "an empty array", value: "[]", path: [] }, + { label: "a blank entry", value: '["main.mjs",""]', path: [1] }, +])( + "rejects --source-paths-json holding $label", + async ({ value, path }, { cli }) => { + const directory = await createTestTempDirectory( + "rea-js-source-paths-json-invalid-", + ); + await writeJavaScriptSourceSelectionFixture(directory); + const response = await cli.run({ + arguments: [ + "analyze-javascript-application", + directory, + "--source-paths-json", + value, + "--format", + "json", + ], + }); + expect(response.exitCode).toBe(1); + expect(response.json).toMatchObject({ + code: "invalid_request", + details: { + issues: [{ path: ["source_paths_json", ...path] }], + }, + }); + }, +); + +cliTest( + "rejects --source-paths-json when an explicit provider chooses the native lane", + async ({ cli }) => { + const directory = await createTestTempDirectory( + "rea-js-source-paths-json-native-refusal-", + ); + await writeJavaScriptSourceSelectionFixture(directory); + const response = await cli.run({ + arguments: [ + "analyze", + directory, + "--provider", + "ghidra", + "--source-paths-json", + '["main.mjs"]', + "--format", + "json", + ], + }); + expect(response.exitCode).toBe(1); + expect(response.json).toMatchObject({ + code: "invalid_request", + details: { + issues: [{ path: ["source_paths"], reason: "invalid_value" }], + }, + }); + }, +); + cliTest( "accepts legal ASAR JSON whitespace while preserving selected-source identity", { timeout: 60_000 }, diff --git a/tests/boundary/filesystem/cliJsonInput.test.ts b/tests/boundary/filesystem/cliJsonInput.test.ts index d2bdcd5403..097811fcd5 100644 --- a/tests/boundary/filesystem/cliJsonInput.test.ts +++ b/tests/boundary/filesystem/cliJsonInput.test.ts @@ -7,6 +7,7 @@ import { describe, expect, it, onTestFinished } from "vitest"; import { createCli } from "../../../src/cli.js"; import { parseCliJsonInput } from "../../../src/cliJsonInput.js"; +import { cliJavaScriptSourcePaths } from "../../../src/cli/javascriptApplicationAnalysis.js"; import { readJsonInputFile } from "../../../src/application/JsonInputFile.js"; import { analysisCliErrorEnvelopeSchema } from "../../../src/contracts/errorSchemas.js"; import { JSON_BYTE_ORDER_MARK_MESSAGE } from "../../../src/application/Utf8JsonInput.js"; @@ -307,3 +308,41 @@ describe("CLI JSON input", () => { }, ); }); + +describe("CLI JSON file cancellation", () => { + const writeSelection = async (): Promise => { + const root = await createTestTempDirectory("rea-json-input-cancel-"); + const path = join(root, "paths.json"); + await writeFile(path, JSON.stringify(["main.js", "package.json"])); + return path; + }; + + it("reports a cancelled file read as cancellation, not a file failure", async () => { + const path = await writeSelection(); + const controller = new AbortController(); + controller.abort(); + expect( + await parseCliJsonInput(path, "test-input", controller.signal), + ).toMatchObject({ ok: false, error: { code: "cancelled" } }); + }); + + it("reports cancellation through --source-paths-json selection", async () => { + const path = await writeSelection(); + const controller = new AbortController(); + controller.abort(); + expect( + await cliJavaScriptSourcePaths(["index.js"], path, controller.signal), + ).toMatchObject({ ok: false, error: { code: "cancelled" } }); + }); + + it("still reads the selection when the signal is not aborted", async () => { + const path = await writeSelection(); + expect( + await cliJavaScriptSourcePaths( + ["index.js"], + path, + new AbortController().signal, + ), + ).toEqual({ ok: true, value: ["index.js", "main.js", "package.json"] }); + }); +});