diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index 27c3142aa2..531a8222fd 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -178,6 +178,10 @@ public final class ReaGhidraBridge extends HeadlessScript { private List procedureInventory; private List stringInventory; private DecompInterface decompiler; + /** Per-function native decompiler limit from the session; 0 leaves it unbounded. */ + private int decompileTimeoutSeconds; + // DecompileProcess converts seconds to an int millisecond timer delay. + private static final int MAX_DECOMPILE_TIMEOUT_SECONDS = Integer.MAX_VALUE / 1000; private final ReaGhidraExtensions analysisExtensions = new ReaGhidraExtensions(); private static AddressSpace sessionDefaultAddressSpace; @@ -189,6 +193,7 @@ public final class ReaGhidraBridge extends HeadlessScript { } Path descriptorPath = Path.of(arguments[0]); SessionDescriptor descriptor = readDescriptor(descriptorPath); + decompileTimeoutSeconds = descriptor.decompileTimeoutSeconds(); // Windows retains a native immutable bearer descriptor lease until // runtime_close; POSIX removes it after this read. Normal cleanup removes // the Windows file, while abrupt owner death can leave private residue. @@ -1557,7 +1562,8 @@ public final class ReaGhidraBridge extends HeadlessScript { } catch (RequestFailure failure) { // The listing, reference and CFG facts below do not depend on the // native decompiler. Cancellation still ends the request. - if (!failure.code.equals("decompile_failed")) throw failure; + if (!failure.code.equals("decompile_failed") && + !failure.code.equals("decompile_timed_out")) throw failure; decompilation = null; decompilerFailure = failure.getMessage().strip(); } @@ -3359,12 +3365,22 @@ public final class ReaGhidraBridge extends HeadlessScript { } DecompileResults results = decompiler.decompileFunction( function, - 0, + decompileTimeoutSeconds, monitor ); if (results.isCancelled()) { throw new RequestFailure("decompile_cancelled", "Ghidra decompilation was cancelled"); } + // DecompInterface disposes the native process at the limit and starts a + // fresh one for the next request, so the session stays usable. + if (results.isTimedOut()) { + throw new RequestFailure( + "decompile_timed_out", + "Ghidra decompilation of " + canonicalAddress(function.getEntryPoint()) + + " exceeded the " + decompileTimeoutSeconds + + "-second limit set by REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS" + ); + } if (!results.decompileCompleted()) { throw new RequestFailure( "decompile_failed", @@ -4714,10 +4730,11 @@ public final class ReaGhidraBridge extends HeadlessScript { }; } + // The host queue owns session lifetime, including concurrent cancellation. private static RequestFailure regexStackExhausted(String expression, String stage) { return new RequestFailure("regex_stack_exhausted", "Ghidra's Java regex engine exhausted its stack while " + stage + - " pattern " + expression + ". Use literal mode or simplify the regex; the session remains usable."); + " pattern " + expression + "."); } private static SessionDescriptor readDescriptor(Path path) throws IOException { @@ -4728,7 +4745,15 @@ public final class ReaGhidraBridge extends HeadlessScript { Set keys = new HashSet<>(DESCRIPTOR_KEYS); if (parsed.isJsonObject() && parsed.getAsJsonObject().has("analysis_extensions")) keys.add("analysis_extensions"); + if (parsed.isJsonObject() && parsed.getAsJsonObject().has("decompile_timeout_seconds")) + keys.add("decompile_timeout_seconds"); JsonObject object = requireObject(parsed, keys); + int decompileTimeout = 0; + if (object.has("decompile_timeout_seconds")) { + decompileTimeout = requireInteger(object, "decompile_timeout_seconds"); + if (decompileTimeout < 1 || decompileTimeout > MAX_DECOMPILE_TIMEOUT_SECONDS) + throw new IllegalArgumentException("REA session descriptor is invalid"); + } return new SessionDescriptor( requireString(object, "transport"), requireString(object, "endpoint_path"), @@ -4737,7 +4762,8 @@ public final class ReaGhidraBridge extends HeadlessScript { requireSha256(object, "target_sha256"), requireString(object, "provider_version"), requireString(object, "profile_digest"), - object.has("analysis_extensions") ? object.getAsJsonArray("analysis_extensions") : new JsonArray() + object.has("analysis_extensions") ? object.getAsJsonArray("analysis_extensions") : new JsonArray(), + decompileTimeout ); } @@ -4942,7 +4968,8 @@ public final class ReaGhidraBridge extends HeadlessScript { String targetSha256, String providerVersion, String profileDigest, - JsonArray analysisExtensions + JsonArray analysisExtensions, + int decompileTimeoutSeconds ) {} private record Request(int id, String method, JsonObject params) {} diff --git a/docs/client-configuration.md b/docs/client-configuration.md index e33bcb7763..1395e2bc9b 100644 --- a/docs/client-configuration.md +++ b/docs/client-configuration.md @@ -182,8 +182,16 @@ startup; an explicit Ghidra startup timeout of 2700000 ms produces a 2820000 ms tool timeout. The startup allowance follows an explicitly selected Ghidra or Hopper provider, and otherwise uses the larger known Ghidra/Hopper deadline. An explicit `REA_JEB_MCP_TIMEOUT_MS` deadline, shared by catalog discovery and -each tools/call, also contributes to that maximum. For `ida` or `auto` selection, a valid IDA registration's parsed +each tools/call, also contributes to that maximum. An explicit `REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS` is added to the Ghidra startup allowance, because a single-decompile call can import the target and then decompile up to that limit; setup forwards it into the registration. For `ida` or `auto` selection, a valid IDA registration's parsed `timeoutMs` also contributes to that maximum; its default is 300000 ms. +The derived value is a single-decompile estimate, not a deadline for an +unbounded batch. For example, startup 330 seconds plus two 600-second +decompiles and 120 seconds of headroom needs an estimated 1650 seconds, +before any additional listing or output work. Select the client tool deadline +for the intended workload, split `batch_decompile`, use `decompile_all` +limit/resume, or reduce `trace_native_values` `max_functions`. The trace can +decompile several functions serially; its depth and graph budgets do not +make the single-decompile allowance a whole-traversal deadline. Existing positive caller deadlines are preserved. Setup forwards these explicitly selected REA deadlines and the input schema profile into the registration; unrelated process values are omitted. An explicit `REA_JADX_OPERATION_TIMEOUT_MS` is forwarded with the JADX @@ -336,7 +344,9 @@ not enforce Codex's 30 seconds as the doctor deadline. The 120000 ms tool-operation headroom and the 64 KiB doctor stderr cap stay fixed. Setup preserves positive caller-selected timeout values in the REA entry, and doctor reports a non-blocking advisory when a tool timeout is shorter than the known provider allowance. An explicitly selected -JEB request budget also contributes to the allowance. A configured IDA +JEB request budget also contributes to the allowance, and an explicit +`REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS` is added to the Ghidra startup +allowance. A configured IDA registration's parsed `timeoutMs` contributes for `ida` or `auto` selection. An explicit `REA_JADX_OPERATION_TIMEOUT_MS` is forwarded with the JADX registration but does not contribute; raise `tool_timeout_sec` to cover a diff --git a/docs/ghidra-provider.md b/docs/ghidra-provider.md index 92ff7385e7..d162054d91 100644 --- a/docs/ghidra-provider.md +++ b/docs/ghidra-provider.md @@ -175,7 +175,7 @@ profile stages before retrying their retained descriptors. | Concern | Ghidra contract | | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Decompiler lifetime | One persistent `DecompInterface` is opened for the imported Program and disposed during bridge shutdown. Native decompilation has no fixed per-function deadline. External functions or functions without bodies return `null`; cancellation and native failure remain distinct. | +| Decompiler lifetime | One persistent `DecompInterface` is opened for the imported Program and disposed during bridge shutdown. Native decompilation has no fixed per-function deadline unless `REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS` sets one; a decompile that reaches it fails only that request with `remote_code` `decompile_timed_out`, the Program and session stay open, and the next decompile uses a fresh decompiler process. External functions or functions without bodies return `null`; cancellation and native failure remain distinct. | | Serialization | A FIFO sends one Program request at a time. Caller cancellation removes queued work promptly and is passed to active socket waits. This is an adapter safety commitment, not a claim that every Ghidra API is thread-safe. | | Function identity | Every function result carries the entry address and Ghidra FunctionManager classification for external, thunk, and resolved thunk target. These are observations, not proof that unresolved targetless calls have been recovered. | | Assembly and pseudocode | Assembly is complete Ghidra Listing text; pseudocode is Ghidra decompiler output. Neither is original source, and cross-provider comparison never treats Hopper and Ghidra text as equal or unequal semantic facts. | diff --git a/docs/installation.md b/docs/installation.md index d14e266d54..915efa7fc6 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1180,7 +1180,28 @@ One session contains exactly one imported Program; use `provider_id: "ghidra"`, One persistent decompiler is owned by the Program, and a serial queue keeps Ghidra API calls on the owning Program thread without a fixed queue length. Operations run until a result, caller cancellation, or provider shutdown; there -is no fixed per-operation or response-size ceiling. Unresolved computed calls +is no fixed per-operation or response-size ceiling. Cancelling an operation that +is already running stops the Ghidra session, so one function whose +decompilation runs past the MCP client's deadline costs the whole import. +`REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS` (an integer from 1 to 2,147,483; absent by +default, which leaves decompilation unbounded) bounds each native decompile +instead. This maximum keeps Ghidra’s signed 32-bit seconds-to-milliseconds +conversion from overflowing. The client must still support a deadline covering +startup, that limit and operation headroom. Ghidra stops that one decompile and starts a fresh decompiler process +for the next request. `analyze_function` still returns the function's assembly, +references and basic blocks, with pseudocode, `native_api` and +`native_value_flow` unavailable for the reason "exceeded the N-second limit", +and that dossier is not saved to analysis snapshots. `procedure_pseudo_code` +fails with `provider_timeout` and that message; other Ghidra tools stay +available. If the caller also cancels while the timeout settles, the session +still closes and the message says so. Session reuse concerns an active MCP or +library session; a standalone CLI command still closes its own session when it +finishes. The limit is read when a Ghidra session +starts, and Codex/Kimi registrations derived by `rea setup` add it to the tool +deadline as a single-decompile estimate. A bulk call can need that limit for +every attempted function plus listing and file work; select the client deadline +for the workload, split batches, use `decompile_all` limit/resume, or reduce +`trace_native_values` `max_functions` to select fewer serial decompiles. Ghidra's own interactive default is 30 seconds. Unresolved computed calls remain unknown, reference-kind provenance is preserved, and provider-specific pseudocode is never treated as original source or Hopper-equivalent text. diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 2089e32064..b0246cf697 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -935,6 +935,13 @@ deadlines remain unchanged. Doctor reports an informational recommendation when a configured deadline is shorter than the known provider allowance. An explicit JEB upstream deadline (`REA_JEB_MCP_TIMEOUT_MS`, shared by catalog discovery and each tools/call) also contributes to that allowance. +An explicit `REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS` is added to the Ghidra startup +allowance as a single-decompile estimate. A multi-function call needs a client +deadline covering every attempted decompile plus other work; the static +registration value does not bound an arbitrary batch or `trace_native_values` +traversal. Split batches, use `decompile_all` limit/resume, or reduce the trace +`max_functions` to select work for the whole-call deadline. Existing caller +deadlines remain unchanged. For an already connected client using the pinned SDK, request options are the **second** argument of `callTool`: diff --git a/docs/user-visible-errors.csv b/docs/user-visible-errors.csv index 517277124a..c69819462f 100644 --- a/docs/user-visible-errors.csv +++ b/docs/user-visible-errors.csv @@ -116,5 +116,6 @@ Then rerun this installer.",implemented,verified by shell branch inspection 90,install.sh:version mismatch,Installed rea reports a different version,installed version ACTUAL does not match EXPECTED.,Low risk; gives exact integrity mismatch,Keep current copy,unchanged,verified by exact installer test 90.5,@modelcontextprotocol/server:tool input validation,Caller sends arguments that violate a declared tool schema,"Canonical invalid_request with the SDK diagnostic plus recovery.kind=inspect_input_contract and a bounded instruction to read the advertised schema or call tools/list",Keeps field/type correction while telling the agent how to recover; diagnostics over 4 KiB stay on the existing retained-error path,Normalize ordinary validation failures and retain oversized diagnostics,verified by real in-memory and stdio MCP client/server calls with wrong argument types 90.6,incur:unknown CLI command,Caller enters an unknown REA command,"'COMMAND' is not a command for 'rea'. Suggested command: rea --help",Low risk; identifies input and exact discovery action,Keep framework-generated copy,unchanged,verified by real packaged dispatcher invocation +90.7,src/ghidra/GhidraProviderClient.ts:projectPrimarySessionError,One procedure's decompile reaches REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS,"Ghidra decompilation of {entry} exceeded the {N}-second limit set by REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS. Only this procedure's decompilation was stopped; the Ghidra session remains usable.",Low risk; names the setting and keeps the session,Keep current copy,implemented,provider projection test and real-provider lane 91,src/application/binaryDiagnostics/ByteEntropyService.ts,Invalid file or window,"Expected a regular file.; Selected file could not be found.; Selected byte window starts beyond the end of the file.; Selected byte window extends beyond the end of the file.; File size exceeds exact numeric offset representation.",Invalid selection cannot produce complete entropy Evidence,Choose a readable regular file and a window within its byte size,implemented,entropy service and MCP boundary tests 92,src/domain/analysisView/byteEntropyView.ts,Non-summary entropy view,Byte entropy Evidence supports the summary view; retrieve parent Evidence for the complete block series.,Unsupported view selection,Use the summary view or retrieve complete parent Evidence,implemented,typed input rejection diff --git a/docs/verification/ghidra-procedure-timeout-2026-10-23.md b/docs/verification/ghidra-procedure-timeout-2026-10-23.md new file mode 100644 index 0000000000..f8ba7318a0 --- /dev/null +++ b/docs/verification/ghidra-procedure-timeout-2026-10-23.md @@ -0,0 +1,269 @@ +# Ghidra per-procedure deadline verification — 2026-10-23 + +The completed MCP task attempts pseudocode for a complex function under an +approximately one-second budget, then obtains pseudocode for a small function. +Two rounds use one client connection. Before the feature, the caller cancels +its slow request; afterwards Ghidra enforces the configured native decompile +limit and returns a typed timeout while keeping the session. + +## Source and artifact + +Baseline main: `9e6c6241d7e595c6f3e90d23f30f3d43bc8c9527`. +Original PR head: `cb754c9258de3bd6a2c9da1bf71da63085a49288`. +The maintenance integration preserves main's centralized startup and environment +owners. The measured feature source is `5a10db4b2`; the natural timeout probe +ran on `043110b85bc5e1af9f3c7de2856733885c67046c`, before the subsequent recovery +wording and client deadline guidance changes. + +The generated Linux x86-64 ELF shared library has 3000 volatile-state branches, +contains `rea_slow` and `rea_small`, and is 174,840 bytes. +Source SHA-256: `f99074876faae8e18e45265dab007b6fcc396fe909d57efab72adc8925a960c7`. +Binary SHA-256: `9c49d2eef9c2083f6f66f38de4ed0ee1bac17ca5b73dc8f3a90d9ba62a8d1d5d`. +The benchmark regenerated the verifier's C recipe and checked both hashes +against the original natural probe. Compilation used existing `cc -shared +-fPIC -O0`. The target was never executed, and its bytes remained unchanged. +Fixture preparation is separate from the analysis task (1.215 s elapsed, +1.127 s waited CPU); both variants analyze the same prepared artifact. + +Both variants used existing Ghidra 12.1.4, Temurin JDK 21.0.12.1+1, Node +24.22.1 and the locked dependencies. The upstream analysis-profile digest is +`ae80ece8c83379eddabbdf379bbffca5da84c7df7fa7c159f45fa810468f85d9`. + +## Real behavior and identity + +The natural provider probe used the authenticated loopback TCP bridge. With +`REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS=1`, `analyze_function` returned a live +partial dossier with 45,007 assembly lines and +9,001 basic blocks. Pseudocode, native API and native value flow were +unavailable. A separate pseudocode request returned `provider_timeout`, with +canonical `retryable: true` and the setting/limit in its explanation. The next +small function decompiled successfully. Fresh authenticated pings before and +after retained the same owner-generated per-launch run ID; this probe does +not accept a caller-fixed launch ID. Cleanup and unchanged target bytes were +verified. The probe took 56.832 s elapsed, 86.017 s waited CPU and 1426.7 MiB +sampled family RSS. + +The public CLI command `rea function slow-decompile.so --procedure 0x1010f9 +--provider ghidra --format json` also exited zero with the same 45,007 assembly +lines and 9,001 basic blocks, and explicit unavailable decompiler facets. +It preserved the target digest and completed in 61.776 s elapsed, 89.656 s +waited CPU, with 2169.9 MiB sampled family RSS. The complete JSON was +25,929,463 bytes; this is a full-evidence CLI output, without a compact-output +claim. A standalone CLI invocation closes its own session when done. + +The public MCP benchmark used production `scripts/rea.mjs mcp` and the pinned +SDK. The baseline's caller deadline armed after the request was written. The +native limit starts inside Ghidra's decompiler, so these are approximate, +phase-specific one-second budgets rather than identical timer locations. +Both slow attempts failed under their budget. The following small pseudocode +was identical across variants and both rounds (64 UTF-8 bytes, SHA-256 +`c947ff22cab1ffef9b962b12d82e12d0e5f5e2a6c5634a2ed5ea3c353e1d5807`). + +Before the feature, the next request automatically re-imported the target; +the driver did not force close/open when that request succeeded. Two JVM +changes were observed. Afterwards both rounds retained one JVM. These +observations combine descendant membership with the existing process-start +identity primitive; PID and executable path alone are not used as identity +proof or authority to signal processes. + +## Completed-task measurements + +| Scope | Baseline | Feature | +| ------------------------------------- | ---------: | ---------: | +| Cold bounded attempt + small function | 76.207 s | 1.540 s | +| Warm bounded attempt + small function | 73.275 s | 1.380 s | +| Full verifier elapsed | 237.602 s | 95.388 s | +| Full waited process-family CPU | 232.779 s | 90.945 s | +| Sampled family peak RSS | 2051.3 MiB | 2008.8 MiB | +| Full task estimated tokens | 983,317 | 989,913 | + +The full task includes catalog discovery, open/inventory, both rounds and +close. Both variants made eight recorded SDK calls, including tools/list. +The baseline also emitted two cancellation notifications. All 221 tools remain; +only the descriptions of `batch_decompile` and `decompile_all` changed, to +explain serialization and whole-call deadlines. Input/output schemas stayed +identical. Catalog estimates were 974,158 → 974,222 tokens. + +Estimated context increased **0.67%**. The timeout now returns canonical +provider diagnostics, including captured startup stdout/stderr, instead of +only a serialized SDK cancellation error. The current shared diagnostic +projection remains a context cost. These measurements establish session reuse +and avoided re-imports, rather than token reduction. + +Counts use tokenx 1.3.0 on actual tool arguments and complete SDK results, +including textual/structured projections, the observed serialized cancellation +error and actual cancellation notifications. They are heuristic estimates, +not model-specific BPE or actual model usage. Handshake framing, prompts, +reasoning and subsequent evidence inspection are unmeasured. + +Cold/warm denote task rounds inside one client. Baseline cancellation loses +provider reuse. Filesystem/JDK caches were not flushed and the baseline ran +first. Initial open/inventory was 84.615 → 89.631 s. +Host load, compilation caches and runtime startup vary; this single-artifact +pair does not establish a general latency, CPU or RSS speedup. Full metrics +include SDK, token estimation and passive identity verification overhead. + +Runs were restricted to two CPU cores, a 2 GiB Node heap and a 1 GiB configured +Java heap, with 6 GiB sampled family RSS, 300 sampled CPU seconds and 600 elapsed +seconds as verifier stop conditions. CPU is waited child usage for the complete +run. RSS is sampled every 50 ms, can double-count shared pages and can miss +short peaks. These guards are not kernel memory limits. Both completed runs +exited zero without a resource stop. Detailed phase samples use independent +timers and can differ slightly from the outer peak. + +## Bounds, caller deadlines and lifecycle + +The positive whole-second maximum is 2,147,483. This follows Ghidra 12.1.4's +signed integer seconds-to-milliseconds conversion, rather than an arbitrary +day cap. [Pinned upstream implementation](https://github.com/NationalSecurityAgency/ghidra/blob/Ghidra_12.1.4_build/Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/DecompileProcess.java#L562). +Configuration tests cover the last safe value and the first overflowing value. +A real MCP export/resume with that maximum succeeded on `/usr/bin/sha256sum`: +two files matched the baseline bytes, one verified file was skipped on resume, +and the target digest stayed unchanged. This validates acceptance and fast +operations under the maximum; it does not test a multi-day decompile or every +client's timer capacity. + +The cap applies per native decompile. Static registration defaults can only +estimate startup + one decompile + headroom. Batch cardinality and other work +are unknown at configuration time; a finite static value cannot guarantee an +arbitrary bulk call. Doctor and both bulk-tool descriptions state that scope. +A two-item 600-second example with 330-second startup and 120-second headroom +needs an estimated 1650 seconds, plus any additional work. Existing positive +client deadlines, whether shorter or longer, remain caller choices. Tests +verify advice, unchanged 10/1800-second Codex selections and no file rewrite +or backup on an aligned registration. Work can be split, or selected with +`decompile_all` limit/resume. Caller cancellation still closes the session. + +The queue retains the observed remote failure as primary when cancellation +also closes the session. One message owner interprets that fact across native +timeouts, ordinary decompiler failures and regex resource errors. Unconfirmed +cleanup does not promise immediate restart. The original projection fails five +of six new regression cases; all six pass after repair. Cleanup-owner and +cancellation-reason preservation are covered through the production queue. + +## Main integration + +Main `bcd18d289845051b0f0513e11bc1aefd3f766902` was merged for admission after +the paired measurements. The two conflicts preserve both the upstream +single-active-provider-read batch description and the new deadline guidance, +and both registration-test imports. The report's measured source remains +pinned above; this integration is not a repeat of the performance pair. + +The first complete `npm run check` on this integration stopped at formatting +errors in six files, all byte-identical to that exact main: third-party +notices, JavaScript artifact graph findings, reconstruction journey findings, +IDA doctor source/CLI tests, and native UI MCP tests. Blob identities are +retained in the raw receipt. This full gate is not reported as passing. + +## Review follow-up verification + +The Java regex error factory now reports the engine failure, stage and caller +pattern without promising a session lifetime or prescribing recovery. The +provider's existing queue-aware continuation owner adds session state for +healthy, concurrently cancelled and cleanup-incomplete outcomes. The boundary +matrix now uses actual producer wording for all three failure families and +covers nine state combinations. + +The timeout remediation distinguishes new Codex/Kimi registrations from +existing positive client deadlines, which setup preserves. It directs callers +to edit an existing deadline explicitly. + +- Updated source/boundary tests: 235 passed; two unchanged main SIGKILL wording + assertions still fail as reproduced on the exact base. +- Full `npm run check` passed on the review follow-up integrated with main + `ba9c5bc6867272d02d2fcc9d5c7c2ea83f234067`. +- The real Ghidra producer probe passed 41 checks with complete cleanup, + including the real regex factory's selector/stage and fact-only wording. + This invokes the actual factory and is not a natural regex stack-overflow + reproduction. The whole probe took 42.416 s elapsed, 55.326 s waited CPU and + 788.5 MiB sampled family RSS. +- Negative controls retained old production source at + `b4e105a48c381f1f1f86e277f771edf98e37ea6a` with updated verification inputs. + The deadline assertion fails on the old hint, and the corrected real Java + control fails specifically at + `regex_failure_leaves_lifecycle_to_provider`. + The initial producer-control harness did not compile and is excluded from + regression proof; the final control's assertion and complete diagnostics + are retained separately. + +The completed-task timing/context pair above was not repeated after these +wording changes or subsequent main integration. Those later costs are +unmeasured; the earlier estimates remain tied to their pinned source. + +## Capability and traversal guidance follow-up + +The static `analyze_function` capability previously described only completion +or caller cancellation. Its producing descriptor now names the optional +per-function setting and describes closure requests, leaving observed session +state to the queue-aware error owner. Every function response receives this +same capability; consumers do not reinterpret the setting. + +The deadline advisory now covers multi-function workflows in general, +including `trace_native_values`. The trace's tool description and client, +installation and MCP guides explain that serial decompiles and other work need +a whole-call allowance; reducing `max_functions` selects less work without +changing existing positive client deadlines. + +At source `7caac1fbc5833f145d9cc0f87efcb6ebf8c2efd6`, including main +`0785223582e5ecef4150e8bb736b9c6857703699`: + +- The full `npm run check` passed after correcting one formatter mismatch in + the advisory. The first attempt stopped at that mismatch and is retained. +- All 42 deadline/registration tests passed. Old production with updated test + inputs fails both Ghidra traversal-advice cases; the Hopper control passes. + Its built capability also fails the new setting-advertisement assertion. +- With the setting explicitly selected as one second, the real producer lane + passed its 41 Java checks and verified that the real healthy provider's + function limitations carry the new control policy. Cleanup completed. + The complete lane took 32.263 s elapsed, 48.337 s waited CPU and 785.3 MiB + sampled family RSS. Degraded behavior in this lane uses failure injection; + this is not a fresh natural large-function timeout run. +- `npm run docs:check` passed. + +The completed-task performance pair remains tied to its earlier pinned source; +these later capability/catalog wording costs were not measured by that pair. + +## Verifier fixture budget + +The optional branch-count override is bounded before source allocation by a +1 MiB generated-source budget and a conservative maximum line size, including +the longest safe-integer decimal representation. The resulting maximum is +11,272 branches; this bounds the line array, joined string and compiler input, +not a claim about Ghidra capacity or compilation memory. Existing command +supervision still bounds compilation time. + +A direct generator check admitted 1, 3,000, 6,000 and 11,272 branches and rejected +zero, negative, fractional, billion-branch and maximum-safe-integer requests +before generation. The maximum generated 854,215 bytes. The 3,000-branch source +remains byte-identical to the measured fixture (SHA-256 `f99074876faae8e18e45265dab007b6fcc396fe909d57efab72adc8925a960c7`). +These checks do not repeat the real-provider measurements above. + +## Validation and limits + +- The initial selected source/boundary run passed 310 tests with 10 platform + skips. After the related cancellation changes, all 174 provider/queue cases + passed. Overlapping executions are not summed. +- Client deadline scope and registration tests passed 41 cases. +- The final pre-refresh source/boundary run passed 351 tests with 10 platform + skips. The schema suite initially could not load its missing generated test + metadata; after generation, all 15 SDK schema cases passed separately. +- The real Java producer probe passed 39 checks, including injected timeout + state, unchanged listing facets, live snapshot policy, timeout explanations + and recovery. This is injected failure verification on the real engine, + separate from the natural timeout and public MCP observations. +- The natural Ghidra probe, public MCP paired task and real maximum-bound + export/resume passed as described above. +- An additional real MCP batch probe requested the same slow entry twice + under a sufficient explicit 600-second client deadline. Both items returned + their own provider_timeout result; a following small function succeeded in + the same observed JVM. The batch phase took 2.976 s. This covers two + requested items, not two distinct slow functions or large-batch capacity. + The separate full probe took 102.887 s elapsed and 145.300 s waited CPU. + +Real-provider coverage is Linux x86-64 Ghidra 12.1.4. Windows, macOS, other +providers, universal performance and very large batch capacity remain +unverified by these runs. The original author's macOS report is separate +from this maintenance verification. Raw exchanges, complete error diagnostics, +process observations, scripts and resource receipts remain in +`/tmp/rea-maintenance-20261023/5662/`; generated binaries and provider projects +are not committed. diff --git a/knip.json b/knip.json index 74f7c04613..9eb05c02dc 100644 --- a/knip.json +++ b/knip.json @@ -16,6 +16,7 @@ "scripts/verify/termux/browser.mjs", "scripts/verify-real-ghidra-noreturn.mjs", "scripts/verify-real-ghidra-decompiler-failure.mjs", + "scripts/verify-real-ghidra-decompile-timeout.mjs", "tests/processOwnershipRunner.ts" ], "project": ["src/**/*.ts", "tests/**/*.ts", "scripts/**/*.mjs"], diff --git a/scripts/verify-real-ghidra-decompile-timeout.mjs b/scripts/verify-real-ghidra-decompile-timeout.mjs new file mode 100644 index 0000000000..822600a136 --- /dev/null +++ b/scripts/verify-real-ghidra-decompile-timeout.mjs @@ -0,0 +1,275 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js"; +import { parseConfig } from "../dist/config/parseConfig.js"; +import { projectAnalysisError } from "../dist/domain/analysisErrorProjection.js"; +import { GhidraClient } from "../dist/ghidra/GhidraClient.js"; +import { inspectVerifierGhidraInstallation } from "./lib/ghidra-installation-inspection.mjs"; +import { SUPPORTED_GHIDRA_VERSION } from "../dist/ghidra/GhidraReleasePolicy.js"; +import { GhidraProvider } from "../dist/ghidra/GhidraProvider.js"; +import { silentLogger } from "../dist/logger.js"; +import { + OwnedCommandFailure, + runOwnedCommand, +} from "../dist/process/OwnedCommand.js"; +import { + assertVerifierCleanupComplete, + completeVerifierRun, + createVerifierRun, +} from "./lib/verifier-run.mjs"; + +// A natural slow decompile: one function with many independent branches on +// volatile state. Ghidra decompiles it correctly but takes far longer than the +// configured limit, so the limit, not an injected result, stops it. +const LIMIT_SECONDS = 1; +// This verifier budgets one MiB for its generated C source. Bound the number +// of lines before allocating them, including the largest admitted integer's +// decimal representation; the joined source and line array remain bounded. +const FIXTURE_SOURCE_BYTES = 1024 * 1024; +const FIXTURE_PREFIX = [ + "volatile unsigned rea_state[64];", + "__attribute__((noinline, used)) unsigned rea_slow(unsigned x) {", + " unsigned a = x;", +]; +const FIXTURE_SUFFIX = [ + " return a;", + "}", + "__attribute__((noinline, used)) int rea_small(int v) {", + " return v + (int)rea_state[0];", + "}", + "", +]; +const MAX_BRANCH_LINE_BYTES = Buffer.byteLength( + ` if (rea_state[63] & 1073741824u) a = a * 31u + ${Number.MAX_SAFE_INTEGER}u; else a ^= rea_state[63];\n`, +); +const FIXTURE_FIXED_BYTES = Buffer.byteLength( + [...FIXTURE_PREFIX, ...FIXTURE_SUFFIX].join("\n"), +); +const MAX_BRANCHES = Math.floor( + (FIXTURE_SOURCE_BYTES - FIXTURE_FIXED_BYTES) / MAX_BRANCH_LINE_BYTES, +); +const BRANCHES = Number(process.env.REA_DECOMPILE_TIMEOUT_BRANCHES ?? 6000); +assert.ok( + Number.isSafeInteger(BRANCHES) && BRANCHES > 0 && BRANCHES <= MAX_BRANCHES, + `REA_DECOMPILE_TIMEOUT_BRANCHES must be an integer from 1 to ${MAX_BRANCHES} (generated source budget: ${FIXTURE_SOURCE_BYTES} bytes)`, +); + +const run = createVerifierRun(); +assert.ok( + process.platform === "linux" || process.platform === "darwin", + "This regression lane requires Linux or macOS", +); +const environment = { + ...process.env, + REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: String(LIMIT_SECONDS), +}; +const config = parseConfig(environment); +if (!config.ok) throw config.error; +assert.equal(config.value.ghidraDecompileTimeoutSeconds, LIMIT_SECONDS); +const installation = await inspectVerifierGhidraInstallation({ + environment, + installDir: config.value.ghidraInstallDir, + javaHome: config.value.ghidraJavaHome, +}); +assert.equal(installation.status, "available", JSON.stringify(installation)); +assert.equal(installation.providerVersion, SUPPORTED_GHIDRA_VERSION); + +const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); +const fixtureSource = () => { + const lines = [...FIXTURE_PREFIX]; + for (let index = 0; index < BRANCHES; index += 1) + lines.push( + ` if (rea_state[${index % 64}] & ${(1 << (index % 31)) >>> 0}u) a = a * 31u + ${index}u; else a ^= rea_state[${(index * 7) % 64}];`, + ); + lines.push(...FIXTURE_SUFFIX); + const source = lines.join("\n"); + assert.ok(Buffer.byteLength(source) <= FIXTURE_SOURCE_BYTES); + return source; +}; + +const workspace = await mkdtemp( + join(tmpdir(), "rea-ghidra-decompile-timeout-"), +); +let cleaned = true; +let report; +try { + const sourcePath = join(workspace, "slow-decompile.c"); + const source = fixtureSource(); + await writeFile(sourcePath, source); + const targetPath = join( + workspace, + process.platform === "darwin" + ? "slow-decompile.dylib" + : "slow-decompile.so", + ); + const compiler = { + command: process.env.REA_CC ?? "cc", + arguments: [ + ...(process.platform === "darwin" ? ["-dynamiclib"] : ["-shared"]), + "-fPIC", + "-O0", + sourcePath, + "-o", + targetPath, + ], + }; + cleaned = false; + try { + await runOwnedCommand( + { ...compiler, runId: run.run_id, expectedCommand: null }, + { timeoutMs: 300_000, diagnosticBytes: 1024 * 1024 }, + ); + cleaned = true; + } catch (cause) { + cleaned = + cause instanceof OwnedCommandFailure && cause.cleanupOwner === undefined; + throw cause; + } + const targetDigest = digest(await readFile(targetPath)); + + // One real provider session over the read-only authenticated TCP bridge. + let bridgeClient; + const provider = new GhidraProvider( + config.value, + silentLogger, + environment, + undefined, + (options) => { + // This verifier has no caller-selected run ID: each new launch receives + // a fresh owner-generated identity, so equality detects session restarts. + assert.equal(options.runId, undefined); + bridgeClient = new GhidraClient({ + ...options, + transport: "authenticated-loopback-tcp", + }); + return bridgeClient; + }, + ); + const target = await parseBinaryTarget(targetPath); + if (!target.ok) throw target.error; + const profile = await provider.resolveAnalysisProfile(target.value); + if (!profile.ok) throw profile.error; + assert.ok(profile.value.profile); + const client = provider.createClient(target.value, profile.value.profile); + cleaned = false; + const failures = []; + const observed = {}; + try { + // Mach-O symbols carry a leading underscore; address the functions by + // their entry instead of guessing the recovered name. + const listed = await client.execute("list_procedures", {}); + if (!listed.ok) throw listed.error; + const entry = (name) => { + const found = listed.value.result.find( + (item) => item.value === name || item.value === `_${name}`, + ); + assert.ok(found, `Missing fixture function ${name}`); + return found.address; + }; + const slowEntry = entry("rea_slow"); + const smallEntry = entry("rea_small"); + assert.ok(bridgeClient); + const initialSession = await bridgeClient.ping(); + if (!initialSession.ok) throw initialSession.error; + assert.equal(initialSession.value.target.sha256, targetDigest); + observed.initial_session_run_id = initialSession.value.run_id; + const limitText = `exceeded the ${String(LIMIT_SECONDS)}-second limit set by REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS`; + let started = performance.now(); + const slow = await client.execute("analyze_function", { + procedure: slowEntry, + }); + observed.analyze_seconds = (performance.now() - started) / 1000; + if (!slow.ok) throw slow.error; + const dossier = slow.value.result; + assert.equal(dossier.facet_availability.pseudocode.available, false); + assert.ok( + dossier.facet_availability.pseudocode.reason.endsWith(limitText), + dossier.facet_availability.pseudocode.reason, + ); + assert.equal(dossier.native_api.available, false); + assert.equal(dossier.native_value_flow.available, false); + assert.ok(dossier.assembly.length > BRANCHES); + assert.ok(dossier.basic_blocks.length > 0); + assert.equal(slow.value.cachePolicy, "live"); + observed.degraded = { + pseudocode: dossier.facet_availability.pseudocode, + assembly_lines: dossier.assembly.length, + basic_blocks: dossier.basic_blocks.length, + cache_policy: slow.value.cachePolicy, + }; + + started = performance.now(); + const pseudocode = await client.execute("procedure_pseudo_code", { + procedure: slowEntry, + }); + observed.pseudocode_seconds = (performance.now() - started) / 1000; + assert.equal(pseudocode.ok, false); + const projected = projectAnalysisError(pseudocode.error); + assert.equal(pseudocode.error._tag, "AnalysisTimeoutError"); + assert.equal(projected.code, "provider_timeout"); + assert.equal(projected.retryable, true); + assert.equal(projected.details.timeout_ms, LIMIT_SECONDS * 1000); + assert.ok(projected.message.includes(limitText), projected.message); + assert.ok( + projected.remediation.action.includes( + "REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS", + ), + ); + observed.pseudocode_error = { + code: projected.code, + retryable: projected.retryable, + message: projected.message, + }; + + // The same session decompiles the next function with a fresh decompiler. + const small = await client.execute("procedure_pseudo_code", { + procedure: smallEntry, + }); + if (!small.ok) throw small.error; + assert.ok(String(small.value.result).includes("rea_state")); + const continuedSession = await bridgeClient.ping(); + if (!continuedSession.ok) throw continuedSession.error; + assert.equal(continuedSession.value.run_id, initialSession.value.run_id); + assert.equal(continuedSession.value.target.sha256, targetDigest); + observed.final_session_run_id = continuedSession.value.run_id; + observed.same_session_next_decompile = "succeeded"; + } catch (cause) { + failures.push(cause); + } + const closed = await client.close(); + if (!closed.ok) failures.push(closed.error); + if (failures.length > 0) + throw new AggregateError(failures, "Decompile timeout verification failed"); + assert.equal(digest(await readFile(targetPath)), targetDigest); + const verifierRun = await completeVerifierRun(run); + try { + assertVerifierCleanupComplete(verifierRun.process_lineage); + } catch (cause) { + throw new Error( + `Verifier cleanup unresolved; retained ${workspace}; run ${run.run_id}`, + { cause }, + ); + } + cleaned = true; + report = { + status: "passed", + limit_seconds: LIMIT_SECONDS, + branches: BRANCHES, + target_sha256: targetDigest, + source_sha256: digest(Buffer.from(source)), + compiler, + provider: profile.value.profile.provider, + analysis_profile_digest: profile.value.profile.digest, + ...observed, + verifier_run: verifierRun, + scope: + "Real Ghidra ELF or Mach-O analysis of a generated slow function through the read-only TCP provider: the decompile limit stops it, analyze_function degrades, procedure_pseudo_code names the setting, and the same session decompiles the next function. No target execution; Windows is unverified.", + }; +} finally { + if (cleaned) await rm(workspace, { recursive: true, force: true }); +} +console.log(JSON.stringify({ ...report, cleanup: "complete" })); diff --git a/scripts/verify-real-ghidra-decompiler-failure.mjs b/scripts/verify-real-ghidra-decompiler-failure.mjs index e6455fa113..c9c6c23ad1 100644 --- a/scripts/verify-real-ghidra-decompiler-failure.mjs +++ b/scripts/verify-real-ghidra-decompiler-failure.mjs @@ -142,6 +142,8 @@ try { assert.equal(probe.target_sha256, targetDigest); assert.equal(probe["ReaGhidraBridge.java"], bridgeDigest); for (const check of [ + "regex_failure_preserves_stage_and_pattern", + "regex_failure_leaves_lifecycle_to_provider", "pseudocode_unavailable_with_failure", "native_api_unavailable_with_failure", "native_value_flow_unavailable_with_failure", @@ -153,6 +155,11 @@ try { "failure_limitation_named", "pseudocode_request_still_fails", "cancellation_still_ends_request", + "timed_out_decompile_is_live", + "timed_out_pseudocode_names_limit", + "timed_out_native_api_names_limit", + "timed_out_listing_unchanged", + "timed_out_pseudocode_request_names_limit", "session_recovers_after_failure", ]) assert.ok(probe.checks.includes(check), check); @@ -221,7 +228,24 @@ try { assert.ok( !result.value.limitations.some((item) => item.includes(injected)), ); + const functionLimitCapability = result.value.limitations.find((item) => + item.includes("REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS"), + ); + assert.ok(functionLimitCapability?.includes("per-function limit")); + assert.ok( + functionLimitCapability.includes( + "caller cancellation requests session closure", + ), + ); + assert.ok( + !result.value.limitations.some((item) => + item.includes( + "lets each decompilation run until it completes or the caller cancels", + ), + ), + ); healthy = { + function_limit_capability: functionLimitCapability, pseudocode_available: true, native_api_available: value.native_api.available, }; diff --git a/src/application/ClientRegistrationEnvironment.ts b/src/application/ClientRegistrationEnvironment.ts index c5f4920d0a..ddd11b406d 100644 --- a/src/application/ClientRegistrationEnvironment.ts +++ b/src/application/ClientRegistrationEnvironment.ts @@ -37,6 +37,7 @@ interface ResolvedClientRegistrationEnvironmentDetails extends ClientRegistratio const TIMEOUT_INPUT_SETTINGS: ReadonlySet = new Set([ "REA_ANALYSIS_PROVIDER", + "REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS", "REA_GHIDRA_STARTUP_TIMEOUT_MS", "REA_HOPPER_STARTUP_TIMEOUT_MS", "REA_JEB_MCP_TIMEOUT_MS", @@ -212,6 +213,12 @@ const environmentDetails = ( : { analysisProvider: configuration.REA_ANALYSIS_PROVIDER, ghidraStartupTimeoutMs: configuration.REA_GHIDRA_STARTUP_TIMEOUT_MS, + ...(configuration.REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS === undefined + ? {} + : { + ghidraDecompileTimeoutMs: + configuration.REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS * 1000, + }), hopperStartupTimeoutMs: configuration.REA_HOPPER_STARTUP_TIMEOUT_MS, ...(configuration.REA_JEB_MCP_TIMEOUT_MS === undefined ? {} diff --git a/src/application/ClientRegistrationStatus.ts b/src/application/ClientRegistrationStatus.ts index 5b1d7c6058..4aa7eb8c03 100644 --- a/src/application/ClientRegistrationStatus.ts +++ b/src/application/ClientRegistrationStatus.ts @@ -492,7 +492,13 @@ const timeoutAdvisory = ( const isKimi = client.name === "kimi"; // A bigint is beyond the safe Number range, so rounding cannot cross the allowance. const toolTimeoutMs = Number(toolTimeout) * (isKimi ? 1 : 1000); - if (toolTimeoutMs >= providerAllowance) return undefined; + const batchScope = + timeoutInputs.ghidraDecompileTimeoutMs !== undefined && + (timeoutInputs.analysisProvider === "ghidra" || + timeoutInputs.analysisProvider === "auto") + ? "The derived Ghidra deadline is a single-decompile estimate. For multi-function workflows such as batch_decompile, decompile_all or trace_native_values, allow startup plus every attempted function's limit and other work; split calls, reduce trace_native_values max_functions, or select a larger client deadline when needed. Explicit caller deadlines remain unchanged." + : undefined; + if (toolTimeoutMs >= providerAllowance) return batchScope; const setting = isKimi ? "toolTimeoutMs" : "tool_timeout_sec"; const currentValue = isKimi ? `${String(toolTimeout)} ms` @@ -503,7 +509,10 @@ const timeoutAdvisory = ( const headroom = isKimi ? `${String(MCP_STARTUP_POLICY.toolOperationHeadroomMs)} ms` : `${String(MCP_STARTUP_POLICY.toolOperationHeadroomMs / 1000)} seconds`; - return `The configured ${setting} (${currentValue}) is shorter than REA's known provider allowance (${String(providerAllowance)} ms), so provider startup or an upstream request may exceed this client deadline. REA recommends ${setting} = ${recommendedValue} to cover that allowance plus ${headroom} of operation headroom; this additional headroom is a recommendation, not a guarantee for every analysis.`; + const recommendation = `The configured ${setting} (${currentValue}) is shorter than REA's known provider allowance (${String(providerAllowance)} ms), so provider startup or an upstream request may exceed this client deadline. REA recommends ${setting} = ${recommendedValue} to cover that allowance plus ${headroom} of operation headroom; this additional headroom is a recommendation, not a guarantee for every analysis.`; + return batchScope === undefined + ? recommendation + : `${recommendation} ${batchScope}`; }; const packageVersionRemediation = ( diff --git a/src/application/SetupRegistrationEnvironment.ts b/src/application/SetupRegistrationEnvironment.ts index 7b5ed4e7df..d0cf289c3c 100644 --- a/src/application/SetupRegistrationEnvironment.ts +++ b/src/application/SetupRegistrationEnvironment.ts @@ -5,6 +5,7 @@ const SELECTED_REGISTRATION_SETTINGS = [ "REA_ANALYSIS_PROVIDER", "REA_BRIDGE_MAX_REQUEST_BYTES", "REA_CAPTURE_DECODER_OLD_GENERATION_MIB", + "REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS", "REA_GHIDRA_STARTUP_TIMEOUT_MS", "REA_HOPPER_STARTUP_TIMEOUT_MS", "REA_JEB_MCP_TIMEOUT_MS", diff --git a/src/config.test.ts b/src/config.test.ts index 1c2017017a..a4261a5faf 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -487,3 +487,46 @@ describe("default macOS Hopper launcher", () => { }); }); }); + +describe("Ghidra decompile limit configuration", () => { + it.each([ + [undefined, undefined], + ["1", 1], + ["30", 30], + ["86400", 86_400], + ["86401", 86_401], + ["2147483", 2_147_483], + ] as const)( + "reads the Ghidra decompile limit %j as %j seconds", + (value, expected) => { + const result = parseConfig( + value === undefined + ? {} + : { REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: value }, + ); + if (!result.ok) throw result.error; + expect(result.value.ghidraDecompileTimeoutSeconds).toBe(expected); + }, + ); + + it.each(["", "0", "-1", "1.5", "30s", "2147484", "2147483647"])( + "rejects malformed Ghidra decompile limit %j with its setting", + (value) => { + const result = parseConfig({ + REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: value, + }); + if (result.ok) throw new Error("Expected invalid decompile limit"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "configuration_invalid", + details: { + settings: [ + { + setting: "REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS", + constraint: expect.any(String), + }, + ], + }, + }); + }, + ); +}); diff --git a/src/config/environment.ts b/src/config/environment.ts index 5a1adc8361..aed0a19f9f 100644 --- a/src/config/environment.ts +++ b/src/config/environment.ts @@ -13,6 +13,7 @@ import { DEFAULT_GHIDRA_STARTUP_TIMEOUT_MS, DEFAULT_HOPPER_STARTUP_TIMEOUT_MS, DEFAULT_SWIFTC_COMPILE_TIMEOUT_MS, + ghidraDecompileTimeoutSchema, providerStartupTimeoutSchema, } from "./providerStartupTimeout.js"; import { mcpInputSchemaProfileSchema } from "./mcpInputSchemaProfile.js"; @@ -77,6 +78,7 @@ const environmentSchema = z.object({ REA_SWIFTC_COMPILE_TIMEOUT_MS: providerStartupTimeoutSchema.default( DEFAULT_SWIFTC_COMPILE_TIMEOUT_MS, ), + REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: ghidraDecompileTimeoutSchema.optional(), REA_GHIDRA_LANGUAGE_ID: ghidraLanguageIdSchema.optional(), REA_GHIDRA_COMPILER_SPEC_ID: ghidraCompilerSpecIdSchema.optional(), REA_GHIDRA_DISABLED_ANALYZERS: ghidraDisabledAnalyzersSchema.optional(), diff --git a/src/config/parseConfig.ts b/src/config/parseConfig.ts index 067d09c7a7..60235532b7 100644 --- a/src/config/parseConfig.ts +++ b/src/config/parseConfig.ts @@ -83,6 +83,12 @@ export const parseConfig = ( ghidraInstallDir: env.GHIDRA_INSTALL_DIR, ghidraJavaHome: env.JAVA_HOME, ghidraStartupTimeoutMs: env.REA_GHIDRA_STARTUP_TIMEOUT_MS, + ...(env.REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS === undefined + ? {} + : { + ghidraDecompileTimeoutSeconds: + env.REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS, + }), ...(env.REA_GHIDRA_LANGUAGE_ID === undefined ? {} : { diff --git a/src/config/providerStartupTimeout.ts b/src/config/providerStartupTimeout.ts index 6b8051b416..603bd9e776 100644 --- a/src/config/providerStartupTimeout.ts +++ b/src/config/providerStartupTimeout.ts @@ -16,6 +16,21 @@ export const DEFAULT_SWIFTC_COMPILE_TIMEOUT_MS = 120_000; */ export const MAX_PROVIDER_STARTUP_TIMEOUT_MS = 2_147_483_647; +/** + * Ghidra 12.1.4 converts whole seconds to a signed int millisecond delay. + * Keep that multiplication positive and exact rather than imposing a day cap. + */ +const MAX_GHIDRA_DECOMPILE_TIMEOUT_SECONDS = Math.floor( + MAX_PROVIDER_STARTUP_TIMEOUT_MS / 1000, +); + +/** Parse REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS; absent keeps Ghidra decompiles unbounded. */ +export const ghidraDecompileTimeoutSchema = z + .string() + .regex(/^[1-9]\d*$/u, "must be a positive decimal integer") + .transform(Number) + .pipe(z.number().int().min(1).max(MAX_GHIDRA_DECOMPILE_TIMEOUT_SECONDS)); + /** Parse the optional setting without reading the ambient process environment. */ export const providerStartupTimeoutSchema = z .string() diff --git a/src/config/types.ts b/src/config/types.ts index a977fb1b3d..5f41299c18 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -18,6 +18,8 @@ export interface AppConfig { readonly ghidraInstallDir: string | undefined; readonly ghidraJavaHome: string | undefined; readonly ghidraStartupTimeoutMs: number; + /** Per-function native decompiler limit; absent leaves decompiles unbounded. */ + readonly ghidraDecompileTimeoutSeconds?: number; readonly ghidraLanguageOverride?: GhidraLanguageOverride; readonly ghidraDisabledAnalyzers?: readonly string[]; readonly ghidraSeedFile?: string; diff --git a/src/contracts/enhancedToolContracts.ts b/src/contracts/enhancedToolContracts.ts index 4d835b1061..e1b19e7ff8 100644 --- a/src/contracts/enhancedToolContracts.ts +++ b/src/contracts/enhancedToolContracts.ts @@ -43,12 +43,12 @@ export const ENHANCED_TOOL_CONTRACTS = [ ), enhanced( "batch_decompile", - "Decompile each explicit procedure symbol or address, with one provider read active per batch. Preserves each input selector in address and reports the observed canonical entry and entry label in procedure; identity or label failures remain explicit without discarding usable pseudocode. Returns ordered per-item success or error results and aggregate counts. The complete reply retains every pseudocode body; use selected small batches to bound retained output.", + "Decompile each explicit procedure symbol or address, with one provider read active per batch. Preserves each input selector in address and reports the observed canonical entry and entry label in procedure; identity or label failures remain explicit without discarding usable pseudocode. Returns ordered per-item success or error results and aggregate counts. The complete reply retains every pseudocode body; use selected small batches to bound retained output. A per-function limit is not a batch deadline; allow for every item or split calls.", enhancedInputSchemas.batch_decompile, ), enhanced( "decompile_all", - "Write provider pseudocode for every analyzed function, or a filtered subset, into a caller-selected directory and return only a manifest. Each file records the provider function name, canonical address, inclusive body ranges when procedure_info supplies them, and that function's procedure_info and pseudocode limitations. Each function's index record keeps its complete limitations. Evidence keeps at most eight first-observed distinct per-function limitations within an encoded-text budget; the omitted count measures observations, not distinct warnings. The manifest reports counts, the output directory, the index path, at most 32 failure details within 2,048 JSON-encoded UTF-8 bytes (names can be omitted; full names remain in the index), provider identity, and the analysis-profile digest. Pseudocode stays in the files. An existing directory is refused unless resume is set and the journal matches the target SHA-256, provider, and analysis profile; verified files are skipped and conflicting files are left unchanged. layout dwarf reports that compilation-unit paths are unavailable and names files by address. A failure of one function does not stop the others. Functions procedure_info classifies as external are counted as external, get no file, and are not decompiled; a resume skips them. Cancellation leaves files already written.", + "Write provider pseudocode for every analyzed function, or a filtered subset, into a caller-selected directory and return only a manifest. Each file records the provider function name, canonical address, inclusive body ranges when procedure_info supplies them, and that function's procedure_info and pseudocode limitations. Each function's index record keeps its complete limitations. Evidence keeps at most eight first-observed distinct per-function limitations within an encoded-text budget; the omitted count measures observations, not distinct warnings. The manifest reports counts, the output directory, the index path, at most 32 failure details within 2,048 JSON-encoded UTF-8 bytes (names can be omitted; full names remain in the index), provider identity, and the analysis-profile digest. Pseudocode stays in the files. An existing directory is refused unless resume is set and the journal matches the target SHA-256, provider, and analysis profile; verified files are skipped and conflicting files are left unchanged. layout dwarf reports that compilation-unit paths are unavailable and names files by address. A failure of one function does not stop the others. Functions procedure_info classifies as external are counted as external, get no file, and are not decompiled; a resume skips them. Cancellation leaves files already written. Per-function limits do not bound the whole call; use limit and resume to select work that fits the client deadline, including import, listing and file work.", enhancedInputSchemas.decompile_all, ), enhanced( @@ -89,7 +89,7 @@ export const ENHANCED_TOOL_CONTRACTS = [ ), enhanced( "trace_native_values", - "Trace a bounded static dependency graph from one explicit native procedure. Includes high-p-code def-use, constants, operators, memory and branch operations, resolved call edges, and derived argument/parameter and return/output bindings. Missing bindings, alias semantics, persistent state and RNG roles remain unknown. Budgets bound depth, decompilations, graph size and payloads; nodes are paginated.", + "Trace a bounded static dependency graph from one explicit native procedure. Includes high-p-code def-use, constants, operators, memory and branch operations, resolved call edges, and derived argument/parameter and return/output bindings. Missing bindings, alias semantics, persistent state and RNG roles remain unknown. Budgets bound depth, decompilations, graph size and payloads; nodes are paginated. A Ghidra per-function timeout does not bound the whole traversal; allow startup, every attempted function and other work, or reduce max_functions.", enhancedInputSchemas.trace_native_values, ), ] as const satisfies readonly ToolContract[]; diff --git a/src/ghidra/GhidraLauncher.ts b/src/ghidra/GhidraLauncher.ts index d716530df7..277676bcc5 100644 --- a/src/ghidra/GhidraLauncher.ts +++ b/src/ghidra/GhidraLauncher.ts @@ -226,6 +226,8 @@ export interface GhidraHeadlessLauncherOptions { readonly languageOverride?: GhidraLanguageOverride; /** Auto-analyzers disabled before default auto-analysis, as configured. */ readonly disabledAnalyzers?: readonly string[]; + /** Per-function native decompiler limit passed to the bridge; absent is unbounded. */ + readonly decompileTimeoutSeconds?: number; /** Caller seeds committed by the analysis profile, applied before analysis. */ readonly analysisSeeds?: GhidraSeedCommitment; readonly analysisExtensions?: readonly GhidraExtension[]; @@ -340,6 +342,11 @@ export class GhidraHeadlessLauncher implements GhidraLauncher { target_sha256: session.targetSha256, provider_version: session.providerVersion, profile_digest: session.profileDigest, + ...(this.options.decompileTimeoutSeconds === undefined + ? {} + : { + decompile_timeout_seconds: this.options.decompileTimeoutSeconds, + }), ...(extensions.length === 0 ? {} : { analysis_extensions: extensions }), diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index c2520d6362..0914b5cec2 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -86,8 +86,9 @@ const installationHost = ( const provider = ( host = installationHost(), clientFactory?: GhidraProviderClientFactory, + environment: Readonly> = {}, ): GhidraProvider => { - const config = parseConfig({ GHIDRA_INSTALL_DIR: INSTALL }); + const config = parseConfig({ GHIDRA_INSTALL_DIR: INSTALL, ...environment }); if (!config.ok) throw config.error; return new GhidraProvider( config.value, @@ -1365,6 +1366,169 @@ describe("Ghidra decompiler failure projection", () => { expect(projected.remediation.action).not.toContain("rea doctor"); } }); + + it("reports one procedure's decompile limit as a request timeout", async () => { + const code = "decompile_timed_out"; + const message = + "Ghidra decompilation of 0x401000 exceeded the 5-second limit set by REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS"; + const ghidra = provider( + installationHost(), + () => ({ + start: () => Promise.resolve(ok(sessionInfo())), + callTool: () => + Promise.resolve( + err( + new GhidraSessionError( + "remote", + message, + { remote_code: code }, + { remoteCode: code }, + ), + ), + ), + close: () => Promise.resolve(ok(null)), + }), + { REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: "5" }, + ); + const client = await createElfClient(ghidra); + const result = await client.execute("procedure_pseudo_code", { + procedure: "main", + }); + // Not a ProviderAdapterError: BinarySession keeps other Ghidra + // operations available after a request-level timeout. + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisTimeoutError", timeoutMs: 5000 }, + }); + if (!result.ok) { + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + code: "provider_timeout", + retryable: true, + message: `${message}. Only this procedure's decompilation was stopped; the Ghidra session remains usable.`, + recovery: { kind: "inspect_timeout_state", timeout_ms: 5000 }, + details: { diagnostics: { remote_code: code } }, + }); + const action = projected.remediation.action; + expect(action).toContain("analyze_function"); + expect(action).toContain("REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS"); + expect(action).toContain("tool-call deadline"); + expect(action).toContain("new Codex/Kimi registrations"); + expect(action).toContain("preserves existing positive values"); + expect(action).toContain("edit existing deadlines yourself"); + expect(action).not.toContain("rea doctor"); + } + }); +}); + +describe("Ghidra failures during cancellation cleanup", () => { + it.each( + [ + { + remoteCode: "decompile_failed", + projectedCode: "execution_failure", + message: "Ghidra decompilation failed: Injected backend failure", + }, + { + remoteCode: "decompile_timed_out", + projectedCode: "provider_timeout", + message: + "Ghidra decompilation of 0x401000 exceeded the 5-second limit set by REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS", + }, + { + remoteCode: "regex_stack_exhausted", + projectedCode: "resource_constraint", + message: + "Ghidra's Java regex engine exhausted its stack while matching pattern (a+)+$.", + }, + ].flatMap((entry) => [ + { ...entry, cancelled: false, cleanupFailed: false }, + { ...entry, cancelled: true, cleanupFailed: false }, + { ...entry, cancelled: true, cleanupFailed: true }, + ]), + )( + "reports observed session state for $remoteCode (cancelled: $cancelled, cleanup failed: $cleanupFailed)", + async ({ + remoteCode, + projectedCode, + message, + cancelled, + cleanupFailed, + }) => { + const cleanup = new ProviderCleanupError("ghidra", ["owned session"], { + reason: "Fixture cleanup still needs retry", + }); + const observed = new GhidraSessionError( + "remote", + message, + { remote_code: remoteCode }, + { remoteCode }, + ); + const remote = cancelled + ? observed.withConcurrentCancellation({ reason: "client deadline" }) + : observed; + const ghidra = provider( + installationHost(), + () => ({ + start: () => Promise.resolve(ok(sessionInfo())), + callTool: () => + Promise.resolve( + err(cleanupFailed ? remote.withCleanupFailure(cleanup) : remote), + ), + close: () => Promise.resolve(ok(null)), + }), + { REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: "5" }, + ); + const client = await createElfClient(ghidra); + const result = + remoteCode === "regex_stack_exhausted" + ? await client.execute("search_procedures", { pattern: "(a+)+$" }) + : await client.execute("procedure_pseudo_code", { + procedure: "main", + }); + if (result.ok) throw new Error("Expected the observed remote failure"); + const projected = projectAnalysisError(result.error); + expect(projected.code).toBe( + cleanupFailed ? "cleanup_incomplete" : projectedCode, + ); + expect(projected.message).toContain(message); + if (!cancelled) { + expect(projected.message).toContain( + remoteCode === "regex_stack_exhausted" + ? "remain available" + : "remains usable", + ); + expect(projected.remediation.action).not.toContain("remain available"); + return; + } + const explanation = JSON.stringify(projected); + expect(explanation).not.toContain("remains usable"); + expect(explanation).not.toContain("remain available"); + expect(explanation).toContain( + cleanupFailed + ? "retry retained cleanup" + : "REA closed the Ghidra session", + ); + if (cleanupFailed) { + expect(explanation).not.toContain( + "the next request starts a new session", + ); + expect(result.error).toMatchObject({ cleanupIncomplete: true }); + } + expect(projected.details).toMatchObject({ + diagnostics: cleanupFailed + ? { + primary_error: { + code: projectedCode, + details: { + diagnostics: { session_closed_by_cancellation: true }, + }, + }, + } + : { session_closed_by_cancellation: true }, + }); + }, + ); }); describe("Ghidra PE CodeView debug symbol limitations", () => { diff --git a/src/ghidra/GhidraProviderCapabilities.ts b/src/ghidra/GhidraProviderCapabilities.ts index cc21d1396c..7eba124dba 100644 --- a/src/ghidra/GhidraProviderCapabilities.ts +++ b/src/ghidra/GhidraProviderCapabilities.ts @@ -181,7 +181,7 @@ export const limitationsFor = (operation: string): readonly string[] => { "The dossier combines Ghidra FunctionManager, Listing, ReferenceManager, BasicBlockModel, and decompiler observations; provider-specific pseudocode and assembly are not cross-provider text invariants.", "Resolved reference metadata identifies computed, indirect, external, call, jump, and data edges; unresolved targetless flows remain unknown, and function classifications distinguish thunks and externals.", "Synthetic Ghidra entry-point references without actionable memory sources are omitted.", - "The Java bridge serializes one function request per Program and lets each decompilation run until it completes or the caller cancels.", + "The Java bridge serializes one function request per Program. Decompilation runs until completion or caller cancellation unless REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS sets a per-function limit. That limit stops the requested decompile without requesting session closure; caller cancellation requests session closure.", ]; default: return common; diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index af79efa4a4..a3d7a01096 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -386,6 +386,9 @@ export const createGhidraProviderClient = (input: { ...(config.ghidraDisabledAnalyzers === undefined ? {} : { disabledAnalyzers: config.ghidraDisabledAnalyzers }), + ...(config.ghidraDecompileTimeoutSeconds === undefined + ? {} + : { decompileTimeoutSeconds: config.ghidraDecompileTimeoutSeconds }), ...(seeds === undefined ? {} : { analysisSeeds: seeds }), platform: installation.platform, ...(extensions.length === 0 ? {} : { analysisExtensions: extensions }), @@ -506,7 +509,7 @@ export const createGhidraProviderClient = (input: { projectSessionError( operation, started.error, - config.ghidraStartupTimeoutMs, + config, lostProviderHealth(), ), ); @@ -535,11 +538,7 @@ export const createGhidraProviderClient = (input: { targetPath: target.path, readSnapshot: client.readTargetSnapshot?.bind(client), projectFailure: (operation, failure) => - projectSessionError( - operation, - failure, - config.ghidraStartupTimeoutMs, - ), + projectSessionError(operation, failure, config), }) : undefined; const projectExecution = ( @@ -566,7 +565,7 @@ export const createGhidraProviderClient = (input: { projectSessionError( operation, called.error, - config.ghidraStartupTimeoutMs, + config, lostProviderHealth(), ), ); @@ -606,12 +605,7 @@ export const createGhidraProviderClient = (input: { measured: observed, client, mapSessionError: (failure) => - projectSessionError( - operation, - failure, - config.ghidraStartupTimeoutMs, - lostProviderHealth(), - ), + projectSessionError(operation, failure, config, lostProviderHealth()), nativeAotResult: derived, signal, }); @@ -678,7 +672,7 @@ export const createGhidraProviderClient = (input: { projectSessionError( operation, called.error, - config.ghidraStartupTimeoutMs, + config, lostProviderHealth(), ), ); @@ -710,7 +704,7 @@ export const createGhidraProviderClient = (input: { projectSessionError( operation, started.error, - config.ghidraStartupTimeoutMs, + config, lostProviderHealth(), ), ); @@ -859,16 +853,22 @@ const ghidraClientPrerequisites = ( }); }; +/** Configured deadlines that Ghidra failure projections name. */ +type GhidraProjectionLimits = Pick< + AppConfig, + "ghidraStartupTimeoutMs" | "ghidraDecompileTimeoutSeconds" +>; + const projectSessionError = ( operation: AnalysisOperation, failure: GhidraSessionError, - startupTimeoutMs: number, + limits: GhidraProjectionLimits, health?: ProviderOperationHealth, ): AnalysisError => { const primary = projectPrimarySessionError( operation, failure, - startupTimeoutMs, + limits, health, ); return failure.cleanupFailure === undefined @@ -883,9 +883,10 @@ const projectSessionError = ( const projectPrimarySessionError = ( operation: AnalysisOperation, failure: GhidraSessionError, - startupTimeoutMs: number, + limits: GhidraProjectionLimits, health?: ProviderOperationHealth, ): AnalysisError => { + const startupTimeoutMs = limits.ghidraStartupTimeoutMs; const context = { cause: failure, diagnostics: failure.diagnostics, @@ -930,10 +931,34 @@ const projectPrimarySessionError = ( if (failure.kind === "remote" && failure.remoteCode === "decompile_failed") return new ProviderAdapterError("ghidra", operation, { ...context, - userMessage: `${failure.message.trim().replace(/\.$/u, "")}. Only the decompiler failed for this procedure; the Ghidra session remains usable.`, + userMessage: `${failure.message.trim().replace(/\.$/u, "")}. ${ghidraSessionContinuationMessage( + failure, + "Only the decompiler failed for this procedure; the Ghidra session remains usable.", + )}`, remediationAction: "Call analyze_function (CLI: `rea function`) for the same procedure. It returns assembly, callers, callees, references and basic blocks from the Ghidra listing and records this decompiler failure under facet_availability.pseudocode. Repeating the unchanged decompile request is not expected to succeed.", }); + // A request-level deadline, not a provider failure: other operations stay + // available. Only promise session reuse when no caller cancellation closed + // the session while the timeout settled. + if ( + failure.kind === "remote" && + failure.remoteCode === "decompile_timed_out" && + limits.ghidraDecompileTimeoutSeconds !== undefined + ) + return new AnalysisTimeoutError( + operation, + limits.ghidraDecompileTimeoutSeconds * 1000, + { + ...context, + userMessage: `${failure.message.trim().replace(/\.$/u, "")}. ${ghidraSessionContinuationMessage( + failure, + "Only this procedure's decompilation was stopped; the Ghidra session remains usable.", + )}`, + remediationAction: + "Use analyze_function (CLI: `rea function`) for listing facts. Raise REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS and restart REA. Set the MCP tool-call deadline to cover startup and the full call; earlier cancellation closes the session. `rea setup` estimates deadlines for new Codex/Kimi registrations and preserves existing positive values; edit existing deadlines yourself.", + }, + ); if (failure.kind === "analysis_timeout") return new AnalysisTimeoutError( operation, @@ -947,12 +972,15 @@ const projectPrimarySessionError = ( return new AnalysisResourceConstraintError( operation, "memory", - failure.message, + `${failure.message.trim().replace(/\.$/u, "")}. ${ghidraSessionContinuationMessage( + failure, + "The active analysis session and annotations remain available.", + )}`, null, { ...context, remediationAction: - "Retry this search in literal mode or simplify the regex. The active analysis session and annotations remain available.", + "Retry this search in literal mode or simplify the regex.", }, ); if ( @@ -1136,6 +1164,17 @@ const projectObservedJvmMemoryFailure = ( ); }; +/** Interpret the queue's observed cancellation and cleanup outcome once. */ +const ghidraSessionContinuationMessage = ( + failure: GhidraSessionError, + healthyMessage: string, +): string => { + if (failure.concurrentCancellation === undefined) return healthyMessage; + return failure.cleanupFailure === undefined + ? "The request was also cancelled before the failure settled, so REA closed the Ghidra session; the next request starts a new session and repeats import and analysis." + : "The request was also cancelled, and Ghidra session shutdown left cleanup unconfirmed. Close the target to retry retained cleanup before starting another Ghidra session."; +}; + const memoryCgroupSchema = z.object({ path: z.string(), oom_kill_before: z.number().int().nonnegative(), diff --git a/src/ghidra/GhidraRequestQueue.test.ts b/src/ghidra/GhidraRequestQueue.test.ts index a8b470c3c1..585dc3f08e 100644 --- a/src/ghidra/GhidraRequestQueue.test.ts +++ b/src/ghidra/GhidraRequestQueue.test.ts @@ -254,6 +254,54 @@ it("joins a completed response with cancellation cleanup before reporting cancel expect(stop).toHaveBeenCalledTimes(1); }); +it.each([false, true])( + "records the session close on a late remote failure (cleanup failed: %s)", + async (cleanupFailed) => { + const execution = deferred(); + const original = new GhidraSessionError( + "remote", + "decompile limit reached", + { remote_code: "decompile_timed_out" }, + { remoteCode: "decompile_timed_out" }, + ); + const controller = new AbortController(); + const reason = new Error("client deadline"); + const cleanupFailure = new ProviderCleanupError( + "ghidra", + ["owned session"], + { + reason: "Fixture cleanup needs retry", + }, + ); + const queue = new GhidraRequestQueue( + () => execution.promise, + failure, + () => Promise.resolve(cleanupFailed ? err(cleanupFailure) : ok(null)), + ); + const pending = queue.run("active", {}, { signal: controller.signal }); + controller.abort(reason); + execution.resolve(err(original)); + const result = await pending; + if (result.ok) throw new Error("Expected the remote failure"); + // The observed failure stays primary; only the session close is added. + expect(result.error).toMatchObject({ + kind: "remote", + remoteCode: "decompile_timed_out", + message: "decompile limit reached", + concurrentCancellation: { reason }, + diagnostics: { + remote_code: "decompile_timed_out", + session_closed_by_cancellation: true, + }, + }); + expect(result.error.cleanupFailure).toBe( + cleanupFailed ? cleanupFailure : undefined, + ); + expect(result.error.cancellation).toBeUndefined(); + expect(original.concurrentCancellation).toBeUndefined(); + }, +); + it("awaits active cancellation cleanup when execution rejects", async () => { const execution = deferred(); const cleanup = deferred(); diff --git a/src/ghidra/GhidraRequestQueue.ts b/src/ghidra/GhidraRequestQueue.ts index eca4b6924f..94c5940a05 100644 --- a/src/ghidra/GhidraRequestQueue.ts +++ b/src/ghidra/GhidraRequestQueue.ts @@ -160,7 +160,15 @@ export class GhidraRequestQueue { ); } if (cancellation === undefined) return result; - const primary = result.ok ? cancellation.failure : result.error; + // An observed bridge reply stays primary, but it must carry the session + // close so its projection does not promise that the session is usable. + const primary = result.ok + ? cancellation.failure + : result.error.kind === "remote" + ? result.error.withConcurrentCancellation({ + reason: entry.options.signal?.reason, + }) + : result.error; const cleanup = await cancellation.cleanup; if (cleanup.kind === "rejected") return err( diff --git a/src/ghidra/GhidraSessionError.ts b/src/ghidra/GhidraSessionError.ts index acc89a9218..aac58d6f56 100644 --- a/src/ghidra/GhidraSessionError.ts +++ b/src/ghidra/GhidraSessionError.ts @@ -39,6 +39,8 @@ export interface GhidraSessionErrorOptions extends ErrorOptions { readonly timeoutMs?: number; readonly remoteCode?: string; readonly cleanupFailure?: AnalysisError; + /** Caller cancellation that closed the session while this failure settled. */ + readonly concurrentCancellation?: { readonly reason: unknown }; } /** Provider-owned failure with bounded, token-redacted local diagnostics. */ @@ -49,6 +51,7 @@ export class GhidraSessionError extends Error { readonly cleanupFailure: AnalysisError | undefined; readonly cancellation: { readonly reason: unknown } | undefined; readonly partialObservation: AnalysisPartialObservation | undefined; + readonly concurrentCancellation: { readonly reason: unknown } | undefined; constructor( readonly kind: GhidraSessionFailureKind, @@ -64,6 +67,7 @@ export class GhidraSessionError extends Error { this.cleanupFailure = options.cleanupFailure; this.cancellation = options.cancellation; this.partialObservation = options.partialObservation; + this.concurrentCancellation = options.concurrentCancellation; } /** Refresh diagnostics without discarding owned failure or cleanup facts. */ @@ -101,11 +105,28 @@ export class GhidraSessionError extends Error { ); } + /** + * Keep this failure primary while recording that caller cancellation closed + * the session before it settled, so projections do not promise reuse. + */ + withConcurrentCancellation(cancellation: { + readonly reason: unknown; + }): GhidraSessionError { + return this.#copy( + { ...this.diagnostics, session_closed_by_cancellation: true }, + this.cleanupFailure, + this.message, + this.partialObservation, + cancellation, + ); + } + #copy( diagnostics: Readonly>, cleanupFailure: AnalysisError | undefined, message = this.message, partialObservation = this.partialObservation, + concurrentCancellation = this.concurrentCancellation, ): GhidraSessionError { return new GhidraSessionError(this.kind, message, diagnostics, { ...(Object.hasOwn(this, "cause") ? { cause: this.cause } : {}), @@ -116,6 +137,9 @@ export class GhidraSessionError extends Error { ...(this.remoteCode === undefined ? {} : { remoteCode: this.remoteCode }), ...(cleanupFailure === undefined ? {} : { cleanupFailure }), ...(partialObservation === undefined ? {} : { partialObservation }), + ...(concurrentCancellation === undefined + ? {} + : { concurrentCancellation }), }); } } diff --git a/src/mcpStartupPolicy.test.ts b/src/mcpStartupPolicy.test.ts new file mode 100644 index 0000000000..6f310cf7d2 --- /dev/null +++ b/src/mcpStartupPolicy.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; + +import { parseClientConfiguration } from "./application/ClientConfigurationDocument.js"; +import { clientRegistrationEnvironmentDetails } from "./application/ClientRegistrationEnvironment.js"; +import { explicitSetupRegistrationEnvironment } from "./application/SetupRegistrationEnvironment.js"; +import { + mcpProviderDeadlineAllowanceMs, + recommendedMcpToolTimeoutSeconds, +} from "./mcpStartupPolicy.js"; + +const inputs = { + ghidraStartupTimeoutMs: 330_000, + hopperStartupTimeoutMs: 120_000, +} as const; + +describe("MCP tool deadline allowance for a Ghidra decompile limit", () => { + it.each([ + ["ghidra", 930_000], + ["auto", 930_000], + ["hopper", 120_000], + ] as const)( + "allows %s cold startup plus the selected decompile limit", + (analysisProvider, expected) => { + expect( + mcpProviderDeadlineAllowanceMs({ + ...inputs, + analysisProvider, + ghidraDecompileTimeoutMs: 600_000, + }), + ).toBe(expected); + }, + ); + + it("keeps the startup-only allowance when no decompile limit is set", () => { + const allowance = { ...inputs, analysisProvider: "ghidra" } as const; + expect(mcpProviderDeadlineAllowanceMs(allowance)).toBe(330_000); + expect(recommendedMcpToolTimeoutSeconds(allowance)).toBe(450); + }); + + it("derives the registration allowance from REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS", () => { + const details = clientRegistrationEnvironmentDetails( + parseClientConfiguration( + JSON.stringify({ + mcpServers: { + rea: { + command: "rea", + args: ["mcp"], + env: { + REA_ANALYSIS_PROVIDER: "ghidra", + REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: "600", + }, + }, + }, + }), + "json", + ), + ); + if (!details.ok) throw details.error; + expect(details.value.timeoutInputs).toMatchObject({ + analysisProvider: "ghidra", + ghidraDecompileTimeoutMs: 600_000, + }); + // 330 s default startup + 600 s limit + 120 s operation headroom. + const timeoutInputs = details.value.timeoutInputs; + if (timeoutInputs === undefined) + throw new Error("Expected resolved decompile timeout inputs"); + expect(recommendedMcpToolTimeoutSeconds(timeoutInputs)).toBe(1050); + }); + + it("forwards an explicit decompile limit into setup registrations", () => { + expect( + explicitSetupRegistrationEnvironment({ + REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: "600", + UNRELATED: "dropped", + }), + ).toEqual({ REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: "600" }); + }); +}); diff --git a/src/mcpStartupPolicy.ts b/src/mcpStartupPolicy.ts index 6ef4657f78..d9c5f3a9bf 100644 --- a/src/mcpStartupPolicy.ts +++ b/src/mcpStartupPolicy.ts @@ -24,6 +24,8 @@ export const mcpDoctorDeadlineRecovery = (budget: McpPhaseBudget): string => export interface McpToolTimeoutInputs { readonly analysisProvider: AnalysisProviderSelector; readonly ghidraStartupTimeoutMs: number; + /** One configured Ghidra decompile allowance; bulk cardinality is unknown here. */ + readonly ghidraDecompileTimeoutMs?: number; readonly hopperStartupTimeoutMs: number; readonly jebMcpTimeoutMs?: number; readonly providerRequestTimeoutMs?: number; @@ -34,15 +36,15 @@ export const mcpProviderDeadlineAllowanceMs = ( inputs: McpToolTimeoutInputs, ): number => { const selectedProvider = inputs.analysisProvider; + // A bounded decompile can run right after a cold import in the same call. + const ghidraAllowance = + inputs.ghidraStartupTimeoutMs + (inputs.ghidraDecompileTimeoutMs ?? 0); const selectedTimeout = selectedProvider === "ghidra" - ? inputs.ghidraStartupTimeoutMs + ? ghidraAllowance : selectedProvider === "hopper" ? inputs.hopperStartupTimeoutMs - : Math.max( - inputs.ghidraStartupTimeoutMs, - inputs.hopperStartupTimeoutMs, - ); + : Math.max(ghidraAllowance, inputs.hopperStartupTimeoutMs); return Math.max( selectedTimeout, inputs.jebMcpTimeoutMs ?? 0, @@ -50,7 +52,7 @@ export const mcpProviderDeadlineAllowanceMs = ( ); }; -/** Add operation headroom to the parsed provider deadline allowance. */ +/** Add headroom to the single-request estimate; this is not a bulk-workflow bound. */ export const recommendedMcpToolTimeoutMs = ( inputs: McpToolTimeoutInputs, ): number => diff --git a/tests/boundary/filesystem/clientConfigurationEnvironment.test.ts b/tests/boundary/filesystem/clientConfigurationEnvironment.test.ts index fcaecc7294..82b9c3856a 100644 --- a/tests/boundary/filesystem/clientConfigurationEnvironment.test.ts +++ b/tests/boundary/filesystem/clientConfigurationEnvironment.test.ts @@ -460,6 +460,7 @@ it.each(["linux", "win32"] as const)( it.each( [ "REA_ANALYSIS_PROVIDER", + "REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS", "REA_GHIDRA_STARTUP_TIMEOUT_MS", "REA_HOPPER_STARTUP_TIMEOUT_MS", "REA_JEB_MCP_TIMEOUT_MS", diff --git a/tests/boundary/filesystem/clientRegistrationStatus.test.ts b/tests/boundary/filesystem/clientRegistrationStatus.test.ts index 37b1766823..79c465b572 100644 --- a/tests/boundary/filesystem/clientRegistrationStatus.test.ts +++ b/tests/boundary/filesystem/clientRegistrationStatus.test.ts @@ -1,4 +1,4 @@ -import { chmod, mkdir, symlink, writeFile } from "node:fs/promises"; +import { chmod, mkdir, readFile, symlink, writeFile } from "node:fs/promises"; import { dirname, join, relative, resolve } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -7,6 +7,7 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { clearClientLocationEnvironment } from "../../fixtures/clientEnvironment.js"; import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; +import { configureClientConfiguration } from "../../../src/application/SetupClientConfiguration.js"; import { classifyClientRegistrationCommand } from "../../../src/application/ClientRegistrationIdentity.js"; import { existingMaintenanceScope } from "../../../src/application/UpdateMaintenance.js"; import { PRODUCT_IDENTITY } from "../../../src/identity.js"; @@ -406,6 +407,46 @@ describe("client activation keys", () => { ); }); +describe("Ghidra multi-function deadline registration advice", () => { + it.each([ + { provider: "ghidra", timeout: 10, hasBatchScope: true }, + { provider: "ghidra", timeout: 1800, hasBatchScope: true }, + { provider: "hopper", timeout: 1800, hasBatchScope: false }, + ])( + "preserves a selected $timeout-second $provider deadline and describes its scope", + async ({ provider, timeout, hasBatchScope }) => { + const home = await createTestTempDirectory("rea-codex-decompile-scope-"); + await mkdir(join(home, ".codex")); + const entry = resolve("scripts/rea.mjs"); + const configPath = join(home, ".codex/config.toml"); + // Two 600-second Ghidra decompiles plus startup/headroom would need an + // estimated 1650 seconds. A caller's shorter or longer choice stays theirs. + const original = `[mcp_servers.rea]\ncommand = ${JSON.stringify(process.execPath)}\nargs = [${JSON.stringify(entry)}, "mcp"]\nstartup_timeout_sec = 30\ntool_timeout_sec = ${timeout}\n[mcp_servers.rea.env]\nREA_ANALYSIS_PROVIDER = "${provider}"\nREA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS = "600"\n`; + await writeFile(configPath, original); + const [status] = await readClientRegistrationStatuses(home, entry, { + environment: {}, + }); + expect(status).toMatchObject({ client: "codex", state: "aligned" }); + if (hasBatchScope) { + expect(status?.advisory).toContain("single-decompile estimate"); + expect(status?.advisory).toContain("every attempted function"); + expect(status?.advisory).toContain("trace_native_values"); + expect(status?.advisory).toContain("max_functions"); + } else expect(status?.advisory).toBeUndefined(); + const configured = await configureClientConfiguration( + { name: "codex", format: "toml", configPath }, + {}, + [process.execPath, entry, "mcp"], + ); + expect(configured).toMatchObject({ status: "unchanged" }); + expect(await readFile(configPath, "utf8")).toBe(original); + await expect(readFile(`${configPath}.rea.backup`)).rejects.toMatchObject({ + code: "ENOENT", + }); + }, + ); +}); + describe("Node-wrapped registration policy", () => { it.each([1, 30])( "accepts Codex caller-selected startup timeout %d independently of launcher", diff --git a/tests/boundary/mcp/sessionAvailabilityRecovery.test.ts b/tests/boundary/mcp/sessionAvailabilityRecovery.test.ts index a214f02d6f..e0f3884718 100644 --- a/tests/boundary/mcp/sessionAvailabilityRecovery.test.ts +++ b/tests/boundary/mcp/sessionAvailabilityRecovery.test.ts @@ -3,12 +3,14 @@ import { expect, it } from "vitest"; import { toolContract } from "../../../src/contracts/toolContracts.js"; import { expandToolAvailability } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; +import { AnalysisTimeoutError } from "../../../src/domain/analysisErrorCore.js"; import { ProviderAdapterError } from "../../../src/domain/providerAdapterError.js"; import { ProviderCleanupError } from "../../../src/domain/providerCleanupError.js"; import { err, ok } from "../../../src/domain/result.js"; import { silentLogger } from "../../../src/logger.js"; import { createServer } from "../../../src/server/createServer.js"; import { observed } from "../../fixtures/analysisExecution.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createBinarySessionTargets, createCacheProvider, @@ -252,3 +254,75 @@ it("keeps a retired run and its lineage visible over MCP until cleanup succeeds" await session.close(); } }); + +it("keeps provider tools available after one request reaches its deadline", async () => { + const [path] = await createBinarySessionTargets(); + const provider = createCacheProvider([]); + let calls = 0; + provider.createClient = () => ({ + execute: (operation) => { + calls += 1; + return Promise.resolve( + calls === 1 + ? err( + new AnalysisTimeoutError(operation, 5000, { + userMessage: + "Fixture decompilation exceeded its 5-second limit. The session remains usable.", + }), + ) + : observed("after-timeout"), + ); + }, + close: () => Promise.resolve(ok(null)), + }); + const session = createTestBinarySession(provider); + const server = createServer( + { kind: "session", session }, + { environment: {}, logger: silentLogger }, + ); + const client = new Client({ name: "request-timeout", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + const availability = async () => { + const reply = await client.callTool({ + name: "binary_session", + arguments: {}, + }); + return expandToolAvailability( + toolContract("binary_session").outputSchema.parse(reply.structuredContent) + .result.tool_availability, + ); + }; + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + expect( + (await client.callTool({ name: "open_binary", arguments: { path } })) + .isError, + ).not.toBe(true); + const before = await availability(); + const query = { address: "0x1000", document: "fixture" }; + const timedOut = await client.callTool({ + name: "address_name", + arguments: query, + }); + expect(timedOut.isError).toBe(true); + expect(parseMcpToolError(timedOut)).toMatchObject({ + error: { code: "provider_timeout", retryable: true }, + }); + // A request-level deadline is not a provider failure: nothing is marked + // unavailable, unlike the ProviderAdapterError case above. + expect(await availability()).toEqual(before); + const retried = await client.callTool({ + name: "address_name", + arguments: query, + }); + expect(retried.structuredContent).toMatchObject({ + normalized_result: "after-timeout", + }); + } finally { + await client.close(); + await server.close(); + await session.close(); + } +}); diff --git a/tests/conformance/ghidra/README.md b/tests/conformance/ghidra/README.md index bf1e1ecfd4..cc9d43570f 100644 --- a/tests/conformance/ghidra/README.md +++ b/tests/conformance/ghidra/README.md @@ -45,6 +45,18 @@ binaries and Ghidra projects are never committed. retry after backend recovery, and keep healthy dossier snapshot reuse. The injection does not establish which natural inputs make the decompiler fail. +- `node scripts/verify-real-ghidra-decompile-timeout.mjs` (after + `npm run build:cached`, Linux or macOS x64/arm64) compiles a generated shared + library whose `rea_slow` function has thousands of branches and analyzes it + with `REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS=1` through the real read-only TCP + provider. Ghidra's own decompiler stops at the limit with no injection: + `analyze_function` returns the listing facets with pseudocode unavailable and + a live cache policy, `procedure_pseudo_code` fails with a message naming the + setting, and the same session then decompiles `rea_small`. Set + `REA_DECOMPILE_TIMEOUT_BRANCHES` to change the fixture size. The verifier + budgets 1 MiB for generated C source and derives the maximum branch count + from the worst-case line size before allocating the source. Larger or + invalid counts are rejected before installation inspection or compilation. `scripts/verify-real-ghidra.mjs` uses `cc`, `clang`, and `lld-link` by default; `REA_CC`, `REA_CLANG`, and `REA_LLD_LINK` can select alternate commands. The diff --git a/tests/conformance/ghidra/ReaDecompilerFailureProbe.java b/tests/conformance/ghidra/ReaDecompilerFailureProbe.java index 6467b3b856..76f0cdba85 100644 --- a/tests/conformance/ghidra/ReaDecompilerFailureProbe.java +++ b/tests/conformance/ghidra/ReaDecompilerFailureProbe.java @@ -105,6 +105,18 @@ public class ReaDecompilerFailureProbe extends GhidraScript { Method initialize = bridge.getClass().getDeclaredMethod("initializeDecompiler"); initialize.setAccessible(true); initialize.invoke(bridge); + + // Exercise the real production error factory, without claiming a natural regex overflow. + Method regexFactory = bridge.getClass().getDeclaredMethod( + "regexStackExhausted", String.class, String.class); + regexFactory.setAccessible(true); + Throwable regexFailure = (Throwable) regexFactory.invoke(null, "(a+)+$", "matching"); + require("regex_failure_preserves_stage_and_pattern", + failureCode(regexFailure).equals("regex_stack_exhausted") && + regexFailure.getMessage().contains("matching pattern (a+)+$.")); + require("regex_failure_leaves_lifecycle_to_provider", + !regexFailure.getMessage().contains("session") && + !regexFailure.getMessage().contains("Use literal")); Class requestType = Class.forName(bridge.getClass().getName() + "$Request", true, bridge.getClass().getClassLoader()); Class descriptorType = Class.forName(bridge.getClass().getName() + "$SessionDescriptor", true, bridge.getClass().getClassLoader()); Constructor requestConstructor = requestType.getDeclaredConstructor(int.class, String.class, JsonObject.class); @@ -189,6 +201,37 @@ public class ReaDecompilerFailureProbe extends GhidraScript { failureCode(failure.getCause()).equals("decompile_cancelled")); } + // A decompile that reaches REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS degrades the + // same way, names the setting, and stays out of snapshots. + Field limit = bridge.getClass().getDeclaredField("decompileTimeoutSeconds"); + limit.setAccessible(true); + limit.setInt(bridge, 5); + String limitText = "exceeded the 5-second limit set by REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS"; + decompiler.set(bridge, failing(target, DisposeState.DISPOSED_ON_TIMEOUT)); + JsonObject timedOut = (JsonObject) analyze.invoke(bridge, request, null); + JsonObject timedOutDossier = timedOut.getAsJsonObject("value"); + require("timed_out_decompile_is_live", + timedOut.has("cache_policy") && timedOut.get("cache_policy").getAsString().equals("live")); + String timedOutReason = reason(timedOutDossier.getAsJsonObject("facet_availability") + .getAsJsonObject("pseudocode")); + require("timed_out_pseudocode_names_limit", timedOutReason != null && + timedOutReason.startsWith("Ghidra decompilation of ") && timedOutReason.endsWith(limitText)); + require("timed_out_native_api_names_limit", + !timedOutDossier.getAsJsonObject("native_api").get("available").getAsBoolean() && + timedOutReason.equals(reason(timedOutDossier.getAsJsonObject("native_api")))); + require("timed_out_listing_unchanged", + timedOutDossier.get("assembly").equals(value.get("assembly")) && + timedOutDossier.get("basic_blocks").equals(value.get("basic_blocks"))); + try { + pseudocode.invoke(bridge, pseudocodeQuery); + throw new IllegalStateException("Timed-out pseudocode request unexpectedly succeeded"); + } catch (InvocationTargetException failure) { + require("timed_out_pseudocode_request_names_limit", + failureCode(failure.getCause()).equals("decompile_timed_out") && + failure.getCause().getMessage().endsWith(limitText)); + } + limit.setInt(bridge, 0); + decompiler.set(bridge, real); JsonObject recovered = (JsonObject) analyze.invoke(bridge, request, null); require("session_recovers_after_failure", recovered.getAsJsonObject("value")