From 0134732fcd59c129ac4ab39f64b12858a5a101aa Mon Sep 17 00:00:00 2001 From: Aayush Bhadbhade <145219884+AayushJB03@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:40:54 +0530 Subject: [PATCH 01/13] feat(capture): decode selected protobuf and gRPC bodies (#5031) Retain exact bounded wire facts and optional supplied-schema interpretations for selected historical captures. Share CLI/MCP views and preserve Evidence, partial coverage, credential exclusions and source provenance. Fixes #4986 Co-authored-by: AayushJB03 Co-authored-by: gensokyo-dev Co-authored-by: Claude Sonnet 5.5 --- .../references/runtime-observation.md | 6 + docs/web-network-captures.md | 111 +++- licenses/README.md | 1 + licenses/npm-dependencies.txt | 7 +- licenses/npm-runtime-notices.txt | 250 ++++++++- licenses/protobuf-es-v2.16.0-LICENSE.txt | 201 +++++++ licenses/upstream-sources.json | 5 + package-lock.json | 7 + package.json | 1 + .../WebNetworkCaptureService.test.ts | 26 + src/application/WebNetworkCaptureService.ts | 48 ++ src/browser/history/CaptureFailures.ts | 82 ++- src/browser/history/CaptureRedaction.ts | 27 +- .../history/HistoricalCaptureDecoder.ts | 96 ++-- src/browser/history/protobuf/ProtobufBody.ts | 194 +++++++ .../history/protobuf/ProtobufCapture.test.ts | 77 +++ .../history/protobuf/ProtobufCapture.ts | 501 +++++++++++++++++ .../history/protobuf/ProtobufFraming.ts | 144 +++++ .../history/protobuf/ProtobufSchema.ts | 299 ++++++++++ .../history/protobuf/ProtobufWire.test.ts | 522 +++++++++++++++++ src/browser/history/protobuf/ProtobufWire.ts | 269 +++++++++ src/cli/webNetworkCaptureCommands.ts | 67 ++- src/contracts/analysisViewToolContracts.ts | 2 +- .../webNetworkCaptureToolContracts.ts | 2 +- src/domain/analysisView/analysisView.ts | 91 +-- .../analysisView/protobufCaptureView.test.ts | 223 ++++++++ .../analysisView/protobufCaptureView.ts | 315 +++++++++++ .../analysisView/webNetworkCaptureView.ts | 52 +- src/domain/explicitSensitiveFailure.test.ts | 14 + src/domain/explicitSensitiveFailure.ts | 12 +- src/domain/protobufWire.ts | 206 +++++++ src/domain/webNetworkCapture.ts | 30 + .../acceptance/tools/protobufCapture.test.ts | 530 ++++++++++++++++++ tests/boundary/browser/sazWebCapture.test.ts | 176 ++++++ tests/fixtures/protobufCapture.ts | 83 +++ 35 files changed, 4502 insertions(+), 175 deletions(-) create mode 100644 licenses/protobuf-es-v2.16.0-LICENSE.txt create mode 100644 src/browser/history/protobuf/ProtobufBody.ts create mode 100644 src/browser/history/protobuf/ProtobufCapture.test.ts create mode 100644 src/browser/history/protobuf/ProtobufCapture.ts create mode 100644 src/browser/history/protobuf/ProtobufFraming.ts create mode 100644 src/browser/history/protobuf/ProtobufSchema.ts create mode 100644 src/browser/history/protobuf/ProtobufWire.test.ts create mode 100644 src/browser/history/protobuf/ProtobufWire.ts create mode 100644 src/domain/analysisView/protobufCaptureView.test.ts create mode 100644 src/domain/analysisView/protobufCaptureView.ts create mode 100644 src/domain/protobufWire.ts create mode 100644 tests/acceptance/tools/protobufCapture.test.ts create mode 100644 tests/fixtures/protobufCapture.ts diff --git a/.agents/skills/reverse-engineer-anything/references/runtime-observation.md b/.agents/skills/reverse-engineer-anything/references/runtime-observation.md index df75bcaf3d..a67648c802 100644 --- a/.agents/skills/reverse-engineer-anything/references/runtime-observation.md +++ b/.agents/skills/reverse-engineer-anything/references/runtime-observation.md @@ -69,6 +69,12 @@ with `inspect_analysis_view` and `view: {kind: "capture", detail: "preview", off Select complete record pages only for needed headers, metadata or payloads; views read the historical parent without repeating capture decoding. +For selected protobuf/gRPC bodies, keep raw bytes as the source and use +`inspect_analysis_view` kind `protobuf`, collection `bodies`, `frames` or +`fields`. Batch exact record ordinals and field numbers/IDs, and select +`schema_value` or `text` only when needed. Supply a local descriptor set and +message type for schema interpretation; compressed payloads remain unknown. + Inspect native `.saz` snapshots with `inspect_web_network_capture` and `format: "saz"`; encrypted archives are unsupported. For live traffic analysis, connect Everywhere's separate MCP server and use its diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index 4a9667bd6a..ce3c8796b7 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -122,29 +122,96 @@ XML node tree; optional `_w.txt` and `_g.txt` files retain opaque WebSocket and gRPC message bytes. Missing members remain null, distinct from empty bodies. See Telerik's [SAZ format reference](https://www.telerik.com/fiddler/fiddler-everywhere/documentation/knowledge-base/fiddler-archives). -The existing ZIP reader verifies read members; expanded archive contents are -limited to 32 MiB in addition to the input/output budgets below. Exceeding this -expanded-byte budget returns `invalid_request` / `invalid_input` through both -CLI and MCP, with an `out_of_range` issue at -`capture_path` and `maximum_expanded_capture_bytes`. Selecting records does not -bypass the complete-archive admission check or return partial Evidence. Encrypted -archives and unrecognized raw session members are unsupported. SAZ requires an -enforced decoder memory bound: Node's fixed heap, or the Windows Job Object -commit limit. Bun on hosts without that Job Object limit returns capability -unavailable before starting the SAZ decoder. Non-session -members are inventoried without decoding their content. Original files remain -unchanged. Linux verification includes independently checked public recorded -Mattermost captures from [FOREST](https://github.com/dfrc-korea/FOREST/tree/405c9154658a128ea203a6c10040cec55111ed03/api-results/dataset) -and the Fiddler author's [encoding corpus](https://github.com/ericlaw1979/Clearinet/blob/b6faf78cde1e72042980364f180ceb34b77d1434/Test/data/Encodings.saz), -plus synthetic malformed/limit fixtures. Their exporting product versions are -not authenticated. An interactive export from a current Fiddler Everywhere -installation and its live MCP server remain unverified. Support here is the -bounded documented raw HTTP/XML SAZ layout, not every product/version profile. +## Protobuf and gRPC bodies -For visual inspection, use Everywhere's **Import Snapshot**, then **Inspectors** -and **Overview**. HAR remains an alternative: select sessions, choose -**Export > HTTPArchive**, and confirm `log.version` is `"1.2"`. Preserve the -original SAZ when exporting. See the [import/export instructions](https://www.telerik.com/fiddler/fiddler-everywhere/documentation/collaboration/exporting-and-importing). +Selecting `record_ordinals` (CLI `--record`) also decodes retained bodies with +recognized protobuf or gRPC content types. Whole-capture inspection does not +perform this extra work. The original binary sidecar, SHA-256 and producer +ordinal remain the source; decoding adds flat body, frame and field facts. + +Native gRPC admits only data-frame flags 0/1; gRPC-web additionally admits its +terminal trailer frame. `grpc-web-text` supports independently padded base64 +chunks. Its offsets refer to **base64-decoded body bytes**; other offsets refer +to retained body bytes. Every field has a body-scoped `field_id` derived from +its byte offset, a `frame_index`, and a `parent_field_id` for schema-driven +nested messages or groups. Repeated field numbers remain separate occurrences. + +Without a schema, varints and fixed-width values retain exact unsigned decimal +representations. Length-delimited UTF-8, nested-message and packed readings +are candidates; REA does not choose one or invent field names or signedness. +Nested candidate ranges remain accessible without recursively expanding them. +Malformed messages retain the parsed prefix and an exact error offset. +Compressed frames and encoded HTTP bodies remain unknown; they are not +silently treated as empty messages. + +For schema interpretation, supply a local compiled `FileDescriptorSet` and its +fully qualified message type. All imports must be included. REA never fetches +schemas or executes the selected application. Descriptor identity and parser +version remain in the body facts. Scalars, repeated numeric packing and nested +messages are interpreted alongside the unchanged wire facts; disagreements +remain explicit. Map semantics, extensions and required-field validation are +unsupported. A supplied schema does not establish the sender's actual schema. +An invalid nested interpretation records `schema_status: "invalid-message"` +and `schema_error` locally; later outer fields remain available. Valid string +contents, including a leading U+FEFF, are preserved. + +```bash +# Compile your own schema with your existing protoc, including local imports: +protoc --include_imports --descriptor_set_out=/evidence/api.pb api.proto +rea inspect-web-network-capture /evidence/session.har har --record 3 \ + --protobuf-schema /evidence/api.pb --protobuf-message example.Response \ + --detail summary --evidence_output /evidence/session-evidence.json --json +``` + +MCP accepts the same choices as `protobuf: {schema: {descriptor_set_path: +"/evidence/api.pb", message_type: "example.Response"}}`, with explicit +`record_ordinals`. Optional `body_pointers` batch-select exact binary pointers; +`framing` can explicitly select `raw`, `grpc`, `grpc-web` or `grpc-web-text` +when MIME metadata is unavailable. CLI exposes `--protobuf-body-pointer` and +`--protobuf-framing`. Filters never invent bytes excluded by source redaction. Decoding an extra +base64 layer also checks explicit exclusions; newly recognized trailer +credentials exclude the entire affected binary sidecar and its reported +encoding, preserving a redaction coordinate instead of a credential digest. +Credential inspection covers every selected body before descriptor reads or +field expansion. Later capacity or schema failures can therefore retain the +sanitized source even with explicit sensitivity options. If that source pass +is interrupted, unchecked bodies are not published as partial Evidence. +Framing overrides apply only to format-owned body coordinates and preserve +exclusions for the transport declared in MIME metadata. +Unknown or explicitly excluded HTTP coding metadata is not treated as absent: +SAZ/native bodies remain opaque until identity coding is established. Raw +header values and retained body bytes remain available in source Evidence. +Skipping wire analysis still runs the independent credential check. If HTTP +coding metadata is unavailable and that check cannot complete, the operation +returns unavailable without publishing unchecked capture bodies as Evidence. + +Use `detail: "summary"`, then `inspect_analysis_view` with its retained parent: + +```json +{"kind":"protobuf","collection":"bodies","offset":0,"limit":16} +{"kind":"protobuf","collection":"frames","record_ordinals":[3],"offset":0,"limit":16} +{"kind":"protobuf","collection":"fields","record_ordinals":[3],"field_numbers":[1,4],"fields":["number","schema_name","schema_value"],"offset":0,"limit":32} +``` + +Filters intersect; an empty filter selects nothing. Field pages also accept +`field_ids`, `frame_indices` and `schema_names`. Default field pages omit `text`, +`schema_value` and enum alias inventories; select these columns when needed. Identity, source offset +basis and body status survive column selection. Complete Evidence retains +all decoded values, and ordinary capture previews expose counts. Page exhaustion +covers retained facts only; a partial source does not establish absence. + +Authenticated immutable parents reuse validation and the most recent field +filter per capture. These caches live only with retained Evidence, hold references +to existing facts and never reopen either file. Cached selector text is bounded +by retained field cardinality; larger selections use an uncached scan. Mutable +input is revalidated. +The existing 96 MiB report budget includes the original decoder reply plus +conservative charges for new facts, strings and descriptor expansion. The explicit +`protobuf_coverage` reports selected and inspected record counts, preserving +partial decoding through composed views. Capacity failure retains the source +observation as partial Evidence; cancellation yields +between bounded work batches. Real elapsed time and process-family memory +still depend on the selected capture and schema. ## Fiddler Everywhere MCP and public skills diff --git a/licenses/README.md b/licenses/README.md index c811a4f767..fa698f449e 100644 --- a/licenses/README.md +++ b/licenses/README.md @@ -8,6 +8,7 @@ upstream texts; they do not change [REA's MIT license](../LICENSE). | ------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `npm-dependencies.txt` | Every package path in the root lockfile, with its exact version, declared license, scope, download URL, and integrity. Includes development and optional platform entries. | | `npm-runtime-notices.txt` | Original license, copyright, and NOTICE texts collected from integrity-verified locked runtime npm archives across all platforms; also the license of Rolldown's generated runtime helpers. The archive manifest's name and version are checked. | +| `protobuf-es-v2.16.0-LICENSE.txt` | Original Apache license omitted from the protobuf-es npm archive, preserved from its v2.16.0 source commit. Its embedded Google BSD notice is retained in `npm-runtime-notices.txt`. | | `nativeaot-parser-MIT.txt` | Original Washi notice for derived NativeAOT parser code; source revision is recorded in the parser and the root notices. | | `dotnet-runtime-MIT.txt` | Original .NET Foundation MIT text from the NativeAOT reference revision `a3b2d40328be0be3f8dd43f0e5cc925b8827b044`. | | `ghidra-12.1.4-{LICENSE,NOTICE,DISCLAIMER}.txt` | Unchanged top-level documents from `Ghidra_12.1.4_build`. Ghidra is not bundled; these reference copies do not replace its component and GPL support-program notices. The US-government-specific disclaimer applies to Ghidra, not to REA. | diff --git a/licenses/npm-dependencies.txt b/licenses/npm-dependencies.txt index 19fb7e1ee6..c2c2280df1 100644 --- a/licenses/npm-dependencies.txt +++ b/licenses/npm-dependencies.txt @@ -2,7 +2,7 @@ REA npm dependency inventory Prepared from the root package-lock.json; refresh this inventory when locked dependencies change. License identifiers are upstream lockfile declarations, not a legal assessment. Runtime includes transitive and optional platform packages; development includes build, test and website tooling. -Locked package entries: 750; runtime: 178. +Locked package entries: 751; runtime: 179. Full runtime copyright/license/NOTICE texts: npm-runtime-notices.txt. @algolia/abtesting@1.25.0 | development | MIT @@ -335,6 +335,11 @@ obug@3.0.0 | development | MIT Package: https://www.npmjs.com/package/@bcoe/v8-coverage/v/1.0.2 Archive: https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz Integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA== +@bufbuild/protobuf@2.16.0 | runtime | (Apache-2.0 AND BSD-3-Clause) + Lockfile path: node_modules/@bufbuild/protobuf + Package: https://www.npmjs.com/package/@bufbuild/protobuf/v/2.16.0 + Archive: https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.16.0.tgz + Integrity: sha512-FWa0sPlqYGJgpTs6OxBRcL/AW4JT0OIO+W1ajerluoTVh8CV0B7XM1qsNCRgnkSCkRCgOICKHRQlY17jrsm0+Q== @cfworker/json-schema@4.1.1 | runtime | MIT Lockfile path: node_modules/@cfworker/json-schema Package: https://www.npmjs.com/package/@cfworker/json-schema/v/4.1.1 diff --git a/licenses/npm-runtime-notices.txt b/licenses/npm-runtime-notices.txt index d9efda90fa..0fc82a1312 100644 --- a/licenses/npm-runtime-notices.txt +++ b/licenses/npm-runtime-notices.txt @@ -1,7 +1,7 @@ REA third-party npm runtime copyright, license and NOTICE texts Collected from integrity-verified locked npm archives. Release-specific supplements and upstream notice gaps are identified in their entries. External engines are separately obtained; see THIRD_PARTY_NOTICES.md. -Runtime dependency fingerprint (SHA-256): 131100b2e1e6ad5e55b10a0baf4be2d7a0838fd724fb307b9bafd73446b0d3ec +Runtime dependency fingerprint (SHA-256): 4125777159e02cb749e17c959e77f59528cfffe3f7a5220b8a21ce3c49ad3061 ===== @babel/helper-string-parser@8.0.6 ===== Declared license: MIT @@ -126,6 +126,254 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +===== @bufbuild/protobuf@2.16.0 ===== +Declared license: (Apache-2.0 AND BSD-3-Clause) +Archive: https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.16.0.tgz +Integrity: sha512-FWa0sPlqYGJgpTs6OxBRcL/AW4JT0OIO+W1ajerluoTVh8CV0B7XM1qsNCRgnkSCkRCgOICKHRQlY17jrsm0+Q== +Scope: npm runtime dependency, including optional platform packages. + +The locked archive omits standalone license files. The Apache license is preserved from its v2.16.0 upstream commit; the BSD notice is embedded in the integrity-verified archive. + +--- licenses/protobuf-es-v2.16.0-LICENSE.txt (see upstream-sources.json) --- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021-2026 Buf Technologies, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--- package/dist/esm/wire/varint.js (unchanged leading BSD notice) --- + +// Copyright 2008 Google Inc. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Code generated by the Protocol Buffer compiler is owned by the owner +// of the input file used when generating it. This code is not +// standalone and requires a support library to be linked with it. This +// support library is itself covered by the above license. + + ===== @cfworker/json-schema@4.1.1 ===== Declared license: MIT Archive: https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz diff --git a/licenses/protobuf-es-v2.16.0-LICENSE.txt b/licenses/protobuf-es-v2.16.0-LICENSE.txt new file mode 100644 index 0000000000..0852eae3ad --- /dev/null +++ b/licenses/protobuf-es-v2.16.0-LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021-2026 Buf Technologies, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/licenses/upstream-sources.json b/licenses/upstream-sources.json index fb1dad9759..2fbf6a8855 100644 --- a/licenses/upstream-sources.json +++ b/licenses/upstream-sources.json @@ -68,5 +68,10 @@ "file": "licenses/LGPL-2.0.txt", "source": "https://www.gnu.org/licenses/old-licenses/lgpl-2.0.txt", "sha256": "cc535c21133c895b56b374c8a1dc1eb948d99003ed2b47372069456b62f42b24" + }, + { + "file": "licenses/protobuf-es-v2.16.0-LICENSE.txt", + "source": "https://raw.githubusercontent.com/bufbuild/protobuf-es/b3b232f7f466eda27c66e8f63c82f930578b6cc6/LICENSE", + "sha256": "c04b4216f1cd4c5a4f7fb2f2a1b0ae70d847e9e0cac7c9dee9bf8cc03177c449" } ] diff --git a/package-lock.json b/package-lock.json index 2841a5d1a7..67a74129f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@babel/parser": "8.0.4", "@babel/types": "8.0.4", + "@bufbuild/protobuf": "2.16.0", "@clack/prompts": "1.7.0", "@electron/asar": "4.2.0", "@jridgewell/trace-mapping": "0.3.31", @@ -1090,6 +1091,12 @@ "node": ">=18" } }, + "node_modules/@bufbuild/protobuf": { + "version": "2.16.0", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.16.0.tgz", + "integrity": "sha512-FWa0sPlqYGJgpTs6OxBRcL/AW4JT0OIO+W1ajerluoTVh8CV0B7XM1qsNCRgnkSCkRCgOICKHRQlY17jrsm0+Q==", + "license": "(Apache-2.0 AND BSD-3-Clause)" + }, "node_modules/@cfworker/json-schema": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", diff --git a/package.json b/package.json index 2032234383..5f77580431 100644 --- a/package.json +++ b/package.json @@ -323,6 +323,7 @@ "dependencies": { "@babel/parser": "8.0.4", "@babel/types": "8.0.4", + "@bufbuild/protobuf": "2.16.0", "@clack/prompts": "1.7.0", "@electron/asar": "4.2.0", "@jridgewell/trace-mapping": "0.3.31", diff --git a/src/application/WebNetworkCaptureService.test.ts b/src/application/WebNetworkCaptureService.test.ts index 7dfc1d8ab7..17510384e2 100644 --- a/src/application/WebNetworkCaptureService.test.ts +++ b/src/application/WebNetworkCaptureService.test.ts @@ -543,3 +543,29 @@ it("reports malformed retained facts while keeping their cleanup owner retryable expect(result.error.cleanupIncomplete).toBe(false); expect(projectAnalysisError(result.error).code).toBe("unreadable_output"); }); + +it("rejects relative descriptor paths before starting a capture decoder", async () => { + const service = new WebNetworkCaptureService({ + inspect: () => { + throw new Error("Invalid input must fail before provider admission"); + }, + }); + const result = await service.inspect({ + capture_path: "/capture.har", + format: "har", + record_ordinals: [0], + protobuf: { + schema: { + descriptor_set_path: "relative.pb", + message_type: "capture.Message", + }, + }, + }); + if (result.ok) throw new Error("Expected typed invalid input"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "invalid_request", + details: { + issues: [{ path: ["protobuf", "schema", "descriptor_set_path"] }], + }, + }); +}); diff --git a/src/application/WebNetworkCaptureService.ts b/src/application/WebNetworkCaptureService.ts index 0c70daccd9..e1c397b51b 100644 --- a/src/application/WebNetworkCaptureService.ts +++ b/src/application/WebNetworkCaptureService.ts @@ -80,6 +80,30 @@ export class WebNetworkCaptureService { }, ]), ); + if (input.protobuf !== undefined && input.record_ordinals === undefined) + return failure( + new AnalysisInputError(OPERATION, undefined, [ + { + path: ["record_ordinals"], + reason: "invalid_format", + message: + "Protobuf options require explicit producer record ordinals.", + }, + ]), + ); + if ( + input.protobuf?.schema !== undefined && + !isAbsolute(input.protobuf.schema.descriptor_set_path) + ) + return failure( + new AnalysisInputError(OPERATION, undefined, [ + { + path: ["protobuf", "schema", "descriptor_set_path"], + reason: "invalid_format", + message: "Expected an absolute descriptor-set path on this host.", + }, + ]), + ); const loaded = await this.port.inspect(input, options); if (!loaded.ok) { const partial = webNetworkCaptureSchema.safeParse( @@ -201,6 +225,30 @@ export class WebNetworkCaptureService { ? {} : { format: input.format }), record_ordinals: selected, + ...(input.protobuf === undefined + ? {} + : { + protobuf: { + framing: input.protobuf.framing, + ...(input.protobuf.body_pointers === undefined + ? {} + : { + body_pointers: input.protobuf.body_pointers.map(redactText), + }), + ...(input.protobuf.schema === undefined + ? {} + : { + schema: { + descriptor_set_path: redactText( + input.protobuf.schema.descriptor_set_path, + ), + message_type: redactText( + input.protobuf.schema.message_type, + ), + }, + }), + }, + }), sensitive_value_count: input.sensitive_values.length, }); return ok( diff --git a/src/browser/history/CaptureFailures.ts b/src/browser/history/CaptureFailures.ts index 6c42ac8e3b..7bb07fb165 100644 --- a/src/browser/history/CaptureFailures.ts +++ b/src/browser/history/CaptureFailures.ts @@ -1,5 +1,7 @@ import type { ExecutionOptions } from "../../application/AnalysisProvider.js"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; +import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js"; +import { bindArtifactFileCleanup } from "../../artifacts/ArtifactFileOperation.js"; import { artifactFileCancellationCause } from "../../artifacts/ArtifactFileOperation.js"; import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { @@ -33,10 +35,18 @@ export const CAPTURE_PROCESS_MEMORY_LIMIT_REASON = export const historicalCaptureFailure = ( input: InspectWebNetworkCaptureInput, cause: unknown, - phase: "capture-read" | "decoder", + phase: "capture-read" | "descriptor-read" | "decoder", options?: ExecutionOptions, ): AnalysisError => { if (cause instanceof AnalysisError) return cause; + const descriptorRead = phase === "descriptor-read"; + const sourcePath = descriptorRead + ? (input.protobuf?.schema?.descriptor_set_path ?? input.capture_path) + : input.capture_path; + const inputPath = descriptorRead + ? ["protobuf", "schema", "descriptor_set_path"] + : ["capture_path"]; + const sourceLabel = descriptorRead ? "descriptor set" : "capture"; if (cause instanceof OwnedCommandFailure) { if (cause.cleanupFailure !== null) return new ProviderCleanupError( @@ -99,18 +109,18 @@ export const historicalCaptureFailure = ( ); } if ( - phase === "capture-read" && + phase !== "decoder" && cause instanceof ArtifactReaderFailure && cause.reason === "integrity" ) return new AnalysisArtifactChangedError( OPERATION, - input.capture_path, + sourcePath, cause.message, { cause }, ); if ( - phase === "capture-read" && + phase !== "decoder" && cause instanceof ArtifactReaderFailure && cause.reason === "cancelled" ) @@ -118,7 +128,7 @@ export const historicalCaptureFailure = ( cause: artifactFileCancellationCause(cause), }); if ( - phase === "capture-read" && + phase !== "decoder" && cause instanceof ArtifactReaderFailure && (cause.reason === "limit" || cause.reason === "format" || @@ -126,44 +136,44 @@ export const historicalCaptureFailure = ( ) return new AnalysisInputError(OPERATION, { cause }, [ { - path: ["capture_path"], + path: inputPath, reason: cause.reason === "limit" ? "out_of_range" : "invalid_format", message: cause.reason === "limit" - ? `${cause.message}. Select a capture within the ${WEB_NETWORK_CAPTURE_LIMITS.inputBytes}-byte input budget.` + ? `${cause.message}. Select a ${sourceLabel} within the ${WEB_NETWORK_CAPTURE_LIMITS.inputBytes}-byte input budget.` : cause.message, ...(cause.reason === "limit" ? { expected: { - maximum_capture_bytes: WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + [descriptorRead + ? "maximum_descriptor_set_bytes" + : "maximum_capture_bytes"]: + WEB_NETWORK_CAPTURE_LIMITS.inputBytes, }, } : {}), }, ]); if ( - phase === "capture-read" && + phase !== "decoder" && cause instanceof Error && "code" in cause && (cause.code === "EACCES" || cause.code === "EPERM") ) - return new AnalysisAccessDeniedError( - OPERATION, - input.capture_path, - cause.code, - { cause }, - ); + return new AnalysisAccessDeniedError(OPERATION, sourcePath, cause.code, { + cause, + }); if ( - phase === "capture-read" && + phase !== "decoder" && cause instanceof Error && "code" in cause && ["ENOENT", "ENOTDIR"].includes(String(cause.code)) ) return new AnalysisInputError(OPERATION, { cause }, [ { - path: ["capture_path"], + path: inputPath, reason: "invalid_value", - message: `Selected capture could not be read (${String(cause.code)}): ${input.capture_path}.`, + message: `Selected ${sourceLabel} could not be read (${String(cause.code)}): ${sourcePath}.`, }, ]); if ( @@ -182,6 +192,9 @@ export const historicalCaptureFailure = ( diagnostics: { phase, capture_path: input.capture_path, + ...(descriptorRead + ? { source_path: sourcePath, input_path: inputPath } + : {}), reason: cause instanceof Error ? cause.message : String(cause), ...(cause instanceof ArtifactReaderFailure ? { failure_kind: cause.reason } @@ -248,3 +261,36 @@ export const captureMemoryFailure = ( : {}), }, ); + +/** Preserve the exact file owner when capture or descriptor acquisition needs cleanup. */ +export const withCaptureReaderCleanup = ( + primary: AnalysisError, + cause: unknown, +): AnalysisError => { + if (!(cause instanceof ArtifactReaderFailure) || cause.cleanup === undefined) + return primary; + if (cause.reason === "unavailable") { + const cleanup = new ProviderCleanupError( + "web-network-capture", + cause.cleanup.resources, + { reason: cause.cleanup.reason }, + { operation: OPERATION, cause }, + ); + bindArtifactFileCleanup(cause, cleanup); + return cleanup; + } + const cleanup = new ProviderCleanupError( + "web-network-capture", + cause.cleanup.resources, + { reason: cause.cleanup.reason }, + { operation: OPERATION, cause }, + ); + bindArtifactFileCleanup(cause, cleanup); + const composed = analysisErrorWithCleanupFailure( + primary, + cleanup, + OPERATION, + ).bindCleanupDependencies([primary, cleanup]); + bindArtifactFileCleanup(cause, composed); + return composed; +}; diff --git a/src/browser/history/CaptureRedaction.ts b/src/browser/history/CaptureRedaction.ts index 63feb4899e..62f4eac469 100644 --- a/src/browser/history/CaptureRedaction.ts +++ b/src/browser/history/CaptureRedaction.ts @@ -9,6 +9,31 @@ const CREDENTIAL_HEADERS = new Set([ "cookie", "set-cookie", ]); +/** HTTP credential names apply equally to recorded headers and gRPC-web trailers. */ +export const isCaptureCredentialHeader = (name: string): boolean => + CREDENTIAL_HEADERS.has(name.toLowerCase()); + +/** Match an ASCII header name without allocating a potentially enormous line. */ +export const isCaptureCredentialHeaderBytes = ( + bytes: Uint8Array, + start: number, + end: number, +): boolean => { + for (const name of CREDENTIAL_HEADERS) { + if (end - start !== name.length) continue; + let matches = true; + for (let index = 0; index < name.length; index++) { + const byte = bytes[start + index]!; + const lower = byte >= 65 && byte <= 90 ? byte + 32 : byte; + if (lower !== name.charCodeAt(index)) { + matches = false; + break; + } + } + if (matches) return true; + } + return false; +}; /** Literal redaction and structural authentication exclusions preserve unrelated local evidence. */ export class CaptureRedaction { @@ -75,7 +100,7 @@ export class CaptureRedaction { return ( (/^\/(request|response)\/headers\/\d+\/value$/.test(pointer) && typeof parent.name === "string" && - CREDENTIAL_HEADERS.has(parent.name.toLowerCase())) || + isCaptureCredentialHeader(parent.name)) || /^\/(request|response)\/cookies\/\d+\/value$/.test(pointer) ); } diff --git a/src/browser/history/HistoricalCaptureDecoder.ts b/src/browser/history/HistoricalCaptureDecoder.ts index 550c96ca67..79e13071b8 100644 --- a/src/browser/history/HistoricalCaptureDecoder.ts +++ b/src/browser/history/HistoricalCaptureDecoder.ts @@ -1,7 +1,13 @@ +import { + decodeCaptureProtobuf, + prepareCaptureProtobuf, +} from "./protobuf/ProtobufCapture.js"; +import { CaptureFormatError } from "./CaptureFormatError.js"; import { CAPTURE_PROCESS_MEMORY_LIMIT_REASON, historicalCaptureFailure, captureMemoryFailure, + withCaptureReaderCleanup, } from "./CaptureFailures.js"; import { NODE_CAPTURE_HEAP_ENFORCED, @@ -17,7 +23,6 @@ import { z } from "zod"; import type { ExecutionOptions } from "../../application/AnalysisProvider.js"; import { readStableArtifact } from "../../artifacts/readStableArtifact.js"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; -import { bindArtifactFileCleanup } from "../../artifacts/ArtifactFileOperation.js"; import { ArtifactResourceScope, type ArtifactCleanupAttempt, @@ -29,6 +34,7 @@ import { AnalysisCancelledError, AnalysisInputError, AnalysisOutputError, + AnalysisResourceConstraintError, } from "../../domain/analysisErrorCore.js"; import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; @@ -104,7 +110,7 @@ export class HistoricalCaptureDecoder { ); return err( redactExplicitFailure( - this.withReaderCleanup(error, cause), + withCaptureReaderCleanup(error, cause), input.sensitive_values, ), ); @@ -118,6 +124,7 @@ export class HistoricalCaptureDecoder { let runtime: Pick | undefined; let result: Result; let phase: "capture-read" | "decoder" = "capture-read"; + let encodedReplyBytes = 0; try { const adapter = this.adapters.find( (candidate) => candidate.format === input.format, @@ -186,6 +193,7 @@ export class HistoricalCaptureDecoder { undefined, this.#resources, ); + encodedReplyBytes = replyFile.bytes.length; reply = replySchema.parse(JSON.parse(replyFile.bytes.toString("utf8"))); } catch (cause: unknown) { if (cause instanceof ArtifactReaderFailure) throw cause; @@ -295,11 +303,58 @@ export class HistoricalCaptureDecoder { ); result = ok({ ...reply.value, ...common, format: input.format }); } + if (result.ok) { + let safePartialSource = false; + try { + const prepared = await prepareCaptureProtobuf( + result.value, + input, + options?.signal, + ); + safePartialSource = true; + await decodeCaptureProtobuf( + result.value, + input, + this.#resources, + prepared, + options?.signal, + encodedReplyBytes + Buffer.byteLength(JSON.stringify(common)) + 16, + ); + } catch (cause) { + const error = + cause instanceof CaptureFormatError && cause.reason === "limit" + ? new AnalysisResourceConstraintError( + OPERATION, + "capacity", + cause.message, + { + maximum_encoded_capture_bytes: + WEB_NETWORK_CAPTURE_LIMITS.outputBytes, + }, + { + retryable: false, + remediationAction: + "Select smaller records, or inspect the retained original source bytes.", + }, + ) + : cause instanceof CaptureFormatError + ? new AnalysisInputError(OPERATION, undefined, [ + { + path: ["protobuf"], + reason: "invalid_format", + message: cause.message, + }, + ]) + : historicalCaptureFailure(input, cause, phase, options); + if (safePartialSource) error.retainPartialObservation(result.value); + throw error; + } + } } catch (cause: unknown) { const error = historicalCaptureFailure(input, cause, phase, options); result = err( redactExplicitFailure( - this.withReaderCleanup(error, cause), + withCaptureReaderCleanup(error, cause), input.sensitive_values, ), ); @@ -358,41 +413,6 @@ export class HistoricalCaptureDecoder { result.error.retainPartialObservation(completedCapture); return result; } - - private withReaderCleanup( - primary: AnalysisError, - cause: unknown, - ): AnalysisError { - if ( - !(cause instanceof ArtifactReaderFailure) || - cause.cleanup === undefined - ) - return primary; - if (cause.reason === "unavailable") { - const cleanup = new ProviderCleanupError( - "web-network-capture", - cause.cleanup.resources, - { reason: cause.cleanup.reason }, - { operation: OPERATION, cause }, - ); - bindArtifactFileCleanup(cause, cleanup); - return cleanup; - } - const cleanup = new ProviderCleanupError( - "web-network-capture", - cause.cleanup.resources, - { reason: cause.cleanup.reason }, - { operation: OPERATION, cause }, - ); - bindArtifactFileCleanup(cause, cleanup); - const composed = analysisErrorWithCleanupFailure( - primary, - cleanup, - OPERATION, - ).bindCleanupDependencies([primary, cleanup]); - bindArtifactFileCleanup(cause, composed); - return composed; - } } /** The Node capture decoder runs on this executable; only Node.js enforces its V8 heap flags. */ diff --git a/src/browser/history/protobuf/ProtobufBody.ts b/src/browser/history/protobuf/ProtobufBody.ts new file mode 100644 index 0000000000..7d040fd9c1 --- /dev/null +++ b/src/browser/history/protobuf/ProtobufBody.ts @@ -0,0 +1,194 @@ +import { + readGrpcFrame, + prepareProtobufBody, + type PreparedProtobufBody, +} from "./ProtobufFraming.js"; +import { CaptureFormatError } from "../CaptureFormatError.js"; +import type { + ProtobufDecoding, + ProtobufFrame, +} from "../../../domain/protobufWire.js"; +import { + decodeProtobufWire, + ProtobufBudget, + type WireSchema, +} from "./ProtobufWire.js"; + +/** Framing is protocol-specific: native gRPC has no in-body trailer frames. */ +export const PROTOBUF_FRAMING = { + "application/grpc": "grpc", + "application/grpc+proto": "grpc", + "application/grpc-web": "grpc-web", + "application/grpc-web+proto": "grpc-web", + "application/grpc-web-text": "grpc-web-text", + "application/grpc-web-text+proto": "grpc-web-text", + "application/x-protobuf": "raw", + "application/protobuf": "raw", + "application/vnd.google.protobuf": "raw", +} as const; + +export const PROTOBUF_LIMITATIONS = [ + "Wire numbers and ranges are observations. Length-delimited string, nested-message and packed readings are candidates without a supplied schema. Raw decoding does not infer field names, signedness or business meaning; nested candidate bytes remain accessible through their ranges.", + "Schema values are interpretations of the caller-supplied descriptor, not proof that the sender used that schema. Map semantics, extensions, required-field validation and unknown schema fields are not reconstructed.", +] as const; + +export const decodeProtobufBody = async ( + body: Uint8Array, + framing: ProtobufDecoding["framing"], + source: Pick, + redact: (value: string) => string, + budget: ProtobufBudget, + schema?: WireSchema, + prepared?: PreparedProtobufBody, +): Promise => { + const result: ProtobufDecoding = { + ...source, + decoder: "rea-protobuf-wire-v1", + framing, + offset_basis: + framing === "grpc-web-text" ? "base64-decoded-body" : "retained-body", + decoded_bytes: body.length, + status: "complete", + examined_bytes: 0, + frames: [], + fields: [], + limitations: [], + }; + const stop = (offset: number, reason: string): ProtobufDecoding => { + result.status = "partial"; + result.error = { offset, reason: redact(reason) }; + return result; + }; + // The production source pass already decoded text. Charge representation expansion + // here only after that pass made the retained source safe on failure. + if (framing === "grpc-web-text") budget.charge(body.length * 6); + const data = + prepared === undefined + ? ( + await prepareProtobufBody( + body, + framing, + { decoded: () => false, header: () => false }, + budget.signal, + ) + ).data + : prepared.data; + if (data === undefined) + return stop( + 0, + "gRPC-web text is not canonical base64 or independently padded base64 chunks", + ); + result.decoded_bytes = data.length; + let at = 0; + const addMessage = async (frame: ProtobufFrame) => { + if (frame.compressed) { + frame.status = "unknown"; + frame.error = { + offset: frame.payload_offset, + reason: + "Compressed gRPC payload requires its negotiated codec; source bytes remain retained", + }; + return; + } + const decoded = await decodeProtobufWire( + data, + frame.payload_offset, + frame.payload_offset + frame.bytes, + frame.index, + budget, + redact, + schema, + result.fields.length, + ); + for (const field of decoded.fields) result.fields.push(field); + frame.examined_bytes = decoded.examined_bytes; + if (decoded.error !== undefined) { + frame.status = "partial"; + frame.error = decoded.error; + } + }; + if (framing === "raw") { + const frame: ProtobufFrame = { + index: 0, + kind: "data", + offset: 0, + payload_offset: 0, + bytes: data.length, + compressed: false, + status: "complete", + examined_bytes: 0, + }; + result.frames.push(frame); + await addMessage(frame); + result.status = frame.status; + result.examined_bytes = frame.examined_bytes; + return result; + } + let trailerSeen = false; + while (at < data.length) { + await budget.checkpoint(); + if (trailerSeen) + return stop(at, "A gRPC-web trailer frame must terminate the body"); + const parsed = readGrpcFrame(data, at, framing); + if (!parsed.ok) return stop(parsed.error.offset, parsed.error.reason); + const { flags, bytes: length } = parsed.value; + budget.charge(1024); + const frame: ProtobufFrame = { + index: result.frames.length, + kind: flags & 128 ? "trailer" : "data", + offset: at, + payload_offset: at + 5, + bytes: length, + compressed: (flags & 1) !== 0, + status: "complete", + examined_bytes: 0, + }; + result.frames.push(frame); + if (frame.kind === "data") await addMessage(frame); + else { + trailerSeen = true; + if (frame.compressed) { + frame.status = "unknown"; + frame.error = { + offset: at + 5, + reason: "Compressed gRPC-web trailers remain opaque", + }; + } else { + budget.charge(length * 8); + try { + const text = new TextDecoder("utf-8", { + fatal: true, + ignoreBOM: true, + }).decode(data.subarray(at + 5, at + 5 + length)); + frame.trailers = []; + for (const line of text.split("\r\n")) { + if (line === "") continue; + const colon = line.indexOf(":"); + if ( + colon <= 0 || + !/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(line.slice(0, colon)) + ) + throw new Error("Malformed trailer header"); + budget.charge(512); + frame.trailers.push({ + name: redact(line.slice(0, colon)), + value: redact(line.slice(colon + 1).replace(/^[ \t]+/, "")), + }); + } + frame.examined_bytes = length; + } catch (cause) { + if (cause instanceof CaptureFormatError) throw cause; + frame.status = "partial"; + frame.error = { + offset: at + 5, + reason: "Invalid gRPC-web trailer header block", + }; + } + } + } + if (frame.status !== "complete") result.status = "partial"; + result.examined_bytes += 5 + frame.examined_bytes; + at += 5 + length; + } + return result; +}; diff --git a/src/browser/history/protobuf/ProtobufCapture.test.ts b/src/browser/history/protobuf/ProtobufCapture.test.ts new file mode 100644 index 0000000000..7e711471be --- /dev/null +++ b/src/browser/history/protobuf/ProtobufCapture.test.ts @@ -0,0 +1,77 @@ +import { createHash } from "node:crypto"; +import { expect, it } from "vitest"; +import { ArtifactResourceScope } from "../../../artifacts/ArtifactResourceScope.js"; +import { + inspectWebNetworkCaptureInputSchema, + type WebNetworkCapture, +} from "../../../domain/webNetworkCapture.js"; +import { + prepareCaptureProtobuf, + decodeCaptureProtobuf, +} from "./ProtobufCapture.js"; + +it("restricts framing overrides to native body facts while preserving header bytes", async () => { + const binary = (pointer: string, bytes: Buffer) => ({ + pointer, + representation: "producer-bytes" as const, + state: "retained" as const, + bytes: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + content_base64: bytes.toString("base64"), + }); + const headers = [ + binary("/response/headers/0/0", Buffer.from("content-type")), + binary("/response/headers/0/1", Buffer.from("application/grpc")), + ]; + const report: WebNetworkCapture = { + artifact: { path: "/fixture.mitm", bytes: 100, sha256: "a".repeat(64) }, + decoder: { id: "fixture", name: "Native producer facts", version: "1" }, + container: { + reported: null, + numeric_literals: [], + redactions: [], + records_pointer: "/flows", + }, + format: "mitmproxy", + total_records: 1, + runtime_attribution: "unknown", + limitations: [], + records: [ + { + ordinal: 0, + location: { kind: "byte-range", offset: 0, bytes: 100 }, + reported: { response: { headers: [[null, null]], content: null } }, + binary_fields: [ + ...headers, + binary("/response/content", Buffer.from([0, 0, 0, 0, 2, 8, 1])), + ], + numeric_literals: [], + redactions: [], + limitations: [], + }, + ], + }; + const input = inspectWebNetworkCaptureInputSchema.parse({ + capture_path: report.artifact.path, + format: report.format, + record_ordinals: [0], + protobuf: { framing: "grpc" }, + }); + const resources = new ArtifactResourceScope(); + try { + const prepared = await prepareCaptureProtobuf(report, input); + await decodeCaptureProtobuf(report, input, resources, prepared); + } finally { + await resources.close(); + } + expect(report.records[0]?.protobuf).toMatchObject([ + { + pointer: "/response/content", + status: "complete", + fields: [{ number: 1, value: "1" }], + }, + ]); + expect(report.records[0]?.protobuf).toHaveLength(1); + expect(report.records[0]?.binary_fields.slice(0, 2)).toEqual(headers); + expect(report.protobuf_coverage?.status).toBe("complete"); +}); diff --git a/src/browser/history/protobuf/ProtobufCapture.ts b/src/browser/history/protobuf/ProtobufCapture.ts new file mode 100644 index 0000000000..0eaaf78806 --- /dev/null +++ b/src/browser/history/protobuf/ProtobufCapture.ts @@ -0,0 +1,501 @@ +import { isAbsolute } from "node:path"; +import { CaptureFormatError } from "../CaptureFormatError.js"; +import { + CaptureRedaction, + isCaptureCredentialHeaderBytes, +} from "../CaptureRedaction.js"; +import { createHash } from "node:crypto"; +import { readStableArtifact } from "../../../artifacts/readStableArtifact.js"; +import { + historicalCaptureFailure, + withCaptureReaderCleanup, +} from "../CaptureFailures.js"; +import type { ArtifactResourceScope } from "../../../artifacts/ArtifactResourceScope.js"; +import { + AnalysisInputError, + AnalysisCapabilityUnavailableError, +} from "../../../domain/analysisErrorCore.js"; +import { redactExplicitText } from "../../../domain/explicitSensitiveValues.js"; +import { + WEB_NETWORK_CAPTURE_LIMITS, + type InspectWebNetworkCaptureInput, + type WebNetworkCapture, + type WebNetworkCaptureRecord, +} from "../../../domain/webNetworkCapture.js"; +import type { JsonValue } from "../../../domain/jsonValue.js"; +import { ProtobufBudget } from "./ProtobufWire.js"; +import type { ProtobufSchema } from "./ProtobufSchema.js"; +import { + PROTOBUF_FRAMING, + PROTOBUF_LIMITATIONS, + decodeProtobufBody, +} from "./ProtobufBody.js"; +import { + prepareProtobufBody, + type PreparedProtobufBody, +} from "./ProtobufFraming.js"; +import type { ProtobufDecoding } from "../../../domain/protobufWire.js"; + +const object = ( + value: JsonValue | undefined, +): Readonly> | undefined => + value !== null && typeof value === "object" && !Array.isArray(value) + ? value + : undefined; + +/** Exclude the original encoded representation as well as its decoding sidecar. */ +const excludeBody = ( + record: WebNetworkCaptureRecord, + pointer: string, + reason: "explicit-sensitive-value" | "transport-credential", +) => { + record.binary_fields = record.binary_fields.map((field) => + field.pointer !== pointer + ? field + : { + pointer: field.pointer, + representation: field.representation, + state: "redacted", + content_base64: null, + bytes: null, + sha256: null, + }, + ); + const parts = pointer + .split("/") + .slice(1) + .map((part) => part.replaceAll("~1", "/").replaceAll("~0", "~")); + let parent: JsonValue | undefined = record.reported; + for (const part of parts.slice(0, -1)) { + if ( + parent === null || + typeof parent !== "object" || + !Object.hasOwn(parent, part) + ) { + parent = undefined; + break; + } + parent = Array.isArray(parent) ? parent[Number(part)] : parent[part]; + } + const key = parts.at(-1); + if ( + parent !== null && + typeof parent === "object" && + key !== undefined && + Object.hasOwn(parent, key) + ) + Object.defineProperty(parent, key, { + value: null, + writable: true, + configurable: true, + enumerable: true, + }); + record.redactions.push({ pointer, reason }); +}; + +/** Format-aware owner of body metadata; callers never reinterpret producer JSON. */ +const bodyMetadata = ( + record: WebNetworkCaptureRecord, + pointer: string, + format: string, + binaryByPointer: ReadonlyMap< + string, + WebNetworkCaptureRecord["binary_fields"][number] + >, +) => { + const side = /^\/(request|response)\//.exec(pointer)?.[1]; + if ( + side === undefined || + ![ + "/request/postData/text", + "/response/content/text", + "/request/body", + "/response/body", + "/request/content", + "/response/content", + ].includes(pointer) + ) + return undefined; + const message = object(object(record.reported)?.[side]); + if (message === undefined) return undefined; + const headers = new Map(); + let headerNamesKnown = Array.isArray(message.headers); + if (Array.isArray(message.headers)) { + for (const [index, item] of message.headers.entries()) { + const pair = object(item); + const text = ( + value: JsonValue | undefined, + coordinate: string, + ): string | undefined => { + const binary = binaryByPointer.get(coordinate); + if (binary !== undefined) + return binary.state === "retained" + ? Buffer.from(binary.content_base64, "base64").toString("latin1") + : undefined; + return typeof value === "string" ? value : undefined; + }; + const name = Array.isArray(item) + ? text(item[0], `/${side}/headers/${index}/0`) + : pair?.name; + const value = Array.isArray(item) + ? text(item[1], `/${side}/headers/${index}/1`) + : pair?.value; + if (typeof name === "string") { + const key = name.toLowerCase(); + const knownValue = typeof value === "string" ? value : undefined; + headers.set( + key, + headers.has(key) && headers.get(key) !== knownValue + ? undefined + : knownValue, + ); + } else headerNamesKnown = false; + } + } + if (format === "har") { + if ( + pointer !== `/${side}/${side === "request" ? "postData" : "content"}/text` + ) + return undefined; + const content = object( + message[side === "request" ? "postData" : "content"], + ); + return { + contentType: + typeof content?.mimeType === "string" + ? content.mimeType + : headers.get("content-type"), + httpIdentityKnown: true, + httpCodingKnown: true, + }; + } + if (pointer !== `/${side}/body` && pointer !== `/${side}/content`) + return undefined; + return { + contentType: headers.get("content-type"), + httpCodingKnown: + headerNamesKnown && + ["content-encoding", "transfer-encoding"].every( + (name) => !headers.has(name) || headers.get(name) !== undefined, + ), + httpIdentityKnown: + headerNamesKnown && + ["content-encoding", "transfer-encoding"].every( + (name) => + // HTTP OWS is SP/HTAB; require the entire identity token, including EOF. + !headers.has(name) || + /^[ \t]*identity[ \t]*(?![\s\S])/i.test(headers.get(name) ?? ""), + ), + }; +}; + +type RetainedBody = Extract< + WebNetworkCaptureRecord["binary_fields"][number], + { state: "retained" } +>; +interface PreparedBody { + readonly field: RetainedBody; + readonly body: Buffer; + readonly contentType: string | undefined; + readonly framing: ProtobufDecoding["framing"]; + readonly prepared: PreparedProtobufBody; +} +export interface PreparedProtobufCapture { + readonly records: readonly { + record: WebNetworkCaptureRecord; + bodies: readonly PreparedBody[]; + }[]; + readonly incomplete: boolean; +} + +/** Sanitize every selected body before any schema acquisition or field-expansion failure. + * Until this stage completes, the caller must not publish the original source as partial Evidence. + */ +export const prepareCaptureProtobuf = async ( + capture: WebNetworkCapture, + input: InspectWebNetworkCaptureInput, + signal?: AbortSignal, +): Promise => { + if (input.record_ordinals === undefined) return undefined; + capture.protobuf_coverage = { + status: "partial", + selected_records: input.record_ordinals.length, + inspected_records: 0, + }; + capture.limitations.push(...PROTOBUF_LIMITATIONS); + let incomplete = false; + const records: { record: WebNetworkCaptureRecord; bodies: PreparedBody[] }[] = + []; + const options = input.protobuf; + const pointers = + options?.body_pointers === undefined + ? undefined + : new Set(options.body_pointers); + const redactions = new CaptureRedaction(input.sensitive_values); + const work = new ProtobufBudget(signal); + for (const ordinal of input.record_ordinals) { + await work.checkpoint(); + const record = capture.records[ordinal]; + if (record === undefined) continue; + const bodies: PreparedBody[] = []; + records.push({ record, bodies }); + const binaryByPointer = new Map( + record.binary_fields.map((field) => [field.pointer, field]), + ); + for (const field of record.binary_fields) { + await work.checkpoint(); + if (pointers !== undefined && !pointers.has(field.pointer)) continue; + const metadata = bodyMetadata( + record, + field.pointer, + input.format, + binaryByPointer, + ); + // Framing overrides interpret bodies; header and extension bytes remain their own facts. + if (metadata === undefined) continue; + if (field.state !== "retained") { + incomplete = true; + continue; + } + const contentType = metadata.contentType; + const essence = contentType?.split(";", 1)[0]?.trim().toLowerCase(); + const automatic = + essence === undefined + ? undefined + : PROTOBUF_FRAMING[essence as keyof typeof PROTOBUF_FRAMING]; + const framing = + options?.framing === undefined || options.framing === "auto" + ? automatic + : options.framing; + if (framing === undefined) continue; + if ( + field.bytes > WEB_NETWORK_CAPTURE_LIMITS.inputBytes || + field.content_base64.length > + Math.ceil(WEB_NETWORK_CAPTURE_LIMITS.inputBytes / 3) * 4 + ) + throw new CaptureFormatError( + "limit", + "Protobuf body exceeds the admitted 32 MiB source bound", + ); + const body = Buffer.from(field.content_base64, "base64"); + if ( + body.length !== field.bytes || + createHash("sha256").update(body).digest("hex") !== field.sha256 + ) + throw new AnalysisInputError("inspect_web_network_capture", undefined, [ + { + path: ["capture_path"], + reason: "invalid_format", + message: + "Retained protobuf bytes contradict their source descriptor", + }, + ]); + const exclusions = { + decoded: (data: Uint8Array) => + redactions.sensitiveBytes( + Buffer.from(data.buffer, data.byteOffset, data.byteLength), + ), + header: isCaptureCredentialHeaderBytes, + }; + const prepared = await prepareProtobufBody( + body, + framing, + exclusions, + signal, + ); + // Caller overrides do not turn off exclusions for the transport declared by the source. + const declared = + prepared.excluded === undefined && + automatic !== undefined && + automatic !== framing + ? await prepareProtobufBody(body, automatic, exclusions, signal) + : prepared; + const excluded = prepared.excluded ?? declared.excluded; + if (excluded !== undefined) { + excludeBody(record, field.pointer, excluded); + binaryByPointer.delete(field.pointer); + incomplete = true; + continue; + } + // Analysis eligibility never bypasses the independent source exclusions. + if ( + !metadata.httpCodingKnown && + (!prepared.credentialsChecked || !declared.credentialsChecked) + ) + throw new AnalysisCapabilityUnavailableError( + input.format, + "inspect_web_network_capture", + "Credential inspection could not complete while HTTP coding metadata was unavailable; unchecked capture bodies were withheld.", + { + diagnostics: { body_pointer: field.pointer }, + userMessage: + "Use a capture with available HTTP coding metadata, or select records whose transport can be inspected. Unchecked source bytes were not published.", + }, + ); + if (!metadata.httpIdentityKnown) { + incomplete = true; + record.limitations.push( + "Selected protobuf body's HTTP content/transfer coding is encoded or unknown, so identity is not established; its exact stored bytes remain opaque.", + ); + continue; + } + bodies.push({ field, body, contentType, framing, prepared }); + } + } + return { records, incomplete }; +}; + +/** Interpret the sanitized source once, inside the capture adapter's owned operation. */ +export const decodeCaptureProtobuf = async ( + capture: WebNetworkCapture, + input: InspectWebNetworkCaptureInput, + resources: ArtifactResourceScope, + prepared: PreparedProtobufCapture | undefined, + signal?: AbortSignal, + encodedSourceBytes = 0, +): Promise => { + if (prepared === undefined || capture.protobuf_coverage === undefined) return; + let incomplete = prepared.incomplete; + const budget = new ProtobufBudget(signal); + budget.charge( + encodedSourceBytes + + 256 + + Buffer.byteLength(JSON.stringify(PROTOBUF_LIMITATIONS)), + ); + for (const entry of prepared.records) + for (const item of entry.bodies) + budget.charge( + item.body.length + + (item.prepared.data === item.body + ? 0 + : (item.prepared.data?.length ?? 0)), + ); + const redact = (text: string) => { + if (input.sensitive_values.length === 0) return text; + if (input.sensitive_values.some((literal) => text.includes(literal))) + budget.charge(text.length * 60); + return redactExplicitText(text, input.sensitive_values); + }; + const options = input.protobuf; + let schema: ProtobufSchema | undefined; + let schemaSource: ProtobufDecoding["schema"]; + if (options?.schema !== undefined) { + const path = options.schema.descriptor_set_path; + if (!isAbsolute(path)) + throw new AnalysisInputError("inspect_web_network_capture", undefined, [ + { + path: ["protobuf", "schema", "descriptor_set_path"], + reason: "invalid_format", + message: "Expected an absolute descriptor-set path", + }, + ]); + const snapshot = await readStableArtifact( + path, + WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + signal, + undefined, + resources, + ).catch((cause: unknown) => { + throw withCaptureReaderCleanup( + historicalCaptureFailure( + input, + cause, + "descriptor-read", + signal === undefined ? undefined : { signal }, + ), + cause, + ); + }); + const { ProtobufSchema } = await import("./ProtobufSchema.js"); + try { + schema = await ProtobufSchema.read( + snapshot.bytes, + options.schema.message_type, + budget, + redact, + ); + } catch (cause) { + if (signal?.aborted === true && cause === signal.reason) throw cause; + if (cause instanceof CaptureFormatError) throw cause; + throw new AnalysisInputError("inspect_web_network_capture", undefined, [ + { + path: ["protobuf", "schema"], + reason: "invalid_format", + message: + "The selected local FileDescriptorSet or message type is invalid or exceeds decoding budgets", + }, + ]); + } + schemaSource = { + decoder: "protobuf-es-2.16.0", + descriptor_set_path: redact(path), + descriptor_set_sha256: snapshot.sha256, + descriptor_set_bytes: snapshot.bytes.length, + message_type: redact(options.schema.message_type), + }; + } + for (const { record, bodies } of prepared.records) { + const decoded: ProtobufDecoding[] = []; + for (const { + field, + body, + contentType, + framing, + prepared: bodySource, + } of bodies) { + await budget.checkpoint(); + if ( + input.sensitive_values.some( + (value) => + field.pointer.includes(value) || + field.sha256.includes(value) || + schemaSource?.descriptor_set_sha256.includes(value), + ) + ) { + incomplete = true; + continue; + } + budget.charge( + 2048 + + 8 * + (field.pointer.length + + (contentType?.length ?? 0) + + (schemaSource?.descriptor_set_path.length ?? 0) + + (schemaSource?.message_type.length ?? 0)), + ); + schema?.reset(); + const result = await decodeProtobufBody( + body, + framing, + { + pointer: field.pointer, + content_type: contentType ?? null, + body_sha256: field.sha256, + }, + redact, + budget, + schema, + bodySource, + ); + if (schemaSource !== undefined) result.schema = schemaSource; + if ( + result.status !== "complete" || + result.fields.some( + (field) => + field.schema_status !== undefined && + field.schema_status !== "decoded", + ) + ) + incomplete = true; + decoded.push(result); + } + if (decoded.length > 0) record.protobuf = decoded; + capture.protobuf_coverage.inspected_records++; + } + if ( + !incomplete && + capture.protobuf_coverage.inspected_records === + capture.protobuf_coverage.selected_records + ) + capture.protobuf_coverage.status = "complete"; +}; diff --git a/src/browser/history/protobuf/ProtobufFraming.ts b/src/browser/history/protobuf/ProtobufFraming.ts new file mode 100644 index 0000000000..6c93feee8f --- /dev/null +++ b/src/browser/history/protobuf/ProtobufFraming.ts @@ -0,0 +1,144 @@ +import { err, ok } from "../../../domain/result.js"; +import type { ProtobufDecoding } from "../../../domain/protobufWire.js"; +import { WEB_NETWORK_CAPTURE_LIMITS } from "../../../domain/webNetworkCapture.js"; +import { CaptureFormatError } from "../CaptureFormatError.js"; +import { ProtobufBudget } from "./ProtobufWire.js"; + +/** One owner for frame boundaries used by exclusion and field interpretation. */ +export const readGrpcFrame = ( + data: Uint8Array, + at: number, + framing: Exclude, +) => { + if (data.length - at < 5) + return err({ offset: at, reason: "Truncated gRPC frame header" }); + const flags = data[at]!; + if ( + (framing === "grpc" && flags !== 0 && flags !== 1) || + (flags & ~0x81) !== 0 + ) + return err({ + offset: at, + reason: "Invalid flags for the selected gRPC framing", + }); + const length = + data[at + 1]! * 2 ** 24 + + data[at + 2]! * 2 ** 16 + + data[at + 3]! * 256 + + data[at + 4]!; + if (length > data.length - at - 5) + return err({ offset: at, reason: "gRPC frame exceeds the retained body" }); + return ok({ + flags, + bytes: length, + payload_offset: at + 5, + end: at + 5 + length, + }); +}; + +/** Decode padded stream chunks into one source-bounded buffer, never a chunk inventory. */ +const decodeText = async ( + body: Uint8Array, + work: ProtobufBudget, +): Promise => { + const text = Buffer.from(body.buffer, body.byteOffset, body.byteLength) + .toString("latin1") + .replace(/[ \r\n\t]/g, ""); + if (/[^A-Za-z0-9+/=]/.test(text)) return undefined; + const output = Buffer.alloc(Math.floor((text.length * 3) / 4)); + let at = 0; + let written = 0; + while (at < text.length) { + await work.checkpoint(); + let end = text.indexOf("=", at); + if (end < 0) end = text.length; + else { + end++; + if (text[end] === "=") end++; + } + const chunk = text.slice(at, end); + if ( + chunk.length % 4 === 1 || + (chunk.includes("=") && chunk.length % 4 !== 0) + ) + return undefined; + const bytes = Buffer.from(chunk, "base64"); + if ( + bytes.toString("base64").replace(/=+$/, "") !== chunk.replace(/=+$/, "") + ) + return undefined; + output.set(bytes, written); + written += bytes.length; + at = end; + } + return output.subarray(0, written); +}; + +export interface PreparedProtobufBody { + readonly data: Uint8Array | undefined; + readonly credentialsChecked: boolean; + readonly excluded?: "explicit-sensitive-value" | "transport-credential"; +} + +/** Complete credential inspection before schema I/O, fact budgets or recoverable errors. + * Its temporary storage is bounded by admitted source bytes, independently of field expansion. + */ +export const prepareProtobufBody = async ( + body: Uint8Array, + framing: ProtobufDecoding["framing"], + exclusions: { + decoded(bytes: Uint8Array): boolean; + header(bytes: Uint8Array, start: number, end: number): boolean; + }, + signal?: AbortSignal, +): Promise => { + if (body.length > WEB_NETWORK_CAPTURE_LIMITS.inputBytes) + throw new CaptureFormatError( + "limit", + "Protobuf body exceeds the admitted 32 MiB source bound", + ); + const work = new ProtobufBudget(signal); + await work.checkpoint(); + const data = + framing === "grpc-web-text" ? await decodeText(body, work) : body; + if (data === undefined) return { data, credentialsChecked: false }; + if (exclusions.decoded(data)) + return { + data, + credentialsChecked: true, + excluded: "explicit-sensitive-value", + }; + if (framing === "raw" || framing === "grpc") + return { data, credentialsChecked: true }; + let at = 0; + let credentialsChecked = true; + while (at < data.length) { + await work.checkpoint(); + const parsed = readGrpcFrame(data, at, framing); + if (!parsed.ok) { + credentialsChecked = false; + break; + } + const frame = parsed.value; + if (frame.flags === 129) credentialsChecked = false; + if (frame.flags === 128) { + let start = frame.payload_offset; + for (let index = start; index <= frame.end; index++) { + // Large/malformed lines remain source-bounded and cancellation stays cooperative. + if ((index - start) % 256 === 0) await work.checkpoint(); + if (index !== frame.end && data[index] !== 10) continue; + let colon = start; + while (colon < index && data[colon] !== 58) colon++; + if (colon < index && exclusions.header(data, start, colon)) + return { + data, + credentialsChecked: true, + excluded: "transport-credential", + }; + start = index + 1; + } + } + at = frame.end; + } + return { data, credentialsChecked }; +}; diff --git a/src/browser/history/protobuf/ProtobufSchema.ts b/src/browser/history/protobuf/ProtobufSchema.ts new file mode 100644 index 0000000000..8135b6c58d --- /dev/null +++ b/src/browser/history/protobuf/ProtobufSchema.ts @@ -0,0 +1,299 @@ +import { + createFileRegistry, + fromBinary, + ScalarType, + type DescField, + type DescEnum, + type DescMessage, +} from "@bufbuild/protobuf"; +import { + FileDescriptorSetSchema, + type FileDescriptorSet, +} from "@bufbuild/protobuf/wkt"; +import { BinaryReader } from "@bufbuild/protobuf/wire"; +import type { ProtobufField } from "../../../domain/protobufWire.js"; +import { WEB_NETWORK_CAPTURE_LIMITS } from "../../../domain/webNetworkCapture.js"; +import { CaptureFormatError } from "../CaptureFormatError.js"; +import { + decodeProtobufWire, + readProtobufVarint, + type ProtobufBudget, + type WireSchema, +} from "./ProtobufWire.js"; + +const scalarWire = (scalar: ScalarType): ProtobufField["wire_type"] => { + if (scalar === ScalarType.STRING || scalar === ScalarType.BYTES) return "len"; + if ( + [ScalarType.DOUBLE, ScalarType.FIXED64, ScalarType.SFIXED64].includes( + scalar, + ) + ) + return "i64"; + if ( + [ScalarType.FLOAT, ScalarType.FIXED32, ScalarType.SFIXED32].includes(scalar) + ) + return "i32"; + return "varint"; +}; + +const floatingValue = (value: number): string => + Object.is(value, -0) ? "-0" : String(value); + +/** Qualified names can expand a small descriptor into a much larger registry. */ +const chargeRegistryExpansion = async ( + descriptor: FileDescriptorSet, + budget: ProtobufBudget, +): Promise => { + for (const file of descriptor.file) { + budget.charge(file.name.length * 8 + 128); + const stack = file.messageType.map((message) => ({ + message, + prefix: file.package.length, + })); + const chargeQualifiedName = (name: string, prefix: number) => + budget.charge((prefix + 1 + name.length) * 8 + 128); + for (const enumeration of file.enumType) + chargeQualifiedName(enumeration.name, file.package.length); + for (const extension of file.extension) + chargeQualifiedName(extension.name, file.package.length); + for (const service of file.service) { + const length = file.package.length + 1 + service.name.length; + budget.charge(length * 8 + 128); + for (const method of service.method) + chargeQualifiedName(method.name, length); + } + while (stack.length > 0) { + await budget.checkpoint(); + const { message, prefix } = stack.pop()!; + const length = prefix + 1 + message.name.length; + budget.charge(length * 8 + 128); + for (const field of message.field) + chargeQualifiedName(field.name, length); + for (const extension of message.extension) + chargeQualifiedName(extension.name, length); + for (const enumeration of message.enumType) + chargeQualifiedName(enumeration.name, length); + for (const nested of message.nestedType) + stack.push({ message: nested, prefix: length }); + } + } +}; + +const scalarValue = (reader: BinaryReader, scalar: ScalarType): string => { + switch (scalar) { + case ScalarType.DOUBLE: + return floatingValue(reader.double()); + case ScalarType.FLOAT: + return floatingValue(reader.float()); + case ScalarType.INT64: + return String(reader.int64()); + case ScalarType.UINT64: + return String(reader.uint64()); + case ScalarType.INT32: + return String(reader.int32()); + case ScalarType.FIXED64: + return String(reader.fixed64()); + case ScalarType.FIXED32: + return String(reader.fixed32()); + case ScalarType.BOOL: + return String(reader.bool()); + case ScalarType.UINT32: + return String(reader.uint32()); + case ScalarType.SFIXED32: + return String(reader.sfixed32()); + case ScalarType.SFIXED64: + return String(reader.sfixed64()); + case ScalarType.SINT32: + return String(reader.sint32()); + case ScalarType.SINT64: + return String(reader.sint64()); + default: + throw new Error("Not a numeric protobuf scalar"); + } +}; + +/** The supplied schema interprets occurrences; it never overwrites raw wire facts. */ +export class ProtobufSchema implements WireSchema { + readonly #fields = new Map>(); + readonly #messages = new Map(); + readonly #enums = new WeakMap< + DescEnum, + ReadonlyMap + >(); + constructor( + readonly message: DescMessage, + readonly budget: ProtobufBudget, + readonly redact: (text: string) => string, + ) {} + + static async read( + bytes: Uint8Array, + typeName: string, + budget: ProtobufBudget, + redact: (text: string) => string, + ): Promise { + // Bound the object expansion of the descriptor before the SDK allocates its message tree. + const checked = await decodeProtobufWire( + bytes, + 0, + bytes.length, + 0, + budget, + redact, + new ProtobufSchema(FileDescriptorSetSchema, budget, redact), + ); + if ( + checked.error !== undefined || + checked.fields.some((field) => field.schema_status === "invalid-message") + ) + throw new Error("Malformed FileDescriptorSet wire format"); + const descriptor = fromBinary(FileDescriptorSetSchema, bytes, { + readUnknownFields: false, + recursionLimit: WEB_NETWORK_CAPTURE_LIMITS.depth, + }); + await chargeRegistryExpansion(descriptor, budget); + const message = createFileRegistry(descriptor).getMessage( + typeName.replace(/^\./, ""), + ); + if (message === undefined) + throw new Error( + "The supplied descriptor set does not define the selected message type", + ); + return new ProtobufSchema(message, budget, redact); + } + + reset(): void { + this.#messages.clear(); + } + + nested(field: ProtobufField): boolean { + return this.#messages.has(field.index); + } + + async annotate( + field: ProtobufField, + bytes: Uint8Array, + parent: ProtobufField | undefined, + ): Promise { + const message = + parent === undefined ? this.message : this.#messages.get(parent.index); + if (message === undefined) return; + let fields = this.#fields.get(message); + if (fields === undefined) { + fields = new Map(message.fields.map((item) => [item.number, item])); + this.#fields.set(message, fields); + } + const desc = fields.get(field.number); + if (desc === undefined) { + field.schema_status = "unknown-field"; + return; + } + this.budget.charge( + 8 * + (desc.name.length + + (desc.message?.typeName.length ?? 0) + + (desc.enum?.typeName.length ?? 0)), + ); + field.schema_name = this.redact(desc.name); + field.schema_status = "decoded"; + if (desc.fieldKind === "map") { + field.schema_type = "map"; + field.schema_status = "unsupported"; + return; + } + if (desc.message !== undefined) { + field.schema_type = this.redact(desc.message.typeName); + const expected = desc.delimitedEncoding ? "group" : "len"; + if (field.wire_type !== expected) + field.schema_status = "wire-type-mismatch"; + else this.#messages.set(field.index, desc.message); + return; + } + const scalar = desc.enum === undefined ? desc.scalar : ScalarType.INT32; + if (scalar === undefined) { + field.schema_status = "unsupported"; + return; + } + field.schema_type = + desc.enum === undefined + ? ScalarType[scalar].toLowerCase() + : this.redact(desc.enum.typeName); + const packed = + desc.fieldKind === "list" && + field.wire_type === "len" && + scalar !== ScalarType.STRING && + scalar !== ScalarType.BYTES; + if (!packed && field.wire_type !== scalarWire(scalar)) { + field.schema_status = "wire-type-mismatch"; + return; + } + const payload = bytes.subarray( + field.payload_offset, + field.payload_offset + field.payload_bytes, + ); + try { + if (scalar === ScalarType.BYTES) return; // Exact bytes already have an accessible source range. + if (scalar === ScalarType.STRING) { + // Wire decoding owns UTF-8 (including BOM) and literal exclusions. + // Reuse that fact; charge only its additional serialized representation. + this.budget.charge(payload.length * 6); + if (field.text === undefined) + field.schema_status = "wire-type-mismatch"; + else field.schema_value = field.text; + return; + } + const reader = new BinaryReader(payload); + const enumValues = new Map(); + const read = () => { + if ( + scalarWire(scalar) === "varint" && + readProtobufVarint(payload, reader.pos) === undefined + ) + throw new Error("Invalid packed varint"); + const value = scalarValue(reader, scalar); + if (desc.enum !== undefined) { + let names = this.#enums.get(desc.enum); + if (names === undefined) { + this.budget.charge(desc.enum.values.length * 256); + const indexed = new Map(); + for (const item of desc.enum.values) { + const number = String(item.number); + const aliases = indexed.get(number) ?? []; + aliases.push(item.name); + indexed.set(number, aliases); + } + names = indexed; + this.#enums.set(desc.enum, names); + } + const aliases = names.get(value); + if (aliases !== undefined && !enumValues.has(value)) { + this.budget.charge( + 256 + aliases.reduce((sum, name) => sum + name.length * 8, 0), + ); + enumValues.set(value, aliases.map(this.redact)); + } + } + return value; + }; + if (packed) { + const values: string[] = []; + while (reader.pos < reader.len) { + await this.budget.checkpoint(); + this.budget.charge(192); + values.push(this.redact(read())); + } + field.schema_value = JSON.stringify(values); + } else field.schema_value = this.redact(read()); + if (enumValues.size > 0) + field.schema_enum_values = [...enumValues].map(([number, names]) => ({ + number: this.redact(number), + names: [...names], + })); + } catch (cause) { + if (cause instanceof CaptureFormatError) throw cause; + if (this.budget.signal?.aborted && cause === this.budget.signal.reason) + throw cause; + field.schema_status = "wire-type-mismatch"; + } + } +} diff --git a/src/browser/history/protobuf/ProtobufWire.test.ts b/src/browser/history/protobuf/ProtobufWire.test.ts new file mode 100644 index 0000000000..1c7c6eb06c --- /dev/null +++ b/src/browser/history/protobuf/ProtobufWire.test.ts @@ -0,0 +1,522 @@ +import { expect, it } from "vitest"; +import { create, toBinary } from "@bufbuild/protobuf"; +import { FileDescriptorSetSchema } from "@bufbuild/protobuf/wkt"; +import { decodeProtobufWire, ProtobufBudget } from "./ProtobufWire.js"; +import { decodeProtobufBody } from "./ProtobufBody.js"; +import { ProtobufSchema } from "./ProtobufSchema.js"; +import { protobufCaptureDescriptor } from "../../../../tests/fixtures/protobufCapture.js"; + +const same = (value: string) => value; +const wire = (hex: string) => { + const bytes = Buffer.from(hex, "hex"); + return decodeProtobufWire( + bytes, + 0, + bytes.length, + 0, + new ProtobufBudget(), + same, + ); +}; +const source = { + pointer: "/response/content/text", + content_type: "application/grpc-web+proto", + body_sha256: "a".repeat(64), +}; +const frame = (bytes: Buffer, flag = 0) => { + const header = Buffer.alloc(5); + header[0] = flag; + header.writeUInt32BE(bytes.length, 1); + return Buffer.concat([header, bytes]); +}; + +it("rejects overflowing uint64 values without masking and preserves the maximum value", async () => { + expect(await wire("0880808080808080808002")).toMatchObject({ + fields: [], + error: { offset: 0 }, + }); + expect((await wire("08ffffffffffffffffff01")).fields[0]?.value).toBe( + "18446744073709551615", + ); + expect(await wire("0880808080808080808080")).toHaveProperty("error"); + expect(await wire("0880")).toHaveProperty("error"); +}); + +it("preserves repeated field occurrences, source ranges, ambiguous bytes and exact fixed values", async () => { + const result = await wire("0896010801120568656c6c6f1a02080125ffffffff"); + expect(result.error).toBeUndefined(); + expect(result.fields.map(({ number }) => number)).toEqual([1, 1, 2, 3, 4]); + expect(result.fields[2]).toMatchObject({ + field_id: "offset:5", + offset: 5, + payload_offset: 7, + payload_bytes: 5, + text: "hello", + interpretations: ["utf8-string", "packed-varint", "bytes"], + }); + expect(result.fields[3]?.interpretations).toContain("nested-message"); + expect(result.fields[3]?.text).toBe("\b\u0001"); + expect(result.fields[4]?.value).toBe("4294967295"); +}); + +it("handles proto2 groups and reports mismatched or missing end groups", async () => { + expect((await wire("0b10010c")).fields).toMatchObject([ + { wire_type: "group", bytes: 4 }, + { parent_field_id: "offset:0", number: 2, value: "1" }, + ]); + expect(await wire("0b100114")).toHaveProperty("error"); + expect(await wire("0b1001")).toHaveProperty("error"); + expect(await wire("0c")).toHaveProperty("error"); +}); + +it("keeps empty and control-containing UTF-8 among ambiguous length-delimited readings", async () => { + const empty = await wire("0a00"); + expect(empty.fields[0]).toMatchObject({ + text: "", + interpretations: [ + "utf8-string", + "nested-message", + "packed-varint", + "packed-fixed32", + "packed-fixed64", + "bytes", + ], + }); + expect((await wire("0a020801")).fields[0]).toMatchObject({ + text: "\b\u0001", + }); +}); + +it("keeps enum numbers separate from every declared alias", async () => { + const bytes = toBinary( + FileDescriptorSetSchema, + create(FileDescriptorSetSchema, { + file: [ + { + name: "aliases.proto", + package: "aliases", + syntax: "proto3", + enumType: [ + { + name: "Mode", + options: { allowAlias: true }, + value: [ + { name: "ZERO", number: 0 }, + { name: "ALSO_ZERO", number: 0 }, + ], + }, + ], + messageType: [ + { + name: "Message", + field: [ + { + name: "mode", + number: 1, + type: 14, + typeName: ".aliases.Mode", + label: 1, + }, + { name: "negative_zero", number: 2, type: 2, label: 1 }, + ], + }, + ], + }, + ], + }), + ); + const budget = new ProtobufBudget(); + const schema = await ProtobufSchema.read( + bytes, + "aliases.Message", + budget, + same, + ); + const decoded = await decodeProtobufBody( + Buffer.from([8, 0, 21, 0, 0, 0, 128]), + "raw", + source, + same, + budget, + schema, + ); + expect(decoded.fields[1]).toMatchObject({ + value: "2147483648", + schema_value: "-0", + }); + expect(decoded.fields[0]).toMatchObject({ + value: "0", + schema_type: "aliases.Mode", + schema_value: "0", + schema_enum_values: [{ number: "0", names: ["ZERO", "ALSO_ZERO"] }], + }); +}); + +it("bounds qualified descriptor-name expansion before creating the SDK registry", async () => { + const bytes = toBinary( + FileDescriptorSetSchema, + create(FileDescriptorSetSchema, { + file: [ + { + name: "expansion.proto", + syntax: "proto3", + messageType: [ + { + name: "A".repeat(10000), + nestedType: Array.from({ length: 2000 }, (_, index) => ({ + name: `Child${index}`, + })), + }, + ], + }, + ], + }), + ); + await expect( + ProtobufSchema.read(bytes, "Missing", new ProtobufBudget(), same), + ).rejects.toMatchObject({ reason: "limit" }); +}); + +it("distinguishes native gRPC flags from web trailers and retains partial messages", async () => { + const body = Buffer.concat([ + frame(Buffer.from("0801", "hex")), + frame(Buffer.from("grpc-status: 0\r\ngrpc-message: ok\r\n"), 128), + ]); + const web = await decodeProtobufBody( + body, + "grpc-web", + source, + same, + new ProtobufBudget(), + ); + expect(web.status).toBe("complete"); + expect(web.frames[1]?.trailers).toEqual([ + { name: "grpc-status", value: "0" }, + { name: "grpc-message", value: "ok" }, + ]); + expect( + (await decodeProtobufBody(body, "grpc", source, same, new ProtobufBudget())) + .error, + ).toMatchObject({ offset: 7 }); + expect( + ( + await decodeProtobufBody( + frame(Buffer.from("0880", "hex")), + "grpc", + source, + same, + new ProtobufBudget(), + ) + ).frames[0], + ).toMatchObject({ status: "partial", error: { offset: 5 } }); + expect( + ( + await decodeProtobufBody( + frame(Buffer.from("0801", "hex"), 2), + "grpc-web", + source, + same, + new ProtobufBudget(), + ) + ).status, + ).toBe("partial"); +}); + +it("decodes independently padded base64 chunks with explicit coordinates and rejects noncanonical text", async () => { + const first = frame(Buffer.from("0801", "hex")); + const second = frame(Buffer.from("0802", "hex")); + const body = Buffer.from( + first.toString("base64") + second.toString("base64"), + ); + const decoded = await decodeProtobufBody( + body, + "grpc-web-text", + source, + same, + new ProtobufBudget(), + ); + expect(decoded).toMatchObject({ + offset_basis: "base64-decoded-body", + decoded_bytes: 14, + status: "complete", + }); + expect( + decoded.fields.map(({ field_id, value }) => [field_id, value]), + ).toEqual([ + ["offset:5", "1"], + ["offset:12", "2"], + ]); + expect(decoded.fields.map(({ index }) => index)).toEqual([0, 1]); + expect( + ( + await decodeProtobufBody( + Buffer.from("AB=="), + "grpc-web-text", + source, + same, + new ProtobufBudget(), + ) + ).status, + ).toBe("partial"); +}); + +it("keeps compressed payloads explicitly unknown and rejects frames beyond the body", async () => { + const compressed = await decodeProtobufBody( + frame(Buffer.from("0801", "hex"), 1), + "grpc", + source, + same, + new ProtobufBudget(), + ); + expect(compressed.frames[0]).toMatchObject({ + status: "unknown", + compressed: true, + examined_bytes: 0, + }); + expect(compressed.fields).toEqual([]); + expect( + ( + await decodeProtobufBody( + Buffer.from("00000000090801", "hex"), + "grpc", + source, + same, + new ProtobufBudget(), + ) + ).error, + ).toBeDefined(); +}); + +it("uses supplied descriptors without replacing wire values and preserves schema disagreements", async () => { + const descriptor = toBinary( + FileDescriptorSetSchema, + create(FileDescriptorSetSchema, { + file: [ + { + name: "sample.proto", + package: "sample", + syntax: "proto3", + messageType: [ + { + name: "Child", + field: [{ name: "count", number: 1, type: 5, label: 1 }], + }, + { + name: "Message", + field: [ + { name: "signed", number: 1, type: 17, label: 1 }, + { name: "text", number: 2, type: 9, label: 1 }, + { + name: "child", + number: 3, + type: 11, + label: 1, + typeName: ".sample.Child", + }, + { name: "values", number: 4, type: 13, label: 3 }, + ], + }, + ], + }, + ], + }), + ); + const budget = new ProtobufBudget(); + const schema = await ProtobufSchema.read( + descriptor, + "sample.Message", + budget, + same, + ); + const body = Buffer.from("0801120568656c6c6f1a020801220201023801", "hex"); + const decoded = await decodeProtobufBody( + body, + "raw", + source, + same, + budget, + schema, + ); + expect(decoded.fields[0]).toMatchObject({ + value: "1", + schema_name: "signed", + schema_type: "sint32", + schema_value: "-1", + }); + expect( + decoded.fields.find(({ schema_name }) => schema_name === "text") + ?.schema_value, + ).toBe("hello"); + expect( + decoded.fields.find(({ schema_name }) => schema_name === "count"), + ).toMatchObject({ parent_field_id: "offset:9", value: "1" }); + expect( + decoded.fields.find(({ schema_name }) => schema_name === "values") + ?.schema_value, + ).toBe('["1","2"]'); + expect(decoded.fields.at(-1)?.schema_status).toBe("unknown-field"); + schema.reset(); + expect( + ( + await decodeProtobufBody( + Buffer.from("1001", "hex"), + "raw", + source, + same, + budget, + schema, + ) + ).fields[0]?.schema_status, + ).toBe("wire-type-mismatch"); +}); + +it("stops before exhausting fact capacity and yields for caller cancellation", async () => { + const budget = new ProtobufBudget(); + budget.charge(96 * 1024 * 1024 - 1023); + await expect( + decodeProtobufWire(Buffer.from([8, 1]), 0, 2, 0, budget, same), + ).rejects.toMatchObject({ reason: "limit" }); + const abort = new AbortController(); + const reason = new Error("caller stopped selected decoding"); + const data = Buffer.alloc(8192); + for (let at = 0; at < data.length; at += 2) { + data[at] = 8; + data[at + 1] = 1; + } + const task = decodeProtobufWire( + data, + 0, + data.length, + 0, + new ProtobufBudget(abort.signal), + same, + ); + setImmediate(() => abort.abort(reason)); + await expect(task).rejects.toBe(reason); +}); + +it("preserves a leading Unicode BOM in both the wire string and schema interpretation", async () => { + const budget = new ProtobufBudget(); + const schema = await ProtobufSchema.read( + protobufCaptureDescriptor(), + "capture.Message", + budget, + same, + ); + const bytes = Buffer.from("0a08efbbbf68656c6c6f", "hex"); + const result = await decodeProtobufWire( + bytes, + 0, + bytes.length, + 0, + budget, + same, + schema, + ); + expect(result.fields[0]).toMatchObject({ + text: "\uFEFFhello", + schema_value: "\uFEFFhello", + }); +}); + +it("keeps bad nested schema interpretations local and preserves subsequent outer fields", async () => { + const descriptor = toBinary( + FileDescriptorSetSchema, + create(FileDescriptorSetSchema, { + file: [ + { + name: "nested.proto", + package: "nested", + syntax: "proto3", + messageType: [ + { + name: "Child", + field: [{ name: "value", number: 1, type: 13, label: 1 }], + }, + { + name: "Message", + field: [ + { + name: "child", + number: 1, + type: 11, + label: 1, + typeName: ".nested.Child", + }, + { name: "value", number: 2, type: 13, label: 1 }, + ], + }, + ], + }, + ], + }), + ); + const budget = new ProtobufBudget(); + const schema = await ProtobufSchema.read( + descriptor, + "nested.Message", + budget, + same, + ); + const bytes = Buffer.from("0a01ff1001", "hex"); + const result = await decodeProtobufWire( + bytes, + 0, + bytes.length, + 0, + budget, + same, + schema, + ); + expect(result.error).toBeUndefined(); + expect(result.examined_bytes).toBe(bytes.length); + expect(result.fields).toMatchObject([ + { + number: 1, + schema_status: "invalid-message", + schema_error: { offset: 2 }, + }, + { number: 2, value: "1", schema_value: "1", parent_field_id: null }, + ]); +}); + +it("yields and preserves the cancellation reason inside one large packed interpretation", async () => { + const abort = new AbortController(); + const budget = new ProtobufBudget(abort.signal); + const schema = await ProtobufSchema.read( + protobufCaptureDescriptor(), + "capture.Message", + budget, + same, + ); + // Field 2, length 100000, followed by 100000 independently expected uint32 ones. + const bytes = Buffer.concat([ + Buffer.from([18, 160, 141, 6]), + Buffer.alloc(100000, 1), + ]); + const parsed = await decodeProtobufWire( + bytes, + 0, + bytes.length, + 0, + new ProtobufBudget(), + same, + ); + const field = parsed.fields[0]; + if (field === undefined) throw new Error("Expected the packed wire field"); + const task = schema.annotate(field, bytes, undefined); + const reason = new Error("stop during packed annotation"); + setImmediate(() => abort.abort(reason)); + await expect(task).rejects.toBe(reason); +}); + +it("rejects malformed descriptor wire even when the SDK would skip the unknown field", async () => { + // File t.proto defines empty M, then unknown field 99 carries an overflowing uint64. + const bytes = Buffer.from( + "0a1a0a07742e70726f746f22030a014d980680808080808080808002", + "hex", + ); + await expect( + ProtobufSchema.read(bytes, "M", new ProtobufBudget(), same), + ).rejects.toThrow("Malformed FileDescriptorSet wire format"); +}); diff --git a/src/browser/history/protobuf/ProtobufWire.ts b/src/browser/history/protobuf/ProtobufWire.ts new file mode 100644 index 0000000000..a1eb384303 --- /dev/null +++ b/src/browser/history/protobuf/ProtobufWire.ts @@ -0,0 +1,269 @@ +import { setImmediate } from "node:timers/promises"; +import type { ProtobufField } from "../../../domain/protobufWire.js"; +import { CaptureFormatError } from "../CaptureFormatError.js"; +import { WEB_NETWORK_CAPTURE_LIMITS } from "../../../domain/webNetworkCapture.js"; + +/** One charge covers all selected records, including temporary interpretation strings. */ +export class ProtobufBudget { + #remaining = WEB_NETWORK_CAPTURE_LIMITS.outputBytes; + #work = 0; + constructor(readonly signal?: AbortSignal) {} + charge(bytes: number): void { + if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > this.#remaining) + throw new CaptureFormatError( + "limit", + "Protobuf decoding exceeds the shared 96 MiB encoded-result and working-set budget.", + ); + this.#remaining -= bytes; + } + async checkpoint(): Promise { + this.signal?.throwIfAborted(); + if (++this.#work % 256 === 0) { + await setImmediate(); + this.signal?.throwIfAborted(); + } + } +} + +/** Reject overflowing uint64s before conversion; never wrap malformed bytes. */ +export const readProtobufVarint = ( + bytes: Uint8Array, + start: number, + end = bytes.length, +): { value: bigint; end: number } | undefined => { + let value = 0n; + for (let index = 0; index < 10 && start + index < end; index++) { + const byte = bytes[start + index]!; + if (index === 9 && byte > 1) return undefined; + value |= BigInt(byte & 127) << BigInt(index * 7); + if (byte < 128) return { value, end: start + index + 1 }; + } + return undefined; +}; + +const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const textOf = (bytes: Uint8Array): string | undefined => { + try { + return utf8.decode(bytes); + } catch { + return undefined; + } +}; + +/** An allocation-free syntactic candidate check, not a claim about the payload's type. */ +const messageCandidate = async ( + bytes: Uint8Array, + start: number, + end: number, + budget: ProtobufBudget, +): Promise => { + if (start === end) return true; + while (start < end) { + await budget.checkpoint(); + const tag = readProtobufVarint(bytes, start, end); + if ( + tag === undefined || + tag.value >> 3n === 0n || + tag.value >> 3n > 0x1fffffffn + ) + return false; + const type = Number(tag.value & 7n); + start = tag.end; + if (type === 0) { + const value = readProtobufVarint(bytes, start, end); + if (value === undefined) return false; + start = value.end; + } else if (type === 1 || type === 5) start += type === 1 ? 8 : 4; + else if (type === 2) { + const size = readProtobufVarint(bytes, start, end); + if (size === undefined || size.value > BigInt(end - size.end)) + return false; + start = size.end + Number(size.value); + } else return false; + } + return start === end; +}; + +const packedVarintCandidate = async ( + bytes: Uint8Array, + start: number, + end: number, + budget: ProtobufBudget, +): Promise => { + while (start < end) { + await budget.checkpoint(); + const value = readProtobufVarint(bytes, start, end); + if (value === undefined) return false; + start = value.end; + } + return true; +}; + +export interface WireSchema { + annotate( + field: ProtobufField, + bytes: Uint8Array, + parent: ProtobufField | undefined, + ): void | Promise; + nested(field: ProtobufField): boolean; +} + +/** Decode one message into flat occurrence facts with offsets in the decoded body. */ +export const decodeProtobufWire = async ( + bytes: Uint8Array, + start: number, + end: number, + frameIndex: number, + budget: ProtobufBudget, + redact: (text: string) => string, + schema?: WireSchema, + baseIndex = 0, +): Promise<{ + fields: ProtobufField[]; + examined_bytes: number; + error?: { offset: number; reason: string }; +}> => { + const fields: ProtobufField[] = []; + const walk = async ( + from: number, + until: number, + parent: ProtobufField | undefined, + depth: number, + group?: number, + ): Promise<{ + end: number; + payloadEnd?: number; + error?: { offset: number; reason: string }; + }> => { + let at = from; + const fail = (reason: string) => ({ + end: at, + error: { offset: at, reason }, + }); + while (at < until) { + await budget.checkpoint(); + const tag = readProtobufVarint(bytes, at, until); + if (tag === undefined) return fail("Truncated or overflowing field tag"); + const number = Number(tag.value >> 3n); + const type = Number(tag.value & 7n); + if (number === 0 || tag.value >> 3n > 0x1fffffffn) + return fail("Invalid field number"); + if (type === 4) + return group === number + ? { end: tag.end, payloadEnd: at } + : fail("Unexpected or mismatched end-group tag"); + let payloadOffset = tag.end; + let next: number; + let value: string | undefined; + if (type === 0) { + const parsed = readProtobufVarint(bytes, tag.end, until); + if (parsed === undefined) + return fail("Truncated or overflowing uint64 varint"); + next = parsed.end; + value = parsed.value.toString(); + } else if (type === 1 || type === 5) { + next = tag.end + (type === 1 ? 8 : 4); + if (next > until) return fail("Truncated fixed-width field"); + let integer = 0n; + for (let p = next - 1; p >= tag.end; p--) + integer = (integer << 8n) | BigInt(bytes[p]!); + value = integer.toString(); + } else if (type === 2) { + const length = readProtobufVarint(bytes, tag.end, until); + if (length === undefined) + return fail("Truncated or overflowing length prefix"); + if (length.value > BigInt(until - length.end)) + return fail("Length-delimited field exceeds its message"); + payloadOffset = length.end; + next = payloadOffset + Number(length.value); + } else if (type === 3) { + if (depth >= WEB_NETWORK_CAPTURE_LIMITS.depth) + return fail("Group nesting exceeds the 64-level decoder budget"); + next = tag.end; + } else return fail(`Invalid wire type ${type}`); + budget.charge(1024); + const field: ProtobufField = { + index: baseIndex + fields.length, + frame_index: frameIndex, + parent_index: parent?.index ?? null, + field_id: `offset:${at}`, + parent_field_id: + parent === undefined ? null : `offset:${parent.offset}`, + number, + wire_type: + type === 0 + ? "varint" + : type === 1 + ? "i64" + : type === 2 + ? "len" + : type === 3 + ? "group" + : "i32", + offset: at, + bytes: next - at, + payload_offset: payloadOffset, + payload_bytes: next - payloadOffset, + ...(value === undefined ? {} : { value: redact(value) }), + }; + fields.push(field); + if (type === 2) { + // Account for UTF-16 allocation and JSON escaping before decoding any string. + budget.charge(field.payload_bytes * 8); + const text = textOf(bytes.subarray(payloadOffset, next)); + const candidate = await messageCandidate( + bytes, + payloadOffset, + next, + budget, + ); + const packed = await packedVarintCandidate( + bytes, + payloadOffset, + next, + budget, + ); + field.interpretations = [ + ...(text === undefined ? [] : ["utf8-string" as const]), + ...(candidate ? ["nested-message" as const] : []), + ...(packed ? ["packed-varint" as const] : []), + ...(field.payload_bytes % 4 === 0 ? ["packed-fixed32" as const] : []), + ...(field.payload_bytes % 8 === 0 ? ["packed-fixed64" as const] : []), + "bytes", + ]; + if (text !== undefined) field.text = redact(text); + } + if (schema !== undefined) await schema.annotate(field, bytes, parent); + if (type === 3) { + const nested = await walk(tag.end, until, field, depth + 1, number); + field.bytes = nested.end - at; + field.payload_bytes = (nested.payloadEnd ?? nested.end) - payloadOffset; + if (nested.error !== undefined) return nested; + next = nested.end; + } else if (type === 2 && schema?.nested(field)) { + const error = + depth >= WEB_NETWORK_CAPTURE_LIMITS.depth + ? { + offset: payloadOffset, + reason: + "Schema message nesting exceeds the 64-level decoder budget", + } + : (await walk(payloadOffset, next, field, depth + 1)).error; + // The outer length is an observed boundary. A bad interpretation of its + // payload must never hide valid fields following that boundary. + if (error !== undefined) { + field.schema_status = "invalid-message"; + field.schema_error = error; + } + } + at = next; + } + return group === undefined ? { end: at } : fail("Missing end-group tag"); + }; + const result = await walk(start, end, undefined, 0); + return { + fields, + examined_bytes: result.end - start, + ...(result.error === undefined ? {} : { error: result.error }), + }; +}; diff --git a/src/cli/webNetworkCaptureCommands.ts b/src/cli/webNetworkCaptureCommands.ts index 81a7ec4c52..5f2306f35e 100644 --- a/src/cli/webNetworkCaptureCommands.ts +++ b/src/cli/webNetworkCaptureCommands.ts @@ -7,7 +7,11 @@ import type { Logger } from "pino"; import type { CliInstance } from "./types.js"; import { cliIntegerArgument } from "./options.js"; import { withCommandCancellation } from "./commandCancellation.js"; -import { finishAnalysisCommand } from "./finishAnalysisCommand.js"; +import { + analysisEvidenceDeliveryOptions, + deliverCliAnalysisEvidence, +} from "./analysisEvidenceDelivery.js"; +import { absoluteCallerPath } from "../filesystem/callerPath.js"; /** Expose offline historical capture inspection with the same selection contract as MCP. */ export const registerWebNetworkCaptureCommands = ( @@ -26,6 +30,23 @@ export const registerWebNetworkCaptureCommands = ( .describe("Producer capture format"), }), options: z.object({ + ...analysisEvidenceDeliveryOptions, + protobufFraming: z + .enum(["auto", "raw", "grpc", "grpc-web", "grpc-web-text"]) + .optional() + .describe("Override framing for selected retained binary bodies"), + protobufBodyPointer: z + .array(z.string()) + .optional() + .describe("Exact body pointers; repeat to select several"), + protobufSchema: z + .string() + .optional() + .describe("Absolute local compiled FileDescriptorSet path"), + protobufMessage: z + .string() + .optional() + .describe("Fully qualified message type in the descriptor set"), record: z .array(cliIntegerArgument(0)) .optional() @@ -43,13 +64,42 @@ export const registerWebNetworkCaptureCommands = ( logger, CLI_COMMANDS.inspectWebNetworkCapture, async () => { - return finishAnalysisCommand( - () => + return deliverCliAnalysisEvidence({ + run: () => service.inspect( { - capture_path: args.capturePath, + capture_path: absoluteCallerPath(args.capturePath), format: args.format, sensitive_values: options.sensitiveValue, + ...([ + options.protobufFraming, + options.protobufBodyPointer, + options.protobufSchema, + options.protobufMessage, + ].every((value) => value === undefined) + ? {} + : { + protobuf: { + framing: options.protobufFraming ?? "auto", + ...(options.protobufBodyPointer === undefined + ? {} + : { body_pointers: options.protobufBodyPointer }), + ...(options.protobufSchema === undefined && + options.protobufMessage === undefined + ? {} + : { + schema: { + descriptor_set_path: + options.protobufSchema === undefined + ? undefined + : absoluteCallerPath( + options.protobufSchema, + ), + message_type: options.protobufMessage, + }, + }), + }, + }), ...(options.record === undefined ? {} : { record_ordinals: options.record }), @@ -57,11 +107,12 @@ export const registerWebNetworkCaptureCommands = ( { signal }, ), service, - "web-network-capture", - "inspect_web_network_capture", + providerId: "web-network-capture", + operation: "inspect_web_network_capture", signal, - "Web network capture resources", - ); + detail: options.detail, + evidenceOutput: options.evidence_output, + }); }, ), ), diff --git a/src/contracts/analysisViewToolContracts.ts b/src/contracts/analysisViewToolContracts.ts index ad6bf4fa89..f042d237e3 100644 --- a/src/contracts/analysisViewToolContracts.ts +++ b/src/contracts/analysisViewToolContracts.ts @@ -17,7 +17,7 @@ export const ANALYSIS_VIEW_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_analysis_view"), kind: "application", description: - 'Inspect selected views of completed binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, decode_interface_builder, inspect_evm_interface, inspect_managed_artifact, inspect_managed_members, inspect_android_package, inspect_dart_aot, inspect_il2cpp_metadata or inspect_web_network_capture Evidence without repeating analysis. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. The environment-state view extracts, from the same parents, the initial state the recorded reads observed: view {kind: "environment-state", template_rules: [...]} with the rules of the manifest it accompanies. It reads response bodies recorded before the first related write and reports entity types with an identity field (proved by a path parameter or inferred from a field name), entities in collection order with per-view values where views differ, documents for non-parameterized non-entity paths written by POST/PUT/PATCH (item-shaped and delete-only paths stay recorded), and a render plan per read tool: item and list views reproduce their latest recordings from the state, and every other read tool is a recorded view with a reason. Types and identity are inferences; values are recorded data, so it refuses parents inspected with sensitive values, and a scenario parent must have selected response bodies; its values keep the capture redaction markers, and a write whose URL it redacted ends the state of every type. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings and carry the source selection and coverage. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods, with exact indices/names, fields and paging; types also select image_indices/namespaces/declaring_types and methods select declaring_types. Rows always keep their original index scoped to the parent artifact digest. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', + 'Inspect selected views of completed binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, decode_interface_builder, inspect_evm_interface, inspect_managed_artifact, inspect_managed_members, inspect_android_package, inspect_dart_aot, inspect_il2cpp_metadata or inspect_web_network_capture Evidence without repeating analysis. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. The environment-state view extracts, from the same parents, the initial state the recorded reads observed: view {kind: "environment-state", template_rules: [...]} with the rules of the manifest it accompanies. It reads response bodies recorded before the first related write and reports entity types with an identity field (proved by a path parameter or inferred from a field name), entities in collection order with per-view values where views differ, documents for non-parameterized non-entity paths written by POST/PUT/PATCH (item-shaped and delete-only paths stay recorded), and a render plan per read tool: item and list views reproduce their latest recordings from the state, and every other read tool is a recorded view with a reason. Types and identity are inferences; values are recorded data, so it refuses parents inspected with sensitive values, and a scenario parent must have selected response bodies; its values keep the capture redaction markers, and a write whose URL it redacted ends the state of every type. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings and carry the source selection and coverage. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Protobuf views use kind protobuf, collection bodies/frames/fields, with record_ordinals, body_pointers and fields; field pages also select frame_indices, field_ids, field_numbers or schema_names. Byte identity, offset basis and body status remain present. Text and schema_value are opt-in page columns. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods, with exact indices/names, fields and paging; types also select image_indices/namespaces/declaring_types and methods select declaring_types. Rows always keep their original index scoped to the parent artifact digest. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', inputSchema: inspectAnalysisViewInputSchema, outputSchema: evidenceResultOf(analysisViewResultSchema), examples: [ diff --git a/src/contracts/webNetworkCaptureToolContracts.ts b/src/contracts/webNetworkCaptureToolContracts.ts index 592dd9b54e..17370b27b4 100644 --- a/src/contracts/webNetworkCaptureToolContracts.ts +++ b/src/contracts/webNetworkCaptureToolContracts.ts @@ -16,7 +16,7 @@ export const WEB_NETWORK_CAPTURE_TOOL_CONTRACTS = [ name: "inspect_web_network_capture", ...toolContractMetadata("inspect_web_network_capture"), description: - "Request detail summary for capture identity and counts; then page retained records with inspect_analysis_view kind capture (preview or complete), avoiding repeated decoding. Default detail complete preserves existing inline delivery. Inspect retained local HAR 1.2, native mitmproxy, Fiddler SAZ or aioquic 1.5.0 JSON qlog 0.3 events without contacting recorded URLs. Returns original ordinals, pointers/byte ranges/archive member names, preserved producer fields, precise numeric representations and exact exposed binary bytes inline. Excludes known transport credentials and explicitly marked values. Decodes every record before optional ordinal selection; malformed or oversized captures return no partial success. Duplicate JSON members are invalid; prototype-named members such as __proto__ are preserved as ordinary own JSON members without prototype mutation. Qlog reports trace-scoped STREAM interval unions and endpoint loss declarations; it does not recover bodies, prove network loss or correlate PCAP. Other qlog layouts are unsupported. Uses pinned upstream parsers, owned temporary files and bounded offline processes. SAZ preserves HTTP fields, XML metadata and encoded body/message bytes without replay or HAR conversion; encrypted archives are unsupported. Native mitmproxy requires caller-supplied mitmdump 12.2.3 on Linux. Browser transaction IDs, runtime attribution and original bytes of HAR Unicode text remain unknown.", + "Explicit record_ordinals also decode retained protobuf/gRPC bodies whose declared content type is recognized. Optional protobuf selects exact body_pointers, framing, and a caller-supplied local FileDescriptorSet with message_type. Native gRPC and gRPC-web framing remain distinct. Wire fields are flat byte-range facts; ambiguous readings stay candidates, compressed payloads stay unknown. Page retained decoding with inspect_analysis_view kind protobuf, collection bodies/frames/fields, exact batch selectors and fields. Large text and schema values require explicit fields selection in those pages. Request detail summary for capture identity and counts; then page retained records with inspect_analysis_view kind capture (preview or complete), avoiding repeated decoding. Default detail complete preserves existing inline delivery. Inspect retained local HAR 1.2, native mitmproxy, Fiddler SAZ or aioquic 1.5.0 JSON qlog 0.3 events without contacting recorded URLs. Returns original ordinals, pointers/byte ranges/archive member names, preserved producer fields, precise numeric representations and exact exposed binary bytes inline. Excludes known transport credentials and explicitly marked values. Decodes every record before optional ordinal selection; malformed or oversized captures return no partial success. Duplicate JSON members are invalid; prototype-named members such as __proto__ are preserved as ordinary own JSON members without prototype mutation. Qlog reports trace-scoped STREAM interval unions and endpoint loss declarations; it does not recover bodies, prove network loss or correlate PCAP. Other qlog layouts are unsupported. Uses pinned upstream parsers, owned temporary files and bounded offline processes. SAZ preserves HTTP fields, XML metadata and encoded body/message bytes without replay or HAR conversion; encrypted archives are unsupported. Native mitmproxy requires caller-supplied mitmdump 12.2.3 on Linux. Browser transaction IDs, runtime attribution and original bytes of HAR Unicode text remain unknown.", kind: "browser-provider", inputSchema: inspectWebNetworkCaptureInputSchema.extend({ detail: analysisDetailSchema, diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 9a30bde79c..04e5cc7826 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -1,3 +1,8 @@ +import { + protobufViewRequestSchema, + protobufViewItemsSchema, + protobufViewContextSchema, +} from "./protobufCaptureView.js"; import { isAbsolute } from "node:path"; import { localPathStringSchema } from "../localPath.js"; @@ -247,6 +252,7 @@ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ environmentManifestViewRequestSchema, environmentStateViewRequestSchema, captureViewRequestSchema, + protobufViewRequestSchema, qlogViewRequestSchema, pcapViewRequestSchema, dartAotViewRequestSchema, @@ -549,6 +555,14 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ il2cppMetadataSummarySchema, ]), }), + z.strictObject({ + kind: z.literal("protobuf"), + view: protobufViewRequestSchema, + items: protobufViewItemsSchema, + ...viewResultBase, + ...protobufViewContextSchema.shape, + parent_operation: z.literal("inspect_web_network_capture"), + }), z.strictObject({ kind: z.literal("capture"), view: captureViewRequestSchema, @@ -710,79 +724,10 @@ export interface AnalysisViewParent { } /** View payload before the content digest is attached. */ -export type UnsignedAnalysisView = - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit, "view_digest"> - | Omit, "view_digest"> - | Omit, "view_digest"> - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit, "view_digest"> - | Omit, "view_digest"> - | Omit< - Extract, - "view_digest" - > - | Omit, "view_digest"> - | Omit, "view_digest"> - | Omit, "view_digest"> - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - >; +type WithoutViewDigest = T extends { readonly view_digest: string } + ? Omit + : never; +export type UnsignedAnalysisView = WithoutViewDigest; /** Attach the content digest that must change when projected bytes change. */ export const sealAnalysisView = ( diff --git a/src/domain/analysisView/protobufCaptureView.test.ts b/src/domain/analysisView/protobufCaptureView.test.ts new file mode 100644 index 0000000000..04d613a5a5 --- /dev/null +++ b/src/domain/analysisView/protobufCaptureView.test.ts @@ -0,0 +1,223 @@ +import { expect, it } from "vitest"; +import { createEvidence } from "../evidence.js"; +import { projectAnalysisView } from "./analysisView.js"; +import { webNetworkCaptureSchema } from "../webNetworkCapture.js"; + +const capture = async () => { + const report = webNetworkCaptureSchema.parse({ + artifact: { path: "/recordings/a.har", bytes: 100, sha256: "b".repeat(64) }, + decoder: { + id: "projection-fixture", + name: "Original projection facts", + version: "1", + }, + container: { + reported: null, + numeric_literals: [], + redactions: [], + records_pointer: "/log/entries", + }, + format: "har", + total_records: 1, + runtime_attribution: "unknown", + limitations: [], + records: [ + { + ordinal: 0, + location: { kind: "json-pointer", pointer: "/log/entries/0" }, + reported: null, + binary_fields: [], + numeric_literals: [], + redactions: [], + limitations: [], + protobuf: [ + { + decoder: "rea-protobuf-wire-v1", + pointer: "/response/content/text", + content_type: "application/grpc-web+proto", + body_sha256: "c".repeat(64), + framing: "grpc-web", + offset_basis: "retained-body", + decoded_bytes: 37, + status: "complete", + examined_bytes: 37, + limitations: [], + frames: [ + { + index: 0, + kind: "data", + offset: 0, + payload_offset: 5, + bytes: 11, + compressed: false, + status: "complete", + examined_bytes: 11, + }, + { + index: 1, + kind: "trailer", + offset: 16, + payload_offset: 21, + bytes: 16, + compressed: false, + status: "complete", + examined_bytes: 16, + }, + ], + fields: [ + { + index: 0, + frame_index: 0, + parent_index: null, + field_id: "offset:5", + parent_field_id: null, + number: 1, + wire_type: "len", + offset: 5, + bytes: 7, + payload_offset: 7, + payload_bytes: 5, + text: "hello", + interpretations: ["utf8-string", "packed-varint", "bytes"], + }, + { + index: 1, + frame_index: 0, + parent_index: null, + field_id: "offset:12", + parent_field_id: null, + number: 2, + wire_type: "len", + offset: 12, + bytes: 4, + payload_offset: 14, + payload_bytes: 2, + }, + ], + }, + ], + }, + ], + }); + const evidence = createEvidence( + { ...report.artifact, sha256: "b".repeat(64), format: "file" }, + report.decoder, + { + operation: "inspect_web_network_capture", + parameters: { record_ordinals: [0] }, + result: report, + confidence: "observed", + authority: "historical-reference", + }, + ); + return { + report, + parent: { + evidenceId: evidence.evidence_id, + operation: evidence.operation, + normalizedResult: report, + subject: evidence.subject, + limitations: evidence.limitations, + }, + }; +}; + +it("keeps coordinate identity and coverage with empty fields, batch filters and offset paging", async () => { + const { parent } = await capture(); + const viewed = projectAnalysisView(parent, { + kind: "protobuf", + collection: "fields", + field_ids: ["offset:5", "offset:12"], + fields: [], + offset: 1, + limit: 1, + }); + if (!viewed.ok) throw viewed.error; + expect(viewed.value).toMatchObject({ + kind: "protobuf", + coverage: { total: 2, examined: 1, exhausted: true }, + items: [ + { + field_id: "offset:12", + record_ordinal: 0, + body_pointer: "/response/content/text", + body_status: "complete", + offset_basis: "retained-body", + }, + ], + }); + const empty = projectAnalysisView(parent, { + kind: "protobuf", + collection: "fields", + field_numbers: [], + offset: 0, + limit: 1, + }); + if (!empty.ok) throw empty.error; + expect(empty.value).toMatchObject({ + items: [], + coverage: { total: 0, exhausted: true }, + }); +}); + +it("omits large values by default, returns requested text inline and preserves parser uncertainty", async () => { + const { parent, report } = await capture(); + const brief = projectAnalysisView(parent, { + kind: "protobuf", + collection: "fields", + offset: 0, + limit: 1, + }); + if (!brief.ok) throw brief.error; + expect(brief.value).toMatchObject({ + items: [{ interpretations: ["utf8-string", "packed-varint", "bytes"] }], + }); + expect(JSON.stringify(brief.value)).not.toContain('"hello"'); + const preview = projectAnalysisView(parent, { + kind: "capture", + detail: "preview", + offset: 0, + limit: 1, + }); + if (!preview.ok) throw preview.error; + expect(preview.value).toMatchObject({ + items: [ + { + protobuf_summary: { + bodies: 1, + frames: 2, + fields: 2, + incomplete_bodies: 0, + }, + }, + ], + }); + expect(JSON.stringify(preview.value)).not.toContain('"field_id"'); + report.records[0]!.protobuf![0]!.status = "partial"; + const full = projectAnalysisView(parent, { + kind: "protobuf", + collection: "fields", + fields: ["text"], + offset: 0, + limit: 1, + }); + if (!full.ok) throw full.error; + expect(full.value).toMatchObject({ + source_status: "partial", + items: [{ text: "hello", body_status: "partial" }], + }); + const changedSubject = projectAnalysisView( + { ...parent, subject: { ...parent.subject!, local_path: "/other.har" } }, + { kind: "protobuf", collection: "bodies", offset: 0, limit: 1 }, + ); + expect(changedSubject.ok).toBe(false); + report.records[0]!.protobuf![0]!.fields[0]!.offset = 1000; + expect( + projectAnalysisView(parent, { + kind: "protobuf", + collection: "fields", + offset: 0, + limit: 1, + }).ok, + ).toBe(false); +}); diff --git a/src/domain/analysisView/protobufCaptureView.ts b/src/domain/analysisView/protobufCaptureView.ts new file mode 100644 index 0000000000..163dc48657 --- /dev/null +++ b/src/domain/analysisView/protobufCaptureView.ts @@ -0,0 +1,315 @@ +import { isImmutableJsonSnapshot } from "../immutableJson.js"; +import { z } from "zod"; +import { + protobufDecodingSchema, + protobufDecodingObjectSchema, + protobufFieldSchema, + protobufFrameSchema, + type ProtobufDecoding, + type ProtobufField, +} from "../protobufWire.js"; +import type { WebNetworkCapture } from "../webNetworkCapture.js"; +import { pageViewCoverage } from "./analysisViewCoverage.js"; + +const body = { + record_ordinal: z.number().int().nonnegative(), + body_pointer: z.string(), + body_sha256: z.string(), + body_status: protobufDecodingSchema.shape.status, + offset_basis: protobufDecodingSchema.shape.offset_basis, +}; +const bodyRow = protobufDecodingObjectSchema + .omit({ + pointer: true, + body_sha256: true, + offset_basis: true, + frames: true, + fields: true, + }) + .extend({ + ...body, + frame_count: z.number().int().nonnegative(), + field_count: z.number().int().nonnegative(), + }); +const frameRow = protobufFrameSchema.extend(body); +const fieldRow = protobufFieldSchema.extend(body); +const page = { + kind: z.literal("protobuf"), + record_ordinals: z.array(z.number().int().nonnegative()).optional(), + body_pointers: z.array(z.string()).optional(), + offset: z.number().int().nonnegative().default(0), + limit: z.number().int().positive().default(64), +}; + +/** Batch selectors intersect; omitted filters select all retained decoding facts. */ +export const protobufViewRequestSchema = z.discriminatedUnion("collection", [ + z.strictObject({ + ...page, + collection: z.literal("bodies"), + fields: z.array(bodyRow.keyof()).optional(), + }), + z.strictObject({ + ...page, + collection: z.literal("frames"), + frame_indices: z.array(z.number().int().nonnegative()).optional(), + fields: z.array(frameRow.keyof()).optional(), + }), + z.strictObject({ + ...page, + collection: z.literal("fields"), + frame_indices: z.array(z.number().int().nonnegative()).optional(), + field_ids: z.array(z.string()).optional(), + field_numbers: z.array(z.number().int().positive()).optional(), + schema_names: z.array(z.string()).optional(), + fields: z.array(fieldRow.keyof()).optional(), + }), +]); +export const protobufViewItemsSchema = z.union([ + z.array( + bodyRow.partial().required({ + record_ordinal: true, + body_pointer: true, + body_sha256: true, + body_status: true, + offset_basis: true, + }), + ), + z.array( + frameRow.partial().required({ + record_ordinal: true, + body_pointer: true, + body_sha256: true, + body_status: true, + offset_basis: true, + index: true, + offset: true, + }), + ), + z.array( + fieldRow.partial().required({ + record_ordinal: true, + body_pointer: true, + body_sha256: true, + body_status: true, + offset_basis: true, + index: true, + frame_index: true, + field_id: true, + parent_field_id: true, + offset: true, + payload_offset: true, + payload_bytes: true, + }), + ), +]); +export const protobufViewContextSchema = z.strictObject({ + source_status: z.enum(["complete", "partial", "unknown"]), + retained_decoded_bodies: z.number().int().nonnegative(), +}); +type Request = z.output; +type Body = { recordOrdinal: number; decoding: ProtobufDecoding }; +const checkedBodies = new WeakMap(); +interface FieldSelection { + readonly key: string; + readonly frames: ReadonlySet | undefined; + readonly ids: ReadonlySet | undefined; + readonly numbers: ReadonlySet | undefined; + readonly names: ReadonlySet | undefined; + readonly rows: WeakMap | undefined; +} +const fieldQueries = new WeakMap(); +const fieldSelection = ( + report: WebNetworkCapture, + bodies: readonly Body[], + view: Extract, + reusable: boolean, +): FieldSelection | undefined => { + if ( + view.frame_indices === undefined && + view.field_ids === undefined && + view.field_numbers === undefined && + view.schema_names === undefined + ) + return undefined; + const key = JSON.stringify([ + view.frame_indices, + view.field_ids, + view.field_numbers, + view.schema_names, + ]); + const cached = reusable ? fieldQueries.get(report) : undefined; + if (cached?.key === key) return cached; + // Selector strings must fit within one 8-byte reference per retained field. + // One filter per parent avoids multiplying large keys/sets by its body count. + const referenceBytes = bodies.reduce( + (sum, body) => sum + body.decoding.fields.length * 8, + 0, + ); + const cache = reusable && key.length * 2 <= referenceBytes; + const selection: FieldSelection = { + key, + frames: + view.frame_indices === undefined + ? undefined + : new Set(view.frame_indices), + ids: view.field_ids === undefined ? undefined : new Set(view.field_ids), + numbers: + view.field_numbers === undefined + ? undefined + : new Set(view.field_numbers), + names: + view.schema_names === undefined ? undefined : new Set(view.schema_names), + rows: cache ? new WeakMap() : undefined, + }; + if (cache) fieldQueries.set(report, selection); + return selection; +}; +const selectFields = ( + decoding: ProtobufDecoding, + selection: FieldSelection | undefined, +): readonly ProtobufField[] => { + if (selection === undefined) return decoding.fields; + const cached = selection.rows?.get(decoding); + if (cached !== undefined) return cached; + const { frames, ids, numbers, names } = selection; + const rows = decoding.fields.filter( + (field) => + (frames === undefined || frames.has(field.frame_index)) && + (ids === undefined || ids.has(field.field_id)) && + (numbers === undefined || numbers.has(field.number)) && + (names === undefined || + (field.schema_name !== undefined && names.has(field.schema_name))), + ); + selection.rows?.set(decoding, rows); + return rows; +}; + +/** Pages select owner-parsed facts; neither payloads nor descriptors are decoded again. */ +export const projectProtobufCapturePage = ( + report: WebNetworkCapture, + view: Request, +) => { + const reusable = isImmutableJsonSnapshot(report); + let bodies = reusable ? checkedBodies.get(report) : undefined; + if (bodies === undefined) { + bodies = report.records.flatMap((record) => + (record.protobuf ?? []).map((decoding) => ({ + recordOrdinal: record.ordinal, + decoding, + })), + ); + if (reusable) checkedBodies.set(report, bodies); + } + const selection = + view.collection === "fields" + ? fieldSelection(report, bodies, view, reusable) + : undefined; + const ordinals = + view.record_ordinals === undefined + ? undefined + : new Set(view.record_ordinals); + const pointers = + view.body_pointers === undefined ? undefined : new Set(view.body_pointers); + const frames = + "frame_indices" in view && view.frame_indices !== undefined + ? new Set(view.frame_indices) + : undefined; + let total = 0; + const items: Record[] = []; + let sourceStatus: "complete" | "partial" | "unknown" = + report.protobuf_coverage?.status === "partial" + ? "partial" + : bodies.length === 0 + ? "unknown" + : "complete"; + for (const entry of bodies) { + const { recordOrdinal, decoding } = entry; + if ( + (ordinals !== undefined && !ordinals.has(recordOrdinal)) || + (pointers !== undefined && !pointers.has(decoding.pointer)) + ) + continue; + if (decoding.status !== "complete") sourceStatus = "partial"; + const rows = + view.collection === "bodies" + ? [decoding] + : view.collection === "frames" + ? frames === undefined + ? decoding.frames + : decoding.frames.filter((frame) => frames.has(frame.index)) + : selectFields(decoding, selection); + const from = Math.max(0, view.offset - total); + const to = Math.min(rows.length, view.offset + view.limit - total); + for (let index = from; index < to; index++) { + const source = rows[index]!; + const row: Record = + view.collection === "bodies" + ? { + ...decoding, + frame_count: decoding.frames.length, + field_count: decoding.fields.length, + } + : source; + const mandatory = [ + "record_ordinal", + "body_pointer", + "body_sha256", + "body_status", + "offset_basis", + ...(view.collection === "fields" + ? [ + "index", + "frame_index", + "field_id", + "parent_field_id", + "offset", + "payload_offset", + "payload_bytes", + ] + : view.collection === "frames" + ? ["index", "offset"] + : []), + ]; + const keys = + view.fields ?? + Object.keys(row).filter( + (key) => + ![ + "frames", + "fields", + "pointer", + "text", + "schema_value", + "schema_enum_values", + ].includes(key), + ); + const chosen = new Set([...mandatory, ...keys]); + const projected: Record = { + record_ordinal: recordOrdinal, + body_pointer: decoding.pointer, + body_sha256: decoding.body_sha256, + body_status: decoding.status, + offset_basis: decoding.offset_basis, + }; + for (const key of keys) + if ( + chosen.has(key) && + Object.hasOwn(row, key) && + !["frames", "fields", "pointer"].includes(key) + ) + projected[key] = row[key]; + for (const key of mandatory) + if (Object.hasOwn(row, key)) projected[key] = row[key]; + items.push(projected); + } + total += rows.length; + } + return { + kind: "protobuf" as const, + view, + items: protobufViewItemsSchema.parse(items), + source_status: sourceStatus, + retained_decoded_bodies: bodies.length, + coverage: pageViewCoverage(view.offset, items.length, total), + }; +}; diff --git a/src/domain/analysisView/webNetworkCaptureView.ts b/src/domain/analysisView/webNetworkCaptureView.ts index f962110c31..83965f7367 100644 --- a/src/domain/analysisView/webNetworkCaptureView.ts +++ b/src/domain/analysisView/webNetworkCaptureView.ts @@ -1,3 +1,4 @@ +import { projectProtobufCapturePage } from "./protobufCaptureView.js"; import { analysisViewInputError } from "./analysisViewErrors.js"; import { createHash } from "node:crypto"; import { z } from "zod"; @@ -11,6 +12,7 @@ import { safeParseOwnedWebNetworkCapture, webNetworkCaptureArtifactSchema, webNetworkCaptureDecoderSchema, + protobufCaptureCoverageSchema, WEB_NETWORK_CAPTURE_FORMATS, type WebNetworkCaptureRecord, type WebNetworkCapture, @@ -38,21 +40,43 @@ export const captureViewRequestSchema = z.strictObject({ limit: z.number().int().positive().default(64), }); -const previewSchema = webNetworkCaptureRecordSchema.extend({ - binary_fields: z.array(webNetworkCaptureBinaryDescriptorSchema), +const protobufSummarySchema = z.strictObject({ + bodies: z.number().int().nonnegative(), + frames: z.number().int().nonnegative(), + fields: z.number().int().nonnegative(), + incomplete_bodies: z.number().int().nonnegative(), }); +const summarizeProtobuf = (records: readonly WebNetworkCaptureRecord[]) => { + const counts = { bodies: 0, frames: 0, fields: 0, incomplete_bodies: 0 }; + for (const record of records) + for (const body of record.protobuf ?? []) { + counts.bodies++; + counts.frames += body.frames.length; + counts.fields += body.fields.length; + counts.incomplete_bodies += Number(body.status !== "complete"); + } + return counts.bodies === 0 ? undefined : counts; +}; +const previewSchema = webNetworkCaptureRecordSchema + .omit({ protobuf: true }) + .extend({ + binary_fields: z.array(webNetworkCaptureBinaryDescriptorSchema), + protobuf_summary: protobufSummarySchema.exactOptional(), + }); export const captureViewItemsSchema = z.array( z.union([webNetworkCaptureRecordSchema, previewSchema]), ); /** Count/identity overview; bodies and the archive member inventory stay in the parent. */ export const captureSummarySchema = z.strictObject({ + protobuf_coverage: protobufCaptureCoverageSchema.exactOptional(), artifact: webNetworkCaptureArtifactSchema, decoder: webNetworkCaptureDecoderSchema, format: z.enum(WEB_NETWORK_CAPTURE_FORMATS), total_records: z.number().int().nonnegative(), retained_records: z.number().int().nonnegative(), runtime_attribution: z.literal("unknown"), + protobuf: protobufSummarySchema.exactOptional(), qlog: qlogSummarySchema.optional(), }); @@ -369,8 +393,13 @@ const preview = (record: WebNetworkCaptureRecord, format: string) => { ); } } + const { protobuf: _protobuf, ...sourceRecord } = record; + const protobufSummary = summarizeProtobuf([record]); return { - ...record, + ...sourceRecord, + ...(protobufSummary === undefined + ? {} + : { protobuf_summary: protobufSummary }), reported: source === null ? record.reported : reported, binary_fields: record.binary_fields.map( ({ content_base64: _content, ...field }) => field, @@ -389,6 +418,7 @@ const parseCaptureParent = ( parent: AnalysisViewParent, ): Result => { const parsed = safeParseOwnedWebNetworkCapture(parent.normalizedResult); + if (!parsed.success) return err( analysisInputErrorFromIssues( @@ -429,6 +459,7 @@ export const projectWebNetworkCaptureView = ( if ( view.kind !== "summary" && view.kind !== "capture" && + view.kind !== "protobuf" && view.kind !== "qlog" ) return err( @@ -463,6 +494,15 @@ export const projectWebNetworkCaptureView = ( : []), ], }; + if (view.kind === "protobuf") + return ok({ + ...shared, + ...projectProtobufCapturePage(report, view), + unknowns: [ + ...shared.unknowns, + "Decoding covers selected retained bodies only. Exhausting wire fields does not establish schema completeness or business meaning; compressed bodies and unsupported schema facets remain unknown.", + ], + }); if (view.kind === "summary") return ok({ kind: "summary", @@ -475,6 +515,12 @@ export const projectWebNetworkCaptureView = ( total_records: report.total_records, retained_records: report.records.length, runtime_attribution: report.runtime_attribution, + ...(report.protobuf_coverage === undefined + ? {} + : { protobuf_coverage: report.protobuf_coverage }), + ...(summarizeProtobuf(report.records) === undefined + ? {} + : { protobuf: summarizeProtobuf(report.records)! }), ...(report.format === "qlog" ? { qlog: summarizeQlogCapture(report.qlog) } : {}), diff --git a/src/domain/explicitSensitiveFailure.test.ts b/src/domain/explicitSensitiveFailure.test.ts index 603097f12b..aedb79d44e 100644 --- a/src/domain/explicitSensitiveFailure.test.ts +++ b/src/domain/explicitSensitiveFailure.test.ts @@ -14,6 +14,7 @@ import { ProviderCleanupError } from "./providerCleanupError.js"; import { ProviderAdapterError } from "./providerAdapterError.js"; import { analysisErrorWithCleanupFailure } from "./analysisErrorCleanup.js"; import { resolvedAnalysisError } from "./analysisErrorBase.js"; +import { createEvidence } from "./evidence.js"; import { redactExplicitFailure, redactExplicitPointer, @@ -55,7 +56,20 @@ it.each([ ), new ProviderAdapterError("provider", "inspect", context), ])("preserves sanitized common context for $._tag", (error) => { + const observation = createEvidence( + undefined, + { id: "fixture", name: "Owner facts", version: "1" }, + { + operation: "inspect", + parameters: {}, + result: { source: "already sanitized owner facts" }, + confidence: "observed", + authority: "historical-reference", + }, + ); + error.retainPartialObservation(observation); const sanitized = redactExplicitFailure(error, ["explicit-secret"]); + expect(sanitized.partialObservation).toBe(observation); expect(sanitized._tag).toBe(error._tag); const projected = projectAnalysisError(sanitized); expect(projected.details).toMatchObject({ diff --git a/src/domain/explicitSensitiveFailure.ts b/src/domain/explicitSensitiveFailure.ts index d14cb443f5..14d680fb2f 100644 --- a/src/domain/explicitSensitiveFailure.ts +++ b/src/domain/explicitSensitiveFailure.ts @@ -41,6 +41,10 @@ export const redactExplicitFailure = ( ): AnalysisError => { if (values.length === 0) return error; const redacted = redactFailure(error, values); + // Observation owners already apply format-aware exclusions. Keep their facts + // and identity across every diagnostic replacement, including input/capacity. + if (redacted !== error && error.partialObservation !== undefined) + redacted.retainPartialObservation(error.partialObservation); return redacted === error ? error : mirrorCleanup(error, redacted, (value) => @@ -118,12 +122,10 @@ const redactFailure = ( return new AnalysisCancelledError(error.operation, { ...context, cause: error.cause, - partialObservation: error.partialObservation, }); if (error instanceof AnalysisTimeoutError) return new AnalysisTimeoutError(error.operation, error.timeoutMs, { ...context, - partialObservation: error.partialObservation, }); if (error instanceof AnalysisAccessDeniedError) return new AnalysisAccessDeniedError( @@ -199,17 +201,11 @@ const redactFailure = ( { ...context, operation: error.operation, - ...(error.partialObservation === undefined - ? {} - : { partialObservation: error.partialObservation }), }, ); if (error instanceof ProviderAdapterError) return new ProviderAdapterError(error.providerId, error.operation, { ...context, - ...(error.partialObservation === undefined - ? {} - : { partialObservation: error.partialObservation }), ...(error.userMessage === undefined ? {} : { userMessage: text(error.userMessage) }), diff --git a/src/domain/protobufWire.ts b/src/domain/protobufWire.ts new file mode 100644 index 0000000000..67c8f86860 --- /dev/null +++ b/src/domain/protobufWire.ts @@ -0,0 +1,206 @@ +import { z } from "zod"; +import { localPathStringSchema } from "./localPath.js"; + +/** Optional decoding applies only to explicitly selected capture records. */ +export const protobufOptionsSchema = z.strictObject({ + framing: z + .enum(["auto", "raw", "grpc", "grpc-web", "grpc-web-text"]) + .default("auto"), + body_pointers: z.array(z.string()).optional(), + schema: z + .strictObject({ + descriptor_set_path: localPathStringSchema, + message_type: z.string().min(1), + }) + .optional(), +}); + +const index = z.number().int().nonnegative(); +const issueSchema = z.strictObject({ offset: index, reason: z.string() }); + +/** Indices are scoped to the capture artifact, producer ordinal and body pointer. */ +export const protobufFieldSchema = z.strictObject({ + index, + frame_index: index, + field_id: z.string(), + parent_field_id: z.string().nullable(), + parent_index: index.nullable(), + number: z.number().int().positive().max(0x1fffffff), + wire_type: z.enum(["varint", "i64", "len", "i32", "group"]), + offset: index, + bytes: index, + payload_offset: index, + payload_bytes: index, + value: z.string().exactOptional(), + text: z.string().exactOptional(), + interpretations: z + .array( + z.enum([ + "utf8-string", + "nested-message", + "packed-varint", + "packed-fixed32", + "packed-fixed64", + "bytes", + ]), + ) + .exactOptional(), + schema_name: z.string().exactOptional(), + schema_type: z.string().exactOptional(), + schema_value: z.string().exactOptional(), + schema_enum_values: z + .array(z.strictObject({ number: z.string(), names: z.array(z.string()) })) + .exactOptional(), + schema_status: z + .enum([ + "decoded", + "unknown-field", + "wire-type-mismatch", + "invalid-message", + "unsupported", + ]) + .exactOptional(), + schema_error: issueSchema.exactOptional(), +}); + +export const protobufFrameSchema = z.strictObject({ + index, + kind: z.enum(["data", "trailer"]), + offset: index, + payload_offset: index, + bytes: index, + compressed: z.boolean(), + status: z.enum(["complete", "partial", "unknown"]), + examined_bytes: index, + error: issueSchema.exactOptional(), + trailers: z + .array(z.strictObject({ name: z.string(), value: z.string() })) + .exactOptional(), +}); + +/** Flat facts avoid recursive advertised schemas and preserve same-number occurrences. */ +export const protobufDecodingObjectSchema = z.strictObject({ + pointer: z.string(), + content_type: z.string().nullable(), + decoder: z.literal("rea-protobuf-wire-v1"), + body_sha256: z.string().regex(/^[0-9a-f]{64}$/), + framing: z.enum(["raw", "grpc", "grpc-web", "grpc-web-text"]), + offset_basis: z.enum(["retained-body", "base64-decoded-body"]), + decoded_bytes: index, + status: z.enum(["complete", "partial", "unknown"]), + examined_bytes: index, + frames: z.array(protobufFrameSchema), + fields: z.array(protobufFieldSchema), + error: issueSchema.exactOptional(), + limitations: z.array(z.string()), + schema: z + .strictObject({ + decoder: z.literal("protobuf-es-2.16.0"), + descriptor_set_path: z.string(), + descriptor_set_sha256: z.string().regex(/^[0-9a-f]{64}$/), + descriptor_set_bytes: index, + message_type: z.string(), + }) + .exactOptional(), +}); +/** Validate coordinate relationships at the owner, before any consumer projects rows. */ +export const protobufDecodingSchema = protobufDecodingObjectSchema.superRefine( + (body, context) => { + const issue = (path: (string | number)[], message: string) => + context.addIssue({ code: "custom", path, message }); + if (body.examined_bytes > body.decoded_bytes) + issue(["examined_bytes"], "Examined bytes exceed the decoded body"); + if ( + body.status === "complete" && + (body.error !== undefined || + body.frames.some((frame) => frame.status !== "complete")) + ) + issue(["status"], "Complete body contradicts frame coverage"); + let expectedOffset = 0; + for (const [index, frame] of body.frames.entries()) { + if ( + frame.offset !== expectedOffset || + frame.payload_offset !== frame.offset + (body.framing === "raw" ? 0 : 5) + ) + issue( + ["frames", index, "offset"], + "Frames must preserve contiguous protocol ranges", + ); + expectedOffset = frame.payload_offset + frame.bytes; + if ( + frame.index !== index || + frame.payload_offset < frame.offset || + frame.payload_offset + frame.bytes > body.decoded_bytes || + frame.examined_bytes > frame.bytes + ) + issue( + ["frames", index], + "Frame identity or range contradicts the body", + ); + if ( + frame.status === "complete" && + (frame.error !== undefined || + frame.examined_bytes !== frame.bytes || + frame.compressed) + ) + issue( + ["frames", index, "status"], + "Complete frame contradicts decoding coverage", + ); + if (body.framing === "grpc" && frame.kind === "trailer") + issue( + ["frames", index, "kind"], + "Native gRPC has no in-body trailer frame", + ); + } + if ( + body.status === "complete" && + (body.examined_bytes !== body.decoded_bytes || + expectedOffset !== body.decoded_bytes) + ) + issue(["status"], "Complete decoding must cover the whole decoded body"); + const ids = new Set(); + for (const [index, field] of body.fields.entries()) { + const frame = body.frames[field.frame_index]; + if ( + field.index !== index || + field.field_id !== `offset:${field.offset}` || + ids.has(field.field_id) + ) + issue(["fields", index], "Field occurrence identity is inconsistent"); + ids.add(field.field_id); + if ( + frame === undefined || + frame.kind !== "data" || + field.offset < frame.payload_offset || + field.offset + field.bytes > frame.payload_offset + frame.bytes || + field.payload_offset < field.offset || + field.payload_offset + field.payload_bytes > field.offset + field.bytes + ) + issue(["fields", index], "Field range contradicts its frame"); + const parent = + field.parent_index === null + ? undefined + : body.fields[field.parent_index]; + if ( + field.parent_index === null + ? field.parent_field_id !== null + : parent === undefined || + parent.index >= index || + parent.field_id !== field.parent_field_id || + parent.frame_index !== field.frame_index || + field.offset < parent.payload_offset || + field.offset + field.bytes > + parent.payload_offset + parent.payload_bytes + ) + issue( + ["fields", index, "parent_field_id"], + "Field parent contradicts its source range", + ); + } + }, +); + +export type ProtobufDecoding = z.output; +export type ProtobufField = z.output; +export type ProtobufFrame = z.output; diff --git a/src/domain/webNetworkCapture.ts b/src/domain/webNetworkCapture.ts index 1f40d3d1b7..99d652bbdc 100644 --- a/src/domain/webNetworkCapture.ts +++ b/src/domain/webNetworkCapture.ts @@ -1,6 +1,10 @@ import { z } from "zod"; import { jsonValueSchema } from "./jsonValue.js"; import { localPathStringSchema } from "./localPath.js"; +import { + protobufDecodingSchema, + protobufOptionsSchema, +} from "./protobufWire.js"; import { qlogCaptureFactsSchema } from "./qlogCapture.js"; import { canBorrowParsedJsonSnapshot } from "./immutableJson.js"; @@ -32,6 +36,7 @@ export const inspectWebNetworkCaptureInputSchema = z.strictObject({ .describe( "Zero-based producer record ordinals, in caller-selected order. Omit to inspect every record.", ), + protobuf: protobufOptionsSchema.optional(), sensitive_values: z .array(z.string().min(1)) .default([]) @@ -128,6 +133,12 @@ export const webNetworkCaptureRecordSchema = z.strictObject({ numeric_literals: z.array(numberSchema), redactions: z.array(redactionSchema), limitations: z.array(z.string()), + protobuf: z + .array(protobufDecodingSchema) + .exactOptional() + .describe( + "Flat decoding facts for explicitly selected records. Field indices are local to a body; offset_basis states whether ranges refer to retained or base64-decoded bytes. Interpretations without a caller-supplied schema are candidates.", + ), }); /** The selected offline parser profile, independent of authenticated producer identity. */ @@ -138,7 +149,26 @@ export const webNetworkCaptureDecoderSchema = z.strictObject({ }); /** Source coverage is shared by every offline decoder. */ +export const protobufCaptureCoverageSchema = z + .strictObject({ + status: z.enum(["complete", "partial"]), + selected_records: z.number().int().nonnegative(), + inspected_records: z.number().int().nonnegative(), + }) + .superRefine((coverage, context) => { + if ( + coverage.inspected_records > coverage.selected_records || + (coverage.status === "complete" && + coverage.inspected_records !== coverage.selected_records) + ) + context.addIssue({ + code: "custom", + path: ["status"], + message: "Protobuf coverage contradicts its inspected selection.", + }); + }); const payloadShape = { + protobuf_coverage: protobufCaptureCoverageSchema.exactOptional(), decoder: webNetworkCaptureDecoderSchema, container: z.strictObject({ reported: jsonValueSchema, diff --git a/tests/acceptance/tools/protobufCapture.test.ts b/tests/acceptance/tools/protobufCapture.test.ts new file mode 100644 index 0000000000..3720c57104 --- /dev/null +++ b/tests/acceptance/tools/protobufCapture.test.ts @@ -0,0 +1,530 @@ +import { z } from "zod"; +import { readFile, unlink, writeFile, mkdir, truncate } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + protobufCaptureDescriptor, + protobufHar, + protobufFrame, + protobufCaptureBody, +} from "../../fixtures/protobufCapture.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { + webNetworkCaptureSchema, + inspectWebNetworkCaptureInputSchema, +} from "../../../src/domain/webNetworkCapture.js"; +import { readEvidenceBundle } from "../../../src/application/EvidenceBundleFiles.js"; +import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; + +type CaptureCall = Awaited>["call"]; +type CaptureRequest = z.input; + +const retainedCaptureDetailsSchema = z.object({ + partial_observation: z.strictObject({ + kind: z.literal("retained-evidence"), + evidence_id: z.string(), + }), +}); + +async function verifyCaptureExclusions( + call: CaptureCall, + args: CaptureRequest, + root: string, +): Promise { + const redacted = await call("inspect_web_network_capture", { + ...args, + sensitive_values: ["hello"], + }); + expect(redacted.isError, JSON.stringify(redacted)).not.toBe(true); + const excluded = webNetworkCaptureSchema.parse( + parseEvidence(redacted.structuredContent).normalized_result, + ); + expect(excluded.records[0]?.protobuf).toBeUndefined(); + expect(excluded.protobuf_coverage).toMatchObject({ + status: "partial", + selected_records: 1, + inspected_records: 1, + }); + expect(JSON.stringify(redacted)).not.toContain("hello"); + const textPath = join(root, "text.har"); + await writeFile( + textPath, + protobufHar( + Buffer.from(protobufCaptureBody().toString("base64")), + "application/grpc-web-text", + ), + ); + const textExcluded = await call("inspect_web_network_capture", { + capture_path: textPath, + format: "har", + record_ordinals: [0], + sensitive_values: ["hello"], + }); + expect(textExcluded.isError, JSON.stringify(textExcluded)).not.toBe(true); + const excludedText = webNetworkCaptureSchema.parse( + parseEvidence(textExcluded.structuredContent).normalized_result, + ); + expect(excludedText.records[0]?.binary_fields[0]).toMatchObject({ + state: "redacted", + content_base64: null, + sha256: null, + }); + expect(excludedText.records[0]?.reported).toMatchObject({ + response: { content: { text: null } }, + }); + expect(excludedText.records[0]?.protobuf).toBeUndefined(); + const credentialPath = join(root, "trailer.har"); + await writeFile( + credentialPath, + protobufHar( + Buffer.concat([ + protobufFrame(Buffer.from("0a0568656c6c6f", "hex")), + protobufFrame( + Buffer.from( + "grpc-status: 0\r\nauthorization: Bearer fixture-only-credential\r\n", + ), + 128, + ), + ]), + ), + ); + const credential = await call("inspect_web_network_capture", { + capture_path: credentialPath, + format: "har", + record_ordinals: [0], + }); + expect(credential.isError, JSON.stringify(credential)).not.toBe(true); + const excludedCredential = webNetworkCaptureSchema.parse( + parseEvidence(credential.structuredContent).normalized_result, + ); + expect(excludedCredential.records[0]?.binary_fields[0]).toMatchObject({ + state: "redacted", + content_base64: null, + }); + expect(excludedCredential.records[0]?.redactions).toContainEqual({ + pointer: "/response/content/text", + reason: "transport-credential", + }); + expect(excludedCredential.records[0]?.protobuf).toBeUndefined(); + expect(JSON.stringify(credential)).not.toContain("fixture-only-credential"); +} + +async function verifyPartialCaptureSources( + call: CaptureCall, + args: CaptureRequest, + root: string, + descriptor: string, +): Promise { + const missing = await call("inspect_web_network_capture", { + ...args, + sensitive_values: ["unrelated fixture exclusion"], + protobuf: { + schema: { + descriptor_set_path: descriptor, + message_type: "capture.Missing", + }, + }, + }); + const missingError = parseMcpToolError(missing).error; + expect(missingError.code).toBe("invalid_request"); + const retainedSource = retainedCaptureDetailsSchema.parse( + missingError.details, + ).partial_observation; + const retainedSummary = await call("inspect_analysis_view", { + source: retainedSource, + view: { kind: "summary" }, + }); + expect(retainedSummary.isError, JSON.stringify(retainedSummary)).not.toBe( + true, + ); + expect( + analysisViewResultSchema.parse( + parseEvidence(retainedSummary.structuredContent).normalized_result, + ), + ).toMatchObject({ + summary: { + protobuf_coverage: { status: "partial", inspected_records: 0 }, + }, + }); + const missingView = await call("inspect_analysis_view", { + source: retainedSource, + view: { kind: "protobuf", collection: "fields", offset: 0, limit: 1 }, + }); + expect(missingView.isError, JSON.stringify(missingView)).not.toBe(true); + expect( + analysisViewResultSchema.parse( + parseEvidence(missingView.structuredContent).normalized_result, + ), + ).toMatchObject({ source_status: "partial", items: [] }); + const capacityPath = join(root, "capacity.har"); + const repeated = Buffer.alloc(200000); + for (let index = 0; index < repeated.length; index += 2) { + repeated[index] = 8; + repeated[index + 1] = 1; + } + await writeFile(capacityPath, protobufHar(protobufFrame(repeated))); + const capacity = await call("inspect_web_network_capture", { + capture_path: capacityPath, + format: "har", + record_ordinals: [0], + sensitive_values: ["unrelated fixture exclusion"], + }); + const capacityError = parseMcpToolError(capacity).error; + expect(capacityError.code).toBe("resource_constraint"); + expect(capacityError.retryable).toBe(false); + const capacitySource = retainedCaptureDetailsSchema.parse( + capacityError.details, + ).partial_observation; + const capacitySummary = await call("inspect_analysis_view", { + source: capacitySource, + view: { kind: "summary" }, + }); + expect(capacitySummary.isError, JSON.stringify(capacitySummary)).not.toBe( + true, + ); + expect( + analysisViewResultSchema.parse( + parseEvidence(capacitySummary.structuredContent).normalized_result, + ), + ).toMatchObject({ + summary: { + protobuf_coverage: { status: "partial", inspected_records: 0 }, + }, + }); + expect(JSON.stringify(capacity)).not.toContain("content_base64"); +} + +async function verifyCredentialPreflight( + call: CaptureCall, + root: string, + descriptor: string, +): Promise { + const repeated = Buffer.alloc(200000); + for (let index = 0; index < repeated.length; index += 2) { + repeated[index] = 8; + repeated[index + 1] = 1; + } + const large = protobufFrame(repeated); + const credential = protobufFrame( + Buffer.from( + "grpc-status: 0\r\nauthorization: fixture-only-trailer-credential\r\n", + ), + 128, + ); + const small = protobufFrame(Buffer.from([8, 1])); + for (const [name, body, mimeType, framing] of [ + [ + "large", + Buffer.concat([large, credential]), + "application/grpc-web+proto", + "auto", + ], + [ + "malformed", + Buffer.concat([ + small, + protobufFrame( + Buffer.from( + "malformed\r\nauthorization: fixture-only-trailer-credential\r\n", + ), + 128, + ), + ]), + "application/grpc-web+proto", + "auto", + ], + [ + "text-override", + Buffer.from(Buffer.concat([small, credential]).toString("base64")), + "application/grpc-web-text", + "raw", + ], + ] as const) { + const path = join(root, `credential-${name}.har`); + await writeFile(path, protobufHar(body, mimeType)); + const response = await call("inspect_web_network_capture", { + capture_path: path, + format: "har", + record_ordinals: [0], + protobuf: { framing }, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const capture = webNetworkCaptureSchema.parse( + parseEvidence(response.structuredContent).normalized_result, + ); + expect(capture.records[0]?.binary_fields[0]).toMatchObject({ + state: "redacted", + content_base64: null, + sha256: null, + }); + expect(capture.records[0]?.protobuf).toBeUndefined(); + expect(capture.protobuf_coverage?.status).toBe("partial"); + } + const path = join(root, "credential-later-record.har"); + await writeFile( + path, + protobufHar(large, "application/grpc-web+proto", [ + Buffer.concat([small, credential]), + ]), + ); + for (const schema of [ + undefined, + { descriptor_set_path: descriptor, message_type: "capture.Missing" }, + ]) { + const response = await call("inspect_web_network_capture", { + capture_path: path, + format: "har", + record_ordinals: [0, 1], + sensitive_values: ["unrelated fixture exclusion"], + ...(schema === undefined ? {} : { protobuf: { schema } }), + }); + const error = parseMcpToolError(response).error; + expect(error.code).toBe( + schema === undefined ? "resource_constraint" : "invalid_request", + ); + const source = retainedCaptureDetailsSchema.parse( + error.details, + ).partial_observation; + const retained = await call("inspect_analysis_view", { + source, + view: { kind: "capture", detail: "complete", offset: 0, limit: 2 }, + }); + expect(retained.isError, JSON.stringify(retained)).not.toBe(true); + const view = analysisViewResultSchema.parse( + parseEvidence(retained.structuredContent).normalized_result, + ); + expect(view).toMatchObject({ + items: [ + expect.anything(), + { + binary_fields: [ + { state: "redacted", content_base64: null, sha256: null }, + ], + }, + ], + }); + } +} + +async function verifyDescriptorReadFailures( + call: CaptureCall, + args: CaptureRequest, + root: string, +): Promise { + const directory = join(root, "descriptor-directory"); + const oversized = join(root, "oversized.pb"); + await mkdir(directory); + await writeFile(oversized, ""); + await truncate(oversized, 32 * 1024 * 1024 + 1); + for (const [path, reason] of [ + [join(root, "missing.pb"), "invalid_value"], + [directory, "invalid_format"], + [oversized, "out_of_range"], + ] as const) { + const response = await call("inspect_web_network_capture", { + ...args, + sensitive_values: ["unrelated fixture exclusion"], + protobuf: { + schema: { descriptor_set_path: path, message_type: "capture.Message" }, + }, + }); + const error = parseMcpToolError(response).error; + expect(error.code).toBe("invalid_request"); + expect(error.details).toMatchObject({ + issues: [{ path: ["protobuf", "schema", "descriptor_set_path"], reason }], + }); + expect( + retainedCaptureDetailsSchema.parse(error.details).partial_observation + .kind, + ).toBe("retained-evidence"); + } +} + +cliTest( + "decodes selected capture bytes with a supplied descriptor and pages exported facts after both sources disappear", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-protobuf-cli-"); + const capture = join(root, "capture.har"); + const descriptor = join(root, "capture.pb"); + const bundle = join(root, "evidence.json"); + await writeFile(capture, protobufHar()); + await writeFile(descriptor, protobufCaptureDescriptor()); + const original = await cli.run({ + arguments: ["inspect-web-network-capture", capture, "har", "--json"], + }); + expect(original.exitCode, original.stderr).toBe(0); + expect( + webNetworkCaptureSchema.parse( + parseEvidence(original.json).normalized_result, + ).records[0], + ).not.toHaveProperty("protobuf"); + const decoded = await cli.run({ + arguments: [ + "inspect-web-network-capture", + capture, + "har", + "--record", + "0", + "--protobuf-schema", + descriptor, + "--protobuf-message", + "capture.Message", + "--detail", + "summary", + "--evidence_output", + bundle, + "--json", + ], + }); + expect(decoded.exitCode, decoded.stdout + decoded.stderr).toBe(0); + const loaded = await readEvidenceBundle(bundle); + if (!loaded.ok) throw loaded.error; + const parent = loaded.value.records.find( + (record) => record.operation === "inspect_web_network_capture", + ); + if (parent === undefined) + throw new Error("Complete capture Evidence must be retained"); + const report = webNetworkCaptureSchema.parse(parent.normalized_result); + expect(report.records[0]?.protobuf?.[0]).toMatchObject({ + status: "complete", + schema: { message_type: "capture.Message" }, + }); + expect(report.records[0]?.protobuf?.[0]?.fields[0]).toMatchObject({ + schema_name: "sentence", + schema_value: "hello", + field_id: "offset:5", + }); + await unlink(capture); + await unlink(descriptor); + const request = join(root, "view.json"); + await writeFile( + request, + JSON.stringify({ + source: { kind: "inline", evidence: parent }, + view: { + kind: "protobuf", + collection: "fields", + field_ids: ["offset:5"], + fields: ["schema_name", "schema_value"], + offset: 0, + limit: 1, + }, + }), + ); + const viewed = await cli.run({ + arguments: ["inspect-analysis-view", request, "--json"], + }); + expect(viewed.exitCode, viewed.stdout + viewed.stderr).toBe(0); + const view = analysisViewResultSchema.parse( + parseEvidence(viewed.json).normalized_result, + ); + expect(view).toMatchObject({ + kind: "protobuf", + parent_evidence_id: parent.evidence_id, + source_status: "complete", + coverage: { total: 1, examined: 1, exhausted: true }, + items: [ + { + record_ordinal: 0, + body_pointer: "/response/content/text", + field_id: "offset:5", + schema_name: "sentence", + schema_value: "hello", + }, + ], + }); + expect(JSON.stringify(view)).not.toContain("content_base64"); + expect((await readFile(bundle)).length).toBeGreaterThan(0); + }, +); + +cliTest( + "uses production MCP schemas, retained pages and CLI-equivalent decoding without reopening sources", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-protobuf-mcp-"); + const capture = join(root, "capture.har"); + const descriptor = join(root, "capture.pb"); + await writeFile(capture, protobufHar()); + await writeFile(descriptor, protobufCaptureDescriptor()); + const { call } = await connectLocalToolsMcp(); + const args: CaptureRequest = { + capture_path: capture, + format: "har", + record_ordinals: [0], + protobuf: { + schema: { + descriptor_set_path: descriptor, + message_type: "capture.Message", + }, + }, + }; + const complete = await call("inspect_web_network_capture", args); + expect(complete.isError, JSON.stringify(complete)).not.toBe(true); + const parent = parseEvidence(complete.structuredContent); + const cliResult = await cli.run({ + arguments: [ + "inspect-web-network-capture", + capture, + "har", + "--record", + "0", + "--protobuf-schema", + descriptor, + "--protobuf-message", + "capture.Message", + "--json", + ], + }); + expect(cliResult.exitCode, cliResult.stdout + cliResult.stderr).toBe(0); + expect(parseEvidence(cliResult.json).evidence_id).toBe(parent.evidence_id); + await verifyCaptureExclusions(call, args, root); + await verifyPartialCaptureSources(call, args, root, descriptor); + await verifyCredentialPreflight(call, root, descriptor); + await verifyDescriptorReadFailures(call, args, root); + const summary = await call("inspect_web_network_capture", { + ...args, + detail: "summary", + }); + expect(summary.isError, JSON.stringify(summary)).not.toBe(true); + const result = analysisViewResultSchema.parse( + parseEvidence(summary.structuredContent).normalized_result, + ); + expect(result).toMatchObject({ + kind: "summary", + parent_evidence_id: parent.evidence_id, + }); + await unlink(capture); + await unlink(descriptor); + const viewed = await call("inspect_analysis_view", { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { + kind: "protobuf", + collection: "fields", + field_numbers: [2], + fields: ["schema_name", "schema_value"], + offset: 0, + limit: 1, + }, + }); + expect(viewed.isError, JSON.stringify(viewed)).not.toBe(true); + expect( + analysisViewResultSchema.parse( + parseEvidence(viewed.structuredContent).normalized_result, + ), + ).toMatchObject({ + kind: "protobuf", + items: [{ schema_name: "values", schema_value: '["1","2"]' }], + coverage: { total: 1, exhausted: true }, + }); + const invalid = await call("inspect_web_network_capture", { + ...args, + record_ordinals: undefined, + }); + expect(invalid.isError).toBe(true); + expect(parseMcpToolError(invalid).error.code).toBe("invalid_request"); + }, +); diff --git a/tests/boundary/browser/sazWebCapture.test.ts b/tests/boundary/browser/sazWebCapture.test.ts index 9a53ff65a8..a325304db7 100644 --- a/tests/boundary/browser/sazWebCapture.test.ts +++ b/tests/boundary/browser/sazWebCapture.test.ts @@ -16,6 +16,7 @@ import { } from "../../../src/domain/webNetworkCapture.js"; import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { protobufFrame } from "../../fixtures/protobufCapture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { @@ -113,6 +114,181 @@ const cli = async (path: string, extra: string[] = []) => { return parseEvidence(JSON.parse(result.stdout)); }; +it("uses HTTP optional whitespace for SAZ identity codings without changing raw headers", async () => { + const body = Buffer.from([0, 0, 0, 0, 2, 8, 1]); + const codings = [ + " \tidentity\t ", + "\tIdEnTiTy ", + " gzip ", + " \u00a0identity", + ]; + const entries: [string, Buffer][] = codings.map((coding, index) => [ + `raw/${index + 1}_s.txt`, + Buffer.concat([ + Buffer.from( + `HTTP/1.1 200 OK\r\nContent-Type: application/grpc\r\nContent-Encoding:${coding}\r\nContent-Length: ${body.length}\r\n\r\n`, + "latin1", + ), + body, + ]), + ]); + const path = await fixture(await archive(entries)); + const evidence = await cli(path, [ + "--record", + "0", + "--record", + "1", + "--record", + "2", + "--record", + "3", + "--protobuf-framing", + "grpc", + ]); + const capture = webNetworkCaptureSchema.parse(evidence.normalized_result); + for (const [index, coding] of codings.entries()) { + expect(capture.records[index]?.reported).toMatchObject({ + response: { + headers: expect.arrayContaining([ + { name: "Content-Encoding", value: coding }, + ]), + }, + }); + if (index < 2) + expect(capture.records[index]?.protobuf).toMatchObject([ + { + pointer: "/response/body", + status: "complete", + fields: [{ number: 1, value: "1" }], + }, + ]); + else expect(capture.records[index]?.protobuf).toBeUndefined(); + } + const { call } = await connectLocalToolsMcp(); + const response = await call("inspect_web_network_capture", { + capture_path: path, + format: "saz", + record_ordinals: [0, 1, 2, 3], + protobuf: { framing: "grpc" }, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(parseEvidence(response.structuredContent).evidence_id).toBe( + evidence.evidence_id, + ); + for (const sensitive of ["gzip", "Content-Encoding"]) { + const excluded = await call("inspect_web_network_capture", { + capture_path: path, + format: "saz", + record_ordinals: [0, 1, 2, 3], + protobuf: { framing: "grpc" }, + sensitive_values: [sensitive], + }); + expect(excluded.isError, JSON.stringify(excluded)).not.toBe(true); + const report = webNetworkCaptureSchema.parse( + parseEvidence(excluded.structuredContent).normalized_result, + ); + expect(report.protobuf_coverage).toMatchObject({ + status: "partial", + inspected_records: 4, + }); + if (sensitive === "gzip") + expect(report.records[2]?.protobuf).toBeUndefined(); + else + expect( + report.records.every((record) => record.protobuf === undefined), + ).toBe(true); + for (const record of report.records) + expect(record.binary_fields).toContainEqual( + expect.objectContaining({ + pointer: "/response/body", + state: "retained", + }), + ); + } +}); + +it("checks trailer credentials before hidden metadata blocks analysis and withholds unchecked source", async () => { + const credential = Buffer.concat([ + protobufFrame(Buffer.from([8, 1])), + protobufFrame(Buffer.from("authorization: Bearer fixture-only\r\n"), 128), + ]); + const responseBytes = (body: Buffer, contentType: string, extra = "") => + Buffer.concat([ + Buffer.from( + `HTTP/1.1 200 OK\r\nContent-Type: ${contentType}\r\nX-Debug: harmless\r\n${extra}\r\n`, + ), + body, + ]); + const bytes = await archive([ + ["raw/1_s.txt", responseBytes(credential, "application/grpc-web+proto")], + [ + "raw/2_s.txt", + responseBytes(Buffer.from([0, 0, 0]), "application/grpc-web+proto"), + ], + [ + "raw/3_s.txt", + responseBytes( + gzipSync(credential), + "application/grpc-web+proto", + "Content-Encoding: gzip\r\n", + ), + ], + [ + "raw/4_s.txt", + responseBytes(Buffer.from("invalid*base64"), "application/grpc-web-text"), + ], + ]); + const path = await fixture(bytes); + const { call } = await connectLocalToolsMcp(); + const base = { + capture_path: path, + format: "saz", + sensitive_values: ["X-Debug"], + }; + const protectedReply = await call("inspect_web_network_capture", { + ...base, + record_ordinals: [0], + }); + expect(protectedReply.isError, JSON.stringify(protectedReply)).not.toBe(true); + const protectedCapture = webNetworkCaptureSchema.parse( + parseEvidence(protectedReply.structuredContent).normalized_result, + ); + expect(protectedCapture.records[0]?.binary_fields).toContainEqual( + expect.objectContaining({ + pointer: "/response/body", + state: "redacted", + content_base64: null, + sha256: null, + }), + ); + expect(protectedCapture.records[0]?.protobuf).toBeUndefined(); + for (const ordinals of [[0, 1], [2], [3]]) { + const response = await call("inspect_web_network_capture", { + ...base, + record_ordinals: ordinals, + }); + const error = parseMcpToolError(response).error; + expect(error.code).toBe("capability_unavailable"); + expect(JSON.stringify(response)).not.toContain("content_base64"); + expect(error.details).not.toHaveProperty("partial_observation"); + } + const opaqueReply = await call("inspect_web_network_capture", { + capture_path: path, + format: "saz", + record_ordinals: [2], + }); + expect(opaqueReply.isError, JSON.stringify(opaqueReply)).not.toBe(true); + const opaque = webNetworkCaptureSchema.parse( + parseEvidence(opaqueReply.structuredContent).normalized_result, + ); + expect(opaque.records[0]?.binary_fields).toContainEqual( + expect.objectContaining({ pointer: "/response/body", state: "retained" }), + ); + expect(opaque.records[0]?.protobuf).toBeUndefined(); + expect(opaque.protobuf_coverage?.status).toBe("partial"); + expect(await readFile(path)).toEqual(bytes); +}); + it("inspects native SAZ through the public CLI and MCP with equal evidence, bytes and archive coordinates", async () => { const bytes = await archive(); const path = await fixture(bytes); diff --git a/tests/fixtures/protobufCapture.ts b/tests/fixtures/protobufCapture.ts new file mode 100644 index 0000000000..183bacecd6 --- /dev/null +++ b/tests/fixtures/protobufCapture.ts @@ -0,0 +1,83 @@ +import { create, toBinary } from "@bufbuild/protobuf"; +import { FileDescriptorSetSchema } from "@bufbuild/protobuf/wkt"; + +export const protobufFrame = (body: Buffer, flag = 0): Buffer => { + const header = Buffer.alloc(5); + header[0] = flag; + header.writeUInt32BE(body.length, 1); + return Buffer.concat([header, body]); +}; + +export const protobufCaptureBody = () => + Buffer.concat([ + protobufFrame(Buffer.from("0a0568656c6c6f12020102", "hex")), + protobufFrame(Buffer.from("grpc-status: 0\r\n"), 128), + ]); + +/** Original fixture schema; independent expected field values come from the literal wire bytes. */ +export const protobufCaptureDescriptor = () => + toBinary( + FileDescriptorSetSchema, + create(FileDescriptorSetSchema, { + file: [ + { + name: "capture.proto", + package: "capture", + syntax: "proto3", + messageType: [ + { + name: "Message", + field: [ + { name: "sentence", number: 1, type: 9, label: 1 }, + { name: "values", number: 2, type: 13, label: 3 }, + ], + }, + ], + }, + ], + }), + ); + +export const protobufHar = ( + body: Buffer = protobufCaptureBody(), + mimeType = "application/grpc-web+proto", + additionalBodies: readonly Buffer[] = [], +) => + JSON.stringify({ + log: { + version: "1.2", + creator: { name: "REA original fixture", version: "1" }, + entries: [body, ...additionalBodies].map((body) => ({ + startedDateTime: "2026-10-20T00:00:00Z", + time: 1, + request: { + method: "POST", + url: "https://example.invalid/api", + httpVersion: "HTTP/1.1", + cookies: [], + headers: [], + queryString: [], + headersSize: -1, + bodySize: 0, + }, + response: { + status: 200, + statusText: "OK", + httpVersion: "HTTP/1.1", + cookies: [], + headers: [], + redirectURL: "", + headersSize: -1, + bodySize: body.length, + content: { + size: body.length, + mimeType, + text: body.toString("base64"), + encoding: "base64", + }, + }, + cache: {}, + timings: { send: 0, wait: 1, receive: 0 }, + })), + }, + }); From 4eaf3a117b3f4863a994b9be743ec5bc1501e16c Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:31:02 +0800 Subject: [PATCH 02/13] fix(mcp): retain protocol framing for oversized tool errors (#5371) Preserve the SDK resultType when bounded error recovery replaces an oversized response. Verify both revisions through the public serving entry, retained Evidence and same-connection recovery. Refs #5273 Co-authored-by: akushonkamen --- docs/mcp-contracts.md | 11 +- src/server/EvidenceMcpServer.test.ts | 230 ++++++++++++++++----------- src/server/EvidenceMcpServer.ts | 10 +- 3 files changed, 152 insertions(+), 99 deletions(-) diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 912d531ea2..409d7d0aa7 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -101,8 +101,9 @@ unresolved relationship reads. Cancellation remains a tool error. REA prepares complete MCP results within the pinned stdio client's 10 MiB receive-buffer budget, including both text and structured representations and -room for the JSON-RPC envelope. Tool errors carry only their text -representation and `isError` flag, so only those count against the budget. Response budget settings are captured at startup; +room for the JSON-RPC envelope. Tool errors omit `structuredContent` from the +wire; their text, `isError`, provenance marker and negotiated protocol framing +count toward the wire budget. Response budget settings are captured at startup; restart or recreate the server to apply changes. If a result cannot fit, REA returns `resource_constraint` with `details.resource: "transport"` before constructing a document-sized string. Analysis Evidence remains complete in the current @@ -112,6 +113,12 @@ that reference for a summary, one section/module, or a stable page, or call `export_evidence_bundle` with a destination path. Complete bundle exports stream canonical JSON into an atomically published file. +On protocol revision `2026-07-28`, completed replies retain the SDK's +`resultType: "complete"`, including the replacement for an oversized error. +This marks completion of the MCP exchange; Evidence coverage, partial facts +and unknowns retain their separate meaning. Legacy replies keep their existing +framing. + For oversized native function Evidence, `details.reported_limits.continuation` contains a ready-to-call `inspect_analysis_view` request for the retained pseudocode's first page. Use its `name` and `arguments` directly, then follow diff --git a/src/server/EvidenceMcpServer.test.ts b/src/server/EvidenceMcpServer.test.ts index ce3e7a2746..18ee8d117e 100644 --- a/src/server/EvidenceMcpServer.test.ts +++ b/src/server/EvidenceMcpServer.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { + isJSONRPCResultResponse, + type JSONRPCResultResponse, +} from "@modelcontextprotocol/server"; +import { serveStdio } from "@modelcontextprotocol/server/stdio"; import { z } from "zod"; import { InvestigationRecords } from "../application/investigation/InvestigationRecords.js"; @@ -43,104 +48,137 @@ describe("MCP request bookkeeping", () => { }); describe("SDK validation error delivery", () => { - it("bounds oversized errors, retains their source, and preserves fitting and valid calls", async () => { - const delivery = new ToolResultDelivery(4096); - const records = new InvestigationRecords(); - const server = new EvidenceMcpServer( - { name: "sdk-validation-budget-test", version: "1" }, - { capabilities: {} }, - (evidence) => records.recordEvidence(evidence), - delivery, - ); - server.registerTool( - "procedure_pseudo_code", - { inputSchema: z.strictObject({ required: z.string() }) }, - async () => ({ content: [{ type: "text", text: "valid call" }] }), - ); - const client = new Client({ - name: "sdk-validation-budget-client", - version: "1", - }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - const oversizedKey = `unexpected_${"x".repeat(6000)}`; - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - - const oversized = await client.callTool({ - name: "procedure_pseudo_code", - arguments: { required: "present", [oversizedKey]: true }, - }); - expect(oversized.isError).toBe(true); - expect(oversized.structuredContent).toBeUndefined(); - expect(oversized._meta).toEqual({ - provenance: { untrusted_content: true, source: "analysis_target" }, - }); - expect(Buffer.byteLength(JSON.stringify(oversized))).toBeLessThan(4096); - const oversizedError = z - .object({ - error: z.object({ - code: z.string(), - details: z.object({ - reported_limits: z.record(z.string(), z.unknown()), - }), - }), - }) - .parse(JSON.parse(firstText(oversized) ?? "null")); - expect(oversizedError.error.code).toBe("resource_constraint"); - const reportedLimits = oversizedError.error.details.reported_limits; - expect(reportedLimits).toMatchObject({ - evidence_reference: { - kind: "retained-evidence", - evidence_id: expect.any(String), - }, - diagnostic_truncated: true, - }); - expect(reportedLimits).not.toHaveProperty("retention"); - const bundle = records.exportEvidenceBundle(); - expect(bundle.records).toHaveLength(1); - expect(bundle.records[0]?.operation).toBe("mcp_tool_error"); - expect(bundle.records[0]?.raw_result).toMatchObject({ - isError: true, - content: [ - { type: "text", text: expect.stringContaining(oversizedKey) }, - ], - }); - const evidenceReference = z - .object({ evidence_id: z.string() }) - .parse(reportedLimits.evidence_reference); - expect(bundle.records[0]?.evidence_id).toBe( - evidenceReference.evidence_id, + it.each(["2025-11-25", "2026-07-28"])( + "bounds and retains errors through the public stdio entry on %s", + async (revision) => { + const delivery = new ToolResultDelivery(4096); + const records = new InvestigationRecords(); + const server = new EvidenceMcpServer( + { name: "sdk-validation-budget-test", version: "1" }, + { capabilities: {} }, + (evidence) => records.recordEvidence(evidence), + delivery, ); + server.registerTool( + "procedure_pseudo_code", + { inputSchema: z.strictObject({ required: z.string() }) }, + async () => ({ content: [{ type: "text", text: "valid call" }] }), + ); + const client = new Client( + { + name: "sdk-validation-budget-client", + version: "1", + }, + revision === "2026-07-28" + ? { versionNegotiation: { mode: { pin: revision } } } + : { supportedProtocolVersions: [revision] }, + ); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + const oversizedKey = `unexpected_${"x".repeat(6000)}`; + const serving = serveStdio(() => server, { transport: serverTransport }); + const replies: JSONRPCResultResponse[] = []; + try { + await client.connect(clientTransport); + expect(client.getNegotiatedProtocolVersion()).toBe(revision); + const receive = clientTransport.onmessage; + clientTransport.onmessage = (message) => { + if (isJSONRPCResultResponse(message)) replies.push(message); + receive?.(message); + }; - const fitting = await client.callTool({ - name: "procedure_pseudo_code", - arguments: { unexpected: true }, - }); - expect(fitting.isError).toBe(true); - expect(firstText(fitting)).toContain("Input validation error"); + // The normal reply and SDK validation error already negotiate and decode + // correctly before oversized recovery replaces an SDK-encoded envelope. + const valid = await client.callTool({ + name: "procedure_pseudo_code", + arguments: { required: "present" }, + }); + expect(valid.content).toContainEqual({ + type: "text", + text: "valid call", + }); + const fitting = await client.callTool({ + name: "procedure_pseudo_code", + arguments: { unexpected: true }, + }); + expect(fitting.isError).toBe(true); + expect(firstText(fitting)).toContain("Input validation error"); - const quotedKey = '"\\'.repeat(350); - const quoted = await client.callTool({ - name: "procedure_pseudo_code", - arguments: { required: "present", [quotedKey]: true }, - }); - expect(quoted.isError).toBe(true); - expect(firstText(quoted)).toContain("Input validation error"); - expect(Buffer.byteLength(JSON.stringify(quoted))).toBeLessThan(4096); - expect(records.exportEvidenceBundle().records).toHaveLength(1); + const oversized = await client.callTool({ + name: "procedure_pseudo_code", + arguments: { required: "present", [oversizedKey]: true }, + }); + expect(oversized.isError).toBe(true); + expect(oversized.structuredContent).toBeUndefined(); + expect(oversized._meta).toEqual({ + provenance: { untrusted_content: true, source: "analysis_target" }, + }); + expect(Buffer.byteLength(JSON.stringify(oversized))).toBeLessThan(4096); + const oversizedError = z + .object({ + error: z.object({ + code: z.string(), + details: z.object({ + reported_limits: z.record(z.string(), z.unknown()), + }), + }), + }) + .parse(JSON.parse(firstText(oversized) ?? "null")); + expect(oversizedError.error.code).toBe("resource_constraint"); + const reportedLimits = oversizedError.error.details.reported_limits; + expect(reportedLimits).toMatchObject({ + evidence_reference: { + kind: "retained-evidence", + evidence_id: expect.any(String), + }, + diagnostic_truncated: true, + }); + expect(reportedLimits).not.toHaveProperty("retention"); + const bundle = records.exportEvidenceBundle(); + expect(bundle.records).toHaveLength(1); + expect(bundle.records[0]?.operation).toBe("mcp_tool_error"); + expect(bundle.records[0]?.raw_result).toMatchObject({ + isError: true, + content: [ + { type: "text", text: expect.stringContaining(oversizedKey) }, + ], + }); + const evidenceReference = z + .object({ evidence_id: z.string() }) + .parse(reportedLimits.evidence_reference); + expect(bundle.records[0]?.evidence_id).toBe( + evidenceReference.evidence_id, + ); - const valid = await client.callTool({ - name: "procedure_pseudo_code", - arguments: { required: "present" }, - }); - expect(valid.content).toContainEqual({ - type: "text", - text: "valid call", - }); - } finally { - await Promise.all([client.close(), server.close()]); - } - }); + const quotedKey = '"\\'.repeat(350); + const quoted = await client.callTool({ + name: "procedure_pseudo_code", + arguments: { required: "present", [quotedKey]: true }, + }); + expect(quoted.isError).toBe(true); + expect(firstText(quoted)).toContain("Input validation error"); + expect(Buffer.byteLength(JSON.stringify(quoted))).toBeLessThan(4096); + expect(records.exportEvidenceBundle().records).toHaveLength(1); + + const recovered = await client.callTool({ + name: "procedure_pseudo_code", + arguments: { required: "present" }, + }); + expect(recovered.content).toContainEqual({ + type: "text", + text: "valid call", + }); + for (const reply of replies) { + expect(reply.result.resultType).toBe( + revision === "2026-07-28" ? "complete" : undefined, + ); + expect( + Buffer.byteLength(JSON.stringify(reply)) + 1, + ).toBeLessThanOrEqual(4096); + } + } finally { + await Promise.all([client.close(), serving.close()]); + } + }, + ); }); diff --git a/src/server/EvidenceMcpServer.ts b/src/server/EvidenceMcpServer.ts index 33ee1f24be..8598afd809 100644 --- a/src/server/EvidenceMcpServer.ts +++ b/src/server/EvidenceMcpServer.ts @@ -451,7 +451,15 @@ class ErrorEvidenceTransport implements Transport { ); return { ...message, - result: this.delivery.toErrorToolResult(failure, targetContent), + result: { + ...this.delivery.toErrorToolResult(failure, targetContent), + // The SDK stamps `resultType` on modern-era results before this + // transport sees them; the replacement must keep the stamp or a + // 2026-07-28 client discards REA's structured error entirely. + ...(typeof message.result.resultType === "string" + ? { resultType: message.result.resultType } + : {}), + }, }; } } From b5d540f3a264206e32c199c40dc17ac8695cf338 Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:54:47 +0800 Subject: [PATCH 03/13] test(capture): split protobuf MCP acceptance scenarios (#5379) Separate independent exclusion, partial-source, credential-preflight and descriptor-failure workflows so their costs do not accumulate under one test deadline. Reuse one worker-owned MCP connection with explicit awaited teardown, preserving sensitivity isolation and retained-Evidence checks. Assisted-by: Reimu (gensokyo-skills:reimu-incident-triage) --- .../acceptance/tools/protobufCapture.test.ts | 656 +++++++++--------- tests/fixtures/localToolsMcp.ts | 3 +- 2 files changed, 344 insertions(+), 315 deletions(-) diff --git a/tests/acceptance/tools/protobufCapture.test.ts b/tests/acceptance/tools/protobufCapture.test.ts index 3720c57104..921c1b13d3 100644 --- a/tests/acceptance/tools/protobufCapture.test.ts +++ b/tests/acceptance/tools/protobufCapture.test.ts @@ -20,9 +20,29 @@ import { analysisViewResultSchema } from "../../../src/domain/analysisView/analy import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; -type CaptureCall = Awaited>["call"]; type CaptureRequest = z.input; +// Acceptance files have isolated workers; the file owns one server and each case owns its inputs. +const captureTest = cliTest.extend<{ + $worker: { mcp: Awaited> }; +}>({ + mcp: [ + // oxlint-disable-next-line no-empty-pattern -- Vitest parses fixture dependencies from object destructuring. + async ({}, use) => { + let close: (() => Promise) | undefined; + try { + const mcp = await connectLocalToolsMcp({}, (cleanup) => { + close = cleanup; + }); + await use(mcp); + } finally { + await close?.(); + } + }, + { scope: "worker" }, + ], +}); + const retainedCaptureDetailsSchema = z.object({ partial_observation: z.strictObject({ kind: z.literal("retained-evidence"), @@ -30,321 +50,332 @@ const retainedCaptureDetailsSchema = z.object({ }), }); -async function verifyCaptureExclusions( - call: CaptureCall, - args: CaptureRequest, - root: string, -): Promise { - const redacted = await call("inspect_web_network_capture", { - ...args, - sensitive_values: ["hello"], - }); - expect(redacted.isError, JSON.stringify(redacted)).not.toBe(true); - const excluded = webNetworkCaptureSchema.parse( - parseEvidence(redacted.structuredContent).normalized_result, - ); - expect(excluded.records[0]?.protobuf).toBeUndefined(); - expect(excluded.protobuf_coverage).toMatchObject({ - status: "partial", - selected_records: 1, - inspected_records: 1, - }); - expect(JSON.stringify(redacted)).not.toContain("hello"); - const textPath = join(root, "text.har"); - await writeFile( - textPath, - protobufHar( - Buffer.from(protobufCaptureBody().toString("base64")), - "application/grpc-web-text", - ), - ); - const textExcluded = await call("inspect_web_network_capture", { - capture_path: textPath, +async function prepareMcpCapture( + call: Awaited>["call"], +) { + const root = await createTestTempDirectory("rea-protobuf-mcp-"); + const capture = join(root, "capture.har"); + const descriptor = join(root, "capture.pb"); + await writeFile(capture, protobufHar()); + await writeFile(descriptor, protobufCaptureDescriptor()); + const args: CaptureRequest = { + capture_path: capture, format: "har", record_ordinals: [0], - sensitive_values: ["hello"], - }); - expect(textExcluded.isError, JSON.stringify(textExcluded)).not.toBe(true); - const excludedText = webNetworkCaptureSchema.parse( - parseEvidence(textExcluded.structuredContent).normalized_result, - ); - expect(excludedText.records[0]?.binary_fields[0]).toMatchObject({ - state: "redacted", - content_base64: null, - sha256: null, - }); - expect(excludedText.records[0]?.reported).toMatchObject({ - response: { content: { text: null } }, - }); - expect(excludedText.records[0]?.protobuf).toBeUndefined(); - const credentialPath = join(root, "trailer.har"); - await writeFile( - credentialPath, - protobufHar( - Buffer.concat([ - protobufFrame(Buffer.from("0a0568656c6c6f", "hex")), - protobufFrame( - Buffer.from( - "grpc-status: 0\r\nauthorization: Bearer fixture-only-credential\r\n", - ), - 128, - ), - ]), - ), - ); - const credential = await call("inspect_web_network_capture", { - capture_path: credentialPath, - format: "har", - record_ordinals: [0], - }); - expect(credential.isError, JSON.stringify(credential)).not.toBe(true); - const excludedCredential = webNetworkCaptureSchema.parse( - parseEvidence(credential.structuredContent).normalized_result, - ); - expect(excludedCredential.records[0]?.binary_fields[0]).toMatchObject({ - state: "redacted", - content_base64: null, - }); - expect(excludedCredential.records[0]?.redactions).toContainEqual({ - pointer: "/response/content/text", - reason: "transport-credential", - }); - expect(excludedCredential.records[0]?.protobuf).toBeUndefined(); - expect(JSON.stringify(credential)).not.toContain("fixture-only-credential"); -} - -async function verifyPartialCaptureSources( - call: CaptureCall, - args: CaptureRequest, - root: string, - descriptor: string, -): Promise { - const missing = await call("inspect_web_network_capture", { - ...args, - sensitive_values: ["unrelated fixture exclusion"], protobuf: { schema: { descriptor_set_path: descriptor, - message_type: "capture.Missing", + message_type: "capture.Message", }, }, - }); - const missingError = parseMcpToolError(missing).error; - expect(missingError.code).toBe("invalid_request"); - const retainedSource = retainedCaptureDetailsSchema.parse( - missingError.details, - ).partial_observation; - const retainedSummary = await call("inspect_analysis_view", { - source: retainedSource, - view: { kind: "summary" }, - }); - expect(retainedSummary.isError, JSON.stringify(retainedSummary)).not.toBe( - true, - ); - expect( - analysisViewResultSchema.parse( - parseEvidence(retainedSummary.structuredContent).normalized_result, - ), - ).toMatchObject({ - summary: { - protobuf_coverage: { status: "partial", inspected_records: 0 }, - }, - }); - const missingView = await call("inspect_analysis_view", { - source: retainedSource, - view: { kind: "protobuf", collection: "fields", offset: 0, limit: 1 }, - }); - expect(missingView.isError, JSON.stringify(missingView)).not.toBe(true); - expect( - analysisViewResultSchema.parse( - parseEvidence(missingView.structuredContent).normalized_result, - ), - ).toMatchObject({ source_status: "partial", items: [] }); - const capacityPath = join(root, "capacity.har"); - const repeated = Buffer.alloc(200000); - for (let index = 0; index < repeated.length; index += 2) { - repeated[index] = 8; - repeated[index + 1] = 1; - } - await writeFile(capacityPath, protobufHar(protobufFrame(repeated))); - const capacity = await call("inspect_web_network_capture", { - capture_path: capacityPath, - format: "har", - record_ordinals: [0], - sensitive_values: ["unrelated fixture exclusion"], - }); - const capacityError = parseMcpToolError(capacity).error; - expect(capacityError.code).toBe("resource_constraint"); - expect(capacityError.retryable).toBe(false); - const capacitySource = retainedCaptureDetailsSchema.parse( - capacityError.details, - ).partial_observation; - const capacitySummary = await call("inspect_analysis_view", { - source: capacitySource, - view: { kind: "summary" }, - }); - expect(capacitySummary.isError, JSON.stringify(capacitySummary)).not.toBe( - true, - ); - expect( - analysisViewResultSchema.parse( - parseEvidence(capacitySummary.structuredContent).normalized_result, - ), - ).toMatchObject({ - summary: { - protobuf_coverage: { status: "partial", inspected_records: 0 }, - }, - }); - expect(JSON.stringify(capacity)).not.toContain("content_base64"); + }; + return { call, args, root, capture, descriptor }; } -async function verifyCredentialPreflight( - call: CaptureCall, - root: string, - descriptor: string, -): Promise { - const repeated = Buffer.alloc(200000); - for (let index = 0; index < repeated.length; index += 2) { - repeated[index] = 8; - repeated[index + 1] = 1; - } - const large = protobufFrame(repeated); - const credential = protobufFrame( - Buffer.from( - "grpc-status: 0\r\nauthorization: fixture-only-trailer-credential\r\n", - ), - 128, - ); - const small = protobufFrame(Buffer.from([8, 1])); - for (const [name, body, mimeType, framing] of [ - [ - "large", - Buffer.concat([large, credential]), - "application/grpc-web+proto", - "auto", - ], - [ - "malformed", - Buffer.concat([ - small, - protobufFrame( - Buffer.from( - "malformed\r\nauthorization: fixture-only-trailer-credential\r\n", - ), - 128, - ), - ]), - "application/grpc-web+proto", - "auto", - ], - [ - "text-override", - Buffer.from(Buffer.concat([small, credential]).toString("base64")), - "application/grpc-web-text", - "raw", - ], - ] as const) { - const path = join(root, `credential-${name}.har`); - await writeFile(path, protobufHar(body, mimeType)); - const response = await call("inspect_web_network_capture", { - capture_path: path, +captureTest( + "excludes Protobuf values in text framing and trailer credentials through MCP", + async ({ mcp }) => { + const { call, root } = await prepareMcpCapture(mcp.call); + const textPath = join(root, "text.har"); + await writeFile( + textPath, + protobufHar( + Buffer.from(protobufCaptureBody().toString("base64")), + "application/grpc-web-text", + ), + ); + const textExcluded = await call("inspect_web_network_capture", { + capture_path: textPath, format: "har", record_ordinals: [0], - protobuf: { framing }, + sensitive_values: ["hello"], }); - expect(response.isError, JSON.stringify(response)).not.toBe(true); - const capture = webNetworkCaptureSchema.parse( - parseEvidence(response.structuredContent).normalized_result, + expect(textExcluded.isError, JSON.stringify(textExcluded)).not.toBe(true); + const excludedText = webNetworkCaptureSchema.parse( + parseEvidence(textExcluded.structuredContent).normalized_result, ); - expect(capture.records[0]?.binary_fields[0]).toMatchObject({ + expect(excludedText.records[0]?.binary_fields[0]).toMatchObject({ state: "redacted", content_base64: null, sha256: null, }); - expect(capture.records[0]?.protobuf).toBeUndefined(); - expect(capture.protobuf_coverage?.status).toBe("partial"); - } - const path = join(root, "credential-later-record.har"); - await writeFile( - path, - protobufHar(large, "application/grpc-web+proto", [ - Buffer.concat([small, credential]), - ]), - ); - for (const schema of [ - undefined, - { descriptor_set_path: descriptor, message_type: "capture.Missing" }, - ]) { - const response = await call("inspect_web_network_capture", { - capture_path: path, + expect(excludedText.records[0]?.reported).toMatchObject({ + response: { content: { text: null } }, + }); + expect(excludedText.records[0]?.protobuf).toBeUndefined(); + const credentialPath = join(root, "trailer.har"); + await writeFile( + credentialPath, + protobufHar( + Buffer.concat([ + protobufFrame(Buffer.from("0a0568656c6c6f", "hex")), + protobufFrame( + Buffer.from( + "grpc-status: 0\r\nauthorization: Bearer fixture-only-credential\r\n", + ), + 128, + ), + ]), + ), + ); + const credential = await call("inspect_web_network_capture", { + capture_path: credentialPath, format: "har", - record_ordinals: [0, 1], - sensitive_values: ["unrelated fixture exclusion"], - ...(schema === undefined ? {} : { protobuf: { schema } }), + record_ordinals: [0], }); - const error = parseMcpToolError(response).error; - expect(error.code).toBe( - schema === undefined ? "resource_constraint" : "invalid_request", + expect(credential.isError, JSON.stringify(credential)).not.toBe(true); + const excludedCredential = webNetworkCaptureSchema.parse( + parseEvidence(credential.structuredContent).normalized_result, ); - const source = retainedCaptureDetailsSchema.parse( - error.details, - ).partial_observation; - const retained = await call("inspect_analysis_view", { - source, - view: { kind: "capture", detail: "complete", offset: 0, limit: 2 }, + expect(excludedCredential.records[0]?.binary_fields[0]).toMatchObject({ + state: "redacted", + content_base64: null, }); - expect(retained.isError, JSON.stringify(retained)).not.toBe(true); - const view = analysisViewResultSchema.parse( - parseEvidence(retained.structuredContent).normalized_result, - ); - expect(view).toMatchObject({ - items: [ - expect.anything(), - { - binary_fields: [ - { state: "redacted", content_base64: null, sha256: null }, - ], - }, - ], + expect(excludedCredential.records[0]?.redactions).toContainEqual({ + pointer: "/response/content/text", + reason: "transport-credential", }); - } -} + expect(excludedCredential.records[0]?.protobuf).toBeUndefined(); + expect(JSON.stringify(credential)).not.toContain("fixture-only-credential"); + }, +); -async function verifyDescriptorReadFailures( - call: CaptureCall, - args: CaptureRequest, - root: string, -): Promise { - const directory = join(root, "descriptor-directory"); - const oversized = join(root, "oversized.pb"); - await mkdir(directory); - await writeFile(oversized, ""); - await truncate(oversized, 32 * 1024 * 1024 + 1); - for (const [path, reason] of [ - [join(root, "missing.pb"), "invalid_value"], - [directory, "invalid_format"], - [oversized, "out_of_range"], - ] as const) { - const response = await call("inspect_web_network_capture", { +captureTest( + "retains partial capture Evidence after schema and capacity failures through MCP", + async ({ mcp }) => { + const { call, args, root, descriptor } = await prepareMcpCapture(mcp.call); + const missing = await call("inspect_web_network_capture", { ...args, sensitive_values: ["unrelated fixture exclusion"], protobuf: { - schema: { descriptor_set_path: path, message_type: "capture.Message" }, + schema: { + descriptor_set_path: descriptor, + message_type: "capture.Missing", + }, }, }); - const error = parseMcpToolError(response).error; - expect(error.code).toBe("invalid_request"); - expect(error.details).toMatchObject({ - issues: [{ path: ["protobuf", "schema", "descriptor_set_path"], reason }], + const missingError = parseMcpToolError(missing).error; + expect(missingError.code).toBe("invalid_request"); + const retainedSource = retainedCaptureDetailsSchema.parse( + missingError.details, + ).partial_observation; + const retainedSummary = await call("inspect_analysis_view", { + source: retainedSource, + view: { kind: "summary" }, }); + expect(retainedSummary.isError, JSON.stringify(retainedSummary)).not.toBe( + true, + ); expect( - retainedCaptureDetailsSchema.parse(error.details).partial_observation - .kind, - ).toBe("retained-evidence"); - } -} + analysisViewResultSchema.parse( + parseEvidence(retainedSummary.structuredContent).normalized_result, + ), + ).toMatchObject({ + summary: { + protobuf_coverage: { status: "partial", inspected_records: 0 }, + }, + }); + const missingView = await call("inspect_analysis_view", { + source: retainedSource, + view: { kind: "protobuf", collection: "fields", offset: 0, limit: 1 }, + }); + expect(missingView.isError, JSON.stringify(missingView)).not.toBe(true); + expect( + analysisViewResultSchema.parse( + parseEvidence(missingView.structuredContent).normalized_result, + ), + ).toMatchObject({ source_status: "partial", items: [] }); + const capacityPath = join(root, "capacity.har"); + const repeated = Buffer.alloc(200000); + for (let index = 0; index < repeated.length; index += 2) { + repeated[index] = 8; + repeated[index + 1] = 1; + } + await writeFile(capacityPath, protobufHar(protobufFrame(repeated))); + const capacity = await call("inspect_web_network_capture", { + capture_path: capacityPath, + format: "har", + record_ordinals: [0], + sensitive_values: ["unrelated fixture exclusion"], + }); + const capacityError = parseMcpToolError(capacity).error; + expect(capacityError.code).toBe("resource_constraint"); + expect(capacityError.retryable).toBe(false); + const capacitySource = retainedCaptureDetailsSchema.parse( + capacityError.details, + ).partial_observation; + const capacitySummary = await call("inspect_analysis_view", { + source: capacitySource, + view: { kind: "summary" }, + }); + expect(capacitySummary.isError, JSON.stringify(capacitySummary)).not.toBe( + true, + ); + expect( + analysisViewResultSchema.parse( + parseEvidence(capacitySummary.structuredContent).normalized_result, + ), + ).toMatchObject({ + summary: { + protobuf_coverage: { status: "partial", inspected_records: 0 }, + }, + }); + expect(JSON.stringify(capacity)).not.toContain("content_base64"); + }, +); -cliTest( +captureTest( + "preflights trailer credentials before parsing and retaining later records through MCP", + async ({ mcp }) => { + const { call, root, descriptor } = await prepareMcpCapture(mcp.call); + const repeated = Buffer.alloc(200000); + for (let index = 0; index < repeated.length; index += 2) { + repeated[index] = 8; + repeated[index + 1] = 1; + } + const large = protobufFrame(repeated); + const credential = protobufFrame( + Buffer.from( + "grpc-status: 0\r\nauthorization: fixture-only-trailer-credential\r\n", + ), + 128, + ); + const small = protobufFrame(Buffer.from([8, 1])); + for (const [name, body, mimeType, framing] of [ + [ + "large", + Buffer.concat([large, credential]), + "application/grpc-web+proto", + "auto", + ], + [ + "malformed", + Buffer.concat([ + small, + protobufFrame( + Buffer.from( + "malformed\r\nauthorization: fixture-only-trailer-credential\r\n", + ), + 128, + ), + ]), + "application/grpc-web+proto", + "auto", + ], + [ + "text-override", + Buffer.from(Buffer.concat([small, credential]).toString("base64")), + "application/grpc-web-text", + "raw", + ], + ] as const) { + const path = join(root, `credential-${name}.har`); + await writeFile(path, protobufHar(body, mimeType)); + const response = await call("inspect_web_network_capture", { + capture_path: path, + format: "har", + record_ordinals: [0], + protobuf: { framing }, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const capture = webNetworkCaptureSchema.parse( + parseEvidence(response.structuredContent).normalized_result, + ); + expect(capture.records[0]?.binary_fields[0]).toMatchObject({ + state: "redacted", + content_base64: null, + sha256: null, + }); + expect(capture.records[0]?.protobuf).toBeUndefined(); + expect(capture.protobuf_coverage?.status).toBe("partial"); + } + const path = join(root, "credential-later-record.har"); + await writeFile( + path, + protobufHar(large, "application/grpc-web+proto", [ + Buffer.concat([small, credential]), + ]), + ); + for (const schema of [ + undefined, + { descriptor_set_path: descriptor, message_type: "capture.Missing" }, + ]) { + const response = await call("inspect_web_network_capture", { + capture_path: path, + format: "har", + record_ordinals: [0, 1], + sensitive_values: ["unrelated fixture exclusion"], + ...(schema === undefined ? {} : { protobuf: { schema } }), + }); + const error = parseMcpToolError(response).error; + expect(error.code).toBe( + schema === undefined ? "resource_constraint" : "invalid_request", + ); + const source = retainedCaptureDetailsSchema.parse( + error.details, + ).partial_observation; + const retained = await call("inspect_analysis_view", { + source, + view: { kind: "capture", detail: "complete", offset: 0, limit: 2 }, + }); + expect(retained.isError, JSON.stringify(retained)).not.toBe(true); + const view = analysisViewResultSchema.parse( + parseEvidence(retained.structuredContent).normalized_result, + ); + expect(view).toMatchObject({ + items: [ + expect.anything(), + { + binary_fields: [ + { state: "redacted", content_base64: null, sha256: null }, + ], + }, + ], + }); + } + }, +); + +captureTest( + "reports descriptor read failures with retained capture Evidence through MCP", + async ({ mcp }) => { + const { call, args, root } = await prepareMcpCapture(mcp.call); + const directory = join(root, "descriptor-directory"); + const oversized = join(root, "oversized.pb"); + await mkdir(directory); + await writeFile(oversized, ""); + await truncate(oversized, 32 * 1024 * 1024 + 1); + for (const [path, reason] of [ + [join(root, "missing.pb"), "invalid_value"], + [directory, "invalid_format"], + [oversized, "out_of_range"], + ] as const) { + const response = await call("inspect_web_network_capture", { + ...args, + sensitive_values: ["unrelated fixture exclusion"], + protobuf: { + schema: { + descriptor_set_path: path, + message_type: "capture.Message", + }, + }, + }); + const error = parseMcpToolError(response).error; + expect(error.code).toBe("invalid_request"); + expect(error.details).toMatchObject({ + issues: [ + { path: ["protobuf", "schema", "descriptor_set_path"], reason }, + ], + }); + expect( + retainedCaptureDetailsSchema.parse(error.details).partial_observation + .kind, + ).toBe("retained-evidence"); + } + }, +); + +captureTest( "decodes selected capture bytes with a supplied descriptor and pages exported facts after both sources disappear", async ({ cli }) => { const root = await createTestTempDirectory("rea-protobuf-cli-"); @@ -442,26 +473,12 @@ cliTest( }, ); -cliTest( +captureTest( "uses production MCP schemas, retained pages and CLI-equivalent decoding without reopening sources", - async ({ cli }) => { - const root = await createTestTempDirectory("rea-protobuf-mcp-"); - const capture = join(root, "capture.har"); - const descriptor = join(root, "capture.pb"); - await writeFile(capture, protobufHar()); - await writeFile(descriptor, protobufCaptureDescriptor()); - const { call } = await connectLocalToolsMcp(); - const args: CaptureRequest = { - capture_path: capture, - format: "har", - record_ordinals: [0], - protobuf: { - schema: { - descriptor_set_path: descriptor, - message_type: "capture.Message", - }, - }, - }; + async ({ cli, mcp }) => { + const { call, args, capture, descriptor } = await prepareMcpCapture( + mcp.call, + ); const complete = await call("inspect_web_network_capture", args); expect(complete.isError, JSON.stringify(complete)).not.toBe(true); const parent = parseEvidence(complete.structuredContent); @@ -481,10 +498,21 @@ cliTest( }); expect(cliResult.exitCode, cliResult.stdout + cliResult.stderr).toBe(0); expect(parseEvidence(cliResult.json).evidence_id).toBe(parent.evidence_id); - await verifyCaptureExclusions(call, args, root); - await verifyPartialCaptureSources(call, args, root, descriptor); - await verifyCredentialPreflight(call, root, descriptor); - await verifyDescriptorReadFailures(call, args, root); + const redacted = await call("inspect_web_network_capture", { + ...args, + sensitive_values: ["hello"], + }); + expect(redacted.isError, JSON.stringify(redacted)).not.toBe(true); + const excluded = webNetworkCaptureSchema.parse( + parseEvidence(redacted.structuredContent).normalized_result, + ); + expect(excluded.records[0]?.protobuf).toBeUndefined(); + expect(excluded.protobuf_coverage).toMatchObject({ + status: "partial", + selected_records: 1, + inspected_records: 1, + }); + expect(JSON.stringify(redacted)).not.toContain("hello"); const summary = await call("inspect_web_network_capture", { ...args, detail: "summary", @@ -497,6 +525,12 @@ cliTest( kind: "summary", parent_evidence_id: parent.evidence_id, }); + const invalid = await call("inspect_web_network_capture", { + ...args, + record_ordinals: undefined, + }); + expect(invalid.isError).toBe(true); + expect(parseMcpToolError(invalid).error.code).toBe("invalid_request"); await unlink(capture); await unlink(descriptor); const viewed = await call("inspect_analysis_view", { @@ -520,11 +554,5 @@ cliTest( items: [{ schema_name: "values", schema_value: '["1","2"]' }], coverage: { total: 1, exhausted: true }, }); - const invalid = await call("inspect_web_network_capture", { - ...args, - record_ordinals: undefined, - }); - expect(invalid.isError).toBe(true); - expect(parseMcpToolError(invalid).error.code).toBe("invalid_request"); }, ); diff --git a/tests/fixtures/localToolsMcp.ts b/tests/fixtures/localToolsMcp.ts index bf9c7617a1..590ef8e27e 100644 --- a/tests/fixtures/localToolsMcp.ts +++ b/tests/fixtures/localToolsMcp.ts @@ -36,6 +36,7 @@ const compileValidator = (schema: unknown): ValidateFunction => { /** Start the built production server without configuring deep binary providers. */ export async function connectLocalToolsMcp( environment: Readonly> = {}, + registerCleanup: (cleanup: () => Promise) => void = onTestFinished, ) { const transport = new StdioClientTransport({ command: process.execPath, @@ -52,7 +53,7 @@ export async function connectLocalToolsMcp( stderr: "pipe", }); const client = new Client({ name: "local-tools-e2e", version: "1" }); - onTestFinished(async () => { + registerCleanup(async () => { await client.close(); await transport.close(); }); From 2538214e2c731ed4d9f0cebdc9a75f84d74a56a4 Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:59:43 +0800 Subject: [PATCH 04/13] feat(analysis-view): add native inventory summaries and pages (#5378) Add opt-in summaries and stable selected-field pages for native inventories through the shared CLI/MCP Evidence layer. Preserve complete source access, row identity, provenance, string qualifications and unknown discovery coverage. Record real true/jq workflows and portable old-Evidence checks, including tokenizer proxy reductions, additional call/schema costs and observed latency/memory regressions. Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- .../references/evidence-workflows.md | 11 + docs/mcp-contracts.md | 42 ++ experiments/analysis-efficiency/README.md | 5 + .../native-inventory-costs.py | 115 ++++ .../native-inventory-portable.mjs | 231 +++++++ .../native-inventory-views.json | 647 ++++++++++++++++++ .../native-inventory-views.md | 201 ++++++ .../native-inventory-views.mjs | 374 ++++++++++ src/application/XrefsAddressResolution.ts | 4 +- .../analysisView/AnalysisViewService.ts | 8 +- .../NativeDispatchMetadataInspection.ts | 6 +- src/cli/analysisViewCommands.ts | 2 +- src/contracts/analysisViewToolContracts.ts | 18 +- src/contracts/officialToolContracts.ts | 29 +- src/contracts/toolOutputSchemaGroups.ts | 39 +- src/contracts/toolOutputSchemaPrimitives.ts | 28 - src/domain/analysisView/analysisView.ts | 72 +- .../analysisView/nativeInventoryView.test.ts | 170 +++++ .../analysisView/nativeInventoryView.ts | 192 ++++++ src/domain/native/nativeInventory.ts | 57 ++ src/server/createServer.ts | 1 + src/server/promptCompletion.ts | 4 +- src/server/registerEvidenceTools.ts | 6 + src/server/registerOfficialTools.ts | 154 +---- src/server/toolResult.ts | 17 +- .../mcp/nativeInventoryViewsMcp.test.ts | 329 +++++++++ 26 files changed, 2545 insertions(+), 217 deletions(-) create mode 100644 experiments/analysis-efficiency/native-inventory-costs.py create mode 100644 experiments/analysis-efficiency/native-inventory-portable.mjs create mode 100644 experiments/analysis-efficiency/native-inventory-views.json create mode 100644 experiments/analysis-efficiency/native-inventory-views.md create mode 100644 experiments/analysis-efficiency/native-inventory-views.mjs create mode 100644 src/domain/analysisView/nativeInventoryView.test.ts create mode 100644 src/domain/analysisView/nativeInventoryView.ts create mode 100644 src/domain/native/nativeInventory.ts create mode 100644 tests/boundary/mcp/nativeInventoryViewsMcp.test.ts diff --git a/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md b/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md index 5d6f374dd1..b266299dce 100644 --- a/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md +++ b/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md @@ -24,6 +24,17 @@ retain its complete source and receive that view immediately. Without that advertised choice, keep the existing complete call. CLI inspection returns portable complete Evidence; save it before projecting a selected inline view. +For native procedure/name/string inventories, select `detail: "summary"` when +the connected tool advertises it and the task needs counts or selected rows. +Use its parent Evidence ID with +`view: {kind: "native-inventory", offset: 0, limit: 32, fields: ["value"]}`. +Search tools retain the existing pattern/mode/case/document selection. Omit +`fields` for complete records; `record.address` and `source_index` remain on +selected rows. Selecting `value` keeps its string/decoding qualifications. +Parent Evidence ID plus source index preserves same-address +alternatives. Follow `coverage.next_offset` only as far as the question needs; +exhausting retained rows does not establish complete program discovery. + Use `record_unknown` to track unresolved questions in the user's investigation and name the authority or environment still required. Supply supporting and contradicting evidence IDs. It updates the session registry directly. diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 409d7d0aa7..4f0dd8e6b5 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -119,6 +119,46 @@ This marks completion of the MCP exchange; Evidence coverage, partial facts and unknowns retain their separate meaning. Legacy replies keep their existing framing. +`list_procedures`, `search_procedures`, `list_strings`, `search_strings` and +`list_names` accept `detail: "summary"`. Complete inline delivery remains the +default. Summary retains the original complete Evidence and returns its row +count, provider identity, analysis-profile digest, original query, parent Evidence +ID, limitations and explicit unknown discovery coverage. Delivery detail is excluded from provider arguments +and original Evidence identity. + +Read selected rows with `inspect_analysis_view`: + +```json +{ + "source": { + "kind": "retained-evidence", + "evidence_id": "ev_" + }, + "view": { + "kind": "native-inventory", + "offset": 0, + "limit": 32, + "fields": ["value"] + } +} +``` + +Each row has `source_index` and `record`; parent Evidence ID plus source index +identifies same-address alternatives. Order is the original retained order. +Omit `fields` for complete records; a selected field list always retains +`record.address`. Selecting `value` also retains available `string` and +`decoding` qualifications, so native display text and unterminated fragments +remain distinguishable. Search tools apply their existing filters before retention. +Follow `coverage.next_offset`; exhaustion describes the retained list, not +complete discovery of all code or strings. An unbound source keeps +`artifact: null` and the derived Evidence keeps `subject: null`. + +The CLI uses the same projector. For an original complete Evidence JSON file, +write `{"source":{"kind":"evidence-file","path":"/absolute/source.json"},"view":{"kind":"native-inventory","limit":32}}` +to `/absolute/view-request.json`, then run +`rea inspect-analysis-view /absolute/view-request.json --json`. This also reads +pre-summary source records without changing their identity; no live provider is started. + For oversized native function Evidence, `details.reported_limits.continuation` contains a ready-to-call `inspect_analysis_view` request for the retained pseudocode's first page. Use its `name` and `arguments` directly, then follow @@ -127,6 +167,8 @@ limit is the view contract's default, not a guarantee that arbitrary source or metadata fits; reduce the limit or export when needed. Continuations are offered for completed `analyze_function` results only after session recording is acknowledged; the complete result and its retention reference remain intact. +Oversized native inventories likewise include a ready-to-call first +`native-inventory` page. Reduce its limit or select fields when a row is large. A broad follow-up or `get_evidence_bundle` can also exceed the response budget; exporting preserves the complete session without sending it through a single MCP frame. diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.md index 27667bac93..2f2f049544 100644 --- a/experiments/analysis-efficiency/README.md +++ b/experiments/analysis-efficiency/README.md @@ -1,5 +1,10 @@ # Analysis efficiency measurements +The [native inventory view checkpoint](native-inventory-views.md) compares +complete delivery with summaries and selected rows on real `true` and jq +inventories, including oversized-response recovery, portable old Evidence, +CLI/MCP parity, tokenizer estimates and independent-engine variation. + The [retained-capture validation checkpoint](retained-capture-validation.md) reconciles the earlier audit queue and measures immutable historical-capture validation reuse on public compact and large qlogs, including first-view costs, diff --git a/experiments/analysis-efficiency/native-inventory-costs.py b/experiments/analysis-efficiency/native-inventory-costs.py new file mode 100644 index 0000000000..b6b45aee4a --- /dev/null +++ b/experiments/analysis-efficiency/native-inventory-costs.py @@ -0,0 +1,115 @@ +"""Count recorded native-inventory delivery, including errors and file recovery. + +Requires tiktoken==0.12.0 in the experiment environment. These are serialization +proxies, not model usage or a claim about a host's schema injection/rendering. +""" + +import argparse +import json +from pathlib import Path +import sys + +import tiktoken + + +def compact(value): + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + + +def inspect_case(path, encoding): + report = json.loads((path / "report.json").read_text()) + transcript = json.loads((path / "transcript.json").read_text()) + catalog_text = (path / "catalog.json").read_text() + catalog = json.loads(catalog_text) + scope = next( + json.loads(line) + for line in reversed(path.with_suffix(".log").read_text().splitlines()) + if line.startswith('{"resource_scope":') + ) + observation = scope.get("payload_resource_observation") or {} + count = lambda value: len(encoding.encode(value, disallowed_special=())) + input_projection = [ + {key: tool[key] for key in ["name", "description", "inputSchema"]} + for tool in catalog["tools"] + ] + rpc = [event for event in transcript if "reply" in event or "failure" in event] + local = [event for event in transcript if event["name"] == "local-bundle-projection"] + arguments = sum(count(compact({"name": event["name"], "arguments": event["input"]})) for event in rpc) + text = sum( + count(item["text"]) + for event in rpc + for item in event.get("reply", {}).get("content", []) + if item["type"] == "text" + ) + full_replies = sum(count(compact(event["reply"])) for event in rpc if "reply" in event) + local_output = sum(count(compact(event["projection"])) for event in local) + schemas = count(compact(input_projection)) + completed = ( + report["status"] == "passed" + and len(report["tasks"]) == 2 + and not report["cleanup_failures"] + and not report["receipt_failures"] + and scope["abort"] is None + and scope.get("payload_resource_observation_verified") is True + and observation.get("payload_exit_code") == 0 + and observation.get("payload_cleanup_verified") is True + ) + return { + "recording": path.name, + "status": report["status"], + "completed_with_verified_cleanup": completed, + "mode": report["mode"], + "catalog_tools": len(catalog["tools"]), + "catalog_bytes": len(catalog_text.encode()), + "rpc_calls": len(rpc), + "rpc_errors_or_rejections": sum(event.get("reply", {}).get("isError") is True or "failure" in event for event in rpc), + "rpc_argument_bytes": sum(len(compact({"name": event["name"], "arguments": event["input"]}).encode()) for event in rpc), + "rpc_reply_bytes": sum(call["reply_bytes"] or 0 for call in report["calls"]), + "local_recoveries": report.get("local_recoveries", []), + "token_estimates": { + "catalog_full_json": count(catalog_text), + "catalog_input_facing_projection": schemas, + "rpc_name_and_argument_json": arguments, + "rpc_text_content": text, + "rpc_complete_result_json": full_replies, + "local_projection_output_json": local_output, + "input_facing_catalog_plus_arguments_text_and_local_output": schemas + arguments + text + local_output, + }, + "whole_scope_wall_seconds": scope["elapsed_seconds"], + "payload_family_cpu_seconds": observation["cpu_stat"]["usage_usec"] / 1_000_000 if observation.get("cpu_stat", {}).get("usage_usec") is not None else None, + "kernel_family_peak_bytes_including_file_cache": observation.get("memory_peak_bytes"), + "payload_exit_observation_verified": scope.get("payload_resource_observation_verified"), + "observed_resource_events": observation.get("resource_events"), + "final_whole_scope_events": scope.get("resource_events_final"), + "tasks": [{"phase": task["phase"], "elapsed_ms": task["elapsed_ms"]} for task in report["tasks"]], + "facts_sha256": report.get("facts_sha256"), + "sources_sha256": report.get("sources_sha256"), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("cases", type=Path, nargs="+") + args = parser.parse_args() + encoding = tiktoken.get_encoding("cl100k_base") + result = { + "python": sys.version, + "tiktoken": tiktoken.__version__, + "encoding": encoding.name, + "actual_model_usage": None, + "unmeasured": [ + "model-host schema selection, wrapping and rendering", + "system prompts, local filesystem tool schema/program generation and follow-up reasoning", + "model input/cached/uncached/output usage", + "initialization and JSON-RPC wire framing", + "isolated correctness-oracle CPU and memory overhead", + ], + "samples": [inspect_case(path, encoding) for path in args.cases], + } + args.output.write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps({"samples": len(result["samples"]), "encoding": encoding.name, "actual_model_usage": None})) + + +if __name__ == "__main__": + main() diff --git a/experiments/analysis-efficiency/native-inventory-portable.mjs b/experiments/analysis-efficiency/native-inventory-portable.mjs new file mode 100644 index 0000000000..b0fc5782d8 --- /dev/null +++ b/experiments/analysis-efficiency/native-inventory-portable.mjs @@ -0,0 +1,231 @@ +/** Verify real pre-summary source Evidence through retained MCP and provider-free CLI views. */ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { createWriteStream } from "node:fs"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { finished } from "node:stream/promises"; +import { parseArgs, promisify } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; +import { createRetainedQueryCaller } from "./retained-query-calls.mjs"; +import { finalizeRetainedQueryReceipt } from "./retained-query-finalization.mjs"; + +const { values } = parseArgs({ + options: Object.fromEntries( + ["runtime", "source", "output"].map((name) => [name, { type: "string" }]), + ), +}); +for (const name of ["runtime", "source", "output"]) + assert.ok(values[name] && values[name] === resolve(values[name]), name); +const hash = (value) => createHash("sha256").update(value).digest("hex"); +const originalBytes = await readFile(join(values.source, "evidence.json")); +const original = JSON.parse(originalBytes); +const sources = JSON.parse( + await readFile(join(values.source, "sources.json"), "utf8"), +).filter((source) => source.phase === "first"); +assert.equal(sources.length, 5); +await mkdir(values.output, { mode: 0o700 }); +const home = await mkdtemp(join(tmpdir(), "rea-portable-native-inventory-")); +const environment = { + HOME: home, + PATH: process.env.PATH, + NODE_OPTIONS: "--max-old-space-size=2048", + REA_LOG_LEVEL: "silent", +}; +const client = new Client({ name: "portable-native-inventory", version: "1" }); +const transport = new StdioClientTransport({ + command: process.execPath, + args: [join(values.runtime, "scripts/rea.mjs"), "mcp"], + env: environment, + stderr: "pipe", +}); +const serverLog = createWriteStream(join(values.output, "server.log")); +const serverLogFinished = finished(serverLog); +serverLogFinished.catch(() => {}); +transport.stderr?.pipe(serverLog); +const report = { + source_bundle_sha256: hash(originalBytes), + calls: [], + cli: [], + rows: {}, +}; +const transcript = []; +const call = createRetainedQueryCaller(client, (receipt, event) => { + report.calls.push(receipt); + transcript.push(event); +}); +const result = async (name, input, phase = name) => { + const reply = await call(name, input, phase); + assert.notEqual(reply.isError, true, JSON.stringify(reply)); + assert.ok(reply.structuredContent); + return reply.structuredContent; +}; +const started = performance.now(); +try { + await client.connect(transport); + const catalog = await client.listTools(); + assert.equal(catalog.nextCursor, undefined); + report.catalog = { + tool_count: catalog.tools.length, + bytes: Buffer.byteLength(JSON.stringify(catalog)), + sha256: hash(JSON.stringify(catalog)), + }; + const definitions = new Map(catalog.tools.map((tool) => [tool.name, tool])); + const originalCall = client.callTool.bind(client); + client.callTool = (input, options) => + originalCall(input, { + ...options, + toolDefinition: definitions.get(input.name), + }); + await result("import_evidence_bundle", { + path: join(values.source, "evidence.json"), + }); + for (const { operation, record } of sources) { + assert.deepEqual( + original.records.find( + (source) => source.evidence_id === record.evidence_id, + ), + record, + ); + const sourcePath = join(values.output, `${operation}.json`); + const sourceBytes = Buffer.from(JSON.stringify(record)); + await writeFile(sourcePath, sourceBytes); + const source = { + kind: "retained-evidence", + evidence_id: record.evidence_id, + }; + const pageView = { + kind: "native-inventory", + offset: 0, + limit: 4, + fields: ["value"], + }; + const expected = await result( + "inspect_analysis_view", + { source, view: pageView }, + `${operation}:selected`, + ); + for (const view of [{ kind: "summary" }, pageView]) { + const inputPath = join( + values.output, + `${operation}-${view.kind}.input.json`, + ); + await writeFile( + inputPath, + JSON.stringify({ + source: { kind: "evidence-file", path: sourcePath }, + view, + }), + ); + const cliStarted = performance.now(); + const { stdout, stderr } = await promisify(execFile)( + process.execPath, + [ + join(values.runtime, "scripts/rea.mjs"), + "inspect-analysis-view", + inputPath, + "--json", + ], + { + env: environment, + encoding: "utf8", + timeout: 60000, + maxBuffer: STDIO_DEFAULT_MAX_BUFFER_SIZE, + }, + ); + const cliElapsed = performance.now() - cliStarted; + const actual = JSON.parse(stdout); + const mcp = + view.kind === "summary" + ? await result( + "inspect_analysis_view", + { source, view }, + `${operation}:summary`, + ) + : expected; + assert.deepEqual(actual, mcp); + report.cli.push({ + operation, + view: view.kind, + elapsed_ms: cliElapsed, + stdout_bytes: Buffer.byteLength(stdout), + stderr_bytes: Buffer.byteLength(stderr), + evidence_id: actual.evidence_id, + }); + } + const all = []; + let offset = 0; + do { + const evidence = await result( + "inspect_analysis_view", + { source, view: { kind: "native-inventory", offset, limit: 128 } }, + `${operation}:page:${offset}`, + ); + const page = evidence.normalized_result; + assert.equal(page.kind, "native-inventory"); + const expectedRows = Math.min( + 128, + record.normalized_result.length - offset, + ); + assert.equal(page.rows.length, expectedRows); + assert.equal(page.coverage.total, record.normalized_result.length); + for (const [index, row] of page.rows.entries()) + assert.equal(row.source_index, offset + index); + all.push(...page.rows.map((row) => row.record)); + assert.equal( + page.coverage.next_offset, + offset + expectedRows >= record.normalized_result.length + ? null + : offset + expectedRows, + ); + offset = page.coverage.next_offset; + } while (offset !== null); + assert.deepEqual(all, record.normalized_result); + report.rows[operation] = all.length; + assert.equal(hash(await readFile(sourcePath)), hash(sourceBytes)); + } + const exportPath = join(values.output, "exported.json"); + await result("export_evidence_bundle", { path: exportPath }); + const exported = JSON.parse(await readFile(exportPath, "utf8")); + for (const { record } of sources) + assert.deepEqual( + exported.records.find( + (source) => source.evidence_id === record.evidence_id, + ), + record, + ); + assert.equal( + hash(await readFile(join(values.source, "evidence.json"))), + report.source_bundle_sha256, + ); + report.status = "passed"; +} catch (cause) { + report.status = "failed"; + report.failure = String(cause?.stack ?? cause); + process.exitCode = 1; +} finally { + report.elapsed_before_cleanup_ms = performance.now() - started; + try { + await writeFile( + join(values.output, "transcript.json"), + JSON.stringify(transcript), + ); + } catch (cause) { + report.status = "failed"; + report.transcript_failure = String(cause); + } + const finalized = await finalizeRetainedQueryReceipt({ + report, + client, + transport, + account: home, + serverLog, + serverLogFinished, + output: values.output, + }); + if (report.status !== "passed" || !finalized) process.exitCode = 1; +} diff --git a/experiments/analysis-efficiency/native-inventory-views.json b/experiments/analysis-efficiency/native-inventory-views.json new file mode 100644 index 0000000000..a9ce76f705 --- /dev/null +++ b/experiments/analysis-efficiency/native-inventory-views.json @@ -0,0 +1,647 @@ +{ + "question": "Counts and first four address/value pairs for list_procedures, list_names, list_strings and literal main search_procedures/search_strings; first and warm repetition.", + "baseline_revision": "58d6043303e722115f1da18b9505c24abeeab423", + "runtime_pins": { + "baseline_manifest_sha256": "dc5e30fa1c75f999a8597608c5f0a3d8e1e16ccc14eb39d3884b5362076985dd", + "candidate_manifest_sha256": "14b4af9117051b722eb528dfdf79668fa188f874ce69853bd1ed609eb704a88d", + "candidate_tracked_patch_sha256": "e988c4f5142363dc2cbf6ab98e11d1ea1e4acccc66ce941d15eb51a945b4fb3a", + "candidate_untracked_sources": { + "experiments/analysis-efficiency/native-inventory-portable.mjs": "262cd4a076ba13936ff744c844892ec108b015f3adc908703120fddd251fe149", + "experiments/analysis-efficiency/native-inventory-views.mjs": "c5de78d1998e254a269e3d1694c96d7034162aac41965a7b842235ae09ac9db8", + "src/domain/analysisView/nativeInventoryView.test.ts": "b0b3a3d14a0d8be2a42bb9ae9e0e5a32eef94bf09ffcce01f89554300eeb97f2", + "src/domain/analysisView/nativeInventoryView.ts": "e1e06ead227c30c7eaa8231b48c934b5f0be57876847b022e75f6a89eadaddbd", + "src/domain/native/nativeInventory.ts": "431e6041cd2dec688ca541dbe343cea2228c9d616af1332338b9a36c73a32320", + "tests/boundary/mcp/nativeInventoryViewsMcp.test.ts": "f6991ba8e6d97922c430512d5bfec16b392e22a9eacab21a5853ea93f6d3a73d" + }, + "note": "Frozen candidate before measurement; later documentation, bounded harness diagnostics and rebases are not relabeled as measured source revisions." + }, + "environment": { + "platform": "Linux-6.1.0-53-cloud-amd64-x86_64-with-glibc2.36", + "node": "24.18.0", + "npm": "11.16.0", + "mcp_sdk": "2.3.1", + "ghidra": "12.1.4", + "java": "Eclipse Adoptium 21.0.12.1+1-LTS", + "java_heap": "768M", + "node_heaps_mib": 2048, + "active_java_processors": 1, + "cpu_affinity": [15], + "cpu_quota_percent": 100, + "family_memory_hard_bytes": 4294967296, + "family_memory_early_stop_bytes": 3221225472, + "swap_bytes": 0, + "task_limit": 192, + "cache_state": "Fresh sessions and Java analysis, then warm repeated calls; OS caches unflushed.", + "accounting": "cgroup v2 family counters at verified payload exit; kernel peak includes file cache and controller work; final whole-scope counters unavailable." + }, + "artifacts": [ + { + "path": "true", + "bytes": 35664, + "sha256": "c79bf44242829108e323378531f4ac839513ca1fba45efd6583643526e1e9fd2" + }, + { + "path": "jq-linux-amd64", + "bytes": 2267912, + "sha256": "b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f" + } + ], + "driver_versions": [ + { + "sha256": "c71957b673824a692efeaebb165a305799b6d7424a2dbb96d706a6f48e283aae", + "used_by": ["baseline-true-1", "baseline-jq-1"], + "path": "experiments/analysis-efficiency/native-inventory-views.mjs" + }, + { + "sha256": "c5de78d1998e254a269e3d1694c96d7034162aac41965a7b842235ae09ac9db8", + "used_by": ["candidate-true-1", "baseline-jq-2", "candidate-jq-1"] + }, + { + "sha256": "0ce2e2a2c02e366f5005f9b1e4563f7cd1a5a378d8461de6750154dccf41995b", + "runs": ["candidate-jq-2"], + "change": "Bounded full-source comparison diagnostic, same strict oracle; candidate-jq-2 has no cross-session full-source control. Compare task facts separately and preserve Ghidra primary alias variance.", + "candidate-jq-1": "Incomplete: scope_memory_early_stop while rendering full source mismatch; facts and source records retained; no completed report or cleanup proof." + } + ], + "token_estimates": { + "python": "3.11.2 (main, Oct 7 2026, 12:35:07) [GCC 12.2.0]", + "tiktoken": "0.12.0", + "encoding": "cl100k_base", + "actual_model_usage": null, + "unmeasured": [ + "model-host schema selection, wrapping and rendering", + "system prompts, local filesystem tool schema/program generation and follow-up reasoning", + "model input/cached/uncached/output usage", + "initialization and JSON-RPC wire framing", + "isolated correctness-oracle CPU and memory overhead" + ], + "samples": [ + { + "recording": "baseline-true-1", + "status": "passed", + "completed_with_verified_cleanup": true, + "mode": "complete", + "catalog_tools": 209, + "catalog_bytes": 3263492, + "rpc_calls": 13, + "rpc_errors_or_rejections": 0, + "rpc_argument_bytes": 854, + "rpc_reply_bytes": 921748, + "local_recoveries": [], + "token_estimates": { + "catalog_full_json": 831584, + "catalog_input_facing_projection": 185367, + "rpc_name_and_argument_json": 214, + "rpc_text_content": 120915, + "rpc_complete_result_json": 247753, + "local_projection_output_json": 0, + "input_facing_catalog_plus_arguments_text_and_local_output": 306496 + }, + "whole_scope_wall_seconds": 53.662, + "payload_family_cpu_seconds": 44.201557, + "kernel_family_peak_bytes_including_file_cache": 1157513216, + "payload_exit_observation_verified": true, + "observed_resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_events": null, + "tasks": [ + { + "phase": "first", + "elapsed_ms": 35533.921138000005 + }, + { + "phase": "warm", + "elapsed_ms": 236.46414299999742 + } + ], + "facts_sha256": "fb6cfbd6e3b9cdff8f649ffc41de54d7de38a003d3ec4f20f7cdad227e38afca", + "sources_sha256": "c87d8c5b12b5244e3777f61a396e2a19b333ce9a9c1cb246db4ab54fb075b13b" + }, + { + "recording": "candidate-true-1", + "status": "passed", + "completed_with_verified_cleanup": true, + "mode": "summary", + "catalog_tools": 209, + "catalog_bytes": 3293844, + "rpc_calls": 23, + "rpc_errors_or_rejections": 0, + "rpc_argument_bytes": 3469, + "rpc_reply_bytes": 171130, + "local_recoveries": [], + "token_estimates": { + "catalog_full_json": 838818, + "catalog_input_facing_projection": 186009, + "rpc_name_and_argument_json": 1086, + "rpc_text_content": 20907, + "rpc_complete_result_json": 43673, + "local_projection_output_json": 0, + "input_facing_catalog_plus_arguments_text_and_local_output": 208002 + }, + "whole_scope_wall_seconds": 36.777, + "payload_family_cpu_seconds": 35.482574, + "kernel_family_peak_bytes_including_file_cache": 1244180480, + "payload_exit_observation_verified": true, + "observed_resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_events": null, + "tasks": [ + { + "phase": "first", + "elapsed_ms": 30791.001210000002 + }, + { + "phase": "warm", + "elapsed_ms": 223.86245000000054 + } + ], + "facts_sha256": "fb6cfbd6e3b9cdff8f649ffc41de54d7de38a003d3ec4f20f7cdad227e38afca", + "sources_sha256": "c87d8c5b12b5244e3777f61a396e2a19b333ce9a9c1cb246db4ab54fb075b13b" + }, + { + "recording": "baseline-jq-2", + "status": "passed", + "completed_with_verified_cleanup": true, + "mode": "complete", + "catalog_tools": 209, + "catalog_bytes": 3263492, + "rpc_calls": 15, + "rpc_errors_or_rejections": 2, + "rpc_argument_bytes": 1139, + "rpc_reply_bytes": 3032150, + "local_recoveries": [ + { + "phase": "first:list_names", + "status": "passed", + "bundle_bytes_read": 7318483, + "projection_bytes": 1186, + "elapsed_ms": 164.94348000001628 + }, + { + "phase": "warm:list_names", + "status": "passed", + "bundle_bytes_read": 7769768, + "projection_bytes": 1186, + "elapsed_ms": 116.13296299998183 + } + ], + "token_estimates": { + "catalog_full_json": 831584, + "catalog_input_facing_projection": 185367, + "rpc_name_and_argument_json": 307, + "rpc_text_content": 403794, + "rpc_complete_result_json": 820583, + "local_projection_output_json": 564, + "input_facing_catalog_plus_arguments_text_and_local_output": 590032 + }, + "whole_scope_wall_seconds": 355.586, + "payload_family_cpu_seconds": 318.976126, + "kernel_family_peak_bytes_including_file_cache": 1571012608, + "payload_exit_observation_verified": true, + "observed_resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_events": null, + "tasks": [ + { + "phase": "first", + "elapsed_ms": 334659.460051 + }, + { + "phase": "warm", + "elapsed_ms": 2520.551972999994 + } + ], + "facts_sha256": "34cf397309b0db4132f1fcfbcaf92daab75b8b823e4c328ac1592f46dff38094", + "sources_sha256": "e8f4f98c0cc9a141f07db695fcc5b2b3f22ae3697bb09d0d8fd03d6a47762260" + }, + { + "recording": "candidate-jq-2", + "status": "passed", + "completed_with_verified_cleanup": true, + "mode": "summary", + "catalog_tools": 209, + "catalog_bytes": 3293844, + "rpc_calls": 21, + "rpc_errors_or_rejections": 0, + "rpc_argument_bytes": 3040, + "rpc_reply_bytes": 172180, + "local_recoveries": [], + "token_estimates": { + "catalog_full_json": 838818, + "catalog_input_facing_projection": 186009, + "rpc_name_and_argument_json": 921, + "rpc_text_content": 21372, + "rpc_complete_result_json": 44505, + "local_projection_output_json": 0, + "input_facing_catalog_plus_arguments_text_and_local_output": 208302 + }, + "whole_scope_wall_seconds": 333.104, + "payload_family_cpu_seconds": 311.318996, + "kernel_family_peak_bytes_including_file_cache": 1774460928, + "payload_exit_observation_verified": true, + "observed_resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_events": null, + "tasks": [ + { + "phase": "first", + "elapsed_ms": 318797.998245 + }, + { + "phase": "warm", + "elapsed_ms": 6916.484328999999 + } + ], + "facts_sha256": "34cf397309b0db4132f1fcfbcaf92daab75b8b823e4c328ac1592f46dff38094", + "sources_sha256": "237f727384546a5323bf39d08d3fe3061b89cb32da22d062493b008e474ba6a2" + } + ] + }, + "equivalence": { + "true_all_original_source_evidence_fields_and_ids_equal": true, + "jq_complete_selected_task_facts_equal": true, + "jq_all_independent_live_source_evidence_equal": false, + "earlier_independent_jq_source_differences": { + "attempt": "candidate-jq-1", + "completed": false, + "reason": "scope_memory_early_stop; no report or payload-exit observation", + "task_facts_exact_equal": true, + "full_source_equal": false, + "different_leaves": { + "symbol.primary": 156, + "[1, 'record', 'normalized_result', 45988, 'value']": 1, + "[1, 'record', 'normalized_result', 46001, 'value']": 1, + "evidence_id": 2, + "[6, 'record', 'normalized_result', 45988, 'value']": 1, + "[6, 'record', 'normalized_result', 46001, 'value']": 1 + }, + "examples": [ + { + "path": [ + 1, + "record", + "normalized_result", + 30995, + "symbol", + "primary" + ], + "baseline": false, + "candidate": true + }, + { + "path": [ + 1, + "record", + "normalized_result", + 31013, + "symbol", + "primary" + ], + "baseline": true, + "candidate": false + }, + { + "path": [ + 1, + "record", + "normalized_result", + 31120, + "symbol", + "primary" + ], + "baseline": false, + "candidate": true + }, + { + "path": [ + 1, + "record", + "normalized_result", + 31121, + "symbol", + "primary" + ], + "baseline": true, + "candidate": false + } + ], + "interpretation": "Independent Ghidra analyses differ in symbol.primary for alias rows and two generated switch-label values (repeated in first/warm records). Root cause is unestablished; no byte-identical full-source or completed-performance claim." + }, + "portable_checks": [ + { + "recording": "portable-true-1", + "status": "passed", + "rows_reconstructed": { + "list_procedures": 231, + "list_names": 1027, + "list_strings": 143, + "search_procedures": 8, + "search_strings": 2 + }, + "actual_cli_mcp_equal_evidence_comparisons": 10, + "rpc_calls_including_import_and_export": 27, + "source_bundle_sha256": "d9f574ff9deb84cef58a714e563590c70184c15ba01c174f47c027b50b1d6bfa", + "whole_scope_wall_seconds": 25.194, + "family_cpu_seconds": 25.015127, + "kernel_family_peak_bytes_including_file_cache": 1025908736, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "payload_cleanup_verified": true, + "final_whole_scope_events": null + }, + { + "recording": "portable-jq-1", + "status": "passed", + "rows_reconstructed": { + "list_procedures": 2279, + "list_names": 49684, + "list_strings": 2915, + "search_procedures": 0, + "search_strings": 24 + }, + "actual_cli_mcp_equal_evidence_comparisons": 10, + "rpc_calls_including_import_and_export": 444, + "source_bundle_sha256": "49ba421c1d7c2bbee5d6d4ae7acec437b076f760097eeae1aae8633d45f8e51c", + "whole_scope_wall_seconds": 77.537, + "family_cpu_seconds": 63.38141, + "kernel_family_peak_bytes_including_file_cache": 2034434048, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "payload_cleanup_verified": true, + "final_whole_scope_events": null + } + ], + "limits": "This qualifies preservation/projection of recorded facts, not every Ghidra interpretation or complete program discovery." + }, + "incomplete_attempts": [ + { + "recording": "baseline-jq-1", + "completed": false, + "reason": "Default list_names exceeds the existing MCP response budget; first driver lacks export recovery.", + "abort": null, + "whole_scope_wall_seconds": 283.113, + "peak_sampled_bytes": 1308225536, + "last_observed_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "payload_resource_observation": { + "phase": "payload-exited-observer-alive", + "ownership_token": "rea-efficiency-2ad56ea678404ef49da951c8f1761822.scope", + "payload_exit_code": 1, + "observer_pid": 3613358, + "remaining_processes": [3613358], + "payload_cleanup_verified": true, + "cpu_stat": { + "usage_usec": 281189241, + "user_usec": 274593631, + "system_usec": 6595610, + "nr_periods": 2829, + "nr_throttled": 585, + "throttled_usec": 218972, + "nr_bursts": 0, + "burst_usec": 0 + }, + "memory_peak_bytes": 1311993856, + "control_group": "user.slice/user-1004.slice/user@1004.service/app.slice/rea-efficiency-2ad56ea678404ef49da951c8f1761822.scope", + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + }, + "final_whole_scope_events": null + }, + { + "recording": "candidate-jq-1", + "completed": false, + "reason": "Independent full-source observations differ; guard stopped during comparison. Assertion diagnostic expansion is source-derived, not a recovered final stack trace.", + "abort": "scope_memory_early_stop", + "whole_scope_wall_seconds": 269.574, + "peak_sampled_bytes": 3222269952, + "last_observed_events": { + "memory.events": { + "low": 0, + "high": 26, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "payload_resource_observation": null, + "final_whole_scope_events": null + } + ], + "validation": { + "focused_files": 24, + "focused_tests": 305, + "focused_command": "npm run test:focused -- src/domain/analysisView/EnvironmentManifestAccumulator.test.ts src/domain/analysisView/WebNetworkApiSurfaceAccumulator.test.ts src/domain/analysisView/analysisView.test.ts src/domain/analysisView/artifactInventoryView.test.ts src/domain/analysisView/environmentManifestRules.test.ts src/domain/analysisView/environmentManifestView.test.ts src/domain/analysisView/environmentStateView.test.ts src/domain/analysisView/environmentStateWriteCutoffs.test.ts src/domain/analysisView/il2cppMetadataView.test.ts src/domain/analysisView/interfaceBuilderView.test.ts src/domain/analysisView/machoView.test.ts src/domain/analysisView/nativeInventoryView.test.ts src/domain/analysisView/pcapCaptureView.test.ts src/domain/analysisView/scenarioCaptureRecords.test.ts src/domain/analysisView/webNetworkApiSurfaceView.test.ts src/domain/analysisView/webNetworkCaptureView.test.ts tests/boundary/mcp/nativeInventoryViewsMcp.test.ts tests/boundary/mcp/analysisViewMcp.test.ts tests/boundary/mcp/officialArguments.test.ts tests/boundary/mcp/officialTools.test.ts tests/boundary/mcp/largeToolResult.test.ts tests/boundary/mcp/ghidraEvidenceMcp.test.ts tests/boundary/mcp/nativeDispatchAvailability.test.ts tests/boundary/mcp/promptCompletion.test.ts", + "static_command": "npm run check", + "static_result": "passed, six tasks plus module/architecture guards", + "docs_result": "npm run docs:check passed", + "intermediate_failures": "Type errors and SDK fixture/error-envelope harness errors were corrected before passing checks. Two host-memory guard stops were retained; neither is a successful static check.", + "provider_scope": "Real Linux x86-64 Ghidra true/jq metadata; other engines/platforms and complete application reconstruction unqualified.", + "post_rebase": { + "base_revision": "0134732fc", + "implementation_revision": "8d1ddce86", + "locked_dependencies": "npm ci passed for updated package-lock.json; frozen measurement runtimes retain their original physical dependency tree", + "focused_files": 28, + "focused_tests": 325, + "focused_command": "npm run test:focused -- src/domain/analysisView/EnvironmentManifestAccumulator.test.ts src/domain/analysisView/WebNetworkApiSurfaceAccumulator.test.ts src/domain/analysisView/analysisView.test.ts src/domain/analysisView/artifactInventoryView.test.ts src/domain/analysisView/environmentManifestRules.test.ts src/domain/analysisView/environmentManifestView.test.ts src/domain/analysisView/environmentStateView.test.ts src/domain/analysisView/environmentStateWriteCutoffs.test.ts src/domain/analysisView/il2cppMetadataView.test.ts src/domain/analysisView/interfaceBuilderView.test.ts src/domain/analysisView/machoView.test.ts src/domain/analysisView/nativeInventoryView.test.ts src/domain/analysisView/pcapCaptureView.test.ts src/domain/analysisView/scenarioCaptureRecords.test.ts src/domain/analysisView/webNetworkApiSurfaceView.test.ts src/domain/analysisView/webNetworkCaptureView.test.ts tests/boundary/mcp/nativeInventoryViewsMcp.test.ts tests/boundary/mcp/analysisViewMcp.test.ts tests/boundary/mcp/officialArguments.test.ts tests/boundary/mcp/officialTools.test.ts tests/boundary/mcp/largeToolResult.test.ts tests/boundary/mcp/ghidraEvidenceMcp.test.ts tests/boundary/mcp/nativeDispatchAvailability.test.ts tests/boundary/mcp/promptCompletion.test.ts src/domain/analysisView/protobufCaptureView.test.ts src/browser/history/protobuf/ProtobufCapture.test.ts src/browser/history/protobuf/ProtobufWire.test.ts tests/acceptance/tools/protobufCapture.test.ts", + "static_check": "npm run check passed after final field-selection normalization", + "docs_check": "npm run docs:check passed, including five README consistency tests", + "final_change": "Normalize duplicate field selection once, preserving original view request; focused two-file/nine-test rerun passed", + "final_projector_sha256": "8795c18fe0a7c97b8a841aa3cc1fc6efe08fa1bf9711c21b0183b2cdd1f1b180", + "verification_runtime_manifest_sha256": "f6a30e4a56fa1f95926e2259fbefe837ed375fb1c608e2298fb88440e1898f2f", + "portable_real_source_checks": [ + { + "recording": "portable-true-final-1", + "status": "passed", + "rows_reconstructed": { + "list_procedures": 231, + "list_names": 1027, + "list_strings": 143, + "search_procedures": 8, + "search_strings": 2 + }, + "cli_mcp_equal_comparisons": 10, + "rpc_calls": 27, + "catalog": { + "tool_count": 209, + "bytes": 3317414, + "sha256": "4c920219321a08102a29673a1cdd48f74d9990e766a313313903525a9f9efa87" + }, + "whole_scope_wall_seconds": 23.489, + "family_cpu_seconds": 23.283072, + "kernel_family_peak_bytes": 1057517568, + "payload_cleanup_verified": true, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + }, + { + "recording": "portable-jq-final-1", + "status": "passed", + "rows_reconstructed": { + "list_procedures": 2279, + "list_names": 49684, + "list_strings": 2915, + "search_procedures": 0, + "search_strings": 24 + }, + "cli_mcp_equal_comparisons": 10, + "rpc_calls": 444, + "catalog": { + "tool_count": 209, + "bytes": 3317414, + "sha256": "4c920219321a08102a29673a1cdd48f74d9990e766a313313903525a9f9efa87" + }, + "whole_scope_wall_seconds": 34.99, + "family_cpu_seconds": 34.589707, + "kernel_family_peak_bytes": 2153742336, + "payload_cleanup_verified": true, + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + } + ], + "performance_limit": "Fixed runtime timing/token pairs remain pinned to the pre-rebase 58d-based runtime. Final source compatibility and full-row preservation were rerun; final main performance pairs and actual model usage were not rerun.", + "guard_stopped_static_retry": { + "recording": "full-check-rebase-1", + "abort": "host_available_memory_early_stop", + "completed": false, + "own_oom_events": 0 + }, + "final_static_and_docs_family": { + "wall_seconds": 93.069, + "cpu_seconds": 92.615364, + "kernel_peak_bytes": 5417869312, + "payload_cleanup_verified": true + } + }, + "protocol_rebase": { + "base_revision": "4eaf3a117", + "implementation_revision": "7fbe79ef8", + "change": "Preserve latest main oversized-error protocol framing and both documentation insertions; native projector unchanged.", + "focused_command": "npm run test:focused -- src/server/EvidenceMcpServer.test.ts tests/boundary/mcp/nativeInventoryViewsMcp.test.ts tests/boundary/mcp/largeToolResult.test.ts", + "focused_files": 3, + "focused_tests": 11, + "result": "passed", + "real_old_source_cli_mcp_verifier": "true all rows and ten actual CLI/MCP comparisons passed again; no provider launched", + "catalog": { + "tool_count": 209, + "bytes": 3317414, + "sha256": "4c920219321a08102a29673a1cdd48f74d9990e766a313313903525a9f9efa87" + }, + "docs_check": "npm run docs:check passed again", + "static_check_scope": "Earlier full static checks remain pinned above; final pre-push/CI checks track the latest branch head." + } + } +} diff --git a/experiments/analysis-efficiency/native-inventory-views.md b/experiments/analysis-efficiency/native-inventory-views.md new file mode 100644 index 0000000000..02af99c180 --- /dev/null +++ b/experiments/analysis-efficiency/native-inventory-views.md @@ -0,0 +1,201 @@ +# Native inventory summaries and retained rows + +Continues [#4249](https://github.com/morluto/rea/issues/4249). The analyst +question is to report observed row counts and the first four address/value +pairs from procedures, names, strings, and literal `main` procedure/string +searches. Complete delivery reads every row even when only those facts are +needed. A real jq name inventory exceeds the existing 10 MiB MCP result budget. + +## Change and preserved semantics + +The five existing tools accept opt-in `detail: "summary"`. Their original +complete Evidence is retained before a summary is returned. Delivery detail +does not enter provider parameters or the original Evidence identity. Default +complete requests keep their existing response. Oversized complete inventory +replies gain an actionable retained-page continuation after recording succeeds. + +`inspect_analysis_view` projects `native-inventory` rows with offsets, limits +and selected fields. Parent Evidence ID and original `source_index` identify +duplicate-address alternatives; original order is preserved. Address is always +retained, and selecting a string's value also retains its string/decoding +qualifications. Counts describe retained provider observations. Exhausting a +page does not establish complete code/string discovery. Unknown source artifact +identity stays unknown rather than acquiring an invented path. + +CLI and MCP use the shared application/domain projector. Its source-format +validation is reused only after successful representation-preserving validation +of an authenticated immutable JSON snapshot. Two weak sets distinguish names +and the stricter strings format. They retain no second parsed inventory or live +provider state; entries disappear with their source arrays. Mutable, foreign, +malformed or transformed values do not acquire this proof. A parent change +rebinds provenance rather than reusing an earlier view envelope. + +This reduces delivery after the provider has produced its full inventory. It +does not reduce Ghidra discovery, eliminate the retained full inventory, or +establish a universal CPU/memory improvement. The catalog becomes larger and +summary-plus-page requires more RPCs than a small complete reply. + +## Real inputs, controls and oracle + +Read-only Linux x86-64 targets, without target execution: + +| Target | Bytes | SHA-256 | Provenance | +| ------ | --------: | ------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------- | +| `true` | 35,664 | `c79bf44242829108e323378531f4ac839513ca1fba45efd6583643526e1e9fd2` | Installed Debian coreutils 9.1-1 | +| jq | 2,267,912 | `b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f` | [Official jq 1.8.2 asset](https://github.com/jqlang/jq/releases/tag/jq-1.8.2), release API size/digest matched | + +Baseline runtime is main `58d6043303e722115f1da18b9505c24abeeab423`. +The candidate runtime is frozen from that base plus this implementation before +measurement. Runtime file manifests, the exact patch, driver revisions, original +Evidence, transcripts, errors and resource observations are retained outside +Git. The curated [receipt](native-inventory-views.json) pins the recordings. +Later rebases do not relabel these timings as measurements of a different base. + +Node 24.18.0/npm 11.16.0, MCP SDK 2.3.1, Ghidra 12.1.4 and Eclipse Adoptium +21.0.12.1+1-LTS. Each pair uses a fresh MCP server and Ghidra analysis, followed +by a warm repetition in that session. OS caches are unflushed. Java heap is +768 MiB and one active processor; Node heaps are 2 GiB. The family guard uses +one logical CPU, 4 GiB hard/3 GiB early-stop memory, no swap, and 192 tasks. +Heavy jobs are serial. Kernel peaks include file cache, controller verification +and receipt work, rather than isolated provider heap. Payload-exit observations +verify owned process cleanup; final whole-scope counters remain unavailable. + +The live named SDK contracts validate replies. Both modes discover the full +catalog, open the same target, complete the question twice, export original +Evidence and close. Summary mode requests four rows only for nonempty results. +Complete mode includes acknowledged export recovery and local JSON projection +when `list_names` exceeds transport capacity. Its errors, export calls, bytes +read and local projection output remain in costs. Model generation of that +local program and filesystem-tool schemas are unmeasured. + +The small pair preserves every field of every original source Evidence, +including IDs, and both phases' complete selected fact vectors. The large +completed pair compares the requested facts exactly; it must not claim identical +independent Ghidra analyses. An earlier candidate attempt found alias-primary +and two generated switch-label differences outside the requested selection. +Their underlying cause is unestablished. To qualify projection independently +of that variation, the portable verifier imports the actual baseline Evidence, +reads every canonical row through MCP, and compares it with the recorded source. +It also compares actual CLI file-source views with retained MCP views, without +Ghidra/JDK configuration, and reexports original source records unchanged. + +## Measurements and limits + +Each recording contains discovery, both first/warm questions, original Evidence +export and cleanup. The token proxy below adds input-facing catalog JSON, +name/argument JSON, text replies and local recovery output; it does not count +both text and structured copies as if every host supplied both to a model. + +| Recording / metric | Before | After | +| ---------------------------------------- | ---------: | ---------: | +| `true`: RPC calls | 13 | 23 | +| `true`: RPC result bytes | 921,748 | 171,130 | +| `true`: combined token proxy | 306,496 | 208,002 | +| `true`: first question | 35.534 s | 30.791 s | +| `true`: warm question | 0.236 s | 0.224 s | +| `true`: whole scope wall | 53.662 s | 36.777 s | +| `true`: family CPU | 44.202 s | 35.483 s | +| `true`: kernel family peak | 1103.9 MiB | 1186.5 MiB | +| jq: RPC calls, including export recovery | 15 | 21 | +| jq: RPC result bytes | 3,032,150 | 172,180 | +| jq: combined token proxy | 590,032 | 208,302 | +| jq: first question | 334.659 s | 318.798 s | +| jq: warm question | 2.521 s | 6.916 s | +| jq: whole scope wall | 355.586 s | 333.104 s | +| jq: family CPU | 318.976 s | 311.319 s | +| jq: kernel family peak | 1498.2 MiB | 1692.3 MiB | + +The catalog remains complete at 209 tools: 3,263,492 → 3,293,844 bytes. +Its input-facing projection grows by 642 proxy tokens, from 185,367 to 186,009. +Including that increase and every call, the combined serialization proxy falls +32.1% for `true` and 64.7% for jq. Actual model usage is unmeasured. + +Preserve the negative results: the large warm question is slower and both +family memory peaks increase. The first native view also pays source-format +validation and SDK schema setup; these phases are not isolated by this driver. +Fresh analysis dominates total wall time and varies between sessions. These +observations do not establish a latency, CPU or memory improvement. The owning +benefit is selected delivery with complete source access and actionable +recovery, rather than eliminating full provider inventory production. + +Individual completed samples, tokenizer estimates and portable checks are in +the receipt. A `cl100k_base` estimate over recorded serialization is not actual +model usage. Counts include discovery, call arguments, text replies, errors, +continuations and local recovery output; SDK result JSON is also reported +separately. Actual client schema selection/rendering, wire framing, model +reasoning and isolated correctness-oracle overhead are unmeasured. Single pairs +are observations rather than latency distributions or evidence of improved +model investigation outcomes. + +Incomplete attempts remain separate. The first baseline jq driver stopped at +the real oversized reply because it lacked export recovery. The first candidate +jq attempt produced its selected facts and source export, then hit the 3 GiB +early-stop threshold while comparing differing full inventories; no final report +or payload-exit cleanup proof was produced. Expanding the enormous assertion +diagnostic is a source-derived explanation, not an observed final stack trace. +The harness now emits a bounded mismatch with source digests while retaining +strict equality and both full files. The subsequent recording explicitly does +not request cross-session full-source equality. + +Two static-check attempts were stopped by the host-memory guard; later checks +passed after host memory recovered. No guard-stopped attempt is a successful +performance sample. Engine-version/platform equivalence, other providers, +complete application reconstruction and post-GC retained heap are unqualified. + +Executed verification: 24 focused files/305 tests passed, including advertised +SDK-converted JSON Schema validation, all five tools, nonretaining refusal, +oversized recovery, same-address alternatives, decoding qualifiers, immutable +validation ownership and unknown-artifact sources. Full `npm run check` passed, +including module/architecture guards; `npm run docs:check` also passed. The portable runs passed with 27/444 MCP +calls, ten actual CLI comparisons each, every row reconstructed, original +records preserved and payload cleanup verified. Their scope times are +25.194/77.537 s and kernel family peaks are 978.4/1940.2 MiB; these additional +correctness runs are not part of the selected-question timing pairs. + +After rebasing onto main `0134732fc` (including Protobuf/gRPC views), 28 focused +files/325 tests passed. Final field-selection normalization removes repeated +per-row scans of duplicate requested fields while retaining the exact request; +the affected two files/nine tests passed again. Final `npm run check` and +`npm run docs:check` passed. Both real old-source portable checks passed again +against the final build, including every canonical row and all 20 CLI/MCP +comparisons. The observed catalog is then 209 tools/3,317,414 bytes. Those final +source and dependency pins are separate from the earlier frozen timing/token +pairs: final-main performance pairs and actual model usage were not rerun. + +## Reproduce + +Build separate baseline/candidate runtimes, then run each mode with the same +absolute artifact, Ghidra and JDK paths: + +```sh +python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \ + node experiments/analysis-efficiency/native-inventory-views.mjs \ + --runtime /absolute/baseline --target /absolute/application \ + --ghidra /absolute/ghidra --java /absolute/jdk \ + --mode complete --output /absolute/before + +python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \ + node experiments/analysis-efficiency/native-inventory-views.mjs \ + --runtime /absolute/candidate --target /absolute/application \ + --ghidra /absolute/ghidra --java /absolute/jdk \ + --mode summary --output /absolute/after + +python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \ + node experiments/analysis-efficiency/native-inventory-portable.mjs \ + --runtime /absolute/candidate --source /absolute/before \ + --output /absolute/portable +``` + +`--control /absolute/before` additionally requires every source Evidence field +to match across the two live analyses. Preserve failure if the engine produces +different observations. `--verify-pages` performs extra live full-page checks; +keep those correctness runs distinct from selected-question timing pairs. +With private `tiktoken==0.12.0`, `native-inventory-costs.py --output FILE CASE...` +counts completed recordings and their sibling `.log` resource observations. + +The subsequent main `4eaf3a117` oversized-error protocol-framing fix was also +merged into the branch. Its producing-server, native inventory and large-result +checks passed (three files/11 tests), preserving both protocol revisions. The +real old `true` source verifier and `npm run docs:check` passed again. The native +projector and earlier performance pins are unchanged; latest-head pre-push/CI +checks provide the final branch status. diff --git a/experiments/analysis-efficiency/native-inventory-views.mjs b/experiments/analysis-efficiency/native-inventory-views.mjs new file mode 100644 index 0000000000..fefe353b6e --- /dev/null +++ b/experiments/analysis-efficiency/native-inventory-views.mjs @@ -0,0 +1,374 @@ +/** Compare complete delivery with selected native inventory views on real Ghidra. */ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { createWriteStream } from "node:fs"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { finished } from "node:stream/promises"; +import { isDeepStrictEqual, parseArgs } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { createRetainedQueryCaller } from "./retained-query-calls.mjs"; +import { finalizeRetainedQueryReceipt } from "./retained-query-finalization.mjs"; + +const { values } = parseArgs({ + options: { + ...Object.fromEntries( + ["runtime", "target", "output", "ghidra", "java", "mode", "control"].map( + (name) => [name, { type: "string" }], + ), + ), + "verify-pages": { type: "boolean", default: false }, + }, +}); +for (const name of ["runtime", "target", "output", "ghidra", "java"]) + assert.ok(values[name] && values[name] === resolve(values[name]), name); +assert.ok(["complete", "summary"].includes(values.mode)); +assert.ok(!values["verify-pages"] || values.mode === "summary"); +const hash = (value) => createHash("sha256").update(value).digest("hex"); +const output = values.output; +await mkdir(output, { mode: 0o700 }); +const targetBytes = await readFile(values.target); +const home = await mkdtemp(join(tmpdir(), "rea-native-inventory-")); +const client = new Client({ name: "native-inventory-views", version: "1" }); +const transport = new StdioClientTransport({ + command: process.execPath, + args: [join(values.runtime, "scripts/rea.mjs"), "mcp"], + env: { + HOME: home, + PATH: process.env.PATH, + NODE_OPTIONS: "--max-old-space-size=2048", + GHIDRA_INSTALL_DIR: values.ghidra, + JAVA_HOME: values.java, + GHIDRA_HEADLESS_MAXMEM: "768M", + JAVA_TOOL_OPTIONS: "-XX:ActiveProcessorCount=1", + REA_GHIDRA_STARTUP_TIMEOUT_MS: "600000", + REA_LOG_LEVEL: "silent", + }, + stderr: "pipe", +}); +const serverLog = createWriteStream(join(output, "server.log")); +const serverLogFinished = finished(serverLog); +serverLogFinished.catch(() => {}); +transport.stderr?.pipe(serverLog); +const report = { + artifact: { + path: values.target, + bytes: targetBytes.length, + sha256: hash(targetBytes), + }, + mode: values.mode, + verify_pages: values["verify-pages"], + question: + "Report retained row counts and the first four address/value pairs for procedures, names, strings, and literal main procedure/string matches.", + calls: [], + tasks: [], + local_recoveries: [], +}; +const transcript = []; +const call = createRetainedQueryCaller(client, (receipt, event) => { + report.calls.push(receipt); + transcript.push(event); +}); +const result = async (name, input, phase) => { + const reply = await call(name, input, phase ?? name); + assert.notEqual(reply.isError, true, JSON.stringify(reply)); + assert.ok(reply.structuredContent); + return reply.structuredContent; +}; +const operations = [ + "list_procedures", + "list_names", + "list_strings", + "search_procedures", + "search_strings", +]; +const selectedRows = (rows) => + rows.slice(0, 4).map(({ address, value }) => ({ address, value })); +const sourceReferences = []; +const started = performance.now(); +let opened = false; +try { + await client.connect(transport); + const discovery = performance.now(); + const catalog = await client.listTools(); + assert.equal(catalog.nextCursor, undefined); + report.catalog = { + elapsed_ms: performance.now() - discovery, + tool_count: catalog.tools.length, + bytes: Buffer.byteLength(JSON.stringify(catalog)), + sha256: hash(JSON.stringify(catalog)), + }; + await writeFile(join(output, "catalog.json"), JSON.stringify(catalog)); + const definitions = new Map(catalog.tools.map((tool) => [tool.name, tool])); + const original = client.callTool.bind(client); + client.callTool = (input, options) => + original(input, { + ...options, + timeout: 600000, + toolDefinition: definitions.get(input.name), + }); + await result("open_binary", { path: values.target, provider_id: "ghidra" }); + opened = true; + for (const phase of ["first", "warm"]) { + const taskStarted = performance.now(); + const facts = []; + for (const operation of operations) { + const parameters = operation.startsWith("search_") + ? { pattern: "main", mode: "literal", case_sensitive: false } + : {}; + const input = + values.mode === "summary" + ? { ...parameters, detail: "summary" } + : parameters; + const reply = await call(operation, input, `${phase}:${operation}`); + let evidence; + if (reply.isError) { + const text = reply.content.find((item) => item.type === "text"); + assert.ok(text); + const error = JSON.parse(text.text).error; + assert.equal(values.mode, "complete"); + assert.equal(error.code, "resource_constraint"); + const reference = error.details?.reported_limits?.evidence_reference; + assert.equal(error.details?.reported_limits?.boundary, "mcp-response"); + assert.equal(reference?.kind, "retained-evidence"); + const recovery = { phase: `${phase}:${operation}`, status: "started" }; + report.local_recoveries.push(recovery); + const path = join( + output, + `recovery-${report.local_recoveries.length}.json`, + ); + await result( + "export_evidence_bundle", + { path }, + `${phase}:${operation}:recovery-export`, + ); + const recoveryStarted = performance.now(); + try { + const bytes = await readFile(path); + const bundle = JSON.parse(bytes); + evidence = bundle.records.find( + (record) => record.evidence_id === reference.evidence_id, + ); + assert.ok(evidence); + assert.equal(evidence.operation, operation); + assert.ok(Array.isArray(evidence.normalized_result)); + // This is local JSON field selection after the documented export recovery, + // not another provider query or an MCP view available in the baseline. + const projection = { + evidence_id: evidence.evidence_id, + operation, + row_count: evidence.normalized_result.length, + rows: selectedRows(evidence.normalized_result), + provider: evidence.provider, + parameters: evidence.parameters, + limitations: evidence.limitations, + }; + recovery.bundle_bytes_read = bytes.length; + recovery.projection_bytes = Buffer.byteLength( + JSON.stringify(projection), + ); + recovery.status = "passed"; + transcript.push({ + name: "local-bundle-projection", + phase: recovery.phase, + input: { + path, + evidence_id: reference.evidence_id, + offset: 0, + limit: 4, + fields: ["address", "value"], + }, + projection, + }); + } finally { + recovery.elapsed_ms = performance.now() - recoveryStarted; + } + } else evidence = reply.structuredContent; + assert.ok(evidence); + assert.equal(evidence.subject.digest.sha256, report.artifact.sha256); + let count, rows, provider, query, parent; + if (values.mode === "complete") { + assert.equal(evidence.operation, operation); + assert.equal(evidence.provider.id, "ghidra"); + assert.ok(Array.isArray(evidence.normalized_result)); + count = evidence.normalized_result.length; + rows = selectedRows(evidence.normalized_result); + provider = evidence.provider; + query = evidence.parameters; + parent = evidence.evidence_id; + } else { + assert.equal(evidence.operation, "inspect_analysis_view"); + const summary = evidence.normalized_result; + assert.equal(summary.kind, "summary"); + assert.equal(summary.parent_operation, operation); + assert.equal(summary.summary.discovery_coverage, "unknown"); + assert.equal(summary.source_provider.id, "ghidra"); + count = summary.summary.row_count; + provider = summary.source_provider; + query = summary.source_parameters; + parent = summary.parent_evidence_id; + assert.deepEqual(evidence.evidence_links, [parent]); + rows = []; + if (count > 0) { + const pageEvidence = await result( + "inspect_analysis_view", + { + source: { kind: "retained-evidence", evidence_id: parent }, + view: { + kind: "native-inventory", + offset: 0, + limit: 4, + fields: ["value"], + }, + }, + `${phase}:${operation}:page`, + ); + const page = pageEvidence.normalized_result; + assert.equal(page.kind, "native-inventory"); + assert.equal(page.parent_evidence_id, parent); + assert.equal(page.coverage.total, count); + assert.equal(page.rows.length, Math.min(4, count)); + for (const [index, row] of page.rows.entries()) + assert.equal(row.source_index, index); + rows = page.rows.map(({ record: { address, value } }) => ({ + address, + value, + })); + } + } + assert.ok(Number.isSafeInteger(count) && count >= 0); + if (operation.startsWith("list_")) assert.ok(count > 0, operation); + assert.equal(provider.version, "12.1.4"); + assert.equal(Object.hasOwn(query, "detail"), false); + assert.equal(rows.length, Math.min(4, count)); + facts.push({ + operation, + row_count: count, + rows, + provider, + parameters: query, + limitations: [...new Set(evidence.limitations)].sort(), + }); + sourceReferences.push({ phase, operation, parent }); + } + report.tasks.push({ + phase, + elapsed_ms: performance.now() - taskStarted, + facts, + }); + } + assert.deepEqual(report.tasks[0].facts, report.tasks[1].facts); + const taskFacts = report.tasks.map(({ phase, facts }) => ({ phase, facts })); + await writeFile(join(output, "facts.json"), JSON.stringify(taskFacts)); + report.facts_sha256 = hash(JSON.stringify(taskFacts)); + const bundlePath = join(output, "evidence.json"); + await result("export_evidence_bundle", { path: bundlePath }); + const bundle = JSON.parse(await readFile(bundlePath, "utf8")); + const sources = sourceReferences.map(({ phase, operation, parent }) => { + const record = bundle.records.find( + (record) => record.evidence_id === parent, + ); + assert.ok(record); + assert.equal(record.operation, operation); + assert.equal(record.subject.digest.sha256, report.artifact.sha256); + const task = report.tasks + .find((task) => task.phase === phase) + .facts.find((fact) => fact.operation === operation); + assert.equal(record.normalized_result.length, task.row_count); + assert.deepEqual(selectedRows(record.normalized_result), task.rows); + assert.deepEqual(record.parameters, task.parameters); + assert.deepEqual(record.provider, task.provider); + return { phase, operation, record }; + }); + await writeFile(join(output, "sources.json"), JSON.stringify(sources)); + report.sources_sha256 = hash(JSON.stringify(sources)); + if (values.control) { + assert.deepEqual( + taskFacts, + JSON.parse(await readFile(join(values.control, "facts.json"), "utf8")), + ); + const control = JSON.parse( + await readFile(join(values.control, "sources.json"), "utf8"), + ); + // Assertion formatting of a large mismatch can itself exhaust the experiment + // scope. Retain both files and report a bounded failure without weakening it. + assert.ok( + isDeepStrictEqual(sources, control), + `Full source Evidence differs from control; inspect retained sources.json files (${report.sources_sha256} vs ${hash(JSON.stringify(control))}).`, + ); + } + if (values["verify-pages"]) { + for (const { operation, parent } of sourceReferences.filter( + (source) => source.phase === "first", + )) { + const source = sources.find( + (source) => source.phase === "first" && source.operation === operation, + ).record; + const all = []; + let offset = 0; + do { + const evidence = await result( + "inspect_analysis_view", + { + source: { kind: "retained-evidence", evidence_id: parent }, + view: { kind: "native-inventory", offset, limit: 128 }, + }, + `verification:${operation}:${offset}`, + ); + const page = evidence.normalized_result; + assert.equal(page.kind, "native-inventory"); + assert.equal( + page.rows.length, + Math.min(128, source.normalized_result.length - offset), + ); + for (const [index, row] of page.rows.entries()) + assert.equal(row.source_index, offset + index); + all.push(...page.rows.map((row) => row.record)); + assert.equal( + page.coverage.next_offset, + offset + page.rows.length >= source.normalized_result.length + ? null + : offset + page.rows.length, + ); + offset = page.coverage.next_offset; + } while (offset !== null); + assert.deepEqual(all, source.normalized_result); + } + } + assert.equal(hash(await readFile(values.target)), report.artifact.sha256); + report.status = "passed"; +} catch (cause) { + report.status = "failed"; + report.failure = String(cause?.stack ?? cause); + process.exitCode = 1; +} finally { + if (opened) + try { + await result("close_binary", {}); + } catch (cause) { + report.status = "failed"; + report.close_failure = String(cause?.stack ?? cause); + } + report.elapsed_before_cleanup_ms = performance.now() - started; + try { + await writeFile( + join(output, "transcript.json"), + JSON.stringify(transcript), + ); + } catch (cause) { + report.status = "failed"; + report.transcript_failure = String(cause); + } + const finalized = await finalizeRetainedQueryReceipt({ + report, + client, + transport, + account: home, + serverLog, + serverLogFinished, + output, + }); + if (report.status !== "passed" || !finalized) process.exitCode = 1; +} diff --git a/src/application/XrefsAddressResolution.ts b/src/application/XrefsAddressResolution.ts index c7eef8bdc8..3a2f5e52ed 100644 --- a/src/application/XrefsAddressResolution.ts +++ b/src/application/XrefsAddressResolution.ts @@ -1,4 +1,4 @@ -import { addressedValue } from "../contracts/toolOutputSchemaPrimitives.js"; +import { addressedValueSchema } from "../domain/native/nativeInventory.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import { AnalysisCapabilityUnavailableError, @@ -54,7 +54,7 @@ export const resolveXrefsAddress = async ( const addresses = new Set(); for (const [index, item] of inventory.value.result.entries()) { - const parsed = addressedValue.safeParse(item); + const parsed = addressedValueSchema.safeParse(item); if (!parsed.success || parsed.data.address.length === 0) return err( new AnalysisOutputError( diff --git a/src/application/analysisView/AnalysisViewService.ts b/src/application/analysisView/AnalysisViewService.ts index cdabb800f8..c0d285f38b 100644 --- a/src/application/analysisView/AnalysisViewService.ts +++ b/src/application/analysisView/AnalysisViewService.ts @@ -149,6 +149,8 @@ export const inspectAnalysisViewValidated = ( { evidenceId: parent.evidence_id, operation: parent.operation, + provider: parent.provider, + analysisProfileDigest: parent.analysis_profile?.digest ?? null, normalizedResult: immutableJsonSnapshotMember(parent, "normalized_result") ?? parent.normalized_result, @@ -175,11 +177,11 @@ export const inspectAnalysisViewValidated = ( if (!projected.ok) return projected; if ( parent.subject !== null && - parent.subject.digest.sha256 !== projected.value.artifact.sha256 + parent.subject.digest.sha256 !== projected.value.artifact?.sha256 ) return err( new EvidenceIntegrityError( - `Parent subject SHA-256 ${parent.subject.digest.sha256} differs from analysis artifact SHA-256 ${projected.value.artifact.sha256}.`, + `Parent subject SHA-256 ${parent.subject.digest.sha256} differs from analysis artifact SHA-256 ${projected.value.artifact?.sha256 ?? "unknown"}.`, ), ); return ok(createAnalysisViewEvidence(input, parent, projected.value)); @@ -337,7 +339,7 @@ const createAnalysisViewEvidence = ( ? { relationshipCoverage: parent.relationship_coverage } : {}), locations: [ - ...(result.artifact.path.length > 0 + ...(result.artifact != null && result.artifact.path.length > 0 ? [{ kind: "artifact-path" as const, path: result.artifact.path }] : []), ...(result.kind === "native" && diff --git a/src/application/native/NativeDispatchMetadataInspection.ts b/src/application/native/NativeDispatchMetadataInspection.ts index da5e312644..c27898bc6c 100644 --- a/src/application/native/NativeDispatchMetadataInspection.ts +++ b/src/application/native/NativeDispatchMetadataInspection.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { addressedValue } from "../../contracts/toolOutputSchemaPrimitives.js"; +import { addressedValueSchema } from "../../domain/native/nativeInventory.js"; import { nativeDispatchMetadataResultSchema, inspectNativeDispatchMetadata, @@ -44,8 +44,8 @@ export const inspectNativeDispatch = async ( const names = z .array( z.union([ - addressedValue.strict(), - addressedValue + addressedValueSchema.strict(), + addressedValueSchema .omit({ value: true }) .extend({ name: z.string() }) .strict(), diff --git a/src/cli/analysisViewCommands.ts b/src/cli/analysisViewCommands.ts index 4d00e5bb13..d75c419365 100644 --- a/src/cli/analysisViewCommands.ts +++ b/src/cli/analysisViewCommands.ts @@ -71,7 +71,7 @@ export const registerAnalysisViewCommands = ( }); cli.command(CLI_COMMANDS.inspectAnalysisView, { description: - "Project a selected view of completed binary-layout, JavaScript application, native function, artifact-inventory, desktop-directory, WASM, Mach-O, Interface Builder, EVM, managed, Android package, Dart AOT, web network capture or pcap Evidence JSON", + "Project a selected view of completed binary-layout, JavaScript application, native function/inventory, artifact-inventory, desktop-directory, WASM, Mach-O, Interface Builder, EVM, managed, Android package, Dart AOT, web network capture or pcap Evidence JSON", args: z.object({ inputJson: z.string().describe("Inline workflow JSON or JSON file path"), }), diff --git a/src/contracts/analysisViewToolContracts.ts b/src/contracts/analysisViewToolContracts.ts index f042d237e3..ade711978b 100644 --- a/src/contracts/analysisViewToolContracts.ts +++ b/src/contracts/analysisViewToolContracts.ts @@ -17,10 +17,26 @@ export const ANALYSIS_VIEW_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_analysis_view"), kind: "application", description: - 'Inspect selected views of completed binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, decode_interface_builder, inspect_evm_interface, inspect_managed_artifact, inspect_managed_members, inspect_android_package, inspect_dart_aot, inspect_il2cpp_metadata or inspect_web_network_capture Evidence without repeating analysis. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. The environment-state view extracts, from the same parents, the initial state the recorded reads observed: view {kind: "environment-state", template_rules: [...]} with the rules of the manifest it accompanies. It reads response bodies recorded before the first related write and reports entity types with an identity field (proved by a path parameter or inferred from a field name), entities in collection order with per-view values where views differ, documents for non-parameterized non-entity paths written by POST/PUT/PATCH (item-shaped and delete-only paths stay recorded), and a render plan per read tool: item and list views reproduce their latest recordings from the state, and every other read tool is a recorded view with a reason. Types and identity are inferences; values are recorded data, so it refuses parents inspected with sensitive values, and a scenario parent must have selected response bodies; its values keep the capture redaction markers, and a write whose URL it redacted ends the state of every type. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings and carry the source selection and coverage. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Protobuf views use kind protobuf, collection bodies/frames/fields, with record_ordinals, body_pointers and fields; field pages also select frame_indices, field_ids, field_numbers or schema_names. Byte identity, offset basis and body status remain present. Text and schema_value are opt-in page columns. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods, with exact indices/names, fields and paging; types also select image_indices/namespaces/declaring_types and methods select declaring_types. Rows always keep their original index scoped to the parent artifact digest. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', + 'Inspect selected views of completed binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, decode_interface_builder, inspect_evm_interface, inspect_managed_artifact, inspect_managed_members, inspect_android_package, inspect_dart_aot, inspect_il2cpp_metadata or inspect_web_network_capture Evidence without repeating analysis. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. The environment-state view extracts, from the same parents, the initial state the recorded reads observed: view {kind: "environment-state", template_rules: [...]} with the rules of the manifest it accompanies. It reads response bodies recorded before the first related write and reports entity types with an identity field (proved by a path parameter or inferred from a field name), entities in collection order with per-view values where views differ, documents for non-parameterized non-entity paths written by POST/PUT/PATCH (item-shaped and delete-only paths stay recorded), and a render plan per read tool: item and list views reproduce their latest recordings from the state, and every other read tool is a recorded view with a reason. Types and identity are inferences; values are recorded data, so it refuses parents inspected with sensitive values, and a scenario parent must have selected response bodies; its values keep the capture redaction markers, and a write whose URL it redacted ends the state of every type. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings and carry the source selection and coverage. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Protobuf views use kind protobuf, collection bodies/frames/fields, with record_ordinals, body_pointers and fields; field pages also select frame_indices, field_ids, field_numbers or schema_names. Byte identity, offset basis and body status remain present. Text and schema_value are opt-in page columns. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods, with exact indices/names, fields and paging; types also select image_indices/namespaces/declaring_types and methods select declaring_types. Rows always keep their original index scoped to the parent artifact digest. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native inventory Evidence from list_names, list/search_procedures or list/search_strings uses view: {kind: "native-inventory", offset: 0, limit: 64, fields: ["value"]}. Rows keep source_index and address even with selected fields. Summary counts describe retained rows; provider discovery coverage remains unknown. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', inputSchema: inspectAnalysisViewInputSchema, outputSchema: evidenceResultOf(analysisViewResultSchema), examples: [ + { + title: + "Page selected native inventory fields without repeating the complete list", + input: { + source: { + kind: "retained-evidence", + evidence_id: RETAINED_EXAMPLE_ID, + }, + view: { + kind: "native-inventory", + offset: 0, + limit: 32, + fields: ["value"], + }, + }, + }, { title: "Read exact inventory paths from retained inspection Evidence", input: { diff --git a/src/contracts/officialToolContracts.ts b/src/contracts/officialToolContracts.ts index 76ee861e93..cde2b9adb5 100644 --- a/src/contracts/officialToolContracts.ts +++ b/src/contracts/officialToolContracts.ts @@ -1,3 +1,4 @@ +import { analysisDetailSchema } from "./analysisDetail.js"; import { z } from "zod"; import { nativeFunctionAnnotationsInputSchema } from "../domain/native/nativeFunctionAnnotations.js"; @@ -109,13 +110,17 @@ export const OFFICIAL_TOOL_CONTRACTS = [ ), official( "list_names", - "List every analyzed memory and external symbol as address/value pairs. Provider metadata distinguishes Ghidra primary, dynamic, external, type, and source facts when available.", - z.object({ document, address: optionalAddress }), + "List every analyzed memory and external symbol as address/value pairs. Provider metadata distinguishes Ghidra primary, dynamic, external, type, and source facts when available. Select detail summary for counts and retained native-inventory pages; complete remains the default.", + z.object({ + document, + address: optionalAddress, + detail: analysisDetailSchema, + }), ), official( "list_procedures", - "List every analyzed procedure as address/value pairs after provider analysis. Ghidra metadata distinguishes thunks and external functions; use returned addresses in later function operations.", - z.object({ document }), + "List every analyzed procedure as address/value pairs after provider analysis. Ghidra metadata distinguishes thunks and external functions; use returned addresses in later function operations. Select detail summary for counts and retained native-inventory pages; complete remains the default.", + z.object({ document, detail: analysisDetailSchema }), ), official( "list_segments", @@ -124,8 +129,12 @@ export const OFFICIAL_TOOL_CONTRACTS = [ ), official( "list_strings", - "List every provider-reported string object or candidate, or filter to one address, as address/value pairs with encoding, terminator status, and byte length when available. Hopper reads complete typed objects and retains its original display text as provider_value; long literals may span adjacent unterminated objects. Kuna scans mapped printable sequences of at least five characters, including unterminated fragments; references are not computed.", - z.object({ document, address: optionalAddress }), + "List every provider-reported string object or candidate, or filter to one address, as address/value pairs with encoding, terminator status, and byte length when available. Hopper reads complete typed objects and retains its original display text as provider_value; long literals may span adjacent unterminated objects. Kuna scans mapped printable sequences of at least five characters, including unterminated fragments; references are not computed. Select detail summary for counts and retained native-inventory pages; complete remains the default.", + z.object({ + document, + address: optionalAddress, + detail: analysisDetailSchema, + }), ), official( "next_address", @@ -191,13 +200,13 @@ export const OFFICIAL_TOOL_CONTRACTS = [ ), official( "search_procedures", - "Search every analyzed procedure name using literal matching by default or regex when requested. Results are deterministic and complete. Hopper regex mode uses ECMAScript Unicode syntax in a cancellable worker; a matching deadline leaves its API usable.", - z.object(analysisSearchInput), + "Search every analyzed procedure name using literal matching by default or regex when requested. Results are deterministic and complete. Hopper regex mode uses ECMAScript Unicode syntax in a cancellable worker; a matching deadline leaves its API usable. Select detail summary for counts and retained native-inventory pages; complete remains the default.", + z.object({ ...analysisSearchInput, detail: analysisDetailSchema }), ), official( "search_strings", - "Search every provider-reported string object or candidate using literal matching by default or regex when requested. Results are deterministic and complete over that provider inventory, not every text sequence in artifact bytes. Kuna scans mapped sequences of at least five characters, including unterminated fragments, without computing references. Ghidra may omit packed or undefined literals, including Rust strings; an empty result does not prove text is absent. If an address is known, read_bytes and xrefs inspect its bytes and references. Hopper retains typed byte decoding, native display, and explicit decoding unknowns. Adjacent long-literal fragments match independently. Hopper regex mode uses ECMAScript Unicode syntax in a cancellable worker; a matching deadline leaves its API usable.", - z.object(analysisSearchInput), + "Search every provider-reported string object or candidate using literal matching by default or regex when requested. Results are deterministic and complete over that provider inventory, not every text sequence in artifact bytes. Kuna scans mapped sequences of at least five characters, including unterminated fragments, without computing references. Ghidra may omit packed or undefined literals, including Rust strings; an empty result does not prove text is absent. If an address is known, read_bytes and xrefs inspect its bytes and references. Hopper retains typed byte decoding, native display, and explicit decoding unknowns. Adjacent long-literal fragments match independently. Hopper regex mode uses ECMAScript Unicode syntax in a cancellable worker; a matching deadline leaves its API usable. Select detail summary for counts and retained native-inventory pages; complete remains the default.", + z.object({ ...analysisSearchInput, detail: analysisDetailSchema }), ), official( "set_address_name", diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index 5d41fbce5b..0073793f45 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -1,3 +1,6 @@ +import { analysisDetailOutputSchemaOf } from "./analysisDetail.js"; +import { nativeInventorySummaryViewResultSchema } from "../domain/analysisView/analysisView.js"; +import { nativeInventoryRowsSchemas } from "../domain/native/nativeInventory.js"; import { z } from "zod"; import { byteSearchResultSchema } from "../domain/native/bytePattern.js"; import { @@ -36,7 +39,6 @@ import { functionInstructionWindowSchema, referenceEdgeSchema, unresolvedCallSchema, - analysisStringSchema, procedureIdentitySchema, } from "../domain/hopperValues.js"; import { nativeApiInspectionResultSchema } from "../domain/native/nativeApiBoundary.js"; @@ -75,7 +77,6 @@ import { nativeDispatchMetadataResultSchema } from "../domain/native/objcSwiftMe import { nativeInvestigationTraceSchema } from "../domain/native/nativeInvestigationGraph.js"; import { addressList, - addressedValue, addressedEntry, containingProcedureResolution, functionDossierOutput, @@ -130,10 +131,25 @@ export const officialOutputSchemas: Readonly> = { inline_comment: evidenceResultOf(nullableText), list_bookmarks: evidenceResultOf(z.array(analysisBookmarkSchema)), list_documents: evidenceResultOf(z.array(z.string())), - list_names: evidenceResultOf(z.array(addressedValue)), - list_procedures: evidenceResultOf(z.array(addressedValue)), + list_names: analysisDetailOutputSchemaOf( + nativeInventoryRowsSchemas.names, + nativeInventorySummaryViewResultSchema.extend({ + parent_operation: z.literal("list_names"), + }), + ), + list_procedures: analysisDetailOutputSchemaOf( + nativeInventoryRowsSchemas.names, + nativeInventorySummaryViewResultSchema.extend({ + parent_operation: z.literal("list_procedures"), + }), + ), list_segments: segmentOutput, - list_strings: evidenceResultOf(z.array(analysisStringSchema)), + list_strings: analysisDetailOutputSchemaOf( + nativeInventoryRowsSchemas.strings, + nativeInventorySummaryViewResultSchema.extend({ + parent_operation: z.literal("list_strings"), + }), + ), next_address: evidenceResultOf(z.string()), prev_address: evidenceResultOf(z.string()), procedure_address: evidenceResultOf(z.string()), @@ -192,10 +208,19 @@ export const officialOutputSchemas: Readonly> = { ), procedure_pseudo_code: evidenceResultOf(nullableText), resolve_containing_procedure: evidenceResultOf(containingProcedureResolution), - search_procedures: evidenceResultOf( + search_procedures: analysisDetailOutputSchemaOf( z.array(z.object({ address: z.string(), value: z.string() })), + nativeInventorySummaryViewResultSchema.extend({ + parent_operation: z.literal("search_procedures"), + }), + ), + + search_strings: analysisDetailOutputSchemaOf( + nativeInventoryRowsSchemas.strings, + nativeInventorySummaryViewResultSchema.extend({ + parent_operation: z.literal("search_strings"), + }), ), - search_strings: evidenceResultOf(z.array(analysisStringSchema)), set_address_name: evidenceResultOf(z.boolean()), set_addresses_names: evidenceResultOf(z.record(z.string(), z.boolean())), set_bookmark: evidenceResultOf(z.boolean()), diff --git a/src/contracts/toolOutputSchemaPrimitives.ts b/src/contracts/toolOutputSchemaPrimitives.ts index 4d27d102a3..6221d57cb4 100644 --- a/src/contracts/toolOutputSchemaPrimitives.ts +++ b/src/contracts/toolOutputSchemaPrimitives.ts @@ -579,34 +579,6 @@ const availableMemoryPermissions = z.object({ source: z.literal("ghidra-memory-block"), }); -export const addressedValue = z.object({ - address: z.string(), - value: z.string(), - symbol: z - .object({ - primary: z.boolean(), - dynamic: z.boolean(), - external: z.boolean(), - type: z.string(), - source: z.enum(["default", "analysis", "ai", "imported", "user_defined"]), - }) - .optional(), - procedure: z - .object({ - external: z.boolean(), - thunk: z.boolean(), - thunk_target: z.string().nullable(), - }) - .optional(), - string: z - .object({ - encoding: z.string(), - termination: z.enum(["missing", "present_or_not_required"]), - byte_length: z.number().int().min(0), - }) - .optional(), -}); - const memoryRegionOutput = z.object({ name: z.string(), start: z.string(), diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 04e5cc7826..2ed46a7a28 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -3,6 +3,17 @@ import { protobufViewItemsSchema, protobufViewContextSchema, } from "./protobufCaptureView.js"; +import { + nativeInventoryViewRequestSchema, + nativeInventoryViewRowSchema, + nativeInventoryViewContext, + nativeInventorySummarySchema, + projectNativeInventoryView, +} from "./nativeInventoryView.js"; +import { + NATIVE_INVENTORY_OPERATIONS, + nativeInventoryOperationSchema, +} from "../native/nativeInventory.js"; import { isAbsolute } from "node:path"; import { localPathStringSchema } from "../localPath.js"; @@ -240,6 +251,7 @@ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ javascriptModuleReferencesRequestSchema, javascriptFactsViewRequestSchema, nativeViewSchema, + nativeInventoryViewRequestSchema, artifactInventoryViewRequestSchema, wasmViewRequestSchema, machoViewRequestSchema, @@ -364,6 +376,7 @@ const viewResultBase = { "inspect_pcap_capture", "inspect_dart_aot", "inspect_il2cpp_metadata", + ...NATIVE_INVENTORY_OPERATIONS, ]), parent_digest: digestSchema, view_digest: digestSchema, @@ -502,6 +515,15 @@ export const il2cppMetadataSummaryViewResultSchema = z.strictObject({ summary: il2cppMetadataSummarySchema, }); +/** Exact inventory summary with the original observation's provider and query. */ +export const nativeInventorySummaryViewResultSchema = z.strictObject({ + ...summaryViewResultBase, + ...nativeInventoryViewContext, + parent_operation: nativeInventoryOperationSchema, + artifact: artifactSchema.nullable(), + summary: nativeInventorySummarySchema, +}); + /** Projected facts for one selected view of retained analysis Evidence. */ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ z.strictObject({ @@ -535,25 +557,32 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ ]), view: environmentStateViewRequestSchema, }), + z.discriminatedUnion("parent_operation", [ + binaryLayoutSummaryViewResultSchema, + desktopApplicationSummaryViewResultSchema, + javascriptApplicationSummaryViewResultSchema, + artifactInventorySummaryViewResultSchema, + wasmArtifactSummaryViewResultSchema, + machoSummaryViewResultSchema, + interfaceBuilderSummaryViewResultSchema, + evmInterfaceSummaryViewResultSchema, + managedArtifactSummaryViewResultSchema, + managedMemberSummaryViewResultSchema, + webNetworkCaptureSummaryViewResultSchema, + androidPackageSummaryViewResultSchema, + pcapCaptureSummaryViewResultSchema, + dartAotSummaryViewResultSchema, + il2cppMetadataSummaryViewResultSchema, + nativeInventorySummaryViewResultSchema, + ]), z.strictObject({ - ...summaryViewResultBase, - summary: z.union([ - layoutSummarySchema, - desktopSummarySchema, - javascriptSummarySchema, - artifactInventorySummarySchema, - wasmSummarySchema, - machoSummarySchema, - interfaceBuilderSummarySchema, - evmSummarySchema, - managedSummarySchema, - managedMemberSummarySchema, - captureSummarySchema, - androidPackageSummarySchema, - pcapSummarySchema, - dartAotSummarySchema, - il2cppMetadataSummarySchema, - ]), + ...viewResultBase, + ...nativeInventoryViewContext, + kind: z.literal("native-inventory"), + parent_operation: nativeInventoryOperationSchema, + artifact: artifactSchema.nullable(), + view: nativeInventoryViewRequestSchema, + rows: z.array(nativeInventoryViewRowSchema), }), z.strictObject({ kind: z.literal("protobuf"), @@ -713,6 +742,8 @@ export interface AnalysisViewParent { readonly evidenceId: string; readonly operation: string; readonly normalizedResult: unknown; + readonly provider?: Evidence["provider"]; + readonly analysisProfileDigest?: string | null; /** The owner proved normalizedResult belongs to a sealed, deep-frozen Evidence snapshot. */ readonly sealedResult?: boolean; /** Preserve the authenticated source identity, including explicit absence. */ @@ -774,7 +805,7 @@ const unsupportedParent = ( parentArtifactPath(parent), parent.operation === "capture_browser_scenario" ? `Parent Evidence operation "capture_browser_scenario" supports only the environment-manifest and environment-state views.` - : `Parent Evidence operation ${JSON.stringify(parent.operation)} does not support analysis views. Use completed binary layout, JavaScript application, native function, artifact inventory/inspection, WASM, Mach-O, Interface Builder, EVM, managed artifact or member, Android package, Dart AOT, IL2CPP metadata, or web network capture Evidence.`, + : `Parent Evidence operation ${JSON.stringify(parent.operation)} does not support analysis views. Use completed binary layout, JavaScript application, native function/inventory, artifact inventory/inspection, WASM, Mach-O, Interface Builder, EVM, managed artifact or member, Android package, Dart AOT, IL2CPP metadata, or web network capture Evidence.`, ); const parentArtifactPath = (parent: AnalysisViewParent): string => { @@ -800,6 +831,9 @@ const parentArtifactPath = (parent: AnalysisViewParent): string => { /** Named source owners for views that need no extra decoder or accumulator. */ const retainedViewProjectors = new Map([ + ...NATIVE_INVENTORY_OPERATIONS.map( + (operation) => [operation, projectNativeInventoryView] as const, + ), ["inspect_desktop_application", projectDesktopApplicationView], ["inspect_wasm_artifact", projectWasmArtifactView], ["inspect_macho", projectMachoView], diff --git a/src/domain/analysisView/nativeInventoryView.test.ts b/src/domain/analysisView/nativeInventoryView.test.ts new file mode 100644 index 0000000000..80dafb012b --- /dev/null +++ b/src/domain/analysisView/nativeInventoryView.test.ts @@ -0,0 +1,170 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { freezeJsonSnapshot } from "../immutableJson.js"; +import { nativeInventoryRowsSchemas } from "../native/nativeInventory.js"; +import { + analysisViewRequestSchema, + projectAnalysisView, + type AnalysisViewParent, +} from "./analysisView.js"; + +afterEach(() => vi.restoreAllMocks()); + +const rows = () => [ + { + address: "0x10", + value: "first", + procedure: { external: false, thunk: false, thunk_target: null }, + }, + { address: "0x10", value: "alias" }, + { address: "0x20", value: "last" }, +]; +const parent = (normalizedResult: unknown): AnalysisViewParent => ({ + evidenceId: `ev_${"a".repeat(64)}`, + operation: "search_procedures", + normalizedResult, + sealedResult: true, + provider: { id: "fixture", name: "Fixture", version: "1" }, + artifact: null, + subject: null, + parameters: { pattern: "a", mode: "literal", case_sensitive: false }, + limitations: ["Provider discovery is partial."], +}); +const project = (source: AnalysisViewParent, view: unknown) => { + const result = projectAnalysisView( + source, + analysisViewRequestSchema.parse(view), + ); + if (!result.ok) throw result.error; + return result.value; +}; + +it("pages exact source order and same-address alternatives, with selectable fields and stable row identity", () => { + const original = rows(); + const source = parent(freezeJsonSnapshot(original)); + const selected = project(source, { + kind: "native-inventory", + offset: 0, + limit: 2, + fields: ["value", "address", "value"], + }); + if (selected.kind !== "native-inventory") + throw new Error("Expected row page"); + expect(selected.rows).toEqual([ + { source_index: 0, record: { address: "0x10", value: "first" } }, + { source_index: 1, record: { address: "0x10", value: "alias" } }, + ]); + expect(selected.coverage).toMatchObject({ + total: 3, + next_offset: 2, + exhausted: false, + }); + expect(selected.view).toMatchObject({ + fields: ["value", "address", "value"], + }); + const complete = [0, 2].flatMap((offset) => { + const page = project(source, { + kind: "native-inventory", + offset, + limit: 2, + }); + if (page.kind !== "native-inventory") throw new Error("Expected row page"); + return page.rows.map((row) => row.record); + }); + expect(complete).toEqual(original); + expect(source.normalizedResult).toEqual(original); +}); + +it("reuses only authenticated immutable validation while rebinding provenance to the current parent", () => { + const source = parent(freezeJsonSnapshot(rows())); + const parse = vi.spyOn(nativeInventoryRowsSchemas.names, "safeParse"); + project(source, { kind: "summary" }); + const next = project( + { + ...source, + evidenceId: `ev_${"b".repeat(64)}`, + provider: { id: "other", name: "Other", version: null }, + }, + { kind: "native-inventory", limit: 1 }, + ); + expect(parse).toHaveBeenCalledOnce(); + expect(next).toMatchObject({ + parent_evidence_id: `ev_${"b".repeat(64)}`, + source_provider: { id: "other", name: "Other", version: null }, + source_parameters: source.parameters, + artifact: null, + limitations: source.limitations, + }); +}); + +it("revalidates mutable and shallow-frozen arrays despite a forged sealing flag", () => { + for (const shallowFreeze of [false, true]) { + const values: unknown[] = rows(); + if (shallowFreeze) Object.freeze(values); + const source = parent(values); + project(source, { kind: "native-inventory", limit: 1 }); + Object.assign(values[0] as object, { value: 123 }); + expect(() => project(source, { kind: "summary" })).toThrow(); + } +}); + +it("does not use names validation as proof of the stricter string observation format", () => { + const source = parent( + freezeJsonSnapshot([ + { + address: "0x10", + value: "x", + string: { encoding: "", termination: "missing", byte_length: 1 }, + }, + ]), + ); + project(source, { kind: "summary" }); + expect(() => + project({ ...source, operation: "list_strings" }, { kind: "summary" }), + ).toThrow(); +}); + +it("preserves string display and decoding facts, while empty results retain unknown discovery coverage", () => { + const original = [ + { + address: "0x10", + value: "native display", + provider_value: "original display", + decoding: { available: false as const, reason: "not decoded" }, + string: { + encoding: "native", + encoding_status: "inferred" as const, + termination: "missing" as const, + byte_length: 8, + }, + }, + ]; + const page = project( + { ...parent(freezeJsonSnapshot(original)), operation: "list_strings" }, + { kind: "native-inventory" }, + ); + if (page.kind !== "native-inventory") throw new Error("Expected row page"); + expect(page.rows.map((row) => row.record)).toEqual(original); + const selected = project( + { ...parent(freezeJsonSnapshot(original)), operation: "list_strings" }, + { kind: "native-inventory", fields: ["value"] }, + ); + if (selected.kind !== "native-inventory") + throw new Error("Expected selected row page"); + expect(selected.rows[0]?.record).toEqual({ + address: original[0]!.address, + value: original[0]!.value, + string: original[0]!.string, + decoding: original[0]!.decoding, + }); + const empty = parent(freezeJsonSnapshot([])); + const summary = project(empty, { kind: "summary" }); + expect(summary).toMatchObject({ + artifact: null, + summary: { row_count: 0, discovery_coverage: "unknown" }, + }); + expect(summary.unknowns.length).toBeGreaterThan(0); + expect( + project(empty, { kind: "native-inventory", offset: 2, fields: [] }) + .coverage, + ).toMatchObject({ status: "empty", total: 0, exhausted: true }); +}); diff --git a/src/domain/analysisView/nativeInventoryView.ts b/src/domain/analysisView/nativeInventoryView.ts new file mode 100644 index 0000000000..73ba8bd6e0 --- /dev/null +++ b/src/domain/analysisView/nativeInventoryView.ts @@ -0,0 +1,192 @@ +import { z } from "zod"; +import { analysisStringSchema } from "../hopperValues.js"; +import { canBorrowParsedJsonSnapshot } from "../immutableJson.js"; +import { jsonValueSchema } from "../jsonValue.js"; +import { + addressedValueSchema, + isNativeInventoryOperation, + nativeInventoryRowsSchemas, +} from "../native/nativeInventory.js"; +import { digestSchema } from "../digests.js"; +import { providerSchema } from "../evidence.js"; +import { err, ok, type Result } from "../result.js"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import type { + AnalysisViewParent, + AnalysisViewRequest, + UnsignedAnalysisView, +} from "./analysisView.js"; +import { analysisViewInputError } from "./analysisViewErrors.js"; +import { parseAnalysisViewResult } from "./analysisViewParent.js"; +import { + boundedViewMessages, + completeWithinViewCoverage, + pageViewCoverage, +} from "./analysisViewCoverage.js"; + +/** Preserve all canonical native row metadata, including string decoding facts. */ +const inventoryRecordSchema = addressedValueSchema.extend({ + provider_value: analysisStringSchema.shape.provider_value, + decoding: analysisStringSchema.shape.decoding, + string: z.union([ + analysisStringSchema.shape.string, + addressedValueSchema.shape.string, + ]), +}); +const fieldSchema = inventoryRecordSchema.keyof(); + +export const nativeInventoryViewRequestSchema = z.strictObject({ + kind: z.literal("native-inventory"), + offset: z.number().int().nonnegative().default(0), + limit: z.number().int().positive().default(64), + fields: z + .array(fieldSchema) + .exactOptional() + .describe( + "Select record fields; address and source_index preserve identity. Selecting value also retains its string and decoding qualifications. Omit for every recorded field. Search tools filter before retention.", + ), +}); + +/** Parent Evidence ID plus source_index identifies even same-address alternatives. */ +export const nativeInventoryViewRowSchema = z.strictObject({ + source_index: z.number().int().nonnegative(), + record: inventoryRecordSchema.partial().required({ address: true }), +}); + +export const nativeInventoryViewContext = { + source_provider: providerSchema.nullable(), + source_analysis_profile_digest: digestSchema.nullable(), + source_parameters: jsonValueSchema, +}; + +export const nativeInventorySummarySchema = z.strictObject({ + collection: z.enum(["procedures", "strings", "names"]), + row_count: z.number().int().nonnegative(), + discovery_coverage: z.literal("unknown"), + selectable_fields: z.array(fieldSchema), +}); + +const checkedNames = new WeakSet(); +const checkedStrings = new WeakSet(); +type InventoryRecord = z.infer; + +/** Cache only a successful, representation-preserving validation of owned immutable facts. */ +const readRows = ( + parent: AnalysisViewParent, + strings: boolean, +): Result => { + const checked = strings ? checkedStrings : checkedNames; + const source = parent.normalizedResult; + if (Array.isArray(source) && checked.has(source)) return ok(source); + const parsed = strings + ? parseAnalysisViewResult(nativeInventoryRowsSchemas.strings, parent) + : parseAnalysisViewResult(nativeInventoryRowsSchemas.names, parent); + if (!parsed.ok) return parsed; + if ( + Array.isArray(source) && + parent.sealedResult === true && + canBorrowParsedJsonSnapshot(source, parsed.value) + ) { + checked.add(source); + return ok(source as readonly InventoryRecord[]); + } + return parsed; +}; + +/** Project already completed provider observations without another provider read or sort. */ +export const projectNativeInventoryView = ( + parent: AnalysisViewParent, + view: AnalysisViewRequest, +): Result => { + if ( + !isNativeInventoryOperation(parent.operation) || + (view.kind !== "summary" && view.kind !== "native-inventory") + ) + return err( + analysisViewInputError([ + { + path: ["view"], + reason: "invalid_value", + message: + "Native inventory Evidence supports summary and native-inventory views.", + }, + ]), + ); + const strings = + parent.operation === "list_strings" || + parent.operation === "search_strings"; + const parsed = readRows(parent, strings); + if (!parsed.ok) return parsed; + const rows = parsed.value; + const shared = { + parent_evidence_id: parent.evidenceId, + parent_operation: parent.operation, + parent_digest: parent.evidenceId.slice(3), + artifact: parent.artifact ?? null, + source_provider: parent.provider ?? null, + source_analysis_profile_digest: parent.analysisProfileDigest ?? null, + source_parameters: jsonValueSchema.parse(parent.parameters ?? null), + limitations: boundedViewMessages( + parent.limitations, + parent.evidenceId, + "limitations", + ), + unknowns: [ + "Counts and paging describe the retained provider-reported rows, not complete discovery of every program object. Missing matches do not establish absence outside that observation.", + ...(parent.artifact == null + ? ["No artifact identity was bound to the source Evidence."] + : []), + ], + }; + if (view.kind === "summary") + return ok({ + ...shared, + kind: "summary", + view, + summary: { + collection: strings + ? "strings" + : parent.operation === "list_names" + ? "names" + : "procedures", + row_count: rows.length, + discovery_coverage: "unknown", + selectable_fields: fieldSchema.options, + }, + coverage: completeWithinViewCoverage(rows.length, rows.length), + }); + // Normalize the selection once: repeated caller fields must not multiply + // work or temporary entries for every retained row. Keep the original view. + const fields = + view.fields === undefined + ? undefined + : [...new Set(view.fields)].filter((field) => field !== "address"); + const selectingValue = fields?.includes("value") === true; + const selected = rows.slice(view.offset, view.offset + view.limit); + return ok({ + ...shared, + kind: "native-inventory", + view, + rows: selected.map((record, index) => ({ + source_index: view.offset + index, + record: + fields === undefined + ? record + : { + address: record.address, + ...(selectingValue && record.string !== undefined + ? { string: record.string } + : {}), + ...(selectingValue && record.decoding !== undefined + ? { decoding: record.decoding } + : {}), + ...Object.fromEntries( + fields + .filter((field) => Object.hasOwn(record, field)) + .map((field) => [field, record[field]]), + ), + }, + })), + coverage: pageViewCoverage(view.offset, selected.length, rows.length), + }); +}; diff --git a/src/domain/native/nativeInventory.ts b/src/domain/native/nativeInventory.ts new file mode 100644 index 0000000000..eee10b55b3 --- /dev/null +++ b/src/domain/native/nativeInventory.ts @@ -0,0 +1,57 @@ +import { z } from "zod"; +import { analysisStringSchema } from "../hopperValues.js"; + +/** Canonical named native observations; same-address alternatives stay separate. */ +export const addressedValueSchema = z.object({ + address: z.string(), + value: z.string(), + symbol: z + .object({ + primary: z.boolean(), + dynamic: z.boolean(), + external: z.boolean(), + type: z.string(), + source: z.enum(["default", "analysis", "ai", "imported", "user_defined"]), + }) + .exactOptional(), + procedure: z + .object({ + external: z.boolean(), + thunk: z.boolean(), + thunk_target: z.string().nullable(), + }) + .exactOptional(), + string: z + .object({ + encoding: z.string(), + termination: z.enum(["missing", "present_or_not_required"]), + byte_length: z.number().int().min(0), + }) + .exactOptional(), +}); + +/** Canonical row arrays shared by complete delivery and retained source validation. */ +export const nativeInventoryRowsSchemas = { + names: z.array(addressedValueSchema), + strings: z.array(analysisStringSchema), +}; + +export const NATIVE_INVENTORY_OPERATIONS = [ + "list_procedures", + "search_procedures", + "list_strings", + "search_strings", + "list_names", +] as const; + +export const nativeInventoryOperationSchema = z.enum( + NATIVE_INVENTORY_OPERATIONS, +); +export type NativeInventoryOperation = z.infer< + typeof nativeInventoryOperationSchema +>; + +export const isNativeInventoryOperation = ( + operation: string, +): operation is NativeInventoryOperation => + NATIVE_INVENTORY_OPERATIONS.some((supported) => supported === operation); diff --git a/src/server/createServer.ts b/src/server/createServer.ts index 985e2e075a..02ae613a71 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -885,6 +885,7 @@ const registerBinaryAnalysisTools = ({ activeTarget, recordEvidence, recordUnknown, + evidenceById, withAdmittedAnalysis, }; registerOfficialTools(server, analysisOptions); diff --git a/src/server/promptCompletion.ts b/src/server/promptCompletion.ts index e6be80ff58..81306aeb23 100644 --- a/src/server/promptCompletion.ts +++ b/src/server/promptCompletion.ts @@ -6,7 +6,7 @@ import type { JsonValue } from "../domain/jsonValue.js"; import type { AnalysisOperationPort } from "../application/AnalysisProvider.js"; import type { BinarySessionPort } from "../application/binary/BinarySessionPort.js"; import type { PromptCompletionKind } from "../contracts/promptContracts.js"; -import { addressedValue } from "../contracts/toolOutputSchemaPrimitives.js"; +import { addressedValueSchema } from "../domain/native/nativeInventory.js"; import { currentMcpRequestSignal } from "./EvidenceMcpServer.js"; import { parseArtifactInventoryEvidence } from "../domain/artifactInventoryEvidence.js"; import { EvidenceIntegrityError } from "../domain/evidenceErrors.js"; @@ -14,7 +14,7 @@ import { parseProcessCapture } from "../domain/process/processCaptureParsing.js" const documentListSchema = z.array(z.string().min(1)); const procedureInventorySchema = z.array( - addressedValue.pick({ address: true, value: true }), + addressedValueSchema.pick({ address: true, value: true }), ); const providerStatusSchema = z.object({ analysis_provider_candidates: z.array( diff --git a/src/server/registerEvidenceTools.ts b/src/server/registerEvidenceTools.ts index b3b2358d1f..8fc3867d64 100644 --- a/src/server/registerEvidenceTools.ts +++ b/src/server/registerEvidenceTools.ts @@ -120,6 +120,12 @@ export const registerEvidenceTools = ( : { analysisProfile: execution.value.analysisProfile }), rawResult: execution.value.rawResult, limitations: execution.value.limitations, + ...(execution.value.relationshipCoverage === undefined + ? {} + : { + relationshipCoverage: + execution.value.relationshipCoverage, + }), locations: execution.value.locations, evidenceLinks: sourceEvidence.map(({ evidence_id: id }) => id), }, diff --git a/src/server/registerOfficialTools.ts b/src/server/registerOfficialTools.ts index 8cc603b8fc..72013fe50a 100644 --- a/src/server/registerOfficialTools.ts +++ b/src/server/registerOfficialTools.ts @@ -1,145 +1,27 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; -import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; -import type { ServerContext } from "@modelcontextprotocol/server"; - -import type { - AnalysisExecution, - AnalysisOperationPort, -} from "../application/AnalysisProvider.js"; -import type { ProgressReporter } from "../application/ProgressReporter.js"; import { OFFICIAL_TOOL_CONTRACTS } from "../contracts/officialToolContracts.js"; -import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; -import type { AnalysisError } from "../domain/analysisErrorBase.js"; -import { createEvidence } from "../domain/evidence.js"; -import { jsonObjectSchema, type JsonValue } from "../domain/jsonValue.js"; -import type { Result } from "../domain/result.js"; -import type { Logger } from "pino"; -import { mcpProgressReporter } from "./mcpProgress.js"; -import { logToolExecution } from "./toolLogging.js"; -import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; +import { NATIVE_INVENTORY_OPERATIONS } from "../domain/native/nativeInventory.js"; +import { registerEvidenceTools } from "./registerEvidenceTools.js"; +import { + retainedSummaryDelivery, + withoutDeliveryDetail, + type EvidenceToolFamilyOptions, +} from "./retainedSummaryDelivery.js"; -/** Optional session services used by direct tool registration. */ -export interface OfficialToolRegistration { - readonly logger: Logger; - readonly activeTarget: (() => BinaryTarget | undefined) | undefined; - readonly recordEvidence: EvidenceWriter["recordEvidence"] | undefined; - readonly withAdmittedAnalysis: WithAdmittedAnalysis; -} +const SUMMARY_OPERATIONS: ReadonlySet = new Set( + NATIVE_INVENTORY_OPERATIONS, +); +const withoutDetail = withoutDeliveryDetail(SUMMARY_OPERATIONS); -/** Register direct bridge proxies, preserving MCP cancellation and typed errors. */ +/** Direct tools share Evidence recording/projection with the other provider-backed tools. */ export const registerOfficialTools = ( server: EvidenceMcpServer, - options: OfficialToolRegistration, + options: EvidenceToolFamilyOptions, ): void => { - for (const contract of OFFICIAL_TOOL_CONTRACTS) { - registerOfficialTool(server, contract, { - logger: options.logger, - activeTarget: options.activeTarget, - recordEvidence: options.recordEvidence, - withAdmittedAnalysis: options.withAdmittedAnalysis, - }); - } -}; - -const registerOfficialTool = ( - server: EvidenceMcpServer, - contract: (typeof OFFICIAL_TOOL_CONTRACTS)[number], - registration: { - readonly logger: Logger; - readonly activeTarget: (() => BinaryTarget | undefined) | undefined; - readonly recordEvidence: EvidenceWriter["recordEvidence"] | undefined; - readonly withAdmittedAnalysis: WithAdmittedAnalysis; - }, -): void => { - server.registerTool( - contract.name, - toolRegistrationOptions(contract), - async (input: unknown, context: ServerContext) => { - const admitted = await registration.withAdmittedAnalysis( - contract.name, - context.mcpReq.signal, - async (admittedAnalysis) => { - const arguments_ = jsonObjectSchema.parse( - contract.inputSchema.parse(input), - ); - const progress = mcpProgressReporter(context); - const result = await runOfficialOperation( - admittedAnalysis, - contract, - arguments_, - { - logger: registration.logger, - signal: context.mcpReq.signal, - progress, - }, - ); - if (!result.ok) { - return server.delivery.toCallToolResult(result, contract); - } - const evidence = createEvidence( - result.value.subject ?? registration.activeTarget?.(), - result.value.provider, - { - operation: contract.name, - parameters: arguments_, - result: result.value.result, - ...(result.value.analysisProfile === undefined - ? {} - : { analysisProfile: result.value.analysisProfile }), - rawResult: result.value.rawResult, - limitations: result.value.limitations, - ...(result.value.relationshipCoverage === undefined - ? {} - : { - relationshipCoverage: result.value.relationshipCoverage, - }), - locations: result.value.locations, - }, - ); - const recorded = registration.recordEvidence?.(evidence); - return server.delivery.toEvidenceToolResult( - evidence, - contract, - recorded, - ); - }, - ); - return admitted.ok - ? admitted.value - : server.delivery.toCallToolResult(admitted, contract); - }, - ); -}; - -const runOfficialOperation = async ( - analysis: AnalysisOperationPort, - contract: (typeof OFFICIAL_TOOL_CONTRACTS)[number], - arguments_: Readonly>, - execution: { - readonly logger: Logger; - readonly signal: AbortSignal; - readonly progress: ProgressReporter; - }, -): Promise> => { - await execution.progress.report({ - phase: contract.name, - completed: 0, - total: 1, - message: "started", + registerEvidenceTools(server, OFFICIAL_TOOL_CONTRACTS, { + ...options, + ...retainedSummaryDelivery(SUMMARY_OPERATIONS, options), + parametersForExecution: withoutDetail, + parametersForEvidence: withoutDetail, }); - const result = await logToolExecution(execution.logger, contract.name, () => - analysis.execute(contract.name, arguments_, { - signal: execution.signal, - progress: execution.progress, - }), - ); - await execution.progress.report({ - phase: contract.name, - completed: 1, - total: 1, - message: result.ok ? "completed" : "failed", - terminal: true, - }); - return result; }; diff --git a/src/server/toolResult.ts b/src/server/toolResult.ts index 63a45f872f..ddaf45ab9b 100644 --- a/src/server/toolResult.ts +++ b/src/server/toolResult.ts @@ -1,3 +1,4 @@ +import { isNativeInventoryOperation } from "../domain/native/nativeInventory.js"; import { INSPECT_ANALYSIS_VIEW_OPERATION } from "../domain/analysisView/analysisViewErrors.js"; import type { CallToolResult } from "@modelcontextprotocol/server"; import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; @@ -26,13 +27,19 @@ export const createToolResultDelivery = ( new ToolResultDelivery(maximumResponseBytes ?? STDIO_DEFAULT_MAX_BUFFER_SIZE); /** Offer a named, schema-validated source page only for retained native analysis. */ -const nativeEvidenceContinuation = ( +const retainedEvidenceContinuation = ( evidence: Evidence, ): Readonly> => { - if (evidence.operation !== "analyze_function") return {}; + const view = + evidence.operation === "analyze_function" + ? { kind: "native", facet: "pseudocode", offset: 0 } + : isNativeInventoryOperation(evidence.operation) + ? { kind: "native-inventory", offset: 0, limit: 64 } + : undefined; + if (view === undefined) return {}; const input = inspectAnalysisViewInputSchema.safeParse({ source: { kind: "retained-evidence", evidence_id: evidence.evidence_id }, - view: { kind: "native", facet: "pseudocode", offset: 0 }, + view, }); if (!input.success) return {}; return { @@ -81,7 +88,7 @@ export class ToolResultDelivery { kind: "retained-evidence", evidence_id: evidence.evidence_id, }, - ...nativeEvidenceContinuation(evidence), + ...retainedEvidenceContinuation(evidence), }), }); } @@ -171,7 +178,7 @@ export class ToolResultDelivery { const retainedEvidence = recovery.evidence_reference !== undefined; const recoveryAdvice = recovery.continuation !== undefined - ? "Call inspect_analysis_view with the supplied continuation arguments to read retained pseudocode. Follow coverage.next_offset for subsequent pages; reduce the page limit if needed, or export the complete retained Evidence." + ? "Call inspect_analysis_view with the supplied continuation arguments to read retained analysis. Follow coverage.next_offset for subsequent pages; reduce the page limit if needed, or export the complete retained Evidence." : retainedEvidence ? "Export retained evidence to consume the complete analysis, or use complete CLI JSON output." : encoded.constraint === "string-length" diff --git a/tests/boundary/mcp/nativeInventoryViewsMcp.test.ts b/tests/boundary/mcp/nativeInventoryViewsMcp.test.ts new file mode 100644 index 0000000000..1f43dc2163 --- /dev/null +++ b/tests/boundary/mcp/nativeInventoryViewsMcp.test.ts @@ -0,0 +1,329 @@ +import { Ajv2020 } from "ajv/dist/2020.js"; +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { expect, it, onTestFinished } from "vitest"; +import { z } from "zod"; +import { createAnalysisExecution } from "../../../src/application/AnalysisProvider.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { inspectAnalysisViewInputSchema } from "../../../src/domain/analysisView/analysisView.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { NATIVE_INVENTORY_OPERATIONS } from "../../../src/domain/native/nativeInventory.js"; +import { ok } from "../../../src/domain/result.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { connectGhidraMcp } from "./ghidraMcpHarness.js"; + +const procedureRows = [ + { + address: "0x1000", + value: "first", + procedure: { external: false, thunk: false, thunk_target: null }, + }, + { + address: "0x2000", + value: "second", + procedure: { external: false, thunk: false, thunk_target: null }, + }, +]; +const stringRows = [ + { + address: "0x3000", + value: "hello", + string: { + encoding: "utf-8", + termination: "present_or_not_required", + byte_length: 6, + }, + }, +]; +const nameRows = [ + { + address: "0x1000", + value: "first", + symbol: { + primary: true, + dynamic: false, + external: false, + type: "function", + source: "analysis", + }, + }, +]; + +it("shares complete/summary source identity and advertised contracts across all five inventory tools", async () => { + const harness = await connectGhidraMcp( + "native-inventory-views", + (operation) => { + if (operation === "search_procedures") + return Promise.resolve( + ok(procedureRows.map(({ address, value }) => ({ address, value }))), + ); + if (operation === "search_strings") + return Promise.resolve( + ok(stringRows.map(({ address, value }) => ({ address, value }))), + ); + const result = + operation === "list_names" + ? nameRows + : operation.endsWith("strings") + ? stringRows + : procedureRows; + return Promise.resolve(ok(result)); + }, + ); + onTestFinished(() => harness.close()); + const tools = (await harness.mcp.listTools()).tools; + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const viewTool = tools.find((tool) => tool.name === "inspect_analysis_view"); + if (viewTool?.outputSchema === undefined) + throw new Error("Missing view schema"); + const viewInputSchema: Record = viewTool.inputSchema; + const viewOutputSchema: Record = viewTool.outputSchema; + expect(ajv.validateSchema(viewInputSchema)).toBe(true); + expect(ajv.validateSchema(viewOutputSchema)).toBe(true); + for (const operation of NATIVE_INVENTORY_OPERATIONS) { + const contract = toolContract(operation); + const advertised = tools.find((tool) => tool.name === operation); + if (advertised?.outputSchema === undefined) + throw new Error("Missing operation schema"); + const inputSchema: Record = advertised.inputSchema; + const outputSchema: Record = advertised.outputSchema; + expect(ajv.validateSchema(inputSchema)).toBe(true); + expect(ajv.validateSchema(outputSchema)).toBe(true); + const parameters = operation.startsWith("search_") + ? { pattern: ".", mode: "regex" } + : {}; + const completeReply = await harness.mcp.callTool({ + name: operation, + arguments: parameters, + }); + if (completeReply.isError) throw new Error(JSON.stringify(completeReply)); + expect( + ajv.validate(outputSchema, completeReply.structuredContent), + JSON.stringify(ajv.errors), + ).toBe(true); + const complete = parseEvidence( + contract.outputSchema.parse(completeReply.structuredContent), + ); + if (!Array.isArray(complete.normalized_result)) + throw new Error("Expected complete row array"); + expect(complete.parameters).not.toHaveProperty("detail"); + const summaryReply = await harness.mcp.callTool({ + name: operation, + arguments: { ...parameters, detail: "summary" }, + }); + if (summaryReply.isError) throw new Error(JSON.stringify(summaryReply)); + expect( + ajv.validate(outputSchema, summaryReply.structuredContent), + JSON.stringify(ajv.errors), + ).toBe(true); + const summary = parseEvidence( + contract.outputSchema.parse(summaryReply.structuredContent), + ); + expect(summary).toMatchObject({ + operation: "inspect_analysis_view", + evidence_links: [complete.evidence_id], + normalized_result: { + parent_evidence_id: complete.evidence_id, + source_provider: complete.provider, + source_analysis_profile_digest: + complete.analysis_profile?.digest ?? null, + source_parameters: complete.parameters, + summary: { + row_count: (complete.normalized_result as unknown[]).length, + discovery_coverage: "unknown", + }, + }, + }); + expect(summary.limitations).toEqual( + expect.arrayContaining(complete.limitations), + ); + expect(harness.session.evidenceById(complete.evidence_id)).toEqual( + complete, + ); + const allRows = []; + for ( + let offset = 0; + offset < (complete.normalized_result as unknown[]).length; + offset++ + ) { + const pageReply = await harness.mcp.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { + kind: "retained-evidence", + evidence_id: complete.evidence_id, + }, + view: { kind: "native-inventory", offset, limit: 1 }, + }, + }); + if (pageReply.isError) throw new Error(JSON.stringify(pageReply)); + expect( + ajv.validate(viewOutputSchema, pageReply.structuredContent), + JSON.stringify(ajv.errors), + ).toBe(true); + const page = toolContract("inspect_analysis_view").outputSchema.parse( + pageReply.structuredContent, + ).normalized_result; + if (page.kind !== "native-inventory") + throw new Error("Expected inventory page"); + expect(page.rows[0]?.source_index).toBe(offset); + allRows.push(...page.rows.map((row) => row.record)); + } + expect(allRows).toEqual(complete.normalized_result); + } + expect( + harness.calls.filter((name) => + NATIVE_INVENTORY_OPERATIONS.some((operation) => operation === name), + ), + ).toHaveLength(10); +}); + +it("recovers an oversized complete inventory through a working retained page without another provider read", async () => { + const items = Array.from({ length: 1000 }, (_, index) => ({ + address: `0x${(0x1000 + index * 0x2000).toString(16)}`, + value: "x".repeat(7000), + string: { + encoding: "utf-8", + termination: "present_or_not_required", + byte_length: 7001, + }, + })); + const harness = await connectGhidraMcp("native-inventory-capacity", () => + Promise.resolve(ok(items)), + ); + onTestFinished(() => harness.close()); + const complete = await harness.mcp.callTool({ + name: "list_strings", + arguments: {}, + }); + expect(complete.isError).toBe(true); + const error = parseMcpToolError(complete).error; + expect(error.code).toBe("resource_constraint"); + const recovery = z + .object({ + reported_limits: z.object({ + boundary: z.literal("mcp-response"), + continuation: z.unknown(), + }), + }) + .parse(error.details).reported_limits; + const continuation = z + .object({ + name: z.literal("inspect_analysis_view"), + arguments: inspectAnalysisViewInputSchema, + }) + .parse(recovery.continuation); + const pageReply = await harness.mcp.callTool(continuation); + if (pageReply.isError) throw new Error(JSON.stringify(pageReply)); + const page = toolContract("inspect_analysis_view").outputSchema.parse( + pageReply.structuredContent, + ).normalized_result; + if (page.kind !== "native-inventory") + throw new Error("Expected inventory page"); + expect(page.rows.map((row) => row.record)).toEqual(items.slice(0, 64)); + expect(page.coverage).toMatchObject({ total: 1000, next_offset: 64 }); + expect(harness.calls).toEqual(["list_strings"]); + const summaryReply = await harness.mcp.callTool({ + name: "list_strings", + arguments: { detail: "summary" }, + }); + if (summaryReply.isError) throw new Error(JSON.stringify(summaryReply)); + expect( + parseEvidence(summaryReply.structuredContent).normalized_result, + ).toMatchObject({ summary: { row_count: 1000 } }); +}); + +it("refuses summary before provider work when no retaining session is configured", async () => { + let calls = 0; + const server = createServer({ + kind: "fixed", + analysis: { + execute: async () => { + calls++; + return ok( + createAnalysisExecution(procedureRows, { + id: "fixture", + name: "Fixture", + version: "1", + }), + ); + }, + }, + }); + const client = new Client({ + name: "unretained-native-summary", + version: "1", + }); + onTestFinished(async () => { + await client.close(); + await server.close(); + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + const reply = await client.callTool({ + name: "list_procedures", + arguments: { detail: "summary" }, + }); + expect(reply.isError).toBe(true); + expect(parseMcpToolError(reply).error.code).toBe("capability_unavailable"); + expect(calls).toBe(0); +}); + +it("projects imported inventories with an unknown artifact without inventing a path or opening a provider", async () => { + const session = createTestBinarySession(() => { + throw new Error("Retained views must not open a provider"); + }); + const source = createEvidence( + undefined, + { id: "borrowed.native", name: "Borrowed engine", version: "1" }, + { + operation: "list_procedures", + parameters: {}, + result: procedureRows, + limitations: [ + "The engine did not bind this observation to an artifact digest.", + ], + }, + ); + const recorded = session.recordEvidence(source); + if (!recorded.ok) throw recorded.error; + const server = createServer({ kind: "session", session }); + const client = new Client({ name: "unbound-native-views", version: "1" }); + onTestFinished(async () => { + await client.close(); + await server.close(); + const closed = await session.close(); + if (!closed.ok) throw closed.error; + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + for (const view of [ + { kind: "summary" }, + { kind: "native-inventory", fields: [] }, + ]) { + const reply = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "retained-evidence", evidence_id: source.evidence_id }, + view, + }, + }); + if (reply.isError) throw new Error(JSON.stringify(reply)); + const evidence = parseEvidence(reply.structuredContent); + expect(evidence).toMatchObject({ + subject: null, + locations: [], + evidence_links: [source.evidence_id], + normalized_result: { + artifact: null, + source_provider: source.provider, + source_parameters: {}, + }, + }); + } +}); From 074768901f3cdc8205b61773fb1faf6f6c7b3ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E4=BD=B3=E6=B5=A9Alben?= <35291372+XiaoBinGan@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:12:55 +0800 Subject: [PATCH 05/13] fix(pwndbg): separate the words in the debugger-context limit message (#5372) The 64 MiB reply-budget refusal rendered as "exceeds64 MiB" because the message word and the limit were joined. The pwntools sibling names the same budget as "exceeds the 64 MiB reply budget"; use that wording. The output-limit path had no producer regression, so the pwndbg suite gains one that drives the real 64 MiB boundary, and the suite is now part of test:python instead of running only when invoked by hand. Co-authored-by: XiaoBinGan --- bridge/pwndbg/core_context.py | 2 +- bridge/pwndbg/test_core_context.py | 13 +++++++++++-- package.json | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/bridge/pwndbg/core_context.py b/bridge/pwndbg/core_context.py index 498c89b3b6..ffdbc72ab4 100644 --- a/bridge/pwndbg/core_context.py +++ b/bridge/pwndbg/core_context.py @@ -95,7 +95,7 @@ def main(): try: encoded = json.dumps(reply, ensure_ascii=True, allow_nan=False).encode("utf-8") if len(encoded) > OUTPUT_BYTES: - encoded = json.dumps({"ok": False, "reason": "output-limit", "message": "Complete debugger context exceeds64 MiB reply budget."}).encode("utf-8") + encoded = json.dumps({"ok": False, "reason": "output-limit", "message": "Complete debugger context exceeds the 64 MiB reply budget."}).encode("utf-8") descriptor = os.open(request["reply_path"], os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) with os.fdopen(descriptor, "wb") as handle: handle.write(encoded) diff --git a/bridge/pwndbg/test_core_context.py b/bridge/pwndbg/test_core_context.py index 95f5611733..3e61455a91 100644 --- a/bridge/pwndbg/test_core_context.py +++ b/bridge/pwndbg/test_core_context.py @@ -14,7 +14,7 @@ BRIDGE = Path(__file__).with_name("core_context.py") class CoreContextTests(unittest.TestCase): - def inspect(self, failure=None, connection="core", symbols=None, architecture="x86_64-little-endian"): + def inspect(self, failure=None, connection="core", symbols=None, architecture="x86_64-little-endian", objfile="recorded-executable"): commands, reads, queries = [], [], [] state = {"executable": None, "symbols": None, "connection": None} gdb = types.ModuleType("gdb") @@ -50,7 +50,7 @@ class CoreContextTests(unittest.TestCase): def maps(): self.assertIsNone(state["executable"], "Helper executable must be discarded before mapping queries") queries.append("maps") - return [types.SimpleNamespace(start=4096, end=8192, offset=0, flags=5, objfile="recorded-executable")] + return [types.SimpleNamespace(start=4096, end=8192, offset=0, flags=5, objfile=objfile)] pwndbg.aglib.vmmap.get = maps modules = {module.__name__: module for module in [ @@ -104,6 +104,15 @@ class CoreContextTests(unittest.TestCase): self.assertEqual(reads, []) self.assertEqual(queries, []) + def test_output_limit_message_separates_the_words_and_names_the_limit(self): + result, _, reads, queries = self.inspect(objfile="m" * (64 * 1024 * 1024)) + self.assertFalse(result["ok"]) + self.assertEqual(result["reason"], "output-limit") + self.assertNotIn("exceeds64", result["message"]) + self.assertEqual(result["message"], "Complete debugger context exceeds the 64 MiB reply budget.") + self.assertEqual(reads, []) + self.assertEqual(queries, ["maps"]) + def test_wrong_connection_or_remaining_symbols_never_queries_mappings(self): for options in [{"connection": "native"}, {"symbols": "unexpected-symbols"}]: with self.subTest(options=options): diff --git a/package.json b/package.json index 5f77580431..e154ff398a 100644 --- a/package.json +++ b/package.json @@ -139,7 +139,7 @@ "test": "npm run test:with-build", "test:with-build": "npm run test:prepare && vitest run", "test:run": "vitest run", - "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py bridge/mitmproxy/test_capture.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v", + "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py bridge/pwndbg/test_core_context.py bridge/mitmproxy/test_capture.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v", "test:local": "node scripts/run-development-tests.mjs local", "test:fast": "node scripts/ci/test-group.mjs source", "test:built": "node scripts/ci/test-group.mjs built", From d532b18123916a9e88f98c7880ba4f984e23d715 Mon Sep 17 00:00:00 2001 From: Liu Jie Date: Tue, 20 Oct 2026 22:15:09 +1300 Subject: [PATCH 06/13] fix(doctor): propagate Kuna cancellation and preserve partial observations (#5241) Fixes #5106. Propagate the owned inspection signal, preserve typed cancellation and earlier failures, and retain completed installation observations through exact-owner cleanup retries. Co-authored-by: Liu Jie Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> --- docs/kuna-provider.md | 8 + src/doctorRuntime.ts | 1 + src/kuna/KunaCommand.ts | 16 +- src/kuna/KunaDoctor.ts | 20 ++- src/kuna/KunaInstallation.ts | 99 +++++++++++- src/kuna/KunaProvider.ts | 9 +- src/kuna/KunaSessionClient.ts | 9 +- tests/boundary/cli/doctorCancellation.test.ts | 78 ++++++--- .../providers/kuna/kunaDoctor.test.ts | 114 ++++++++++++- .../kuna/kunaHashOwnerPropagation.test.ts | 150 +++++++++++++++++- .../providers/kuna/kunaInstallation.test.ts | 112 ++++++++++++- .../kuna/kunaInstallationLifetime.test.ts | 8 +- 12 files changed, 569 insertions(+), 55 deletions(-) diff --git a/docs/kuna-provider.md b/docs/kuna-provider.md index 0a37b98c81..05e5a633bf 100644 --- a/docs/kuna-provider.md +++ b/docs/kuna-provider.md @@ -13,6 +13,14 @@ rea doctor --provider kuna Select it explicitly with `--provider kuna`, MCP's `open_binary.provider_id: "kuna"`, or `REA_ANALYSIS_PROVIDER=kuna`. With Kuna and another provider configured for the same target, `auto` reports `ambiguous` instead of choosing one. +Ordinary readiness cancellation returns `cancelled` (CLI SIGINT/SIGTERM exits +130/143); an earlier recorded timeout or physical failure keeps its diagnostic. +Completed version, hash and script observations remain in `kuna_installation` +with `status: "partial"` and `completed_hash` when available. Typed errors carry +this in `details.diagnostics`; doctor cleanup checks carry it in their `details`. +Omitted fields remain unobserved; availability remains unknown until readiness +completes. Cleanup retries keep both the observations and the exact file owner. + | Setting | Default | Meaning | | ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `REA_KUNA_COMMAND` | none (required) | Absolute path of the `kuna` executable. | diff --git a/src/doctorRuntime.ts b/src/doctorRuntime.ts index 2b3c1e1c08..54c320bd8d 100644 --- a/src/doctorRuntime.ts +++ b/src/doctorRuntime.ts @@ -63,6 +63,7 @@ export const createSystemDoctorHost = ( ), await inspectKunaProvider(environment, process.platform, { fileCloseResources, + signal: inspectionSignal, }), ], linuxDemoRuntimeCheck: () => diff --git a/src/kuna/KunaCommand.ts b/src/kuna/KunaCommand.ts index b4cc5a3de8..283a9aae7f 100644 --- a/src/kuna/KunaCommand.ts +++ b/src/kuna/KunaCommand.ts @@ -31,6 +31,7 @@ import type { } from "../process/ProviderProcess.js"; import { kunaChildEnvironment, + kunaInstallationFailureDiagnostics, type KunaInstallation, } from "./KunaInstallation.js"; import type { KunaDeadline } from "./KunaConfiguration.js"; @@ -99,21 +100,23 @@ export const kunaFileCleanupError = ( failure: RegularFileCleanupFailure, operation: string, ): AnalysisError => { - const cleanup = new ProviderCleanupError( - KUNA_PROVIDER_ID, - [failure.path], - { reason: message(failure.cleanupCause) }, - { cause: failure, operation }, - ); + const diagnostics = kunaInstallationFailureDiagnostics(failure); const cancellation = regularFileCancellation(failure); if (cancellation !== undefined) return new AnalysisCancelledError(operation, { cause: cancellation.reason, + diagnostics, cleanup: { reason: message(failure.cleanupCause), resources: [failure.path], }, }); + const cleanup = new ProviderCleanupError( + KUNA_PROVIDER_ID, + [failure.path], + { reason: message(failure.cleanupCause), ...diagnostics }, + { cause: failure, operation }, + ); if (failure.outcome.kind === "completed") return cleanup; const primaryCause = failure.outcome.cause; const primary = @@ -121,6 +124,7 @@ export const kunaFileCleanupError = ( ? primaryCause : new ProviderAdapterError(KUNA_PROVIDER_ID, operation, { cause: primaryCause, + diagnostics, userMessage: `Kuna ${operation} failed: ${message(primaryCause)}`, }); return analysisErrorWithCleanupFailure(primary, cleanup, operation); diff --git a/src/kuna/KunaDoctor.ts b/src/kuna/KunaDoctor.ts index 8a89b4e4a4..7c6e8b1840 100644 --- a/src/kuna/KunaDoctor.ts +++ b/src/kuna/KunaDoctor.ts @@ -9,8 +9,11 @@ import type { import { readKunaConfiguration } from "./KunaConfiguration.js"; import type { FileCloseResourceOwner } from "../application/AnalysisProvider.js"; import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js"; +import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import { inspectKunaInstallation, + kunaInstallationCancellation, + kunaInstallationFailureDiagnostics, type KunaReadinessFailure, } from "./KunaInstallation.js"; import { @@ -37,6 +40,7 @@ export const inspectKunaProvider = async ( platform: NodeJS.Platform = process.platform, options: { readonly fileCloseResources: FileCloseResourceOwner; + readonly signal?: AbortSignal; }, ): Promise => { let installation: Awaited>; @@ -44,9 +48,18 @@ export const inspectKunaProvider = async ( installation = await inspectKunaInstallation(environment, { platform, fileCloseResources: options.fileCloseResources, + ...(options.signal === undefined ? {} : { signal: options.signal }), }); } catch (cause: unknown) { - if (!(cause instanceof RegularFileCleanupFailure)) throw cause; + if (!(cause instanceof RegularFileCleanupFailure)) { + const cancellation = kunaInstallationCancellation(cause); + if (cancellation !== undefined) + throw new AnalysisCancelledError("doctor", { + cause: cancellation.reason, + diagnostics: cancellation.diagnostics, + }); + throw cause; + } return { id: KUNA_PROVIDER_ID, configured: environment.REA_KUNA_COMMAND !== undefined, @@ -61,7 +74,10 @@ export const inspectKunaProvider = async ( detail: `Kuna installation inspection could not confirm file cleanup: ${cause instanceof Error ? cause.message : String(cause)}`, remediation: null, classification: "config_drift", - details: { cleanup_incomplete: true }, + details: { + cleanup_incomplete: true, + ...kunaInstallationFailureDiagnostics(cause), + }, }, ], }; diff --git a/src/kuna/KunaInstallation.ts b/src/kuna/KunaInstallation.ts index 5c6604627d..aedab42f64 100644 --- a/src/kuna/KunaInstallation.ts +++ b/src/kuna/KunaInstallation.ts @@ -1,9 +1,12 @@ -import { assertRegularFileActive } from "../filesystem/RegularFile.js"; +import { + assertRegularFileActive, + RegularFileOperationCancelled, +} from "../filesystem/RegularFile.js"; import { constants } from "node:fs"; import { access, realpath, stat } from "node:fs/promises"; import { join } from "node:path"; -import type { JsonValue } from "../domain/jsonValue.js"; +import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import { err, ok, type Result } from "../domain/result.js"; import { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js"; @@ -14,12 +17,14 @@ import { import type { FileCloseResourceOwner } from "../application/AnalysisProvider.js"; import { RegularFileCleanupFailure, + RegularFileCancelled, regularFileCancellation, withOwnedRegularFile, } from "../filesystem/OwnedRegularFile.js"; import { execFileOutput, execFileOutputFailure, + ExecFileOutputCancelledError, } from "../process/ExecFileOutput.js"; import { readKunaConfiguration, @@ -75,6 +80,39 @@ export interface KunaInstallationOptions { readonly fileCloseResources: FileCloseResourceOwner; } +/** Cancellation facts normalized by the installation inspection that observed them. */ +interface KunaInstallationCancellation { + readonly reason: unknown; + readonly diagnostics?: Readonly>; +} + +// Annotations follow the exact failure's lifetime. Descriptor ownership stays +// with the supplied resource scope, independently of this weak metadata. +const installationFailureDiagnostics = new WeakMap< + Error, + Readonly> +>(); + +export const kunaInstallationFailureDiagnostics = ( + cause: unknown, +): Readonly> | undefined => + cause instanceof Error + ? installationFailureDiagnostics.get(cause) + : undefined; + +/** Consumers project the owner's facts without reinterpreting generic file outcomes. */ +export const kunaInstallationCancellation = ( + cause: unknown, +): KunaInstallationCancellation | undefined => { + const cancellation = regularFileCancellation(cause); + if (cancellation === undefined) return undefined; + const diagnostics = kunaInstallationFailureDiagnostics(cause); + return { + reason: cancellation.reason, + ...(diagnostics === undefined ? {} : { diagnostics }), + }; +}; + /** One installation inspection's file owners and their originating errors. */ export interface KunaFileCloseScope { readonly owner: FileCloseResourceOwner; @@ -150,6 +188,7 @@ export const inspectKunaInstallation = async ( : { setting: configuration.error.setting }), }); const { command } = configuration.value; + assertRegularFileActive(options.signal); let executable: string; try { @@ -162,7 +201,6 @@ export const inspectKunaInstallation = async ( { command }, ); } catch (cause: unknown) { - assertRegularFileActive(options.signal); return failure( "executable_missing", `Cannot execute REA_KUNA_COMMAND ${command}: ${message(cause)}. ${KUNA_COMMAND_REMEDIATION}`, @@ -170,6 +208,8 @@ export const inspectKunaInstallation = async ( ); } + assertRegularFileActive(options.signal); + const deadline = options.probeTimeoutMs ?? VERSION_PROBE_TIMEOUT_MS; let stdout: string; try { @@ -182,7 +222,8 @@ export const inspectKunaInstallation = async ( ...(options.signal === undefined ? {} : { signal: options.signal }), })); } catch (cause: unknown) { - assertRegularFileActive(options.signal); + if (cause instanceof ExecFileOutputCancelledError) + throw new RegularFileOperationCancelled(cause.cause); const output = execFileOutputFailure(cause); return failure( "version_unresolved", @@ -208,6 +249,11 @@ export const inspectKunaInstallation = async ( let fingerprint: { readonly sha256: string; readonly bytes: number }; let script: boolean; + const observed: { + phase: "fingerprint" | "shebang"; + completed_hash?: { readonly sha256: string; readonly bytes: number }; + script?: boolean; + } = { phase: "fingerprint" }; try { ({ fingerprint, script } = await options.fileCloseResources.run( async () => { @@ -216,20 +262,57 @@ export const inspectKunaInstallation = async ( retainFailedClose: (owner) => options.fileCloseResources.retain(owner, executable), }); + observed.completed_hash = fingerprint; + assertRegularFileActive(options.signal); + observed.phase = "shebang"; const script = await startsWithShebang( executable, options.fileCloseResources, + options.signal, ); + observed.script = script; assertRegularFileActive(options.signal); return { fingerprint, script }; }, )); } catch (cause: unknown) { + const cancellation = regularFileCancellation(cause); if ( - cause instanceof RegularFileCleanupFailure || - regularFileCancellation(cause) !== undefined - ) + cause instanceof Error && + (cause instanceof RegularFileCleanupFailure || cancellation !== undefined) + ) { + const outcome = + cause instanceof RegularFileCancelled || + cause instanceof RegularFileCleanupFailure + ? cause.outcome + : undefined; + const completed = + outcome?.kind === "completed" ? outcome.value : undefined; + const completedHash = + observed.completed_hash ?? + (observed.phase === "fingerprint" && outcome?.kind === "completed" + ? jsonValueSchema.parse(completed) + : undefined); + const completedScript = + observed.script ?? + (observed.phase === "shebang" && typeof completed === "boolean" + ? completed + : undefined); + installationFailureDiagnostics.set(cause, { + kuna_installation: { + status: "partial", + command, + executable, + version, + verified: version === KUNA_VERIFIED_VERSION, + ...(completedHash === undefined + ? {} + : { completed_hash: completedHash }), + ...(completedScript === undefined ? {} : { script: completedScript }), + }, + }); throw cause; + } return failure( "executable_missing", `Cannot fingerprint ${executable}: ${message(stableFileHashCause(cause))}`, @@ -250,11 +333,13 @@ export const inspectKunaInstallation = async ( const startsWithShebang = ( path: string, fileCloseResources: FileCloseResourceOwner, + signal?: AbortSignal, ): Promise => withOwnedRegularFile( path, { symlinks: "follow", + ...(signal === undefined ? {} : { signal }), retainFailedClose: (owner) => fileCloseResources.retain(owner, path), }, async (handle) => { diff --git a/src/kuna/KunaProvider.ts b/src/kuna/KunaProvider.ts index a66c5b7a43..02f6ca8db5 100644 --- a/src/kuna/KunaProvider.ts +++ b/src/kuna/KunaProvider.ts @@ -15,10 +15,7 @@ import { } from "../domain/analysisProfile.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { err, ok } from "../domain/result.js"; -import { - RegularFileCleanupFailure, - regularFileCancellation, -} from "../filesystem/OwnedRegularFile.js"; +import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js"; import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import { kunaFileCleanupError, @@ -31,6 +28,7 @@ import { import { createKunaFileCloseScope, inspectKunaInstallation, + kunaInstallationCancellation, } from "./KunaInstallation.js"; import { KunaProjectCache } from "./KunaProjectCache.js"; import { @@ -193,7 +191,7 @@ export class KunaProvider implements AnalysisProviderCandidate { fileCloseResources: fileCloseScope.owner, }); } catch (cause: unknown) { - const cancellation = regularFileCancellation(cause); + const cancellation = kunaInstallationCancellation(cause); if ( !(cause instanceof RegularFileCleanupFailure) && cancellation !== undefined @@ -201,6 +199,7 @@ export class KunaProvider implements AnalysisProviderCandidate { return err( new AnalysisCancelledError("kuna:profile", { cause: cancellation.reason, + diagnostics: cancellation.diagnostics, }), ); if (!(cause instanceof RegularFileCleanupFailure)) throw cause; diff --git a/src/kuna/KunaSessionClient.ts b/src/kuna/KunaSessionClient.ts index cc75416f4d..a95d4845da 100644 --- a/src/kuna/KunaSessionClient.ts +++ b/src/kuna/KunaSessionClient.ts @@ -44,6 +44,7 @@ import { kunaFunctionDeadline } from "./KunaConfiguration.js"; import { createKunaFileCloseScope, inspectKunaInstallation, + kunaInstallationCancellation, type KunaInstallation, } from "./KunaInstallation.js"; import { @@ -76,10 +77,7 @@ import { type KunaProjectFileSpan, } from "./KunaProjectReader.js"; import { kunaUnavailable } from "./KunaUnavailable.js"; -import { - RegularFileCleanupFailure, - regularFileCancellation, -} from "../filesystem/OwnedRegularFile.js"; +import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js"; import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import { kunaQueryFailureForCaller } from "./KunaQueryErrors.js"; @@ -1410,7 +1408,7 @@ export class KunaSessionClient implements AnalysisClient { ); return ok(inspected.value); } catch (cause: unknown) { - const cancellation = regularFileCancellation(cause); + const cancellation = kunaInstallationCancellation(cause); if ( !(cause instanceof RegularFileCleanupFailure) && cancellation !== undefined @@ -1418,6 +1416,7 @@ export class KunaSessionClient implements AnalysisClient { return err( new AnalysisCancelledError("health", { cause: cancellation.reason, + diagnostics: cancellation.diagnostics, }), ); if (!(cause instanceof RegularFileCleanupFailure)) diff --git a/tests/boundary/cli/doctorCancellation.test.ts b/tests/boundary/cli/doctorCancellation.test.ts index 6b8c861dac..7904d0aa5c 100644 --- a/tests/boundary/cli/doctorCancellation.test.ts +++ b/tests/boundary/cli/doctorCancellation.test.ts @@ -11,40 +11,57 @@ import { type OwnedProcessGroup, } from "../../../src/process/ProcessOwnership.js"; -cliTest.skipIf(!["darwin", "linux"].includes(process.platform)).for([ - ["SIGINT", 130], - ["SIGTERM", 143], -] as const)( - "doctor preserves %s after logging and joined prerequisite cleanup", - async ([signal, expected], { processes }) => { +cliTest.skipIf(!["darwin", "linux"].includes(process.platform)).for( + (["jadx", "kuna"] as const).flatMap((provider) => + ( + [ + ["SIGINT", 130], + ["SIGTERM", 143], + ] as const + ).map(([signal, expected]) => ({ + provider, + signal, + expected, + })), + ), +)( + "doctor preserves $signal for $provider after logging and joined prerequisite cleanup", + async ({ provider, signal, expected }, { processes }) => { const root = await createTestTempDirectory("rea-doctor-cancel-"); const jdk = join(root, "jdk"); const jar = join(root, "engine.jar"); const ready = join(root, "ready.json"); + const release = join(root, "release"); await mkdir(join(jdk, "bin"), { recursive: true }); await mkdir(join(root, "home")); await writeJadxJarInventory(jar); - // No JVM/engine runs. Announce only the selected JADX module-listing probe; - // informational Ghidra checks can call the same configured executable first. + // No real provider runs. Announce only the selected prerequisite probe; + // informational Ghidra checks can call the same configured Java first. + const executable = + provider === "kuna" ? join(root, "kuna") : join(jdk, "bin", "java"); + const probeArgument = provider === "kuna" ? "--version" : "--list-modules"; await writeFile( - join(jdk, "bin", "java"), + executable, `#!${process.execPath} -if (!process.argv.slice(2).includes("--list-modules")) process.exit(7); +if (!process.argv.slice(2).includes(${JSON.stringify(probeArgument)})) process.exit(7); require("node:fs").writeFileSync(${JSON.stringify(ready)}, JSON.stringify({pid: process.pid, runId: process.env.REA_PROCESS_RUN_ID})); process.on("SIGTERM", () => {}); +setInterval(() => { if (require("node:fs").existsSync(${JSON.stringify(release)})) process.exit(0); }, 25).unref(); setTimeout(() => process.exit(99), 20000); `, { mode: 0o700 }, ); const child = processes.spawn( process.execPath, - [resolve("scripts/rea.mjs"), "doctor", "--provider", "jadx", "--json"], + [resolve("scripts/rea.mjs"), "doctor", "--provider", provider, "--json"], { env: { PATH: process.env.PATH, HOME: join(root, "home"), - JAVA_HOME: jdk, - REA_JADX_MCP_JAR: jar, + JAVA_HOME: provider === "jadx" ? jdk : join(root, "missing-jdk"), + ...(provider === "jadx" + ? { REA_JADX_MCP_JAR: jar } + : { REA_KUNA_COMMAND: executable }), GHIDRA_INSTALL_DIR: join(root, "missing-ghidra"), REA_LOG_LEVEL: "silent", }, @@ -80,27 +97,42 @@ setTimeout(() => process.exit(99), 20000); const observed = z .strictObject({ pid: z.number().int().positive(), - runId: z.string().uuid(), + runId: z.string().uuid().optional(), }) .parse(JSON.parse(await readFile(ready, "utf8"))); - ownership = { - leaderPid: observed.pid, - processGroupId: observed.pid, - runId: observed.runId, - }; + if (provider === "jadx") { + if (observed.runId === undefined) + throw new Error("Missing JADX run token"); + ownership = { + leaderPid: observed.pid, + processGroupId: observed.pid, + runId: observed.runId, + }; + } child.kill(signal); await expect .poll(() => child.exitCode, { timeout: 10_000 }) .not.toBeNull(); const result = await closed; expect(JSON.parse(stdout)).toMatchObject({ code: "cancelled" }); - const groups = await observeOwnedProcessGroups(ownership.runId, [ - ownership.processGroupId, - ]); - expect(groups.get(ownership.processGroupId)).toEqual({ state: "empty" }); + if (ownership !== undefined) { + const groups = await observeOwnedProcessGroups(ownership.runId, [ + ownership.processGroupId, + ]); + expect(groups.get(ownership.processGroupId)).toEqual({ + state: "empty", + }); + } else { + // Kuna's version probe uses a joined direct execFile child, without a + // detached run-token group. Do not invent a group owner from its PID. + expect(() => process.kill(observed.pid, 0)).toThrowError( + expect.objectContaining({ code: "ESRCH" }), + ); + } expect(result, stderr).toEqual({ code: expected, signal: null }); } finally { try { + await writeFile(release, "release"); if (ownership !== undefined) expect((await cleanupOwnedProcessGroup(ownership)).cleaned).toBe( true, diff --git a/tests/boundary/providers/kuna/kunaDoctor.test.ts b/tests/boundary/providers/kuna/kunaDoctor.test.ts index 0e63a81f14..a5124f235e 100644 --- a/tests/boundary/providers/kuna/kunaDoctor.test.ts +++ b/tests/boundary/providers/kuna/kunaDoctor.test.ts @@ -5,7 +5,10 @@ import { expect, it as test } from "vitest"; import { inspectKunaProviderOwned as inspectKunaProvider } from "./kunaFileCloseOwner.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; -import { writeFakeKuna } from "./fakeKuna.js"; +import { fakeKunaCalls, writeFakeKuna } from "./fakeKuna.js"; +import { createSystemDoctorHost } from "../../../../src/doctorRuntime.js"; +import { AnalysisCancelledError } from "../../../../src/domain/analysisErrorCore.js"; +import { STARTUP_OWNER_CLOSED } from "../../../../src/process/SharedStartupAttempt.js"; const it = test.skipIf(process.platform === "win32"); @@ -134,3 +137,112 @@ it("reports a cache root REA cannot create under its nearest existing parent", a detail: expect.stringContaining("not a directory"), }); }); + +const doctorEnvironment = (root: string, kuna: string): NodeJS.ProcessEnv => ({ + HOME: root, + PATH: "", + JAVA_HOME: join(root, "missing-jdk"), + GHIDRA_INSTALL_DIR: join(root, "missing-ghidra"), + REA_KUNA_COMMAND: kuna, + REA_KUNA_CACHE_DIR: join(root, "cache"), +}); + +// Exercise the production composition, not just Kuna's installation helper: +// both the caller signal and the host's own close signal must reach the probe. +it.each(["caller", "host"] as const)( + "cancels and joins the composed Kuna doctor probe on %s cancellation", + async (source) => { + const root = await createTestTempDirectory("rea-kuna-doctor-cancel-"); + const gate = join(root, "release-version"); + const kuna = await writeFakeKuna(root, { + version: { stdout: "kuna 1.833\n", gate }, + }); + const controller = new AbortController(); + const reason = { request: "cancel Kuna doctor" }; + const host = createSystemDoctorHost( + doctorEnvironment(root, kuna), + controller.signal, + ); + let settled = false; + const inspection = host.providerInspections!().then( + () => { + settled = true; + return undefined; + }, + (cause: unknown) => { + settled = true; + return cause; + }, + ); + let closing: ReturnType> | undefined; + try { + await expect.poll(async () => (await fakeKunaCalls(kuna)).length).toBe(1); + const probe = (await fakeKunaCalls(kuna))[0]; + if (probe === undefined) throw new Error("Missing Kuna version probe"); + expect(probe.argv).toEqual(["--version"]); + if (source === "caller") controller.abort(reason); + else closing = host.close!(); + // The unreleased probe normally waits for its ten-second deadline. + await expect.poll(() => settled, { timeout: 2_000 }).toBe(true); + const failure = await inspection; + expect(failure).toBeInstanceOf(AnalysisCancelledError); + if (!(failure instanceof AnalysisCancelledError)) + throw new Error("Expected Kuna inspection cancellation"); + expect(failure.operation).toBe("doctor"); + expect(failure.cause).toBe( + source === "caller" ? reason : STARTUP_OWNER_CLOSED, + ); + // Cancellation only settles once execFile has reaped its direct child. + expect(() => process.kill(probe.pid, 0)).toThrowError( + expect.objectContaining({ code: "ESRCH" }), + ); + await closing; + expect(await fakeKunaCalls(kuna)).toHaveLength(1); + } finally { + await writeFile(gate, "release"); + await inspection; + await closing; + await host.close!(); + } + }, +); + +it("does not launch the composed Kuna probe for an already cancelled caller", async () => { + const root = await createTestTempDirectory("rea-kuna-doctor-preabort-"); + const kuna = await writeFakeKuna(root); + const controller = new AbortController(); + controller.abort({ request: "cancel before inspection" }); + const host = createSystemDoctorHost( + doctorEnvironment(root, kuna), + controller.signal, + ); + try { + await expect(host.providerInspections!()).rejects.toBeDefined(); + expect(await fakeKunaCalls(kuna)).toEqual([]); + } finally { + await host.close!(); + } +}); + +it("keeps a healthy composed Kuna inspection when the caller stays active", async () => { + const root = await createTestTempDirectory("rea-kuna-doctor-active-"); + const kuna = await writeFakeKuna(root); + const host = createSystemDoctorHost( + doctorEnvironment(root, kuna), + new AbortController().signal, + ); + try { + expect(await host.providerInspections!()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "kuna", + available: true, + providerVersion: "1.833", + }), + ]), + ); + expect(await fakeKunaCalls(kuna)).toHaveLength(1); + } finally { + await host.close!(); + } +}); diff --git a/tests/boundary/providers/kuna/kunaHashOwnerPropagation.test.ts b/tests/boundary/providers/kuna/kunaHashOwnerPropagation.test.ts index b09c1e15dd..616e51aad5 100644 --- a/tests/boundary/providers/kuna/kunaHashOwnerPropagation.test.ts +++ b/tests/boundary/providers/kuna/kunaHashOwnerPropagation.test.ts @@ -1,4 +1,12 @@ -import { chmod, mkdir, readFile, writeFile } from "node:fs/promises"; +import { + chmod, + mkdir, + readFile, + realpath, + symlink, + writeFile, +} from "node:fs/promises"; +import { createHash } from "node:crypto"; import { join } from "node:path"; import { afterEach, expect, it, vi } from "vitest"; @@ -16,6 +24,9 @@ const injected = vi.hoisted(() => ({ failCloseReadStage: undefined as "fingerprint" | "shebang" | undefined, failReadStage: undefined as "fingerprint" | "shebang" | undefined, abortOnReadFailure: undefined as (() => void) | undefined, + abortOnClose: undefined as + | { stage: "fingerprint" | "shebang"; abort: () => void } + | undefined, handles: [] as import("node:fs/promises").FileHandle[], })); @@ -27,6 +38,7 @@ afterEach(async () => { injected.failCloseReadStage = undefined; injected.failReadStage = undefined; injected.abortOnReadFailure = undefined; + injected.abortOnClose = undefined; for (const handle of injected.handles.splice(0)) if (handle.fd >= 0) await handle.close(); }); @@ -45,16 +57,19 @@ vi.mock("../../../../src/filesystem/RegularFile.js", async (importOriginal) => { const failNextClose = injected.failNextClose; const failCloseReadStage = injected.failCloseReadStage; const failReadStage = injected.failReadStage; + const abortOnClose = injected.abortOnClose; if ( !failNextClose && !injected.holdCloseFailures && !failCloseReadStage && - !failReadStage + !failReadStage && + !abortOnClose ) return handle; injected.failNextClose = false; let readStageMatched = false; let failedClose = false; + let abortCloseMatched = false; return { get fd() { return handle.fd; @@ -67,6 +82,11 @@ vi.mock("../../../../src/filesystem/RegularFile.js", async (importOriginal) => { length: number, position: number, ) => { + if ( + (abortOnClose?.stage === "fingerprint" && length > 2) || + (abortOnClose?.stage === "shebang" && length === 2) + ) + abortCloseMatched = true; if ( (failReadStage === "fingerprint" && length > 2) || (failReadStage === "shebang" && length === 2) @@ -83,6 +103,7 @@ vi.mock("../../../../src/filesystem/RegularFile.js", async (importOriginal) => { }, close: async () => { injected.closeCalls += 1; + if (abortCloseMatched) abortOnClose?.abort(); if ( (failNextClose && !failedClose) || (readStageMatched && !failedClose) || @@ -628,3 +649,128 @@ it.skipIf(process.platform === "win32")( } }, ); + +it.skipIf(process.platform === "win32").each( + (["doctor", "profile", "health"] as const).flatMap((consumer) => + (["fingerprint", "shebang"] as const).flatMap((stage) => + [false, true].map((closeRefusal) => ({ + consumer, + stage, + closeRefusal, + })), + ), + ), +)( + "preserves completed installation facts for $consumer during $stage close, refusal=$closeRefusal", + async ({ consumer, stage, closeRefusal }) => { + const fixture = await session("rea-kuna-completed-installation-cancel-"); + const bytes = await readFile(fixture.kuna); + const executable = await realpath(fixture.kuna); + const command = + consumer === "doctor" + ? join(fixture.root, "selected-kuna") + : fixture.kuna; + if (consumer === "doctor") await symlink(fixture.kuna, command); + const fingerprint = { + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.length, + }; + const controller = new AbortController(); + const reason = { request: "cancel completed Kuna installation read" }; + const resources = new ArtifactResourceScope(); + const fileCloseResources: FileCloseResourceOwner = { + retain: (owner, resource) => + resources.retain({ + kind: "lease", + resource, + lease: { release: () => owner.close() }, + }), + run: (operation) => resources.run(operation), + }; + const client = consumer === "health" ? fixture.open() : undefined; + injected.abortOnClose = { stage, abort: () => controller.abort(reason) }; + injected.failCloseReadStage = closeRefusal ? stage : undefined; + let failure: AnalysisError | undefined; + try { + let diagnostics: AnalysisError["diagnostics"]; + if (consumer === "doctor") { + await inspectKunaProvider( + { + REA_KUNA_COMMAND: command, + REA_KUNA_CACHE_DIR: fixture.cacheRoot, + }, + process.platform, + { fileCloseResources, signal: controller.signal }, + ).then( + (inspection) => { + expect(closeRefusal).toBe(true); + expect(inspection.checks[0]).toMatchObject({ + code: "cleanup_incomplete", + }); + diagnostics = inspection.checks[0]?.details; + }, + (cause: unknown) => { + if (!(cause instanceof AnalysisError)) throw cause; + failure = cause; + }, + ); + } else { + const result = + consumer === "profile" + ? await fixture.provider.resolveAnalysisProfile(fixture.target, { + fileCloseResources, + signal: controller.signal, + }) + : await client!.execute( + "health", + {}, + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected installation cancellation"); + failure = result.error; + } + if (failure !== undefined) { + expect(failure).toMatchObject({ + _tag: "AnalysisCancelledError", + cleanupIncomplete: closeRefusal, + }); + expect(failure.cause).toBe(reason); + diagnostics = failure.diagnostics; + expect(projectAnalysisError(failure)).toMatchObject({ + code: closeRefusal ? "cleanup_incomplete" : "cancelled", + details: { + ...(closeRefusal ? { execution_failure: "cancelled" } : {}), + diagnostics: { + kuna_installation: { + status: "partial", + completed_hash: fingerprint, + }, + }, + }, + }); + } + expect(diagnostics).toMatchObject({ + kuna_installation: { + status: "partial", + command, + executable, + version: "1.833", + completed_hash: fingerprint, + ...(stage === "shebang" ? { script: true } : {}), + }, + }); + if (stage === "fingerprint") + expect(diagnostics?.["kuna_installation"]).not.toHaveProperty("script"); + expect(injected.handles.some((handle) => handle.fd >= 0)).toBe( + closeRefusal, + ); + } finally { + injected.abortOnClose = undefined; + injected.failCloseReadStage = undefined; + await resources.close(); + if (client !== undefined) expect((await client.close()).ok).toBe(true); + expect(failure?.cleanupIncomplete ?? false).toBe(false); + expect(injected.handles.every((handle) => handle.fd < 0)).toBe(true); + } + }, +); diff --git a/tests/boundary/providers/kuna/kunaInstallation.test.ts b/tests/boundary/providers/kuna/kunaInstallation.test.ts index 6e5401dd5f..c4f7b42cb1 100644 --- a/tests/boundary/providers/kuna/kunaInstallation.test.ts +++ b/tests/boundary/providers/kuna/kunaInstallation.test.ts @@ -1,9 +1,16 @@ import { createHash } from "node:crypto"; -import { chmod, readFile, realpath, symlink } from "node:fs/promises"; +import { + chmod, + readFile, + realpath, + symlink, + writeFile, +} from "node:fs/promises"; import { join } from "node:path"; import { expect, it as test } from "vitest"; +import { kunaInstallationCancellation } from "../../../../src/kuna/KunaInstallation.js"; import { readKunaConfiguration } from "../../../../src/kuna/KunaConfiguration.js"; import { inspectKunaInstallationOwned as inspectKunaInstallation } from "./kunaFileCloseOwner.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; @@ -219,3 +226,106 @@ it("stops a version probe that ignores SIGTERM instead of waiting on it", async const pid = (await fakeKunaCalls(kuna))[0]?.pid ?? 0; expect(() => process.kill(pid, 0)).toThrow(); }, 20_000); + +it("keeps a version deadline ahead of cancellation during its joined stop", async () => { + const root = await createTestTempDirectory("rea-kuna-deadline-first-"); + const command = join(root, "probe.cjs"); + const stopped = join(root, "stopped"); + const release = join(root, "release"); + await ( + await import("node:fs/promises") + ).writeFile( + command, + `#!${process.execPath} +const fs = require("node:fs"); +process.on("SIGTERM", () => fs.writeFileSync(${JSON.stringify(stopped)}, "timeout stop")); +const gate = setInterval(() => { + if (fs.existsSync(${JSON.stringify(release)})) { clearInterval(gate); process.exit(0); } +}, 5); +setTimeout(() => process.exit(9), 10000).unref(); +`, + { mode: 0o700 }, + ); + const controller = new AbortController(); + const pending = inspectKunaInstallation( + { + REA_KUNA_COMMAND: command, + REA_KUNA_CACHE_DIR: join(root, "cache"), + }, + { signal: controller.signal, probeTimeoutMs: 1000 }, + ); + const outcome = pending.then( + (value) => ({ kind: "result" as const, value }), + (cause: unknown) => ({ kind: "thrown" as const, cause }), + ); + try { + await expect + .poll( + () => + readFile(stopped).then( + () => true, + () => false, + ), + { timeout: 5000 }, + ) + .toBe(true); + controller.abort({ request: "late doctor cancellation" }); + await writeFile(release, "release"); + expect(await outcome).toMatchObject({ + kind: "result", + value: { + ok: false, + error: { + code: "version_unresolved", + diagnostics: { deadline_ms: 1000, killed: true }, + }, + }, + }); + } finally { + await writeFile(release, "release"); + await outcome; + } +}); + +it("retains an admitted lookup failure when cancellation arrives during settlement", async () => { + const root = await createTestTempDirectory("rea-kuna-lookup-failure-"); + const command = join(root, "missing-kuna"); + const controller = new AbortController(); + const pending = inspectKunaInstallation( + { + REA_KUNA_COMMAND: command, + REA_KUNA_CACHE_DIR: join(root, "cache"), + }, + { signal: controller.signal }, + ); + controller.abort({ request: "cancel in-flight lookup" }); + const outcome = await pending.then( + (value) => ({ kind: "result" as const, value }), + (cause: unknown) => ({ kind: "thrown" as const, cause }), + ); + expect(outcome).toMatchObject({ + kind: "result", + value: { + ok: false, + error: { code: "executable_missing", diagnostics: { command } }, + }, + }); +}); + +it("preserves an already-cancelled caller before admitting a lookup", async () => { + const root = await createTestTempDirectory("rea-kuna-preabort-lookup-"); + const controller = new AbortController(); + const reason = { request: "cancel before lookup" }; + controller.abort(reason); + const cause: unknown = await inspectKunaInstallation( + { + REA_KUNA_COMMAND: join(root, "missing-kuna"), + REA_KUNA_CACHE_DIR: join(root, "cache"), + }, + { signal: controller.signal }, + ).then( + () => undefined, + (error: unknown) => error, + ); + expect(kunaInstallationCancellation(cause)?.reason).toBe(reason); +}); diff --git a/tests/boundary/providers/kuna/kunaInstallationLifetime.test.ts b/tests/boundary/providers/kuna/kunaInstallationLifetime.test.ts index ed70d6d0ff..8ae23a75db 100644 --- a/tests/boundary/providers/kuna/kunaInstallationLifetime.test.ts +++ b/tests/boundary/providers/kuna/kunaInstallationLifetime.test.ts @@ -1,4 +1,3 @@ -import { regularFileCancellation } from "../../../../src/filesystem/OwnedRegularFile.js"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; const gates = vi.hoisted(() => { @@ -63,7 +62,10 @@ vi.mock("../../../../src/filesystem/RegularFile.js", async (importOriginal) => { import { ArtifactResourceScope } from "../../../../src/artifacts/ArtifactResourceScope.js"; import type { FileCloseResourceOwner } from "../../../../src/application/AnalysisProvider.js"; -import { inspectKunaInstallation } from "../../../../src/kuna/KunaInstallation.js"; +import { + inspectKunaInstallation, + kunaInstallationCancellation, +} from "../../../../src/kuna/KunaInstallation.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; import { writeFakeKuna } from "./fakeKuna.js"; import { session } from "./kunaSessionFixture.js"; @@ -138,7 +140,7 @@ it const failure: unknown = await inspection.catch( (cause: unknown) => cause, ); - const cancellation = regularFileCancellation(failure); + const cancellation = kunaInstallationCancellation(failure); expect(cancellation).toBeDefined(); expect(cancellation?.reason).toBe(controller.signal.reason); await expect(cleanup).resolves.toBeUndefined(); From e9fc8fb30cc18434475d6ab7cacd2916599b6eff Mon Sep 17 00:00:00 2001 From: ShuaixinHuang <1836656514@qq.com> Date: Tue, 20 Oct 2026 17:25:19 +0800 Subject: [PATCH 07/13] fix(javascript): preserve runtime path bytes and Electron ownership (#4941) * fix(javascript): preserve literal percent runtime paths * fix(javascript): ignore resolve prefixes before the final anchor * fix(electron): retain application roots for loadFile declarations * fix(electron): resolve loadFile from enclosing application package * fix(electron): retain outer application root across package scopes * fix(electron): derive application ownership from inventoried packages * fix(electron): retain resolved package entrypoint ownership * fix(electron): preserve competing application ownership * fix(electron): retain invalid and omitted runtime ownership * fix(electron): preserve omitted module ownership and runtime paths * fix(electron): preserve proven roots before confining page paths * fix(electron): retain possible owners behind missing imports * fix(electron): distinguish absent imports and owned page candidates * fix(electron): keep unavailable JSON out of owner gaps * fix(electron): retain exact loader formats and default entry status * fix(electron): retain unavailable entrypoint ownership candidates * test(ghidra): preserve Windows Node probe prerequisites * test(ghidra): assert cancellation during profile shutdown * fix(electron): separate examined imports from frontend ownership * fix(electron): retain unavailable directory entrypoint owners * test(jeb): retain Linux ownership prerequisite in isolation * fix(javascript): retain unchecked loader ownership and literal main paths * fix(javascript): preserve runtime paths and call-site module aliases * fix(javascript): bound module alias traversal and preserve projections * fix(javascript): retain host module and nested artifact path boundaries * fix(javascript): preserve ESM and archive module loader provenance * test(javascript): distinguish unmapped containers from missing exports * fix: retain default module aliases and Electron builtin imports * fix: preserve host file URL and inward ASAR ownership * fix: preserve CommonJS file URL classification in CI fixtures * fix: retain bound module loaders and Windows entrypoint uncertainty * fix: retain ASAR uncertainty and invalidate changed Electron app roots * fix(javascript): retain ownership uncertainty across loader boundaries * fix(javascript): preserve loader sources and bound alias queries * fix(javascript): retain destructured loader sources * fix(javascript): preserve loader uncertainty at resolution boundaries * fix(javascript): preserve callable and package launch ownership * fix(javascript): preserve captured values across member reads * test(process): assert tagged filesystem cancellation failures * fix(javascript): preserve inventory bases and invocation read timing * fix(javascript): preserve ownership across indirect and unread loads * test(javascript): order fixture directory creation on Windows * test(javascript): share ordered filesystem fixture writer * fix(javascript): unify loader provenance and package validation * fix(javascript): preserve captured Node require loader values * fix(javascript): preserve loader invocation authority * fix(javascript): verify generic loader invocation authority * fix(javascript): preserve conditional loader method alternatives * fix(javascript): require proven factories for source association * fix(javascript): honor replaced Electron app path setters * fix(javascript): prove path and assignment source ownership * fix(javascript): retain native delegate eligibility through shared writes * fix(process): settle stale baseline errors after confirmed absence * fix(javascript): preserve assignment and path boundary evidence * fix(javascript): prove source-directory resolve anchors * fix: preserve application path provenance and portable schemas * fix(javascript): retain CommonJS exported setter effects * fix: isolate Linux identity observations and retain native module exports * fix(javascript): preserve published app and helper provenance * fix(javascript): normalize typed stores and native path fixtures --- ...ScriptArtifactAnalysis.packageMain.test.ts | 64 + .../javascript/JavaScriptArtifactAnalysis.ts | 9 + .../JavaScriptArtifactAnalysisTypes.ts | 5 + .../JavaScriptArtifactGraphBuilder.ts | 65 +- .../JavaScriptArtifactGraphContext.ts | 128 +- .../JavaScriptArtifactGraphFindings.ts | 202 ++- .../JavaScriptArtifactGraphStructure.test.ts | 230 +++ .../JavaScriptArtifactGraphStructure.ts | 106 +- ...PathResolution.syntheticRootEscape.test.ts | 67 + .../JavaScriptArtifactPathResolution.test.ts | 1220 +++++++++++++- .../JavaScriptArtifactPathResolution.ts | 506 ++++-- .../JavaScriptSemanticGraphBuilder.test.ts | 31 + .../JavaScriptSemanticGraphBuilder.ts | 4 +- src/contracts/canonicalBase64Metadata.test.ts | 68 + .../javascript/electronStaticAnalysis.ts | 27 +- ...tronStaticAnalysisAppPathSelection.test.ts | 127 ++ .../electronStaticAnalysisBrowser.ts | 25 +- ...ctronStaticAnalysisExportedAppPath.test.ts | 136 ++ .../electronStaticAnalysisIdentity.test.ts | 124 ++ ...electronStaticAnalysisPublishedApp.test.ts | 112 ++ .../electronStaticAnalysisReturnedApp.test.ts | 231 +++ .../javascript/electronStaticAnalysisTypes.ts | 9 + ...avascriptApplicationProjectionResources.ts | 1 + .../javascriptCommonJsAssignAppPath.test.ts | 141 ++ .../javascriptCommonJsExportReceivers.ts | 244 ++- .../javascriptElectronAppPathChanges.ts | 390 +++++ .../javascriptIntrinsicInvocation.test.ts | 46 + .../javascriptIntrinsicInvocation.ts | 120 +- .../javascript/javascriptModuleDeclaration.ts | 31 + .../javascript/javascriptModuleInterop.ts | 17 +- .../javascriptModuleResolutionMode.ts | 8 + .../javascriptPackageImportAliases.test.ts | 42 + .../javascriptPackageImportAliases.ts | 9 +- .../javascriptPackageSubpathPatterns.test.ts | 30 + .../javascriptPackageSubpathPatterns.ts | 33 + .../javascriptRuntimeStaticCandidates.ts | 44 +- .../javascriptSemanticAliasSelections.ts | 126 ++ .../javascriptSemanticAmbientWrites.test.ts | 26 + .../javascriptSemanticAmbientWrites.ts | 57 +- .../javascript/javascriptSemanticAnalysis.ts | 139 +- .../javascriptSemanticCallResolution.test.ts | 87 +- .../javascriptSemanticCallResolution.ts | 21 +- .../javascript/javascriptSemanticCoverage.ts | 5 + .../javascriptSemanticDataEffectHelpers.ts | 60 +- .../javascriptSemanticDataEffects.ts | 46 +- .../javascriptSemanticFunctionEscapes.test.ts | 47 + .../javascriptSemanticIterationSources.ts | 75 +- .../javascriptSemanticMemberMutations.ts | 244 ++- .../javascriptSemanticMutationInitializers.ts | 35 +- .../javascriptSemanticObjectReferences.ts | 64 + .../javascriptSemanticProjectedSources.ts | 403 +++++ .../javascriptSemanticProjection.ts | 69 +- ...criptSemanticProvenanceAssignments.test.ts | 87 + .../javascriptSemanticReferenceSources.ts | 162 +- .../javascriptSemanticRequireCaptures.test.ts | 151 ++ .../javascriptSemanticRequireOrigin.ts | 1187 ++++++++++++-- .../javascriptSemanticResourceLimits.ts | 4 + ...ascriptSemanticResponseAssignments.test.ts | 81 + ...riptSemanticRestAssignmentCaptures.test.ts | 110 ++ .../javascriptSemanticReturnedReferences.ts | 644 +++++++- ...tSemanticReturnedReferencesBounded.test.ts | 32 + ...ptSemanticReturnedReferencesCopies.test.ts | 83 + ...ptSemanticReturnedReferencesWrites.test.ts | 203 +++ ...avascriptSemanticSharedChildWrites.test.ts | 223 +++ .../javascript/javascriptSemanticState.ts | 71 +- .../javascriptSemanticValueTypes.ts | 3 +- .../javascript/javascriptSemanticValues.ts | 79 +- ...vascriptSemanticWrapperAssignments.test.ts | 119 ++ .../javascript/javascriptStaticAnalysis.ts | 28 +- ...criptStaticAnalysisAssignedRequire.test.ts | 158 ++ ...iptStaticAnalysisAssignmentLoaders.test.ts | 88 ++ ...iptStaticAnalysisCallableReceivers.test.ts | 99 ++ ...scriptStaticAnalysisCalleeBoundary.test.ts | 202 +++ .../javascriptStaticAnalysisCalls.ts | 74 +- ...criptStaticAnalysisCapturedGeneric.test.ts | 98 ++ ...criptStaticAnalysisCapturedRequire.test.ts | 49 + ...ascriptStaticAnalysisCopiedRequire.test.ts | 75 + ...tStaticAnalysisDelegateEligibility.test.ts | 60 + ...tStaticAnalysisFactoryAlternatives.test.ts | 50 + .../javascriptStaticAnalysisHelpers.ts | 139 +- ...tStaticAnalysisInvocationIteration.test.ts | 122 ++ ...ptStaticAnalysisInvocationReadSite.test.ts | 56 + ...iptStaticAnalysisLoaderInvocations.test.ts | 290 ++++ ...riptStaticAnalysisLoaderMayTargets.test.ts | 70 + ...aticAnalysisLoaderMutationControls.test.ts | 79 + ...javascriptStaticAnalysisMainModule.test.ts | 90 ++ ...iptStaticAnalysisProcessMainModule.test.ts | 49 + ...taticAnalysisProjectedModuleWrites.test.ts | 144 ++ ...ptStaticAnalysisReturnedInvocation.test.ts | 180 +++ ...criptStaticAnalysisReturnedRequire.test.ts | 285 ++++ ...ptStaticAnalysisSharedHelperWrites.test.ts | 83 + .../javascriptStaticAnalysisState.ts | 18 + ...iptStaticAnalysisTypedMemberStores.test.ts | 55 + .../javascriptStaticPathIntrinsics.ts | 73 + .../javascriptStaticPathResolution.test.ts | 80 +- .../javascriptStaticPathResolveOwners.test.ts | 166 ++ .../javascriptStaticPathSourceAnchors.test.ts | 121 ++ src/domain/referenceSourceRequireBindings.ts | 12 +- .../JavaScriptAnalysisWorkerSchemas.test.ts | 35 + .../JavaScriptAnalysisWorkerSchemas.ts | 7 +- .../ProcessOwnership.baselineAbsence.test.ts | 124 ++ .../capture/ProcessCaptureLifecycle.test.ts | 32 +- .../capture/ProcessCaptureObservation.test.ts | 43 +- ...avascriptCapturedGenericInvocation.test.ts | 69 + .../javascriptDependencyMainHostPaths.test.ts | 68 + .../javascriptEscapedArtifactImports.test.ts | 63 + .../javascriptExportedAppPathChanges.test.ts | 81 + .../javascriptFactoryCompletion.test.ts | 7 +- .../javascriptIndirectLoaderOwners.test.ts | 280 ++++ .../javascriptLiteralPercentPaths.test.ts | 1403 +++++++++++++++++ .../javascriptLoaderReviewBoundaries.test.ts | 594 +++++++ .../javascriptNonStringMainControls.test.ts | 67 + ...vascriptPackageExportPatternOwners.test.ts | 121 ++ .../javascriptPackageLaunchBoundaries.test.ts | 165 ++ .../javascriptPossibleSetterOwners.test.ts | 68 + .../javascriptPreloadResolveOwners.test.ts | 126 ++ .../javascriptPublishedAppOwners.test.ts | 99 ++ .../javascriptReturnedAppPathChanges.test.ts | 124 ++ .../javascriptReturnedReferenceCycles.test.ts | 39 +- .../javascriptUnreadAppPathChanges.test.ts | 109 ++ .../javascriptWindowsLoadFilePaths.test.ts | 68 + .../javascriptWrapperElectronCaptures.test.ts | 100 ++ .../applications/packageSubpathsCli.test.ts | 16 +- .../applications/packageUrlUnicodeCli.test.ts | 16 +- .../boundary/cli/cliIntegerArguments.test.ts | 98 +- .../cli/electronWrappedCallPaths.test.ts | 4 +- .../artifactLiteralBackslash.test.ts | 190 ++- .../javascriptArtifactReconstruction.test.ts | 398 +++++ .../javascriptFileUrlPaths.test.ts | 30 +- .../javascriptModuleRelationships.test.ts | 53 + .../javascriptModuleSpecifierPaths.test.ts | 23 +- .../javascriptNonStringPackageMain.test.ts | 141 ++ .../javascriptRuntimeReconciliation.test.ts | 75 + ...riptArtifactGraphObservationLookup.test.ts | 2 + .../loaderInvocationMethodAuthority.test.ts | 50 + .../javascript/mainModuleSource.test.ts | 50 + .../packageConditionalExports.test.ts | 98 +- .../javascript/packageImportAliases.test.ts | 25 +- .../javascript/packageMainHostPaths.test.ts | 71 + .../process/processCaptureLifecycle.test.ts | 28 +- .../javascriptAliasSelectionBudget.ts | 10 + .../javascriptCapturedGenericInvocations.ts | 26 + ...systemSnapshotFallbackCancellation.test.ts | 7 +- .../filesystemSnapshotRaces.test.ts | 10 +- .../javascriptRuntimeObservationCli.test.ts | 2 +- .../processSnapshotCleanupMcp.test.ts | 10 +- tests/support/cli/javascriptApplication.ts | 19 +- tests/support/javascriptApplicationFixture.ts | 17 +- tests/support/workspace/writeFixtureFiles.ts | 20 + 149 files changed, 16881 insertions(+), 988 deletions(-) create mode 100644 src/application/javascript/JavaScriptArtifactAnalysis.packageMain.test.ts create mode 100644 src/application/javascript/JavaScriptArtifactPathResolution.syntheticRootEscape.test.ts create mode 100644 src/contracts/canonicalBase64Metadata.test.ts create mode 100644 src/domain/javascript/electronStaticAnalysisAppPathSelection.test.ts create mode 100644 src/domain/javascript/electronStaticAnalysisExportedAppPath.test.ts create mode 100644 src/domain/javascript/electronStaticAnalysisPublishedApp.test.ts create mode 100644 src/domain/javascript/electronStaticAnalysisReturnedApp.test.ts create mode 100644 src/domain/javascript/javascriptCommonJsAssignAppPath.test.ts create mode 100644 src/domain/javascript/javascriptElectronAppPathChanges.ts create mode 100644 src/domain/javascript/javascriptPackageSubpathPatterns.test.ts create mode 100644 src/domain/javascript/javascriptPackageSubpathPatterns.ts create mode 100644 src/domain/javascript/javascriptSemanticAliasSelections.ts create mode 100644 src/domain/javascript/javascriptSemanticFunctionEscapes.test.ts create mode 100644 src/domain/javascript/javascriptSemanticProjectedSources.ts create mode 100644 src/domain/javascript/javascriptSemanticProvenanceAssignments.test.ts create mode 100644 src/domain/javascript/javascriptSemanticRequireCaptures.test.ts create mode 100644 src/domain/javascript/javascriptSemanticResponseAssignments.test.ts create mode 100644 src/domain/javascript/javascriptSemanticRestAssignmentCaptures.test.ts create mode 100644 src/domain/javascript/javascriptSemanticReturnedReferencesCopies.test.ts create mode 100644 src/domain/javascript/javascriptSemanticReturnedReferencesWrites.test.ts create mode 100644 src/domain/javascript/javascriptSemanticSharedChildWrites.test.ts create mode 100644 src/domain/javascript/javascriptSemanticWrapperAssignments.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisAssignedRequire.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisAssignmentLoaders.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisCallableReceivers.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisCapturedGeneric.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisCapturedRequire.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisCopiedRequire.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisDelegateEligibility.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisFactoryAlternatives.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisInvocationIteration.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisInvocationReadSite.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisLoaderInvocations.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisLoaderMayTargets.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisLoaderMutationControls.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisMainModule.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisProcessMainModule.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisProjectedModuleWrites.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisReturnedInvocation.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisReturnedRequire.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisSharedHelperWrites.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisTypedMemberStores.test.ts create mode 100644 src/domain/javascript/javascriptStaticPathIntrinsics.ts create mode 100644 src/domain/javascript/javascriptStaticPathResolveOwners.test.ts create mode 100644 src/domain/javascript/javascriptStaticPathSourceAnchors.test.ts create mode 100644 src/process/ProcessOwnership.baselineAbsence.test.ts create mode 100644 tests/acceptance/applications/javascriptCapturedGenericInvocation.test.ts create mode 100644 tests/acceptance/applications/javascriptDependencyMainHostPaths.test.ts create mode 100644 tests/acceptance/applications/javascriptEscapedArtifactImports.test.ts create mode 100644 tests/acceptance/applications/javascriptExportedAppPathChanges.test.ts create mode 100644 tests/acceptance/applications/javascriptIndirectLoaderOwners.test.ts create mode 100644 tests/acceptance/applications/javascriptLiteralPercentPaths.test.ts create mode 100644 tests/acceptance/applications/javascriptLoaderReviewBoundaries.test.ts create mode 100644 tests/acceptance/applications/javascriptNonStringMainControls.test.ts create mode 100644 tests/acceptance/applications/javascriptPackageExportPatternOwners.test.ts create mode 100644 tests/acceptance/applications/javascriptPackageLaunchBoundaries.test.ts create mode 100644 tests/acceptance/applications/javascriptPossibleSetterOwners.test.ts create mode 100644 tests/acceptance/applications/javascriptPreloadResolveOwners.test.ts create mode 100644 tests/acceptance/applications/javascriptPublishedAppOwners.test.ts create mode 100644 tests/acceptance/applications/javascriptReturnedAppPathChanges.test.ts create mode 100644 tests/acceptance/applications/javascriptUnreadAppPathChanges.test.ts create mode 100644 tests/acceptance/applications/javascriptWindowsLoadFilePaths.test.ts create mode 100644 tests/acceptance/applications/javascriptWrapperElectronCaptures.test.ts create mode 100644 tests/boundary/filesystem/javascript-applications/javascriptNonStringPackageMain.test.ts create mode 100644 tests/boundary/javascript/loaderInvocationMethodAuthority.test.ts create mode 100644 tests/boundary/javascript/mainModuleSource.test.ts create mode 100644 tests/boundary/javascript/packageMainHostPaths.test.ts create mode 100644 tests/fixtures/javascriptAliasSelectionBudget.ts create mode 100644 tests/fixtures/javascriptCapturedGenericInvocations.ts create mode 100644 tests/support/workspace/writeFixtureFiles.ts diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.packageMain.test.ts b/src/application/javascript/JavaScriptArtifactAnalysis.packageMain.test.ts new file mode 100644 index 0000000000..e06f2e0cc7 --- /dev/null +++ b/src/application/javascript/JavaScriptArtifactAnalysis.packageMain.test.ts @@ -0,0 +1,64 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptArtifactFiles } from "./JavaScriptArtifactAnalysis.js"; + +const packageObservation = (text: string) => + analyzeJavaScriptArtifactFiles({ + files: [ + { + path: "app/package.json", + container_sha256: "a".repeat(64), + sha256: "b".repeat(64), + bytes: Buffer.byteLength(text), + inventory_artifact_id: "package-main-fixture", + kind: "package-json", + unpacked: false, + text: { included: true, value: text }, + }, + ], + unavailable_relevant_files: [], + containers: [], + text_bytes_read: Buffer.byteLength(text), + invalid_utf8_files: 0, + }).packages[0]; + +it.each([null, false, 123, {}, []].map((main) => ({ main })))( + "retains non-string main $main evidence separately from Node index fallback", + ({ main }) => { + expect(packageObservation(JSON.stringify({ main }))).toMatchObject({ + status: "included", + main: null, + main_status: "invalid", + main_entrypoint: "", + }); + }, +); + +it.each([ + ["{}", "default", ""], + ['{"main":""}', "default", ""], + ['{"main":"main.cjs"}', "declared", "main.cjs"], + ['{"main":"sub\\\\main.cjs"}', "declared", "sub\\main.cjs"], + ['{"main":"C:main.cjs"}', "declared", "C:main.cjs"], +] as const)( + "preserves package launch declarations from %s", + (text, status, entry) => { + expect(packageObservation(text)).toMatchObject({ + status: "included", + main_status: status, + main_entrypoint: entry, + }); + }, +); + +it.each(["{", "null", "[]", "[{}]", "123", '"root"', "false"])( + "keeps malformed package metadata %s out of known launch fallback", + (text) => { + expect(packageObservation(text)).toMatchObject({ + status: "invalid", + main: null, + main_status: "unavailable", + main_entrypoint: null, + }); + }, +); diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts index c0e3bdd7c3..b4187a52f8 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysis.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts @@ -512,6 +512,13 @@ const parsePackage = ( name: optionalString(value.name), version: optionalString(value.version), main: optionalString(value.main), + main_status: + value.main === undefined || value.main === "" + ? "default" + : typeof value.main === "string" + ? "declared" + : "invalid", + main_entrypoint: typeof value.main === "string" ? value.main : "", renderer: optionalString(value.renderer) ?? optionalString(value.browser) ?? @@ -538,6 +545,8 @@ const unavailablePackage = ( name: null, version: null, main: null, + main_status: "unavailable", + main_entrypoint: null, renderer: null, limitation, }); diff --git a/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts b/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts index 500fb57cf6..f3387b9eb0 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts @@ -20,6 +20,9 @@ interface JavaScriptPackageValues { readonly name: string | null; readonly version: string | null; readonly main: string | null; + readonly main_status: "default" | "declared" | "invalid"; + /** Node launch declaration; an empty string requests the package index fallback. */ + readonly main_entrypoint: string; readonly renderer: string | null; } @@ -35,6 +38,8 @@ export type JavaScriptPackageObservation = JavaScriptStructuredObservation & readonly name: null; readonly version: null; readonly main: null; + readonly main_status: "unavailable"; + readonly main_entrypoint: null; readonly renderer: null; readonly limitation: string; } diff --git a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts index 352838948a..d4ec631045 100644 --- a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts +++ b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts @@ -12,6 +12,8 @@ import { HTML_INLINE_SCRIPT_LOCATION_COVERAGE_LIMIT, omittedHtmlInlineScriptLocations, selfReferenceOmissions, + dependencyManifest, + applicationContainerIdentity, type JavaScriptArtifactGraphContext, } from "./JavaScriptArtifactGraphContext.js"; import { @@ -93,6 +95,57 @@ function* buildJavaScriptArtifactGraphInputSteps( ): Generator { const accumulator = new JavaScriptArtifactGraphAccumulator(); const root = createJavaScriptArtifactRootNode(accumulator, snapshot); + const parsedPackages = new Set( + analysis.packages + .filter((metadata) => metadata.status === "included") + .map(({ path }) => path), + ); + const applicationOwnershipUnknownContainers = new Set(); + const applicationModuleGapSources = new Map>(); + const applicationModuleReferencesRetained = new Set(); + const recordModuleGap = (file: { + path: string; + container_sha256: string; + }) => { + const container = applicationContainerIdentity(fileSet, file); + const sources = + applicationModuleGapSources.get(container) ?? new Set(); + sources.add(file.path); + applicationModuleGapSources.set(container, sources); + }; + for (const files of [fileSet.files, fileSet.unavailable_relevant_files]) { + for (const file of files) { + if ( + file.kind === "package-json" && + !dependencyManifest(file.path) && + !parsedPackages.has(file.path) + ) + applicationOwnershipUnknownContainers.add( + applicationContainerIdentity(fileSet, file), + ); + if ( + file.kind === "javascript" && + (!("text" in file) || !file.text.included) + ) + recordModuleGap(file); + } + } + // Omitted module relationships can hide another launch owner. Do not expand + // retained facts beyond the projection guard merely to recover ownership. + for (const analyzed of analysis.files) { + if ( + analyzed.application_projection_failure !== undefined || + analyzed.analysis_failure !== undefined || + analyzed.javascript?.parse_status === "failed" || + (analyzed.javascript?.parse_error_count ?? 0) > 0 + ) + recordModuleGap(analyzed.file); + else if (analyzed.javascript !== null) + applicationModuleReferencesRetained.add(analyzed.file.path); + } + const unavailableFilesByPath = new Map( + fileSet.unavailable_relevant_files.map((file) => [file.path, file]), + ); const context: JavaScriptArtifactGraphContext = { accumulator, snapshot, @@ -105,13 +158,23 @@ function* buildJavaScriptArtifactGraphInputSteps( ), root, filesByPath: new Map(fileSet.files.map((file) => [file.path, file])), - unanalyzedFiles: fileSet.unanalyzed_files, + unanalyzedFiles: { + get: (path) => + unavailableFilesByPath.get(path) ?? fileSet.unanalyzed_files?.get(path), + hasDirectory: (path, container) => + fileSet.unanalyzed_files?.hasDirectory(path, container) ?? false, + }, fileNodes: new Map(), assetNodes: new Map(), sourceModuleNodes: new Map(), chunkNodes: new Map(), moduleNodes: new Map(), containerNodes: new Map([[snapshot.manifest.root_sha256, root]]), + applicationRootsBySource: new Map(), + applicationOwnershipUnknownContainers, + applicationModuleGapSources, + applicationModuleReferencesRetained, + applicationPossibleRootsByContainer: new Map(), }; addJavaScriptArtifactContainers(context); yield; diff --git a/src/application/javascript/JavaScriptArtifactGraphContext.ts b/src/application/javascript/JavaScriptArtifactGraphContext.ts index db35cdca2a..7dc8b4b1bb 100644 --- a/src/application/javascript/JavaScriptArtifactGraphContext.ts +++ b/src/application/javascript/JavaScriptArtifactGraphContext.ts @@ -1,6 +1,9 @@ import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js"; import type { JsonValue } from "../../domain/jsonValue.js"; -import { compositeKey } from "../../domain/unicodeCodePointOrder.js"; +import { + compositeKey, + compareUnicodeCodePoints, +} from "../../domain/unicodeCodePointOrder.js"; import { completeApplicationCoverage, partialApplicationCoverage, @@ -51,8 +54,112 @@ export interface JavaScriptArtifactGraphContext { readonly chunkNodes: Map; readonly moduleNodes: Map; readonly containerNodes: Map; + /** Resolved entrypoints/imports retain unique or competing application owners. */ + readonly applicationRootsBySource: Map>; + /** Incomplete package facts can hide competing launch declarations. */ + readonly applicationOwnershipUnknownContainers: ReadonlySet; + /** Gap origins retain possible owners separately from proven source owners. */ + readonly applicationModuleGapSources: Map>; + readonly applicationPossibleRootsByContainer: Map>; + /** Sources with retained references; unchecked loads carry their own gaps. */ + readonly applicationModuleReferencesRetained: ReadonlySet; } +/** Application manifests exclude dependency packages but retain nested app roots. */ +export const dependencyManifest = (path: string): boolean => + path.split("/").slice(0, -1).includes("node_modules"); + +/** Two distinct roots prove ambiguity; further roots cannot restore uniqueness. */ +export const addApplicationRootCandidate = ( + roots: Set, + root: string, +): void => { + if (roots.size < 2) roots.add(root); +}; + +const applicationContainerRoot = ( + fileSet: JavaScriptArtifactFileSet, + source: Pick, +): string => + fileSet.containers.find( + ({ sha256, path }) => + sha256 === source.container_sha256 && source.path.startsWith(`${path}/`), + )?.path ?? ""; + +/** Equal archive bytes at different logical paths are distinct ownership scopes. */ +export const applicationContainerIdentity = ( + fileSet: JavaScriptArtifactFileSet, + source: Pick, +): string => + compositeKey([ + source.container_sha256, + applicationContainerRoot(fileSet, source), + ]); + +/** Select the inventory's display root; this does not establish runtime ownership. */ +export const applicationManifestPath = ( + context: JavaScriptArtifactGraphContext, +): string | undefined => + context.analysis.packages + .map(({ path }) => path) + .filter((path) => { + const file = context.filesByPath.get(path); + return file !== undefined && !dependencyManifest(path); + }) + .sort( + (left, right) => + left.split("/").length - right.split("/").length || + compareUnicodeCodePoints(left, right), + )[0]; + +/** Inspect inventoried ancestor packages; absent parsing is not absent ownership. + * Entry-bearing packages are candidates; multiple candidates stay unknown. + */ +export const applicationRootPath = ( + context: JavaScriptArtifactGraphContext, + source: JavaScriptArtifactFile, +): string | null => { + // Unread launch metadata can name a main in another inventoried container. + // Archive occurrence identity does not prove isolation from the Node loader. + if (context.applicationOwnershipUnknownContainers.size > 0) return null; + const entryRoots = context.applicationRootsBySource.get(source.path); + const container = applicationContainerIdentity(context.fileSet, source); + for (const root of context.applicationPossibleRootsByContainer.get( + container, + ) ?? []) + if (!entryRoots?.has(root)) return null; + if ( + entryRoots === undefined && + context.applicationModuleGapSources.has(container) + ) + return null; + const roots = new Set(entryRoots); + const directories = source.path.split("/").slice(0, -1); + for (let depth = directories.length; depth >= 0; depth -= 1) { + const root = directories.slice(0, depth).join("/"); + const prefix = root === "" ? "" : `${root}/`; + const path = `${prefix}package.json`; + if (dependencyManifest(path)) continue; + const file = + context.filesByPath.get(path) ?? context.unanalyzedFiles?.get(path); + if (file?.container_sha256 !== source.container_sha256) continue; + const metadata = context.analysis.packages.find( + (value) => value.path === path, + ); + if (metadata?.status !== "included") return null; + const defaultEntry = context.filesByPath.get(`${prefix}index.js`); + if ( + entryRoots === undefined && + (metadata.main_status === "declared" || + defaultEntry?.container_sha256 === source.container_sha256) + ) + addApplicationRootCandidate(roots, root); + } + if (roots.size > 1) return null; + for (const root of roots) return root; + return applicationContainerRoot(context.fileSet, source); +}; + /** Evidence coverage shape shared by reconstruction graph helpers. */ export type JavaScriptArtifactGraphCoverage = ApplicationGraphEvidence["coverage"]; @@ -302,13 +409,25 @@ export const artifactFileNodeKind = ( export const createElectronRoleNode = ( context: JavaScriptArtifactGraphContext, input: RoleNodeInput, -): ApplicationNode => - context.accumulator.addNode({ +): ApplicationNode => { + const target = + input.resolution.resolved_path ?? + (input.resolution.runtime_path_base_proven === true + ? input.resolution.candidate_path + : undefined) ?? + input.resolution.declared_path; + // Raw application-relative declarations keep their source occurrence even + // when identical source bytes appear under separate application prefixes. + const identityPath = + input.resolution.runtime_path_base === "application-root" + ? compositeKey([input.anchor.path, target]) + : target; + return context.accumulator.addNode({ kind: input.kind, identity: artifactLocalIdentity( input.anchor.sha256, "electron-role", - `${input.kind}:${input.resolution.resolution_context}:${input.resolution.resolved_path ?? input.resolution.declared_path}`, + `${input.kind}:${input.resolution.resolution_context}:${identityPath}`, ), observations: [ { @@ -333,6 +452,7 @@ export const createElectronRoleNode = ( }, ], }); +}; /** Resolve an Electron role to an inventoried asset when available. */ export const linkElectronRoleToAsset = ( diff --git a/src/application/javascript/JavaScriptArtifactGraphFindings.ts b/src/application/javascript/JavaScriptArtifactGraphFindings.ts index 823c81f7dd..7794a7768c 100644 --- a/src/application/javascript/JavaScriptArtifactGraphFindings.ts +++ b/src/application/javascript/JavaScriptArtifactGraphFindings.ts @@ -6,6 +6,9 @@ import type { JavaScriptStaticAnalysis } from "../../domain/javascript/javascrip import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; import { addStaticInferenceEdge, + applicationRootPath, + applicationContainerIdentity, + addApplicationRootCandidate, artifactLocalIdentity, chunkLookupKey, createElectronRoleNode, @@ -26,7 +29,10 @@ import { resolveArtifactPathByContext, type ArtifactPathResolution, } from "./JavaScriptArtifactPathResolution.js"; -import { moduleResolutionModeForJavaScriptReference } from "../../domain/javascript/javascriptModuleResolutionMode.js"; +import { + moduleResolutionModeForJavaScriptReference, + isJsonModuleTarget, +} from "../../domain/javascript/javascriptModuleResolutionMode.js"; type StaticReference = JavaScriptStaticAnalysis["references"][number]; type StaticEndpoint = JavaScriptStaticAnalysis["endpoints"][number]; @@ -42,11 +48,11 @@ interface FindingInput { } interface ResolvedReference { + readonly targetImportsUnexamined?: true; readonly path: string | null; readonly file: JavaScriptArtifactFile | null; readonly node: ApplicationNode | null; readonly limitations: readonly string[]; - readonly resolutionStatus?: string; readonly resolution?: ArtifactPathResolution; } @@ -54,7 +60,10 @@ interface ResolvedReference { export function* addJavaScriptStaticFindingsSteps( context: JavaScriptArtifactGraphContext, ): Generator { - const omissions = { selfReferences: 0 }; + const omissions = { + selfReferences: 0, + imports: new Map>(), + }; for (const analyzed of context.analysis.files) { yield; const { file, javascript } = analyzed; @@ -67,11 +76,42 @@ export function* addJavaScriptStaticFindingsSteps( addEndpoint(context, { file, asset, value, coverage }); for (const value of javascript.storage) addStorage(context, { file, asset, value, coverage }); - for (const value of javascript.role_paths) - addDiscoveredRole(context, { file, asset, value, coverage }); for (const value of javascript.source_map_urls) addSourceMapEdge(context, { file, asset, value, coverage }); } + yield* propagateApplicationOwners(context, omissions.imports); + const possible = new Set(); + for (const roots of context.applicationPossibleRootsByContainer.values()) + for (const root of roots) addApplicationRootCandidate(possible, root); + for (const gaps of context.applicationModuleGapSources.values()) { + for (const gap of gaps) { + yield; + for (const root of context.applicationRootsBySource.get(gap) ?? []) + addApplicationRootCandidate(possible, root); + } + } + // An unknown loader may cross an inventoried archive boundary in either + // direction. Retain possible owners in every container it could reach. + for (const file of context.fileSet.files) { + yield; + context.applicationPossibleRootsByContainer.set( + applicationContainerIdentity(context.fileSet, file), + possible, + ); + } + const changedRoots = changedApplicationRoots(context); + for (const { file, javascript } of context.analysis.files) { + yield; + const asset = context.assetNodes.get(file.path); + if (javascript === null || asset === undefined) continue; + const coverage = javascriptAnalysisCoverage(javascript); + for (const value of javascript.role_paths) + addDiscoveredRole( + context, + { file, asset, value, coverage }, + changedRoots, + ); + } return selfReferenceOmissions( omissions.selfReferences, ["static reference", "static references"], @@ -79,10 +119,78 @@ export function* addJavaScriptStaticFindingsSteps( ); } +/** A setter changes process-wide runtime state, not package ownership. */ +const changedApplicationRoots = ( + context: JavaScriptArtifactGraphContext, +): ReadonlySet | null => { + const roots = new Set(); + // An unchecked load can execute a setter even when the imported source has + // no retained AST. Package ownership alone does not prove its runtime path + // unchanged. Only launch owners that can reach the gap are affected; an + // unrelated unselected source is not evidence of a running setter. + for (const sources of context.applicationModuleGapSources.values()) { + for (const source of sources) { + const owners = context.applicationRootsBySource.get(source); + if (owners === undefined) continue; + if (owners.size >= 2) return null; + for (const root of owners) roots.add(root); + } + } + for (const { file, javascript } of context.analysis.files) { + if (!javascript?.electron.app_path_changes?.length) continue; + const owners = context.applicationRootsBySource.get(file.path); + // A setter whose launching application is unknown cannot establish an + // unchanged root for any package-based fallback. + if (owners === undefined) return null; + // Owner sets stop at two distinct roots once uniqueness is disproven. + // They cannot enumerate every application affected by a shared setter. + if (owners.size >= 2) return null; + for (const root of owners) roots.add(root); + const possibleOwners = context.applicationPossibleRootsByContainer.get( + applicationContainerIdentity(context.fileSet, file), + ); + // An unchecked loader can execute this setter in another application's + // process even when only one entrypoint has a proven import of this source. + // Possible-owner sets have the same two-root saturation as proven owners. + if (possibleOwners !== undefined) { + if (possibleOwners.size >= 2) return null; + for (const root of possibleOwners) roots.add(root); + } + } + return roots; +}; + +/** Carry every entrypoint owner through already-resolved imports before roles. + * Sets and change-driven traversal retain competition and terminate on cycles. + */ +function* propagateApplicationOwners( + context: JavaScriptArtifactGraphContext, + imports: ReadonlyMap>, +): Generator { + const pending = [...context.applicationRootsBySource.keys()]; + while (pending.length > 0) { + yield; + const source = pending.pop(); + if (source === undefined) break; + const roots = context.applicationRootsBySource.get(source); + if (roots === undefined) continue; + for (const target of imports.get(source) ?? []) { + yield; + const candidates = + context.applicationRootsBySource.get(target) ?? new Set(); + const previousSize = candidates.size; + for (const root of roots) addApplicationRootCandidate(candidates, root); + if (candidates.size === previousSize) continue; + context.applicationRootsBySource.set(target, candidates); + pending.push(target); + } + } +} + const addReference = ( context: JavaScriptArtifactGraphContext, input: FindingInput, - omissions: { selfReferences: number }, + omissions: { selfReferences: number; imports: Map> }, ): void => { const source = sourceNodeFor( @@ -95,6 +203,32 @@ const addReference = ( input.value.specifier === null ? null : resolveReference(context, input.file.path, input.value, input.coverage); + if ( + (input.value.kind === "static-import" || + input.value.kind === "dynamic-import" || + input.value.kind === "require") && + (input.value.specifier === null || + resolved === null || + resolved.targetImportsUnexamined === true) + ) { + const container = applicationContainerIdentity(context.fileSet, input.file); + const gaps = + context.applicationModuleGapSources.get(container) ?? new Set(); + gaps.add(input.file.path); + context.applicationModuleGapSources.set(container, gaps); + } + if ( + resolved !== null && + resolved.path !== null && + resolved.file !== null && + (input.value.kind === "static-import" || + input.value.kind === "dynamic-import" || + input.value.kind === "require") + ) { + const targets = omissions.imports.get(input.file.path) ?? new Set(); + targets.add(resolved.path); + omissions.imports.set(input.file.path, targets); + } if (input.value.kind === "worker" || input.value.kind === "service-worker") { const worker = context.accumulator.addNode({ kind: input.value.kind === "worker" ? "worker" : "service-worker", @@ -110,7 +244,7 @@ const addReference = ( declared_specifier: input.value.specifier, expression: input.value.expression, resolved_path: resolved?.path ?? null, - resolution_status: resolved?.resolutionStatus ?? null, + resolution_status: resolved?.resolution?.resolution_status ?? null, ...artifactPathResolutionProperties(resolved?.resolution), }, evidence: staticInferenceEvidence({ @@ -133,7 +267,7 @@ const addReference = ( relation: "loads", properties: { kind: input.value.kind, - resolution_status: resolved?.resolutionStatus ?? null, + resolution_status: resolved?.resolution?.resolution_status ?? null, ...artifactPathResolutionProperties(resolved?.resolution), }, limitations: resolved?.limitations ?? [], @@ -182,7 +316,7 @@ const addReference = ( ) : null, resolved_path: resolved?.path ?? null, - resolution_status: resolved?.resolutionStatus ?? null, + resolution_status: resolved?.resolution?.resolution_status ?? null, ...artifactPathResolutionProperties(resolved?.resolution), }, ...(resolved === null ? {} : { limitations: resolved.limitations }), @@ -344,7 +478,9 @@ const addStorage = ( const addDiscoveredRole = ( context: JavaScriptArtifactGraphContext, input: FindingInput, + changedRoots: ReadonlySet | null, ): void => { + const root = applicationRootPath(context, input.file); const resolution = resolveArtifactPathByContext({ declaredPath: input.value.path, sourcePath: input.file.path, @@ -352,14 +488,25 @@ const addDiscoveredRole = ( files: context.filesByPath, directoryPaths: context.directoryPaths, unanalyzedFiles: context.unanalyzedFiles, - ...(input.value.role === "preload" || - (input.value.role === "renderer" && - input.value.resolution_context === "filesystem-expression") + ...(input.value.role === "preload" ? { runtimePathKind: "exact-file" as const } : input.value.role === "renderer" && - input.value.resolution_context === "module-specifier" - ? { runtimePathKind: "url-resource" as const } - : {}), + input.value.resolution_context === "filesystem-expression" + ? { + // Raw loadFile literals belong to the application, not its caller. + runtimePathKind: "application-file" as const, + applicationRootPath: + changedRoots === null || (root !== null && changedRoots.has(root)) + ? null + : root, + applicationRootIsProven: context.applicationRootsBySource.has( + input.file.path, + ), + } + : input.value.role === "renderer" && + input.value.resolution_context === "module-specifier" + ? { runtimePathKind: "url-resource" as const } + : {}), }); const role = createElectronRoleNode(context, { kind: @@ -533,6 +680,23 @@ const resolveReference = ( : {}), }); const path = resolution.resolved_path; + const targetPath = path ?? resolution.candidate_path; + const targetKind = + targetPath === undefined + ? undefined + : ( + context.filesByPath.get(targetPath) ?? + context.unanalyzedFiles?.get(targetPath) + )?.kind; + // Resolving bytes does not prove that the loader's module imports were read. + // Only an exact JSON loader target can be inert without JavaScript analysis. + const targetIsJson = + targetPath !== undefined && isJsonModuleTarget(targetPath, targetKind); + const targetImportsUnexamined = + !targetIsJson && + (resolution.resolution_status === "unavailable" || + (path !== null && + !context.applicationModuleReferencesRetained.has(path))); if (path === null) { const source = context.filesByPath.get(sourcePath); return source === undefined @@ -550,7 +714,9 @@ const resolveReference = ( resolution, ), limitations: resolution.limitations, - resolutionStatus: resolution.resolution_status, + ...(targetImportsUnexamined + ? { targetImportsUnexamined: true as const } + : {}), resolution, }; } @@ -563,7 +729,9 @@ const resolveReference = ( file, node, limitations: resolution.limitations, - resolutionStatus: resolution.resolution_status, + ...(targetImportsUnexamined + ? { targetImportsUnexamined: true as const } + : {}), resolution, }; }; diff --git a/src/application/javascript/JavaScriptArtifactGraphStructure.test.ts b/src/application/javascript/JavaScriptArtifactGraphStructure.test.ts index dbfc19d765..224d3631e0 100644 --- a/src/application/javascript/JavaScriptArtifactGraphStructure.test.ts +++ b/src/application/javascript/JavaScriptArtifactGraphStructure.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { createHash } from "node:crypto"; import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js"; import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; @@ -14,6 +15,8 @@ import { addJavaScriptArtifactFiles, createJavaScriptArtifactRootNode, } from "./JavaScriptArtifactGraphStructure.js"; +import { addJavaScriptStaticFindingsSteps } from "./JavaScriptArtifactGraphFindings.js"; +import { applicationContainerIdentity } from "./JavaScriptArtifactGraphContext.js"; const SHA256 = "a".repeat(64); const REASON = @@ -98,6 +101,11 @@ const contextFor = ( chunkNodes: new Map(), moduleNodes: new Map(), containerNodes: new Map(), + applicationRootsBySource: new Map(), + applicationOwnershipUnknownContainers: new Set(), + applicationModuleGapSources: new Map(), + applicationPossibleRootsByContainer: new Map(), + applicationModuleReferencesRetained: new Set(), }; }; @@ -146,3 +154,225 @@ describe("skipped JavaScript parse reporting", () => { expect(fileObservation(context).parse_status).toBe(javascript.parse_status); }); }); + +it.each([false, true])( + "retains possible setter owners independently of proven imports (possible owner: %s)", + (possibleOwner) => { + const window: JavaScriptArtifactFile = { + ...skippedFile(), + path: "b/window.cjs", + sha256: "b".repeat(64), + }; + const setter: JavaScriptArtifactFile = { + ...skippedFile(), + path: "shared/change.cjs", + sha256: "c".repeat(64), + }; + const page: JavaScriptArtifactFile = { + ...skippedFile(), + path: "b/index.html", + sha256: "d".repeat(64), + kind: "html", + }; + const analysis = analysisFor([ + { + file: setter, + javascript: analyzeJavaScriptStaticSource( + 'const { app } = require("electron"); app.setAppPath("/changed");', + ), + semantic: null, + }, + { + file: window, + javascript: analyzeJavaScriptStaticSource( + 'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");', + ), + semantic: null, + }, + { file: page, javascript: null, semantic: null }, + ]); + const base = contextFor(window, analysis); + const files = [setter, window, page]; + const context: JavaScriptArtifactGraphContext = { + ...base, + fileSet: { ...base.fileSet, files }, + filesByPath: new Map(files.map((file) => [file.path, file])), + }; + context.applicationRootsBySource.set(setter.path, new Set(["a"])); + context.applicationRootsBySource.set(window.path, new Set(["b"])); + if (possibleOwner) + context.applicationPossibleRootsByContainer.set( + applicationContainerIdentity(context.fileSet, setter), + new Set(["b"]), + ); + addJavaScriptArtifactFiles(context); + const steps = addJavaScriptStaticFindingsSteps(context); + let step = steps.next(); + while (!step.done) step = steps.next(); + const role = context.accumulator + .nodes() + .find(({ kind }) => kind === "electron-renderer"); + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: possibleOwner ? null : "b/index.html", + resolution_status: possibleOwner ? "unavailable" : "resolved", + }), + }), + ]), + ); + }, +); + +it.each([ + "unselected", + "transitive-unselected", + "invalid-utf8", + "worker-failure", + "parse-failed", + "opaque", + "unknown-loader", + "read-inert", + "unreferenced", + "json", + "missing", +] as const)( + "does not prove an unchanged application path across a reachable %s load gap", + (gap) => { + const makeFile = ( + path: string, + kind: JavaScriptArtifactFile["kind"] = "javascript", + ): JavaScriptArtifactFile => ({ + ...skippedFile(), + path, + sha256: createHash("sha256").update(path).digest("hex"), + kind, + }); + const main = makeFile("a/main.cjs"); + const other = makeFile("b/main.cjs"); + const target = makeFile( + gap === "json" ? "a/change.json" : "a/change.cjs", + gap === "json" ? "json" : "javascript", + ); + const page = makeFile("a/index.html", "html"); + const otherPage = makeFile("b/index.html", "html"); + const window = + 'const {BrowserWindow}=require("electron");new BrowserWindow().loadFile("index.html");'; + const statement = + gap === "unknown-loader" + ? "require(process.env.CHANGE);" + : gap === "transitive-unselected" + ? 'require("./middle.cjs");' + : gap === "unreferenced" + ? "" + : `require("./${gap === "json" ? "change.json" : "change.cjs"}");`; + const entry: AnalyzedJavaScriptArtifactFile = + gap === "worker-failure" + ? skippedEntry(target) + : { + file: + gap === "invalid-utf8" + ? { + ...target, + text: { included: false, reason: "invalid-utf8" }, + } + : target, + javascript: + gap === "read-inert" + ? analyzeJavaScriptStaticSource("module.exports=42;") + : gap === "parse-failed" + ? { + ...analyzeJavaScriptStaticSource(""), + parse_status: "failed", + parse_error_count: 1, + } + : null, + semantic: null, + }; + const entries: AnalyzedJavaScriptArtifactFile[] = [ + { + file: main, + javascript: analyzeJavaScriptStaticSource(statement + window), + semantic: null, + }, + { + file: other, + javascript: analyzeJavaScriptStaticSource(window), + semantic: null, + }, + ...(gap === "transitive-unselected" + ? [ + { + file: makeFile("a/middle.cjs"), + javascript: analyzeJavaScriptStaticSource( + 'require("./change.cjs");', + ), + semantic: null, + }, + ] + : []), + ...(gap === "missing" || gap === "opaque" ? [] : [entry]), + { file: page, javascript: null, semantic: null }, + { file: otherPage, javascript: null, semantic: null }, + ]; + const analysis = analysisFor(entries); + const base = contextFor(main, analysis); + const files = entries.map(({ file }) => file); + const context: JavaScriptArtifactGraphContext = { + ...base, + fileSet: { ...base.fileSet, files }, + filesByPath: new Map(files.map((file) => [file.path, file])), + unanalyzedFiles: + gap === "opaque" + ? { + get: (path) => (path === target.path ? target : undefined), + hasDirectory: () => false, + } + : undefined, + applicationModuleReferencesRetained: new Set([ + main.path, + other.path, + ...(gap === "transitive-unselected" ? ["a/middle.cjs"] : []), + ...(gap === "read-inert" ? [target.path] : []), + ]), + }; + context.applicationRootsBySource.set(main.path, new Set(["a"])); + context.applicationRootsBySource.set(other.path, new Set(["b"])); + addJavaScriptArtifactFiles(context); + const steps = addJavaScriptStaticFindingsSteps(context); + while (!steps.next().done) {} + const roles = context.accumulator + .nodes() + .filter(({ kind }) => kind === "electron-renderer"); + expect(roles).toHaveLength(2); + const changed = !["read-inert", "unreferenced", "json", "missing"].includes( + gap, + ); + for (const [path, expected] of [ + [main.path, changed ? null : "a/index.html"], + // Existing loader ownership also allows an unchecked source to import + // the other app's module; no process isolation has been established. + [other.path, changed ? null : "b/index.html"], + ] as const) { + const role = roles.find(({ observations }) => + observations.some( + ({ evidence }) => + evidence.location.available && + evidence.location.value.kind === "source-range" && + evidence.location.value.source === path, + ), + ); + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: expected, + resolution_status: expected === null ? "unavailable" : "resolved", + }), + }), + ]), + ); + } + }, +); diff --git a/src/application/javascript/JavaScriptArtifactGraphStructure.ts b/src/application/javascript/JavaScriptArtifactGraphStructure.ts index 6ec343af15..4594f1589b 100644 --- a/src/application/javascript/JavaScriptArtifactGraphStructure.ts +++ b/src/application/javascript/JavaScriptArtifactGraphStructure.ts @@ -1,4 +1,5 @@ import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js"; +import { isJsonModuleTarget } from "../../domain/javascript/javascriptModuleResolutionMode.js"; import type { ApplicationNode } from "../../domain/javascript/javascriptApplicationGraphSchemas.js"; import { htmlInlineScriptLimitation } from "./JavaScriptArtifactAnalysis.js"; import { @@ -18,6 +19,10 @@ import type { JavaScriptArtifactGraphAccumulator } from "./JavaScriptArtifactGra import { isSkippedJavaScriptAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; import { addArtifactContainsEdge, + applicationManifestPath, + applicationContainerIdentity, + addApplicationRootCandidate, + dependencyManifest, addUnavailableStaticParseScope, artifactFileNodeKind, artifactLocalIdentity, @@ -36,8 +41,8 @@ import { import { artifactPathResolutionProperties, resolveArtifactPathByContext, + type ArtifactPathResolution, } from "./JavaScriptArtifactPathResolution.js"; -import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; interface PackageRoleInput { readonly packageNode: ApplicationNode; @@ -480,6 +485,7 @@ export const addJavaScriptPackageNodes = ( name: packageValue.name, version: packageValue.version, main: packageValue.main, + main_status: packageValue.main_status, renderer: packageValue.renderer, parse_status: packageValue.status, }, @@ -516,6 +522,24 @@ export const addJavaScriptPackageNodes = ( operation: "inventory-package", }); if (dependencyManifest(file.path)) continue; + if ( + packageValue.status === "included" && + packageValue.main_entrypoint === "" + ) { + registerPackageResolutionOwner( + context, + file, + resolveArtifactPathByContext({ + declaredPath: packageValue.main_entrypoint, + sourcePath: file.path, + context: "package-entrypoint", + runtimePathKind: "node-entrypoint", + files: context.filesByPath, + directoryPaths: context.directoryPaths, + unanalyzedFiles: context.unanalyzedFiles, + }), + ); + } addPackageRole(context, { packageNode: node, packageFile: file, @@ -532,24 +556,6 @@ export const addJavaScriptPackageNodes = ( return roots; }; -/** Dependency-only artifacts retain their inventory root. */ -const applicationManifestPath = ( - context: JavaScriptArtifactGraphContext, -): string | undefined => - context.analysis.packages - .map(({ path }) => path) - .filter( - (path) => context.filesByPath.has(path) && !dependencyManifest(path), - ) - .sort( - (left, right) => - left.split("/").length - right.split("/").length || - compareUnicodeCodePoints(left, right), - )[0]; - -const dependencyManifest = (path: string): boolean => - path.split("/").slice(0, -1).includes("node_modules"); - const createFileTarget = ( context: JavaScriptArtifactGraphContext, file: JavaScriptArtifactFile, @@ -645,10 +651,17 @@ const addPackageRole = ( declaredPath: input.declaredPath, sourcePath: input.packageFile.path, context: "package-entrypoint", + // Launch main uses Node's filesystem lookup, independently of renderer + // metadata. Literal percent bytes are filename bytes at this boundary. + ...(input.kind === "electron-main" + ? { runtimePathKind: "node-entrypoint" as const } + : {}), files: context.filesByPath, directoryPaths: context.directoryPaths, unanalyzedFiles: context.unanalyzedFiles, }); + if (input.kind === "electron-main") + registerPackageResolutionOwner(context, input.packageFile, resolution); const role = createElectronRoleNode(context, { kind: input.kind, anchor: input.packageFile, @@ -685,6 +698,61 @@ const addPackageRole = ( }); }; +const registerPackageResolutionOwner = ( + context: JavaScriptArtifactGraphContext, + packageFile: JavaScriptArtifactFile, + resolution: ArtifactPathResolution, +): void => { + if (resolution.resolution_status === "resolved") + registerPackageEntrypointOwner( + context, + packageFile, + resolution.resolved_path, + ); + else if ( + resolution.resolution_status === "unavailable" && + !( + resolution.candidate_path !== undefined && + isJsonModuleTarget( + resolution.candidate_path, + context.unanalyzedFiles?.get(resolution.candidate_path)?.kind, + ) + ) + ) + registerPossiblePackageOwner(context, packageFile); +}; + +const registerPackageEntrypointOwner = ( + context: JavaScriptArtifactGraphContext, + packageFile: JavaScriptArtifactFile, + sourcePath: string, +): void => { + const roots = + context.applicationRootsBySource.get(sourcePath) ?? new Set(); + addApplicationRootCandidate(roots, packageRoot(packageFile)); + context.applicationRootsBySource.set(sourcePath, roots); + if ( + !context.applicationModuleReferencesRetained.has(sourcePath) && + !isJsonModuleTarget(sourcePath, context.filesByPath.get(sourcePath)?.kind) + ) + registerPossiblePackageOwner(context, packageFile); +}; + +const packageRoot = (file: JavaScriptArtifactFile): string => + file.path.slice(0, -"package.json".length).replace(/\/$/u, ""); + +const registerPossiblePackageOwner = ( + context: JavaScriptArtifactGraphContext, + file: JavaScriptArtifactFile, +): void => { + const container = applicationContainerIdentity(context.fileSet, file); + const roots = + context.applicationPossibleRootsByContainer.get(container) ?? + new Set(); + addApplicationRootCandidate(roots, packageRoot(file)); + context.applicationPossibleRootsByContainer.set(container, roots); +}; + const containerProperties = (container: JavaScriptArtifactContainer) => ({ path: container.path, format: "asar", diff --git a/src/application/javascript/JavaScriptArtifactPathResolution.syntheticRootEscape.test.ts b/src/application/javascript/JavaScriptArtifactPathResolution.syntheticRootEscape.test.ts new file mode 100644 index 0000000000..ec58e8b1aa --- /dev/null +++ b/src/application/javascript/JavaScriptArtifactPathResolution.syntheticRootEscape.test.ts @@ -0,0 +1,67 @@ +import { expect, it } from "vitest"; + +import { resolveArtifactPathByContext } from "./JavaScriptArtifactPathResolution.js"; +import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; + +const files = new Map( + ["app/main.mjs", "shared/window.cjs", "rea-artifact/shared/window.cjs"].map( + (path) => [ + path, + { + path, + container_sha256: "a".repeat(64), + sha256: "b".repeat(64), + bytes: 0, + inventory_artifact_id: path, + kind: "javascript", + unpacked: false, + text: { included: true, value: "" }, + } satisfies JavaScriptArtifactFile, + ], + ), +); + +it.each([ + "../../rea-artifact/shared/window.cjs", + "../%2e%2e/rea-artifact/shared/window.cjs", + "%2e%2e/%2e%2e/rea-artifact/shared/window.cjs", + "../../rea-artifact/shared/../shared/window.cjs", +])( + "retains above-root traversal before synthetic URL normalization: %s", + (declaredPath) => { + const actual = new URL(declaredPath, "file:///selected-root/app/main.mjs"); + expect(actual.pathname.startsWith("/selected-root/")).toBe(false); + expect( + resolveArtifactPathByContext({ + declaredPath, + sourcePath: "app/main.mjs", + context: "module-specifier", + moduleKind: "import", + files, + }), + ).toMatchObject({ resolution_status: "rejected", resolved_path: null }); + }, +); + +it.each([ + ["../shared/window.cjs", "shared/window.cjs"], + ["./discard%2Fsegment/../../shared/window.cjs", "shared/window.cjs"], + ["./discard%/../../shared/window.cjs", "shared/window.cjs"], + ["./%252e%252e/../../shared/window.cjs", "shared/window.cjs"], + ["../rea-artifact/shared/window.cjs", "rea-artifact/shared/window.cjs"], +])( + "retains an inside-root URL without banning synthetic-looking names: %s", + (declaredPath, target) => { + const actual = new URL(declaredPath, "file:///selected-root/app/main.mjs"); + expect(actual.pathname).toBe(`/selected-root/${target}`); + expect( + resolveArtifactPathByContext({ + declaredPath, + sourcePath: "app/main.mjs", + context: "module-specifier", + moduleKind: "import", + files, + }), + ).toMatchObject({ resolution_status: "resolved", resolved_path: target }); + }, +); diff --git a/src/application/javascript/JavaScriptArtifactPathResolution.test.ts b/src/application/javascript/JavaScriptArtifactPathResolution.test.ts index 2689324561..e5ece75e21 100644 --- a/src/application/javascript/JavaScriptArtifactPathResolution.test.ts +++ b/src/application/javascript/JavaScriptArtifactPathResolution.test.ts @@ -9,7 +9,396 @@ import { artifactLocalIdentity } from "./JavaScriptArtifactGraphContext.js"; import { applicationNodeIdentitySchema } from "../../domain/javascript/javascriptApplicationEvidenceSchemas.js"; import { analyzeJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js"; +describe("Windows CommonJS request classification", () => { + it.each([ + "..\\shared\\window.cjs", + ".\\window.cjs", + "fixture\\window.cjs", + "C:shared\\window.cjs", + ])( + "retains unknown Windows CommonJS request semantics for %s", + (declaredPath) => { + expect( + resolve({ + declaredPath, + sourcePath: "app/main.cjs", + context: "module-specifier", + moduleKind: "require", + files: fileMap([ + file("app/main.cjs", "root"), + file("app/window.cjs", "root"), + file("shared/window.cjs", "root"), + ]), + }), + ).toMatchObject({ + declared_path: declaredPath, + resolution_status: "unavailable", + resolved_path: null, + }); + }, + ); + + it.each([ + "node:\\fs", + "node:fs\\extra", + "https:\\example.invalid\\window.cjs", + ])( + "does not treat the rejected scheme request %s as an unknown Windows path", + (declaredPath) => { + expect( + resolve({ + declaredPath, + sourcePath: "app/main.cjs", + context: "module-specifier", + moduleKind: "require", + files: fileMap([file("app/main.cjs", "root")]), + }), + ).toMatchObject({ + declared_path: declaredPath, + resolution_status: "rejected", + resolved_path: null, + }); + }, + ); + + it("keeps rejected and URL-owned backslash requests distinct from CommonJS host syntax", () => { + const files = fileMap([ + file("app/main.cjs", "root"), + file("app/window.cjs", "root"), + ]); + for (const [declaredPath, moduleKind, status, resolvedPath] of [ + [".\\window\0.cjs", "require", "rejected", null], + [".\\window.cjs", "import", "rejected", null], + ["./window.cjs", "require", "resolved", "app/window.cjs"], + ["node:fs", "require", "external", null], + ] as const) + expect( + resolve({ + declaredPath, + sourcePath: "app/main.cjs", + context: "module-specifier", + moduleKind, + files, + }), + ).toMatchObject({ + resolution_status: status, + resolved_path: resolvedPath, + }); + }); +}); + +describe("literal POSIX inventory base segments", () => { + it("validates the final URL after a parent reference removes the encoded base segment", () => { + expect( + resolve({ + declaredPath: "../b.js", + sourcePath: "literal\\dir/a.mjs", + context: "module-specifier", + moduleKind: "import", + files: fileMap([file("b.js", "container")]), + }), + ).toMatchObject({ resolution_status: "resolved", resolved_path: "b.js" }); + }); + it.each( + ["literal\\dir", "resources/app.asar/literal\\dir"].flatMap((directory) => + (["import", "require", "compiler-dependent"] as const).map( + (moduleKind) => ({ directory, moduleKind }), + ), + ), + )( + "preserves loader-specific admission from $directory with mode $moduleKind", + ({ directory, moduleKind }) => { + const sourcePath = `${directory}/a.js`; + const target = `${directory}/b.js`; + const files = fileMap([ + file(sourcePath, "container"), + file(target, "container"), + file(`${directory.replaceAll("\\", "/")}/b.js`, "container"), + ]); + expect( + resolve({ + declaredPath: "./b.js", + sourcePath, + context: "module-specifier", + moduleKind, + files, + }), + ).toMatchObject({ + resolution_status: + moduleKind === "require" + ? "resolved" + : moduleKind === "import" + ? "rejected" + : "unavailable", + resolved_path: moduleKind === "require" ? target : null, + ...(moduleKind === "compiler-dependent" + ? { + conditional_resolutions: [ + expect.objectContaining({ + mode: "import", + resolution_status: "rejected", + resolved_path: null, + }), + expect.objectContaining({ + mode: "require", + resolution_status: "resolved", + resolved_path: target, + }), + ], + } + : {}), + }); + }, + ); + + it.each([ + "./bad%5Cb.js", + "./bad%2Fb.js", + "./bad%00b.js", + "./bad%xxb.js", + ".\\b.js", + ])( + "still rejects newly declared unsafe module URL syntax %s inside a literal inventory directory", + (declaredPath) => { + expect( + resolve({ + declaredPath, + sourcePath: "literal\\dir/a.js", + context: "module-specifier", + moduleKind: "import", + files: fileMap([file("literal\\dir/b.js", "container")]), + }), + ).toMatchObject({ resolution_status: "rejected", resolved_path: null }); + }, + ); + + it.each([ + "./discard%5C/../b.js", + "./discard%2F/../b.js", + "./discard%00/../b.js", + "./discard%xx/../b.js", + ])( + "still rejects inherited encoded separators after normalizing discarded request segments: %s", + (declaredPath) => { + expect( + resolve({ + declaredPath, + sourcePath: "literal\\dir/a.js", + context: "module-specifier", + moduleKind: "import", + files: fileMap([file("literal\\dir/b.js", "container")]), + }), + ).toMatchObject({ + resolution_status: "rejected", + resolved_path: null, + }); + }, + ); +}); + +describe("runtime host path classification", () => { + it.each([ + "/discard%2F/../shared/window.cjs", + "/discard%5C/../shared/window.cjs", + "/discard%00/../shared/window.cjs", + "/discard%xx/../shared/window.cjs", + ])( + "normalizes discarded absolute URL segments before pathname validation: %s", + (declaredPath) => { + expect( + resolve({ + declaredPath, + sourcePath: "app/main.mjs", + context: "module-specifier", + moduleKind: "import", + files: new Map(), + }), + ).toMatchObject({ + resolution_status: "unavailable", + resolved_path: null, + }); + }, + ); + it("normalizes discarded relative URL segments before pathname validation", () => { + expect( + resolve({ + declaredPath: "./discard%2F/../window.cjs", + sourcePath: "app/main.mjs", + context: "module-specifier", + moduleKind: "import", + files: fileMap([file("app/window.cjs", "root")]), + }), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "app/window.cjs", + }); + }); + it("retains unknown URL identity for an unpaired source directory segment", () => { + expect( + resolve({ + declaredPath: "./window.cjs", + sourcePath: "app/\ud800/main.mjs", + context: "module-specifier", + moduleKind: "import", + files: new Map(), + }), + ).toMatchObject({ resolution_status: "unavailable", resolved_path: null }); + }); + it.each([ + "/shared%2Fwindow.mjs", + "/shared%5cwindow.mjs", + "/shared%00window.mjs", + "/shared%xxwindow.mjs", + ])( + "rejects invalid absolute ESM URL path syntax before host lookup: %s", + (declaredPath) => { + expect( + resolve({ + declaredPath, + sourcePath: "app/main.mjs", + context: "module-specifier", + moduleKind: "import", + files: new Map(), + }), + ).toMatchObject({ resolution_status: "rejected", resolved_path: null }); + expect( + resolve({ + declaredPath, + sourcePath: "app/main.cjs", + context: "module-specifier", + moduleKind: "require", + files: new Map(), + }), + ).toMatchObject({ + resolution_status: "unavailable", + resolved_path: null, + }); + }, + ); + it("keeps encoded query bytes outside ESM pathname admission", () => { + expect( + resolve({ + declaredPath: "/shared/window.mjs?token=%2F#fragment", + sourcePath: "app/main.mjs", + context: "module-specifier", + moduleKind: "import", + files: new Map(), + }), + ).toMatchObject({ resolution_status: "unavailable", resolved_path: null }); + }); + it.each(["exact-file", "application-file", "node-entrypoint"] as const)( + "keeps POSIX and Windows absolute %s paths unavailable before canonical admission", + (runtimePathKind) => { + for (const declaredPath of [ + "/opt/app/main.cjs", + "C:/app/main.cjs", + "C:\\app\\main.cjs", + "\\\\server\\share\\main.cjs", + "//server/share/main.cjs", + "\\app\\main.cjs", + ]) + expect( + resolve({ + declaredPath, + sourcePath: "app/package.json", + context: "filesystem-expression", + runtimePathKind, + files: fileMap([ + file("opt/app/main.cjs", "root"), + file("app/main.cjs", "root"), + ]), + }), + ).toMatchObject({ + resolution_status: "unavailable", + resolved_path: null, + declared_path: declaredPath, + }); + expect( + resolve({ + declaredPath: "C:/app/\0main.cjs", + sourcePath: "app/main.cjs", + context: "filesystem-expression", + runtimePathKind, + files: new Map(), + }), + ).toMatchObject({ resolution_status: "rejected", resolved_path: null }); + }, + ); +}); + describe("artifact URI scheme classification", () => { + it.each(["entry:prod.cjs", "entry:prod%2Fname.cjs"])( + "retains package main filename bytes for %s without URI classification", + (declaredPath) => { + const files = fileMap([ + file("app/package.json", "root"), + file(`app/${declaredPath}`, "root"), + ]); + expect( + resolve({ + declaredPath, + sourcePath: "app/package.json", + context: "package-entrypoint", + runtimePathKind: "node-entrypoint", + files, + }), + ).toMatchObject({ + declared_path: declaredPath, + resolution_status: "resolved", + resolved_path: `app/${declaredPath}`, + }); + expect( + resolve({ + declaredPath, + sourcePath: "app/package.json", + context: "module-specifier", + moduleKind: "require", + files, + }), + ).toMatchObject({ resolution_status: "external", resolved_path: null }); + expect( + resolve({ + declaredPath, + sourcePath: "app/package.json", + context: "package-entrypoint", + runtimePathKind: "node-entrypoint", + files: fileMap([file("app/package.json", "root")]), + }), + ).toMatchObject({ + resolution_status: "not-found", + resolved_path: null, + candidate_path: `app/${declaredPath}`, + }); + }, + ); + + it.each(["require", "import"] as const)( + "resolves a colon-bearing dependency package main with the %s loader", + (moduleKind) => { + expect( + resolve({ + declaredPath: "fixture", + sourcePath: "app/main.cjs", + context: "module-specifier", + moduleKind, + files: fileMap([ + file("app/main.cjs", "root"), + file( + "app/node_modules/fixture/package.json", + "root", + '{"main":"entry:prod.cjs"}', + ), + file("app/node_modules/fixture/entry:prod.cjs", "root"), + file("app/node_modules/fixture/index.js", "root"), + ]), + }), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "app/node_modules/fixture/entry:prod.cjs", + }); + }, + ); + it.each(["html-reference", "module-specifier", "url-reference"] as const)( "keeps digit-bearing schemes external in %s even when a local file matches", (context) => { @@ -226,10 +615,13 @@ describe("loader-specific module suffix lookup", () => { files, unanalyzedFiles: { ...unanalyzedFiles, - get: () => ({ - ...opaque.get("app/core.jsc")!, - container_sha256: "b".repeat(64), - }), + get: (path: string) => + path === "app/core.jsc" + ? { + ...opaque.get(path)!, + container_sha256: "b".repeat(64), + } + : undefined, }, }), ).toMatchObject({ resolution_status: "not-found", resolved_path: null }); @@ -569,6 +961,37 @@ describe("legacy package main lookup", () => { }); describe("legacy package main metadata validation and fallback", () => { + it.each( + ["[]", "[{}]", "123", '"root"', "null", "false"].flatMap((text) => + (["import", "require", "compiler-dependent"] as const).map( + (moduleKind) => ({ text, moduleKind }), + ), + ), + )( + "blocks index fallback for non-object package root $text in $moduleKind mode", + ({ text, moduleKind }) => { + expect( + resolve({ + declaredPath: "fixture", + sourcePath: "app/entry.js", + context: "module-specifier", + moduleKind, + files: fileMap([ + file("app/entry.js", "root"), + file("app/node_modules/fixture/package.json", "root", text), + file("app/node_modules/fixture/index.js", "root"), + ]), + }), + ).toMatchObject({ + resolution_status: "unavailable", + resolved_path: null, + limitations: expect.arrayContaining([ + expect.stringContaining("not valid package JSON"), + ]), + }); + }, + ); + it("still rejects non-string package main metadata", () => { expect( resolve({ @@ -980,7 +1403,7 @@ describe("ESM module URL path decoding", () => { }); describe("JavaScript file URL resolution", () => { - it("resolves file URL paths without interpreting query bytes as path syntax", () => { + it("keeps absolute file URL modules unavailable without a host mapping", () => { const files = fileMap([file("assets/app.js", "root")]); for (const declaredPath of [ "file:///assets/app.js#entry", @@ -996,8 +1419,8 @@ describe("JavaScript file URL resolution", () => { }), ).toMatchObject({ declared_path: declaredPath, - resolution_status: "resolved", - resolved_path: "assets/app.js", + resolution_status: "unavailable", + resolved_path: null, }); } @@ -1024,7 +1447,7 @@ describe("JavaScript file URL resolution", () => { moduleKind: "import", files, }), - ).toMatchObject({ resolution_status: "external", resolved_path: null }); + ).toMatchObject({ resolution_status: "unavailable", resolved_path: null }); }); }); @@ -1447,7 +1870,118 @@ describe("scoped package directory and specifier resolution", () => { }); }); +const packageExportPatternCases = [ + { + name: "matching", + exports: { "./*": "./*.cjs" }, + request: "fixture/window", + status: "unavailable", + }, + { + name: "nonmatching", + exports: { "./other/*": "./*.cjs" }, + request: "fixture/window", + status: "external", + }, + { + name: "suffix mismatch", + exports: { "./*.js": "./*.cjs" }, + request: "fixture/window.cjs", + status: "external", + }, + { + name: "empty match", + exports: { "./*end": "./*.cjs" }, + request: "fixture/end", + status: "external", + }, + { + name: "multiple stars", + exports: { "./*/*": "./*.cjs" }, + request: "fixture/deep/window", + status: "external", + }, + { + name: "exact before pattern", + exports: { "./*": "./*.cjs", "./window": "./window.cjs" }, + request: "fixture/window", + status: "resolved", + }, + { + name: "exact null before pattern", + exports: { "./*": "./*.cjs", "./window": null }, + request: "fixture/window", + status: "external", + }, + { + name: "matching null", + exports: { "./*": null }, + request: "fixture/window", + status: "external", + }, + { + name: "overlapping null prefix", + exports: { "./*": "./*.cjs", "./private/*": null }, + request: "fixture/private/window", + status: "external", + }, + { + name: "overlapping null suffix", + exports: { "./*": "./*.cjs", "./*.hidden": null }, + request: "fixture/window.hidden", + status: "external", + }, + { + name: "overlapping value prefix", + exports: { "./*": null, "./public/*": "./*.cjs" }, + request: "fixture/public/window", + status: "unavailable", + }, + { + name: "inactive pattern condition", + exports: { "./*": { browser: "./*.cjs" } }, + request: "fixture/window", + status: "external", + }, + { + name: "conditional null", + exports: { "./*": { node: null, default: "./*.cjs" } }, + request: "fixture/window", + status: "external", + }, +]; + describe("package exports subpath encapsulation", () => { + it.each(packageExportPatternCases)( + "keeps $name exports patterns distinct from deterministic encapsulation", + ({ exports, request, status }) => { + for (const moduleKind of ["import", "require"] as const) { + expect( + resolve({ + declaredPath: request, + sourcePath: "app/main.cjs", + context: "module-specifier", + moduleKind, + files: fileMap([ + file("app/main.cjs", "root"), + file( + "app/node_modules/fixture/package.json", + "root", + JSON.stringify({ exports }), + ), + file("app/node_modules/fixture/window.cjs", "root"), + ]), + }), + ).toMatchObject({ + resolution_status: status, + resolved_path: + status === "resolved" + ? "app/node_modules/fixture/window.cjs" + : null, + }); + } + }, + ); it("keeps subpaths encapsulated when only a root target is exported", () => { const files = fileMap([ file("app/src/consumer.js", "root"), @@ -2148,7 +2682,7 @@ describe("package import alias refusals", () => { files, }); expect(outcome).toMatchObject({ - resolution_status: "external", + resolution_status: "unavailable", resolved_path: null, }); expect(outcome.limitations.join(" ")).toMatch(/wildcard/i); @@ -2177,12 +2711,12 @@ describe("package import alias refusals", () => { files, }); expect(recursive).toMatchObject({ - resolution_status: "external", + resolution_status: "unavailable", resolved_path: null, }); expect(recursive.limitations.join(" ")).toMatch(/recursive/i); expect(external).toMatchObject({ - resolution_status: "external", + resolution_status: "unavailable", resolved_path: null, }); expect(external.limitations.join(" ")).toMatch(/external-package/i); @@ -2225,10 +2759,10 @@ describe("package import alias refusals", () => { files, }); expect(outcome).toMatchObject({ - resolution_status: "external", + resolution_status: "unavailable", resolved_path: null, }); - expect(outcome.limitations.join(" ")).toContain("package scopes"); + expect(outcome.limitations.join(" ")).toContain("container boundary"); }); }); @@ -2379,6 +2913,8 @@ describe("contextual JavaScript module identity", () => { const analysis = analyzeJavaScriptStaticSource( ` import { BrowserWindow } from "electron"; + import path, { dirname } from "node:path"; + import { fileURLToPath } from "node:url"; new BrowserWindow({ webPreferences: { preload: path.resolve("ignored", __dirname, "preload.cjs") }}); @@ -2911,6 +3447,122 @@ describe("url-reference JavaScript artifact path resolution", () => { }); }); +describe("module-backed package main host paths", () => { + it.each([ + ["sub\\entry.cjs", "unavailable", null], + ["sub/entry.cjs", "resolved", "app/node_modules/fixture/sub/entry.cjs"], + ] as const)( + "preserves dependency main host semantics for %s", + (main, status, expected) => { + expect( + resolve({ + declaredPath: "fixture", + sourcePath: "app/main.cjs", + context: "module-specifier", + moduleKind: "require", + files: fileMap([ + file("app/main.cjs", "root"), + file( + "app/node_modules/fixture/package.json", + "root", + JSON.stringify({ main }), + ), + file("app/node_modules/fixture/sub/entry.cjs", "root"), + file("app/node_modules/fixture/index.js", "root"), + ]), + }), + ).toMatchObject({ resolution_status: status, resolved_path: expected }); + }, + ); + + it.each( + [ + "/shared/window.cjs", + "C:/shared/window.cjs", + "\\\\host\\share\\window.cjs", + ].flatMap((main) => [true, false].map((direct) => ({ main, direct }))), + )( + "retains host main $main before logical lookup (direct=$direct)", + ({ main, direct }) => { + expect( + resolve({ + declaredPath: direct ? main : "fixture", + sourcePath: direct + ? "app/node_modules/fixture/package.json" + : "app/main.cjs", + context: direct ? "package-entrypoint" : "module-specifier", + moduleKind: "require", + files: fileMap([ + file("app/main.cjs", "root"), + file( + "app/node_modules/fixture/package.json", + "root", + JSON.stringify({ main }), + ), + file("shared/window.cjs", "root"), + file("app/node_modules/fixture/index.js", "root"), + ]), + }), + ).toMatchObject({ + declared_path: direct ? main : "fixture", + resolution_status: "unavailable", + resolved_path: null, + }); + }, + ); + + it.each(["entry.cjs", "./entry.cjs"])( + "preserves relative package main lookup for %s", + (main) => { + expect( + resolve({ + declaredPath: "fixture", + sourcePath: "app/main.cjs", + context: "module-specifier", + moduleKind: "require", + files: fileMap([ + file("app/main.cjs", "root"), + file( + "app/node_modules/fixture/package.json", + "root", + JSON.stringify({ main }), + ), + file("app/node_modules/fixture/entry.cjs", "root"), + file("app/node_modules/fixture/index.js", "root"), + ]), + }), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "app/node_modules/fixture/entry.cjs", + }); + }, + ); + + it.each([true, false])( + "keeps NUL package mains rejected (direct=%s)", + (direct) => { + const main = "/shared/\0window.cjs"; + expect( + resolve({ + declaredPath: direct ? main : "fixture", + sourcePath: direct ? "app/package.json" : "app/main.cjs", + context: direct ? "package-entrypoint" : "module-specifier", + moduleKind: "require", + files: fileMap([ + file("app/main.cjs", "root"), + file( + "app/node_modules/fixture/package.json", + "root", + JSON.stringify({ main }), + ), + file("app/node_modules/fixture/index.js", "root"), + ]), + }), + ).toMatchObject({ resolution_status: "rejected", resolved_path: null }); + }, + ); +}); + const resolve = (input: ResolveArtifactPathInput) => resolveArtifactPathByContext(input); @@ -2994,7 +3646,7 @@ describe("exact HTML resource resolution", () => { }); it.each(["require", "import"] as const)( - "rejects a literal trailing backslash package main before %s probing", + "retains unknown Windows trailing-backslash package main semantics before %s probing", (moduleKind) => { const files = fileMap([ file("entry.js", "root"), @@ -3013,10 +3665,240 @@ it.each(["require", "import"] as const)( moduleKind, files, }), - ).toMatchObject({ resolution_status: "rejected", resolved_path: null }); + ).toMatchObject({ resolution_status: "unavailable", resolved_path: null }); }, ); +describe("literal-percent runtime path resolution", () => { + it.each([ + ["exact-file", "filesystem-expression"], + ["exact-file", "module-specifier"], + ["node-entrypoint", "filesystem-expression"], + ] as const)( + "preserves literal percent bytes for %s in %s", + (runtimePathKind, context) => { + const declaredPath = "./preload%2F%5C%2E.cjs"; + expect( + resolve({ + declaredPath, + sourcePath: "app/main.cjs", + context, + runtimePathKind, + files: fileMap([ + file("app/main.cjs", "root"), + file("app/preload%2F%5C%2E.cjs", "root"), + file("app/preload/entry.cjs", "root"), + ]), + }), + ).toMatchObject({ + declared_path: declaredPath, + resolution_context: context, + resolution_status: "resolved", + resolved_path: "app/preload%2F%5C%2E.cjs", + limitations: [], + }); + }, + ); + + it("keeps a file URL's decoded percent bytes literal at runtime lookup", () => { + const analysis = analyzeJavaScriptStaticSource(` + import { BrowserWindow } from "electron"; + import { fileURLToPath } from "node:url"; + new BrowserWindow({ webPreferences: { + preload: fileURLToPath(new URL("./preload%252Fname.cjs", import.meta.url)) + }}); + `); + const preload = analysis.role_paths.find(({ role }) => role === "preload"); + if (preload === undefined) + throw new Error("Expected a static preload path"); + expect(analysis.parse_status).toBe("complete"); + expect(preload).toMatchObject({ + path: "./preload%2Fname.cjs", + resolution_context: "filesystem-expression", + }); + expect( + resolve({ + declaredPath: preload.path, + sourcePath: "app/main.mjs", + context: preload.resolution_context, + runtimePathKind: "exact-file", + files: fileMap([ + file("app/main.mjs", "root"), + file("app/preload%2Fname.cjs", "root"), + file("app/preload/name.cjs", "root"), + file("app/preload%252Fname.cjs", "root"), + ]), + }), + ).toMatchObject({ + declared_path: "./preload%2Fname.cjs", + resolution_status: "resolved", + resolved_path: "app/preload%2Fname.cjs", + limitations: [], + }); + }); + + it.each(["exact-file", "node-entrypoint"] as const)( + "confines literal-percent filesystem paths for %s", + (runtimePathKind) => { + expect( + resolve({ + declaredPath: "../../outside%2Fname.cjs", + sourcePath: "app/main.cjs", + context: "filesystem-expression", + runtimePathKind, + files: fileMap([ + file("app/main.cjs", "root"), + file("outside%2Fname.cjs", "root"), + ]), + }), + ).toMatchObject({ + resolution_status: "rejected", + resolved_path: null, + limitations: [ + expect.stringContaining("escapes the canonical artifact root"), + ], + }); + }, + ); + + it.each(["%2F", "%5C"])( + "keeps %s rejected in a URL-owned runtime resource", + (separator) => { + expect( + resolve({ + declaredPath: `file:///app/preload${separator}name.cjs`, + sourcePath: "app/main.cjs", + context: "module-specifier", + runtimePathKind: "url-resource", + files: fileMap([ + file("app/main.cjs", "root"), + file(`app/preload${separator}name.cjs`, "root"), + file("app/preload/name.cjs", "root"), + ]), + }), + ).toMatchObject({ + resolution_status: "rejected", + resolved_path: null, + }); + }, + ); +}); + +describe("application-root Electron loadFile resolution", () => { + it.each([ + "sub\\index.html", + ".\\index.html", + "C:index.html", + "C:sub\\index.html", + ])( + "retains unknown Windows loadFile host semantics for %s", + (declaredPath) => { + expect( + resolve({ + declaredPath, + sourcePath: "app/main.cjs", + context: "filesystem-expression", + runtimePathKind: "application-file", + applicationRootPath: "app", + files: fileMap([ + file("app/main.cjs", "root"), + file("app/sub/index.html", "root"), + file("app/index.html", "root"), + ]), + }), + ).toMatchObject({ + declared_path: declaredPath, + resolution_status: "unavailable", + resolved_path: null, + }); + }, + ); + + it.each([ + ["sub/index.html", "resolved", "app/sub/index.html"], + ["sub\\index\0.html", "rejected", null], + ] as const)( + "keeps loadFile admission controls for %s", + (declaredPath, status, path) => { + expect( + resolve({ + declaredPath, + sourcePath: "app/main.cjs", + context: "filesystem-expression", + runtimePathKind: "application-file", + applicationRootPath: "app", + files: fileMap([ + file("app/main.cjs", "root"), + file("app/sub/index.html", "root"), + ]), + }), + ).toMatchObject({ resolution_status: status, resolved_path: path }); + }, + ); + + it("confines a sibling path after applying a nested application root", () => { + expect( + resolve({ + declaredPath: "../shared/index.html", + sourcePath: "app/main.cjs", + context: "filesystem-expression", + runtimePathKind: "application-file", + applicationRootPath: "app", + files: fileMap([ + file("app/main.cjs", "root"), + file("shared/index.html", "root"), + ]), + }), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "shared/index.html", + }); + }); + it.each(["", "resources/app.asar"])( + "resolves loadFile within application root %j instead of the calling script", + (applicationRootPath) => { + const prefix = + applicationRootPath === "" ? "" : `${applicationRootPath}/`; + const files = fileMap([ + file(`${prefix}sub/main.cjs`, "app"), + file(`${prefix}dist%2Findex.html`, "app"), + file(`${prefix}sub/dist%2Findex.html`, "app"), + file(`${prefix}sub/absent.html`, "app"), + file(`${prefix}fallback/index.html`, "app"), + file(`${prefix}foreign.html`, "other-container"), + file("outside.html", "other-container"), + ]); + const loadFile = (declaredPath: string) => + resolve({ + declaredPath, + sourcePath: `${prefix}sub/main.cjs`, + context: "filesystem-expression", + runtimePathKind: "application-file", + applicationRootPath, + files, + }); + expect(loadFile("dist%2Findex.html")).toMatchObject({ + runtime_path_base: "application-root", + resolved_path: `${prefix}dist%2Findex.html`, + resolution_status: "resolved", + }); + expect(loadFile("/dist%2Findex.html")).toMatchObject({ + resolved_path: null, + resolution_status: "unavailable", + }); + for (const path of ["absent.html", "fallback", "foreign.html"]) + expect(loadFile(path)).toMatchObject({ + resolved_path: null, + resolution_status: "not-found", + }); + expect(loadFile("../../../../outside.html")).toMatchObject({ + resolved_path: null, + resolution_status: "rejected", + }); + }, + ); +}); + describe("exact Electron runtime entrypoint resolution", () => { it("requires a protocol for Electron URL resources", () => { const files = fileMap([ @@ -3177,7 +4059,7 @@ it("resolves Node runtime entrypoints without substituting emitted TypeScript so file("directory/index.js", "root"), ]); for (const [declaredPath, status, path, candidatePath] of [ - ["./worker.js", "not-found", null, "worker.js"], + ["./worker.js", "unavailable", null, "worker.js"], ["./runtime", "resolved", "runtime.js", undefined], ["./directory", "resolved", "directory/index.js", undefined], ] as const) @@ -3195,3 +4077,309 @@ it("resolves Node runtime entrypoints without substituting emitted TypeScript so ...(candidatePath === undefined ? {} : { candidate_path: candidatePath }), }); }); + +describe("Node launch package main ownership", () => { + it.each(["index.js", "index.json", "index.cjs"])( + "uses the same package index producer for a default launch with %s", + (fallback) => { + expect( + resolve({ + declaredPath: "", + sourcePath: "app/package.json", + context: "package-entrypoint", + runtimePathKind: "node-entrypoint", + files: fileMap([ + file("app/package.json", "root"), + file(`app/${fallback}`, "root"), + file("app.js", "root"), + ]), + }), + ).toMatchObject({ + resolution_status: fallback === "index.cjs" ? "not-found" : "resolved", + resolved_path: fallback === "index.cjs" ? null : `app/${fallback}`, + }); + }, + ); + + it("keeps unavailable main bytes ahead of a present package index", () => { + const source = file("app/package.json", "root"); + expect( + resolve({ + declaredPath: "declared.cjs", + sourcePath: source.path, + context: "package-entrypoint", + runtimePathKind: "node-entrypoint", + files: fileMap([source, file("app/index.js", "root")]), + unanalyzedFiles: { + get: (path) => + path === "app/declared.cjs" + ? { + path, + container_sha256: source.container_sha256, + sha256: null, + bytes: null, + inventory_artifact_id: null, + kind: "javascript", + hash_status: "unavailable", + limitations: ["main readback unavailable"], + } + : undefined, + hasDirectory: () => false, + }, + }), + ).toMatchObject({ + resolution_status: "unavailable", + resolved_path: null, + candidate_path: "app/declared.cjs", + }); + }); + + it.each([ + ["missing.cjs", "index.js", "app/index.js"], + ["sub/missing", "index.json", "app/index.json"], + ["directory", "index.js", "app/index.js"], + ["declared.cjs", "index.js", "app/declared.cjs"], + ["missing.cjs", "index.cjs", null], + ] as const)( + "uses Node package main lookup for %s with %s", + (declaredPath, fallback, expected) => { + expect( + resolve({ + declaredPath, + sourcePath: "app/package.json", + context: "package-entrypoint", + runtimePathKind: "node-entrypoint", + files: fileMap([ + file("app/package.json", "root"), + file(`app/${fallback}`, "root"), + file("app/declared.cjs", "root"), + file("app/directory/package.json", "root", '{"main":"nested.cjs"}'), + file("app/directory/nested.cjs", "root"), + ]), + }), + ).toMatchObject({ + resolution_status: expected === null ? "not-found" : "resolved", + resolved_path: expected, + }); + }, + ); + + it.each([ + [ + "app/package.json", + "../resources/CODE.ASAR/window.cjs", + "resources/CODE.ASAR/window.cjs", + "package-entrypoint", + ], + [ + "resources/APP.ASAR/package.json", + "../window.cjs", + "resources/window.cjs", + "package-entrypoint", + ], + [ + "main.cjs", + "./resources/CODE.ASAR/window.cjs", + "resources/CODE.ASAR/window.cjs", + "filesystem-expression", + ], + ] as const)( + "retains archive launch lookup for %s loading %s", + (sourcePath, declaredPath, target, context) => { + expect( + resolve({ + sourcePath, + declaredPath, + context, + runtimePathKind: "node-entrypoint", + files: fileMap([file(sourcePath, "source"), file(target, "target")]), + }), + ).toMatchObject({ resolution_status: "resolved", resolved_path: target }); + }, + ); + + it("does not use package fallback for a utility entrypoint or an unproven main container", () => { + for (const [context, declaredPath, status] of [ + ["filesystem-expression", "./missing.cjs", "not-found"], + ["package-entrypoint", "declared.cjs", "unavailable"], + ["package-entrypoint", "../../escape.cjs", "rejected"], + ] as const) + expect( + resolve({ + declaredPath, + sourcePath: "app/package.json", + context, + runtimePathKind: "node-entrypoint", + files: fileMap([ + file("app/package.json", "source"), + file("app/index.js", "source"), + file("app/declared.cjs", "foreign"), + ]), + }), + ).toMatchObject({ resolution_status: status, resolved_path: null }); + }); +}); + +it.each(["sub\\main.cjs", ".\\main.cjs", "C:main.cjs"])( + "retains unknown host semantics for the Windows-relative entrypoint %s", + (declaredPath) => { + expect( + resolve({ + declaredPath, + sourcePath: "app/package.json", + context: "filesystem-expression", + runtimePathKind: "node-entrypoint", + files: fileMap([ + file("app/package.json", "root"), + file("app/main.cjs", "root"), + file("app/sub/main.cjs", "root"), + ]), + }), + ).toMatchObject({ + declared_path: declaredPath, + resolution_status: "unavailable", + resolved_path: null, + }); + }, +); + +describe("package metadata at archive module boundaries", () => { + it.each(["APP.ASAR", "app.AsAr"])( + "preserves archive component bytes while admitting %s runtime crossings", + (archive) => { + const directory = `resources/${archive}`; + const files = fileMap([ + file("main.cjs", "outer"), + file(`${directory}/main.cjs`, "archive"), + file( + `${directory}/pkg/package.json`, + "archive", + '{"main":"actual.cjs"}', + ), + file(`${directory}/pkg/actual.cjs`, "archive"), + file(`${directory}/pkg/index.js`, "archive"), + file("resources/pkg/package.json", "outer", '{"main":"actual.cjs"}'), + file("resources/pkg/actual.cjs", "outer"), + file("resources/pkg/index.js", "outer"), + ]); + for (const [sourcePath, declaredPath, target] of [ + ["main.cjs", `./${directory}/pkg`, `${directory}/pkg/actual.cjs`], + [`${directory}/main.cjs`, "../pkg", "resources/pkg/actual.cjs"], + ] as const) + expect( + resolve({ + sourcePath, + declaredPath, + context: "module-specifier", + moduleKind: "require", + files, + }), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: target, + }); + }, + ); + + it("does not admit same-archive or similarly named foreign containers", () => { + for (const directory of ["resources/APP.ASAR", "resources/APP.ASAR-extra"]) + expect( + resolve({ + sourcePath: `${directory}/main.cjs`, + declaredPath: "./window.cjs", + context: "module-specifier", + moduleKind: "require", + files: fileMap([ + file(`${directory}/main.cjs`, "source"), + file(`${directory}/window.cjs`, "foreign"), + ]), + }), + ).toMatchObject({ + resolution_status: "unavailable", + resolved_path: null, + }); + }); + it.each([ + ["resources/app.asar/main.cjs", "../pkg", "resources/pkg"], + ["app/main.cjs", "../resources/code.asar/pkg", "resources/code.asar/pkg"], + [ + "resources/app.asar/main.cjs", + "fixture", + "resources/node_modules/fixture", + ], + ])( + "uses the selected package main for %s loading %s", + (sourcePath, declaredPath, directory) => { + expect( + resolve({ + sourcePath, + declaredPath, + context: "module-specifier", + moduleKind: "require", + files: fileMap([ + file(sourcePath, "source"), + file( + `${directory}/package.json`, + "target", + '{"main":"actual.cjs"}', + ), + file(`${directory}/actual.cjs`, "target"), + file(`${directory}/index.js`, "target"), + ]), + }), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: `${directory}/actual.cjs`, + }); + }, + ); + + it("keeps metadata-less ancestor package directories after an archive escape", () => { + expect( + resolve({ + sourcePath: "resources/app.asar/main.cjs", + declaredPath: "fixture/parser", + context: "module-specifier", + moduleKind: "require", + files: fileMap([ + file("resources/app.asar/main.cjs", "source"), + file("resources/node_modules/fixture/parser.js", "target"), + ]), + }), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "resources/node_modules/fixture/parser.js", + }); + }); + + it.each(["resources/app.asar/main.cjs", "resources/main.cjs"])( + "keeps unread metadata blocking a decoy index for %s", + (sourcePath) => { + const path = "resources/pkg/package.json"; + const target = file(path, "target", null); + const files = fileMap([ + file(sourcePath, "source"), + file("resources/pkg/index.js", "source"), + ]); + for (const opaque of [false, true]) { + expect( + resolve({ + sourcePath, + declaredPath: sourcePath.includes(".asar/") ? "../pkg" : "./pkg", + context: "module-specifier", + moduleKind: "require", + files: opaque ? files : fileMap([...files.values(), target]), + unanalyzedFiles: { + get: (requested) => + opaque && requested === path ? target : undefined, + hasDirectory: () => false, + }, + }), + ).toMatchObject({ + resolution_status: "unavailable", + resolved_path: null, + }); + } + }, + ); +}); diff --git a/src/application/javascript/JavaScriptArtifactPathResolution.ts b/src/application/javascript/JavaScriptArtifactPathResolution.ts index b536b0a337..fc0a390920 100644 --- a/src/application/javascript/JavaScriptArtifactPathResolution.ts +++ b/src/application/javascript/JavaScriptArtifactPathResolution.ts @@ -1,5 +1,6 @@ import { isBuiltin } from "node:module"; -import { posix } from "node:path"; +import { posix, win32 } from "node:path"; +import { ELECTRON_MODULE } from "../../domain/javascript/javascriptElectronMemberWrites.js"; import { normalizeJoinedLogicalPath, @@ -27,6 +28,7 @@ import { type TypeScriptEmittedSourceCandidate, } from "../TypeScriptEmittedSourceCandidates.js"; import { isRelativeJavaScriptModuleSpecifier } from "../../domain/javascript/javascriptModuleSpecifier.js"; +import { selectPackageSubpathPattern } from "../../domain/javascript/javascriptPackageSubpathPatterns.js"; import type { JavaScriptModuleResolutionMode } from "../../domain/javascript/javascriptModuleResolutionMode.js"; import type { JsonValue } from "../../domain/jsonValue.js"; import { err, ok, type Result } from "../../domain/result.js"; @@ -46,6 +48,8 @@ type UnresolvedArtifactPathStatus = interface ArtifactPathResolutionBase { readonly declared_path: string; + readonly runtime_path_base?: "application-root"; + readonly runtime_path_base_proven?: true; readonly resolution_context: ArtifactPathResolutionContext; readonly limitations: readonly string[]; readonly conditional_resolutions?: readonly ConditionalModuleResolution[]; @@ -87,7 +91,14 @@ export interface ResolveArtifactPathInput { /** Import/require semantics; runtime resource paths omit this provenance. */ readonly moduleKind?: JavaScriptModuleResolutionMode; /** Runtime paths retain their loader owner independently of source imports. */ - readonly runtimePathKind?: "exact-file" | "node-entrypoint" | "url-resource"; + readonly runtimePathKind?: + | "exact-file" + | "application-file" + | "node-entrypoint" + | "url-resource"; + /** Logical root of an inventoried nested application container. */ + readonly applicationRootPath?: string | null; + readonly applicationRootIsProven?: boolean; } /** Preserve an explicitly retained loader mode on graph relationships. */ @@ -193,6 +204,14 @@ export const resolveArtifactPathByContext = ( const resolution = resolvePath(input); const facts = { declared_path: input.declaredPath, + ...(input.runtimePathKind === "application-file" + ? { runtime_path_base: "application-root" as const } + : {}), + ...(input.runtimePathKind === "application-file" && + input.applicationRootIsProven === true && + input.applicationRootPath !== null + ? { runtime_path_base_proven: true as const } + : {}), resolution_context: input.context, limitations: resolution.limitations, ...(resolution.candidatePath === undefined @@ -215,6 +234,12 @@ export const resolveArtifactPathByContext = ( export const artifactPathResolutionProperties = ( resolution: ArtifactPathResolution | undefined, ): Readonly> => ({ + ...(resolution?.runtime_path_base_proven === undefined + ? {} + : { runtime_path_base_proven: resolution.runtime_path_base_proven }), + ...(resolution?.runtime_path_base === undefined + ? {} + : { runtime_path_base: resolution.runtime_path_base }), ...(resolution?.candidate_path === undefined ? {} : { candidate_path: resolution.candidate_path }), @@ -248,7 +273,50 @@ const conditionalResolution = ( : { candidate_path: result.candidatePath }), }); +/** Host path admission is shared by top-level and metadata-derived inputs. */ +const resolveRuntimeHostPath = ( + input: ResolveArtifactPathInput, +): CandidateResolution | null => { + if ( + (input.runtimePathKind !== undefined || isNodeFilesystemLookup(input)) && + input.runtimePathKind !== "url-resource" && + (posix.isAbsolute(input.declaredPath) || + win32.isAbsolute(input.declaredPath) || + (input.context === "module-specifier" && + input.moduleKind !== "require" && + /^file:/iu.test(input.declaredPath))) + ) { + const esm = + input.context === "module-specifier" && input.moduleKind === "import"; + if (esm) { + const normalized = URL.parse( + htmlUrlText(input.declaredPath), + PACKAGE_ARTIFACT_ROOT_URL, + ); + if (normalized === null) + return unresolvedCandidate("rejected", [ + "The module URL cannot be parsed.", + ]); + const admitted = decodeModuleUrlPath(normalized.pathname); + if (typeof admitted !== "string") return admitted; + } + // Host syntax is not a canonical logical path. In particular, Windows + // separators and drive letters must not be reclassified as invalid/URLs. + return input.declaredPath.includes("\0") || + (!esm && fileUrlPath(input.declaredPath) === null) + ? unresolvedCandidate("rejected", [ + "NUL or unsafe file URL path syntax is not admitted in a runtime path.", + ]) + : unresolvedCandidate("unavailable", [ + "An absolute runtime host path has no established mapping to the selected logical artifact root.", + ]); + } + return null; +}; + const resolvePath = (input: ResolveArtifactPathInput): CandidateResolution => { + const hostPath = resolveRuntimeHostPath(input); + if (hostPath !== null) return hostPath; if (input.runtimePathKind === "node-entrypoint") return resolveNodeRuntimeEntrypoint(input); if (input.moduleKind === "compiler-dependent") { @@ -286,7 +354,11 @@ const resolvePath = (input: ResolveArtifactPathInput): CandidateResolution => { } if (input.runtimePathKind === "url-resource") return resolveExactUrlResource(input); - if (input.runtimePathKind === "exact-file" && isRuntimePathDeclaration(input)) + if ( + (input.runtimePathKind === "exact-file" || + input.runtimePathKind === "application-file") && + isRuntimePathDeclaration(input) + ) return resolveRuntimeEntrypoint(input); // Package import keys are literal lookup identities, not artifact paths. // The selected target is admitted separately by resolveExportTarget. @@ -365,18 +437,37 @@ const isRuntimePathDeclaration = (input: ResolveArtifactPathInput): boolean => const resolveNodeRuntimeEntrypoint = ( input: ResolveArtifactPathInput, ): CandidateResolution => { - const { runtimePathKind: _runtimePathKind, ...sourceInput } = input; - const runtimeInput = { ...sourceInput, moduleKind: "require" as const }; - if (!isRuntimePathDeclaration(runtimeInput)) return resolvePath(runtimeInput); + const windowsPath = unknownWindowsRuntimePath(input.declaredPath); + if (windowsPath !== null) return windowsPath; + const runtimeInput = { ...input, moduleKind: "require" as const }; + if (input.context === "package-entrypoint" && input.declaredPath === "") + return resolveNodePackageMain( + runtimeInput, + posix.dirname(input.sourcePath), + undefined, + ); + if ( + runtimeInput.context !== "package-entrypoint" && + !isRuntimePathDeclaration(runtimeInput) + ) { + const { runtimePathKind: _runtimePathKind, ...moduleInput } = runtimeInput; + return resolvePath(moduleInput); + } const candidate = contextualCandidate(runtimeInput); if (typeof candidate !== "string") return candidate; const confined = confineCandidate(candidate); if (typeof confined !== "string") return confined; // Node entrypoints use runtime file/directory probes. They do not originate // in emitted TypeScript import syntax and cannot select a source substitute. - const resolution = resolveCandidate(runtimeInput, confined, new Set()); - // Runtime entrypoint analysis has no extension or directory fallback, but - // retain the normalized loader path when that exact target is absent. + const resolution = + input.context === "package-entrypoint" + ? resolveNodePackageMain( + runtimeInput, + posix.dirname(input.sourcePath), + confined, + ) + : resolveCandidate(runtimeInput, confined, new Set()); + // Retain the normalized declaration when Node's owned probes are exhausted. return resolution.status === "not-found" && resolution.candidatePath === undefined ? { ...resolution, candidatePath: confined } @@ -386,14 +477,30 @@ const resolveNodeRuntimeEntrypoint = ( const resolveRuntimeEntrypoint = ( input: ResolveArtifactPathInput, ): CandidateResolution => { - const rejected = rejectDeclaration(input.declaredPath); + if (input.runtimePathKind === "application-file") { + const windowsPath = unknownWindowsRuntimePath(input.declaredPath); + if (windowsPath !== null) return windowsPath; + } + const rejected = rejectDeclaration( + input.declaredPath, + declaresLiteralPercent(input), + ); if (rejected !== null) return rejected; if (hasScheme(input.declaredPath) || looksExternal(input.declaredPath)) return unresolvedCandidate("external", [ "Electron runtime entrypoints must identify local paths, not external URLs.", ]); + if ( + input.runtimePathKind === "application-file" && + input.applicationRootPath === null + ) + return unresolvedCandidate("unavailable", [ + "Inventoried package metadata does not establish one application root for this source; the loadFile base is unknown.", + ]); const candidate = confineCandidate( - relativeCandidate(input.sourcePath, input.declaredPath), + input.runtimePathKind === "application-file" + ? posix.join(input.applicationRootPath ?? "", input.declaredPath) + : relativeCandidate(input.sourcePath, input.declaredPath), ); if (typeof candidate !== "string") return candidate; return ( @@ -408,6 +515,17 @@ const resolveRuntimeEntrypoint = ( ); }; +/** Host filesystem spellings need target-platform context before admission. */ +const unknownWindowsRuntimePath = ( + declared: string, +): CandidateResolution | null => + !declared.includes("\0") && + (declared.includes("\\") || /^[a-z]:/iu.test(declared)) + ? unresolvedCandidate("unavailable", [ + "Windows-relative runtime paths require target-platform and drive context that the selected logical artifact inventory does not establish.", + ]) + : null; + const resolveRuntimeSourceCandidates = ( input: ResolveArtifactPathInput, exactPath: string, @@ -453,14 +571,19 @@ const rejectDeclaration = ( }; /** - * Node's CommonJS loader never percent-decodes a request or a legacy package - * main, so `%2e` and `%2f` there are literal filename bytes, not encoded dots - * or separators. Root confinement still applies to the joined candidate. + * Filesystem expressions, exact runtime paths, and CommonJS requests retain + * literal filename bytes. URL-to-path conversion belongs to the producer; + * these consumers must not decode or reject the resulting percent spelling. + * Root confinement still applies to the joined candidate. */ const declaresLiteralPercent = (input: ResolveArtifactPathInput): boolean => - input.moduleKind === "require" && - (input.context === "module-specifier" || - input.context === "package-entrypoint"); + input.context === "filesystem-expression" || + (input.runtimePathKind === "exact-file" && + isRuntimePathDeclaration(input) && + !looksExternal(input.declaredPath)) || + (input.moduleKind === "require" && + (input.context === "module-specifier" || + input.context === "package-entrypoint")); const contextualCandidate = ( input: ResolveArtifactPathInput, @@ -497,6 +620,15 @@ const contextualCandidate = ( "File URL authorities or invalid file URL forms cannot be mapped to artifact-local paths.", ]); } + if ( + context === "module-specifier" && + input.moduleKind === "require" && + declared.includes("\\") && + (!hasScheme(declared) || /^[a-z]:/iu.test(declared)) + ) { + const windowsPath = unknownWindowsRuntimePath(declared); + if (windowsPath !== null) return windowsPath; + } const hasUrlPathSemantics = context === "module-specifier" && input.moduleKind === "import"; if (hasUrlPathSemantics) declared = htmlUrlText(declared); @@ -516,15 +648,59 @@ const contextualCandidate = ( ) return bareModuleCandidate(input); if (input.moduleKind !== "require") { - if (/%(?:2f|5c)/iu.test(stripQueryAndFragment(declared))) - return unresolvedCandidate("rejected", [ - "ES module URL paths cannot contain percent-encoded slash or backslash bytes.", - ]); + if (hasUrlPathSemantics) { + // URL normalization can leave and then re-enter the synthetic namespace. + // Preserve the shared URL owner's above-root markers before that loss. + if ( + applyUrlReference( + urlDirectoryState(posix.dirname(input.sourcePath)), + stripQueryAndFragment(declared), + true, + ).escapes > 0 + ) + return unresolvedCandidate("rejected", [ + "The module URL escapes the selected logical artifact root.", + ]); + const sourceUrl = packageMetadataUrl(input.sourcePath, declared); + const base = sourceUrl.ok + ? sourceUrl.value + : sourceUrl.error.structuralUrl; + const normalized = URL.parse(declared, base); + if (normalized === null) + return unresolvedCandidate("rejected", [ + "The module URL cannot be parsed.", + ]); + if (!normalized.pathname.startsWith(PACKAGE_ARTIFACT_ROOT_URL.pathname)) + return unresolvedCandidate("rejected", [ + "The module URL escapes the selected logical artifact root.", + ]); + // Node validates the complete normalized file URL, including encoded + // separator bytes inherited from an inventoried source directory. + const decoded = decodeModuleUrlPath( + normalized.pathname.slice(PACKAGE_ARTIFACT_ROOT_URL.pathname.length), + ); + if (typeof decoded !== "string") return decoded; + if ( + !sourceUrl.ok && + decoded + .split("/") + .some((segment) => sourceUrl.error.structuralSegments.has(segment)) + ) + return unresolvedCandidate("unavailable", [ + sourceUrl.error.limitation, + ]); + const candidate = urlRelativeCandidate("", `/${decoded}`); + return typeof candidate === "string" + ? exactUrlPathCandidate(decoded, candidate, "module URL") + : candidate; + } const decoded = decodeModuleUrlPath(stripQueryAndFragment(declared)); if (typeof decoded !== "string") return decoded; declared = decoded; } - } else if (looksExternal(declared)) + } else if (context !== "package-entrypoint" && looksExternal(declared)) + // Package main is a filesystem declaration. Scheme-like filename bytes + // remain literal; URI classification belongs to module and URL owners. return unresolvedCandidate("external", [ "URL schemes and protocol-relative URLs are outside this local artifact path context.", ]); @@ -574,6 +750,10 @@ const contextualCandidate = ( const decodeModuleUrlPath = ( declared: string, ): string | CandidateResolution => { + if (/%(?:2f|5c)/iu.test(declared)) + return unresolvedCandidate("rejected", [ + "ES module URL paths cannot contain percent-encoded slash or backslash bytes.", + ]); let decoded: string; try { decoded = decodeURIComponent(declared); @@ -618,17 +798,19 @@ const urlDirectoryState = (directory: string): UrlPathState => ({ const applyUrlReference = ( base: UrlPathState, reference: string, + encodedDotSegments = false, ): UrlPathState => { const absolute = reference.startsWith("/"); const segments = absolute ? [] : [...base.segments]; let escapes = absolute ? 0 : base.escapes; const parts = (absolute ? reference.slice(1) : reference).split("/"); for (const [index, part] of parts.entries()) { - if (part !== "." && part !== "..") { + const dot = encodedDotSegments ? part.replace(/%2e/giu, ".") : part; + if (dot !== "." && dot !== "..") { segments.push(part); continue; } - if (part === "..") + if (dot === "..") if (segments.length > 0) segments.pop(); else escapes += 1; if (index === parts.length - 1) segments.push(""); @@ -704,7 +886,6 @@ const bareModuleCandidate = ( if (typeof confined !== "string") return confined; dependencyDirectory = posix.dirname(confined); } - const source = input.files.get(input.sourcePath); let nearestNotFound: CandidateResolution | null = null; for (const directory of packageLookupDirectories(input)) { const candidate = posix.join(directory, dependencyDirectory); @@ -716,14 +897,7 @@ const bareModuleCandidate = ( (!request.dependencyExportsEligible || metadata.exports === undefined))); const packageResolution = - metadata.kind !== "absent" || - hasContainerDirectory( - input.files, - candidate, - source?.container_sha256, - input.directoryPaths, - input.unanalyzedFiles, - ) + metadata.kind !== "absent" || hasContainerDirectory(input, candidate) ? resolvePackageDependency(input, candidate, request, metadata) : request.packageSubpath === "." && input.moduleKind !== "import" ? resolveFileCandidates( @@ -795,10 +969,6 @@ const resolvePackageImportAlias = ( return unresolvedCandidate("rejected", [ `The package import specifier ${JSON.stringify(specifier)} is invalid; "#" and specifiers that end with "/" are not valid import aliases.`, ]); - if (classified.kind === "pattern") - return unresolvedCandidate("external", [ - `Wildcard and directory package import aliases remain unresolved; ${JSON.stringify(specifier)} is not an exact imports key.`, - ]); const metadata = enclosingPackageMetadata(input); if (metadata.kind === "none") return unresolvedCandidate("external", [ @@ -816,7 +986,7 @@ const resolvePackageImportAlias = ( `Package metadata ${metadata.path} does not define package import alias ${specifier}; it declares ${lookup.declared.join(", ") || "no imports keys"}.`, ]); if (lookup.kind === "pattern-unresolved") - return unresolvedCandidate("external", [ + return unresolvedCandidate("unavailable", [ `Wildcard package import aliases remain unresolved; package metadata ${metadata.path} has no exact imports key ${specifier}.`, ]); const selected = selectExportTarget( @@ -839,11 +1009,18 @@ const resolvePackageImportAlias = ( ]); if (selected.kind === "non-relative") { const target = classifyPackageImportTargetString(selected.value); - return unresolvedCandidate("external", [ - target.kind === "recursive" - ? `Recursive package import aliases remain unresolved; package metadata ${metadata.path} maps ${specifier} to ${JSON.stringify(target.target)}.` - : `External-package import targets remain unresolved; package metadata ${metadata.path} maps ${specifier} to ${JSON.stringify(target.target)}.`, - ]); + // Unsupported lookup may still reach a selected module and another launch + // owner. Only a known builtin establishes that no local imports follow. + return unresolvedCandidate( + isBuiltin(selected.value) || ELECTRON_MODULE.test(selected.value) + ? "external" + : "unavailable", + [ + target.kind === "recursive" + ? `Recursive package import aliases remain unresolved; package metadata ${metadata.path} maps ${specifier} to ${JSON.stringify(target.target)}.` + : `External-package import targets remain unresolved; package metadata ${metadata.path} maps ${specifier} to ${JSON.stringify(target.target)}.`, + ], + ); } return resolveExportTarget(input, metadata.path, selected, "imports"); }; @@ -958,16 +1135,7 @@ const resolveLegacyEsmSubpath = ( return unresolvedCandidate("unavailable", [unrepresentableBase.limitation]); const file = resolveFileCandidates(input, [candidate]); if (file !== null) return file; - const source = input.files.get(input.sourcePath); - if ( - hasContainerDirectory( - input.files, - candidate, - source?.container_sha256, - input.directoryPaths, - input.unanalyzedFiles, - ) - ) + if (hasContainerDirectory(input, candidate)) return unresolvedCandidate("rejected", [ `ESM package subpath ${candidate} names a directory; directory imports are unsupported and package main or index lookup was not attempted.`, ]); @@ -998,6 +1166,10 @@ const resolvePackageExports = ( return unresolvedCandidate("external", [ `Directory package metadata ${packagePath} blocks subpath ${subpath} with ${exported.cause === "null" ? "an explicit null exports target" : "an empty exports target array"}.`, ]); + if (exported.status === "pattern-unresolved") + return unresolvedCandidate("unavailable", [ + `Package metadata ${packagePath} selects wildcard exports key ${JSON.stringify(exported.pattern)} for subpath ${subpath}; wildcard target expansion remains unresolved.`, + ]); return resolveExportTarget(input, packagePath, exported); }; @@ -1080,37 +1252,35 @@ const parseBarePackageRequest = ( }; const hasContainerDirectory = ( - files: ReadonlyMap, + input: ResolveArtifactPathInput, candidate: string, - containerSha256: string | undefined, - directoryPaths: ReadonlySet | undefined, - unanalyzedFiles?: JavaScriptUnanalyzedArtifactLookup, ): boolean => { - if (directoryPaths?.has(candidate) === true) return true; + if (input.directoryPaths?.has(candidate) === true) return true; if ( [...indexCandidates(candidate), posix.join(candidate, "package.json")].some( (path) => { - const file = files.get(path); + const file = input.files.get(path); return ( file !== undefined && - (containerSha256 === undefined || - file.container_sha256 === containerSha256) + admitsRuntimeContainer(input, path, file.container_sha256) ); }, ) ) return true; - // Inventory has files rather than directory entries. Any same-container - // descendant establishes a package directory even without package.json. + // Inventory has files rather than directory entries. An admitted descendant + // establishes a package directory even without package.json. const prefix = `${candidate}/`; - for (const [path, file] of files) + for (const [path, file] of input.files) if ( path.startsWith(prefix) && - (containerSha256 === undefined || - file.container_sha256 === containerSha256) + admitsRuntimeContainer(input, path, file.container_sha256) ) return true; - return unanalyzedFiles?.hasDirectory(candidate, containerSha256) ?? false; + const container = crossesRuntimeArchiveBoundary(input, `${candidate}/`) + ? undefined + : input.files.get(input.sourcePath)?.container_sha256; + return input.unanalyzedFiles?.hasDirectory(candidate, container) ?? false; }; const htmlCandidate = ( @@ -1228,12 +1398,12 @@ const readPackageMetadata = ( ): PackageMetadata => { const path = posix.join(directory, "package.json"); const file = input.files.get(path); - const source = input.files.get(input.sourcePath); if (file === undefined) { const unavailable = input.unanalyzedFiles?.get(path); if ( - source !== undefined && - unavailable?.container_sha256 === source.container_sha256 + unavailable !== undefined && + (admitsRuntimeContainer(input, path, unavailable.container_sha256) || + input.moduleKind !== undefined) ) return { kind: "unavailable", @@ -1243,8 +1413,13 @@ const readPackageMetadata = ( }; return { kind: "absent" }; } - if (source !== undefined && file.container_sha256 !== source.container_sha256) - return { kind: "absent" }; + if (!admitsRuntimeContainer(input, path, file.container_sha256)) + return input.moduleKind === undefined + ? { kind: "absent" } + : { + kind: "unavailable", + limitation: `Inventoried package metadata ${path} crosses an artifact container boundary without an established runtime loader mapping.`, + }; if (!file.text.included) return { kind: "unavailable", @@ -1260,7 +1435,7 @@ const readPackageMetadata = ( limitation: `Directory package metadata ${path} is not valid package JSON.`, }; } - if (typeof value !== "object" || value === null) + if (typeof value !== "object" || value === null || Array.isArray(value)) return { kind: "unavailable", limitation: `Directory package metadata ${path} is not valid package JSON.`, @@ -1351,12 +1526,12 @@ const resolveDirectory = ( ? metadata.module : undefined; if (entry === undefined) - return ( - resolveFileCandidates( - input, - indexCandidates(directory, input.moduleKind), - ) ?? notFoundCandidate() - ); + return input.moduleKind !== undefined + ? resolveNodePackageMain(input, directory, undefined) + : (resolveFileCandidates( + input, + indexCandidates(directory, input.moduleKind), + ) ?? notFoundCandidate()); if (typeof entry !== "string" || entry.length === 0) return unresolvedCandidate("unavailable", [ `Directory package metadata ${metadata.path} is not valid package JSON.`, @@ -1374,6 +1549,12 @@ const resolveDirectory = ( : { unanalyzedFiles: input.unanalyzedFiles }), ...(input.moduleKind === undefined ? {} : { moduleKind: input.moduleKind }), }; + const hostPath = resolveRuntimeHostPath(entryInput); + if (hostPath !== null) return hostPath; + if (entryInput.moduleKind !== undefined) { + const windowsPath = unknownWindowsRuntimePath(entry); + if (windowsPath !== null) return windowsPath; + } const candidate = contextualCandidate(entryInput); if (typeof candidate !== "string") return candidate; const confined = confineCandidate(candidate); @@ -1397,55 +1578,47 @@ const resolveDirectory = ( const resolveNodePackageMain = ( input: ResolveArtifactPathInput, directory: string, - mainPath: string, + mainPath: string | undefined, ): CandidateResolution => { - const normalizedMain = mainPath.replace(/\/$/u, ""); - for (const suffix of NODE_PACKAGE_MAIN_SUFFIXES) { - const isDirectoryIndex = suffix.startsWith("/"); - if (normalizedMain === "" && !isDirectoryIndex) continue; - const selected = resolveFileCandidates(input, [ - isDirectoryIndex - ? posix.join(normalizedMain, suffix.slice(1)) - : posix.normalize(`${normalizedMain}${suffix}`), - ]); - if (selected === null) continue; - if (input.moduleKind === "require") return selected; - // ESM probes the normalized main, then appends the selected suffix to - // the original declaration before finalizing the resulting file path. - const finalized = confineCandidate( - relativeCandidate(input.sourcePath, `${input.declaredPath}${suffix}`), - ); - if (typeof finalized !== "string") return finalized; - const finalizedFile = resolveFileCandidates(input, [ - finalized.replace(/\/$/u, ""), - ]); - if (finalizedFile !== null) return finalizedFile; - const source = input.files.get(input.sourcePath); - if ( - hasContainerDirectory( - input.files, - finalized, - source?.container_sha256, - input.directoryPaths, - input.unanalyzedFiles, - ) - ) - return { - ...unresolvedCandidate("rejected", [ - `The selected legacy ESM package main finalizes to directory ${finalized}; directory imports are unsupported and no later main or index candidate was tried.`, - ]), - candidatePath: finalized, - }; - return unresolvedCandidate("not-found", [ - `The selected legacy ESM package main finalizes to ${finalized}, which is not an inventoried file; no later main or index candidate was tried.`, - ]); - } + const normalizedMain = mainPath?.replace(/\/$/u, ""); + if (normalizedMain !== undefined) + for (const suffix of NODE_PACKAGE_MAIN_SUFFIXES) { + const isDirectoryIndex = suffix.startsWith("/"); + if (normalizedMain === "" && !isDirectoryIndex) continue; + const selected = resolveFileCandidates(input, [ + isDirectoryIndex + ? posix.join(normalizedMain, suffix.slice(1)) + : posix.normalize(`${normalizedMain}${suffix}`), + ]); + if (selected === null) continue; + if (input.moduleKind === "require") return selected; + // ESM probes the normalized main, then appends the selected suffix to + // the original declaration before finalizing the resulting file path. + const finalized = confineCandidate( + relativeCandidate(input.sourcePath, `${input.declaredPath}${suffix}`), + ); + if (typeof finalized !== "string") return finalized; + const finalizedFile = resolveFileCandidates(input, [ + finalized.replace(/\/$/u, ""), + ]); + if (finalizedFile !== null) return finalizedFile; + if (hasContainerDirectory(input, finalized)) + return { + ...unresolvedCandidate("rejected", [ + `The selected legacy ESM package main finalizes to directory ${finalized}; directory imports are unsupported and no later main or index candidate was tried.`, + ]), + candidatePath: finalized, + }; + return unresolvedCandidate("not-found", [ + `The selected legacy ESM package main finalizes to ${finalized}, which is not an inventoried file; no later main or index candidate was tried.`, + ]); + } const fallback = resolveFileCandidates( input, indexCandidates(directory, input.moduleKind), ); if (fallback !== null) return fallback; - return input.moduleKind === "require" + return input.moduleKind === "require" && mainPath !== undefined ? { ...notFoundCandidate(), stopsAncestorLookup: true } : notFoundCandidate(); }; @@ -1472,24 +1645,83 @@ const resolveFileCandidates = ( return null; }; +/** Classify the extension as inventory does; retain exact archive occurrence bytes. */ +const runtimeArchiveRoot = (path: string): string | undefined => { + const components = path.split("/"); + const archive = components.findLastIndex( + (component, index) => + index < components.length - 1 && + component.toLowerCase().endsWith(".asar"), + ); + return archive < 0 + ? undefined + : `${components.slice(0, archive + 1).join("/")}/`; +}; + +/** One archive-path admission rule serves files, directories and their metadata. */ +const isNodeFilesystemLookup = (input: ResolveArtifactPathInput): boolean => + input.runtimePathKind === "node-entrypoint" || + (input.moduleKind !== undefined && + (input.context === "module-specifier" || + input.context === "package-entrypoint")); + +/** Runtime launch and module lookup share archive occurrence admission. */ +const crossesRuntimeArchiveBoundary = ( + input: ResolveArtifactPathInput, + path: string, +): boolean => { + // Electron's archive filesystem normalizes a relative module path before + // opening it, including entry into an inventoried archive occurrence and + // escape from one, after the existing artifact-root confinement check. + const sourceArchive = runtimeArchiveRoot(input.sourcePath); + const targetArchive = runtimeArchiveRoot(path); + return ( + isNodeFilesystemLookup(input) && + ((sourceArchive !== undefined && !path.startsWith(sourceArchive)) || + (targetArchive !== undefined && + !input.sourcePath.startsWith(targetArchive))) + ); +}; + +const admitsRuntimeContainer = ( + input: ResolveArtifactPathInput, + path: string, + containerSha256: string, +): boolean => { + const source = input.files.get(input.sourcePath); + return ( + crossesRuntimeArchiveBoundary(input, path) || + (input.runtimePathKind === "application-file" && + input.applicationRootIsProven === true) || + source === undefined || + containerSha256 === source.container_sha256 + ); +}; + /** Own candidate presence and container identity independently of loader policy. */ const findFileCandidate = ( input: ResolveArtifactPathInput, path: string, ): CandidateResolution | null => { - const source = input.files.get(input.sourcePath); const target = input.files.get(path); if ( target !== undefined && - (source === undefined || - target.container_sha256 === source.container_sha256) + admitsRuntimeContainer(input, path, target.container_sha256) ) return { resolvedPath: path, status: "resolved", limitations: [] }; + if (target !== undefined && isNodeFilesystemLookup(input)) + return { + resolvedPath: null, + status: "unavailable", + candidatePath: path, + limitations: [ + "The inventoried module crosses an artifact container boundary; its runtime loader mapping has not been established.", + ], + }; const unanalyzed = input.unanalyzedFiles?.get(path); if ( unanalyzed !== undefined && - (source === undefined || - unanalyzed.container_sha256 === source.container_sha256) + admitsRuntimeContainer(input, path, unanalyzed.container_sha256) ) return { resolvedPath: null, @@ -1542,9 +1774,10 @@ type PackageExportOutcome = | ({ readonly status: "value" } & PackageFileTarget) | { readonly status: "rejected"; readonly limitation: string } | { readonly status: "blocked"; readonly cause: PackageTargetBlockCause } + | { readonly status: "pattern-unresolved"; readonly pattern: string } | { readonly status: "unmatched"; readonly declared: readonly string[] }; -/** Only exact non-directory keys participate; wildcard expansion is unsupported. */ +/** Exact keys precede the best matching pattern; pattern expansion remains unknown. */ const selectPackageExport = ( value: unknown, subpath: string, @@ -1552,6 +1785,7 @@ const selectPackageExport = ( packagePath: string, ): PackageExportOutcome => { let target = value; + let pattern: string | undefined; let declared: string[] = []; if (typeof value === "object" && value !== null && !Array.isArray(value)) { const keys = Object.keys(value); @@ -1564,13 +1798,14 @@ const selectPackageExport = ( }; declared = keys; if (hasSubpaths) { - if ( - subpath.endsWith("/") || - subpath.includes("*") || - !Object.hasOwn(value, subpath) - ) - return { status: "unmatched", declared }; - target = Reflect.get(value, subpath); + if (subpath.endsWith("/")) return { status: "unmatched", declared }; + if (!subpath.includes("*") && Object.hasOwn(value, subpath)) + target = Reflect.get(value, subpath); + else { + pattern = selectPackageSubpathPattern(keys, subpath); + if (pattern === undefined) return { status: "unmatched", declared }; + target = Reflect.get(value, pattern); + } } else if (subpath !== ".") return { status: "unmatched", declared }; } else if (subpath !== ".") return { status: "unmatched", declared }; const selected = selectExportTarget( @@ -1592,6 +1827,7 @@ const selectPackageExport = ( status: "rejected", limitation: `The package exports target ${JSON.stringify(selected.value)} must start with "./".`, }; + if (pattern !== undefined) return { status: "pattern-unresolved", pattern }; return { status: "value", value: selected.value, diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts index 14312fc85e..c3c4fea0fc 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts @@ -2,6 +2,7 @@ import { primitiveByteExpansionSource } from "../../../tests/fixtures/javascript import { graphForJavaScript as graphFor } from "../../../tests/fixtures/javascriptSemanticGraph.js"; import * as t from "@babel/types"; import { expect, it } from "vitest"; +import { aliasSelectionBudgetSource } from "../../../tests/fixtures/javascriptAliasSelectionBudget.js"; import { SEMANTIC_GRAPH_FILE_NODE_CEILING, @@ -26,6 +27,36 @@ import { getterExpansionSource } from "../../../tests/fixtures/javascriptGetterE const SHA256 = "a".repeat(64); +it("projects bounded alias selection as a resource-limit unknown", () => { + const source = aliasSelectionBudgetSource(); + const ir = analyzeJavaScriptSemantics(source); + const graph = graphFor(source, ir); + expect(ir.coverage).toMatchObject({ + status: "partial", + resourceLimits: ["alias-selections"], + }); + const unknown = graph.unknowns.find( + ({ reason, detail }) => + reason === "resource-limit" && /Alias selection expansion/u.test(detail), + ); + expect(unknown).toBeDefined(); + expect( + graph.evidence_contexts.find( + ({ context_id }) => context_id === unknown?.evidence.context_id, + ), + ).toMatchObject({ + coverage: { + limits: expect.arrayContaining([ + { + name: "javascript_semantic_alias_selections", + value: 512, + unit: "items", + }, + ]), + }, + }); +}); + it.each([ ["plain export", "const answer = 42; export { answer };", 0], ["aliased export", "const answer = 42; export { answer as result };", 0], diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts index d39a7ef27c..3417fbb895 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts @@ -660,10 +660,10 @@ function* projectFileSteps( const projectFileResourceLimits = ( context: Pick, ): void => { - // Getter limits describe query effects without tagged values; other limits + // Query limits describe effects without tagged values; other limits // already project from binding or return-site values and must not be duplicated. for (const limit of semanticCoverageResourceLimits(context.ir.coverage)) { - if (limit !== "getter-expansion") continue; + if (limit !== "getter-expansion" && limit !== "alias-selections") continue; addSemanticGraphUnknown(context.state, { node_id: context.moduleNode.node_id, family: "data-flow", diff --git a/src/contracts/canonicalBase64Metadata.test.ts b/src/contracts/canonicalBase64Metadata.test.ts new file mode 100644 index 0000000000..fb5aea42fe --- /dev/null +++ b/src/contracts/canonicalBase64Metadata.test.ts @@ -0,0 +1,68 @@ +import { Ajv } from "ajv"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { expect, it } from "vitest"; +import { z } from "zod"; + +import { canonicalBase64Schema } from "../domain/stringPatterns.js"; +import { digestCanonicalValue } from "../domain/canonicalDigest.js"; +import { toolContract } from "./toolContracts.js"; +import { + toolInputSchemaWithMetadata, + toolOutputSchemaWithMetadata, +} from "./toolSchemaMetadata.js"; + +it.each(["draft-2020-12", "draft-07"] as const)( + "retains native base64 metadata and intersections in both MCP directions: %s", + (target) => { + const literal = { pattern: "^(?!literal)annotation$" }; + const canonical = z.strictObject({ + encoded: z.intersection(canonicalBase64Schema, z.string().max(4)), + literal: z + .strictObject({ pattern: z.literal(literal.pattern) }) + .meta({ default: literal, examples: [literal] }), + }); + const contract = { + ...toolContract("open_binary"), + inputSchema: canonical, + outputSchema: canonical, + }; + const input = toolInputSchemaWithMetadata(contract); + const output = toolOutputSchemaWithMetadata(contract); + const projectedInput = input["~standard"].jsonSchema.input({ target }); + const projectedOutput = output["~standard"].jsonSchema.output({ target }); + const { $id, ...withoutId } = projectedOutput; + expect($id).toBe( + `urn:rea:tool-output:sha256:${digestCanonicalValue({ target, schema: withoutId }, "Tool output schema")}`, + ); + for (const projected of [projectedInput, projectedOutput]) { + const properties = projected.properties as Record; + const literalProperty = properties.literal as Record; + const literalSchema = + typeof literalProperty.$ref === "string" + ? literalProperty.$ref + .slice(2) + .split("/") + .reduce( + (value, key) => (value as Record)[key], + projected, + ) + : literalProperty; + expect(literalSchema).toMatchObject({ + default: literal, + examples: [literal], + }); + const Validator = target === "draft-2020-12" ? Ajv2020 : Ajv; + const validate = new Validator({ strict: false }).compile(projected); + for (const [encoded, valid] of [ + ["AA==", true], + ["AB==", false], + ["AAAAAA==", false], + ] as const) { + const value = { encoded, literal }; + expect(validate(value), encoded).toBe(valid); + expect(input.safeParse(value).success, encoded).toBe(valid); + expect(output.safeParse(value).success, encoded).toBe(valid); + } + } + }, +); diff --git a/src/domain/javascript/electronStaticAnalysis.ts b/src/domain/javascript/electronStaticAnalysis.ts index d850a8458c..366aa7161b 100644 --- a/src/domain/javascript/electronStaticAnalysis.ts +++ b/src/domain/javascript/electronStaticAnalysis.ts @@ -1,9 +1,12 @@ -import type * as t from "@babel/types"; +import * as t from "@babel/types"; import { inspectElectronBrowserNode } from "./electronStaticAnalysisBrowser.js"; import { inspectElectronIpcNode } from "./electronStaticAnalysisIpc.js"; import { inspectElectronNativeNode } from "./electronStaticAnalysisNative.js"; import type { JavaScriptFindingContext } from "./javascriptStaticAnalysisState.js"; +import { electronCalleePath } from "./electronStaticAnalysisValues.js"; +import { addLocatedFinding } from "./javascriptStaticAnalysisFindings.js"; +import { range } from "./javascriptStaticAnalysisHelpers.js"; /** Collect Electron-specific syntax during the shared inert AST traversal. */ export const inspectElectronStaticNode = ( @@ -13,4 +16,26 @@ export const inspectElectronStaticNode = ( inspectElectronBrowserNode(node, context); inspectElectronIpcNode(node, context); inspectElectronNativeNode(node, context); + const path = + t.isCallExpression(node) || t.isOptionalCallExpression(node) + ? (electronCalleePath(node.callee, context) ?? + context.electronInvocationPath?.(node)) + : undefined; + if ( + (path !== undefined && + path[0] === "app" && + path[1] === "setAppPath" && + path + .slice(2) + .every((member) => ["call", "apply", "bind"].includes(member))) || + context.electronAppPathChangePossible?.(node) === true + ) + // A bound or exported setter may escape this source before it changes the + // shared app path. Keep that possibility without claiming an invocation. + addLocatedFinding(context, { + collection: context.accumulator.appPathChanges, + key: "electron:app-path-change", + node, + value: { module_key: null, location: range(node) }, + }); }; diff --git a/src/domain/javascript/electronStaticAnalysisAppPathSelection.test.ts b/src/domain/javascript/electronStaticAnalysisAppPathSelection.test.ts new file mode 100644 index 0000000000..697d545316 --- /dev/null +++ b/src/domain/javascript/electronStaticAnalysisAppPathSelection.test.ts @@ -0,0 +1,127 @@ +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +const prefix = "const {app,BrowserWindow}=require('electron');"; +const cases = [ + { + name: "assigned native setter callee", + body: "let change;(change=app.setAppPath)('/changed');", + possible: true, + calls: [1], + }, + { + name: "assigned local setter callee", + body: "let change;(change=()=>{})('/changed');", + possible: false, + calls: [0], + }, + { + name: "assigned native app receiver", + body: "let runtime;(runtime=app).setAppPath('/changed');", + possible: true, + calls: [1], + }, + { + name: "assigned local app receiver", + body: "let runtime;(runtime={setAppPath(){}}).setAppPath('/changed');", + possible: false, + calls: [0], + }, + { + name: "logical-or assigned app receiver", + body: "let runtime=flag?{setAppPath(){}}:null;(runtime||=app).setAppPath('/changed');", + possible: true, + calls: [1, 0], + }, + { + name: "logical-and assigned app receiver", + body: "let runtime=flag?app:null;(runtime&&=app)?.setAppPath('/changed');", + possible: true, + calls: [0, 1], + }, + { + name: "nullish assigned app receiver", + body: "let runtime=flag?{setAppPath(){}}:null;(runtime??=app).setAppPath('/changed');", + possible: true, + calls: [1, 0], + }, + { + name: "definite local setter replacement", + body: "app.setAppPath=()=>{};app.setAppPath('/changed');", + possible: false, + calls: [0], + }, + { + name: "unchanged native setter", + body: "app.setAppPath('/changed');", + possible: true, + calls: [1], + }, + { + name: "conditional setter replacement", + body: "if(flag)app.setAppPath=()=>{};app.setAppPath('/changed');", + possible: true, + calls: [1, 0], + }, + { + name: "setter captured before replacement", + body: "const change=app.setAppPath;app.setAppPath=()=>{};change('/changed');", + possible: true, + calls: [1], + }, + { + name: "setter captured after replacement", + body: "app.setAppPath=()=>{};const change=app.setAppPath;change('/changed');", + possible: false, + calls: [0], + }, + { + name: "setter invoked before replacement", + body: "app.setAppPath('/changed');app.setAppPath=()=>{};", + possible: true, + calls: [1], + }, +]; + +it.each(cases)( + "selects the app-path callable for $name", + ({ body, possible, calls }) => { + const source = `${prefix}${body}new BrowserWindow().loadFile('index.html');`; + for (const [index, expected] of calls.entries()) { + let changed = 0; + const rendered: string[] = []; + // Record JavaScript member/capture behavior without executing Electron. + runInNewContext(source, { + flag: index === 1, + require: () => ({ + app: { + setAppPath() { + changed += 1; + }, + }, + BrowserWindow: class { + loadFile(file: string) { + rendered.push(file); + } + }, + }), + }); + expect(changed).toBe(expected); + expect(rendered).toEqual(["index.html"]); + } + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.electron.app_path_changes ?? []).toEqual( + possible + ? [ + expect.objectContaining({ + module_key: null, + location: expect.any(Object), + }), + ] + : [], + ); + }, +); diff --git a/src/domain/javascript/electronStaticAnalysisBrowser.ts b/src/domain/javascript/electronStaticAnalysisBrowser.ts index c4a2ba193e..6bc5de9867 100644 --- a/src/domain/javascript/electronStaticAnalysisBrowser.ts +++ b/src/domain/javascript/electronStaticAnalysisBrowser.ts @@ -23,6 +23,7 @@ import { range, staticPath, staticPathResolutionContext, + type JavaScriptStaticPathIntrinsicQuery, } from "./javascriptStaticAnalysisHelpers.js"; import { semanticStaticPropertyKey, @@ -133,6 +134,7 @@ const inspectBrowserWindow = ( context.source, lookup.status, resolvedPreferences, + context.nodePathResolve, ); context.accumulator.unknownFindings += collected.unknown; const finding: ElectronBrowserWindowFinding = { @@ -163,6 +165,7 @@ const collectWindowOptions = ( source: string, lookupStatus: "explicit" | "missing" | "dynamic", preferences: t.Node | undefined, + nodePathResolve?: JavaScriptStaticPathIntrinsicQuery, ): { readonly status: ElectronBrowserWindowFinding["web_preferences_status"]; readonly preferences: readonly ElectronWebPreference[]; @@ -176,12 +179,13 @@ const collectWindowOptions = ( preload: { preload_path: null, preload_resolution_context: null }, unknown: lookupStatus === "missing" ? 0 : 1, }; - return collectWebPreferences(source, preferences); + return collectWebPreferences(source, preferences, nodePathResolve); }; const collectWebPreferences = ( source: string, object: t.ObjectExpression, + nodePathResolve?: JavaScriptStaticPathIntrinsicQuery, ): { readonly status: "object-literal"; readonly preferences: readonly ElectronWebPreference[]; @@ -193,7 +197,7 @@ const collectWebPreferences = ( const effectivePreload = staticObjectProperty(object, "preload"); const preloadPath = effectivePreload.status === "explicit" - ? staticPath(effectivePreload.property.value) + ? staticPath(effectivePreload.property.value, nodePathResolve) : undefined; const preload: ElectronBrowserWindowPreload = effectivePreload.status === "explicit" && preloadPath !== undefined @@ -201,6 +205,7 @@ const collectWebPreferences = ( preload_path: preloadPath, preload_resolution_context: staticPathResolutionContext( effectivePreload.property.value, + nodePathResolve, ), } : { preload_path: null, preload_resolution_context: null }; @@ -230,7 +235,10 @@ const collectWebPreferences = ( }); continue; } - const path = name === "preload" ? staticPath(property.value) : undefined; + const path = + name === "preload" + ? staticPath(property.value, nodePathResolve) + : undefined; const value: ElectronStaticValue = path === undefined ? electronStaticValue(source, property.value) @@ -305,12 +313,14 @@ const inspectUtilityProcess = ( return; const moduleNode = argumentNode(node.arguments[0]); const modulePath = - moduleNode === undefined ? undefined : staticPath(moduleNode); + moduleNode === undefined + ? undefined + : staticPath(moduleNode, context.nodePathResolve); const options = argumentNode(node.arguments[2]); const serviceNameProperty = staticObjectProperty(options, "serviceName"); const serviceName = serviceNameProperty.status === "explicit" - ? staticPath(serviceNameProperty.property.value) + ? staticPath(serviceNameProperty.property.value, context.nodePathResolve) : undefined; if (serviceNameProperty.status !== "missing" && serviceName === undefined) context.accumulator.unknownFindings += 1; @@ -324,7 +334,10 @@ const inspectUtilityProcess = ( } : { module_path: modulePath, - module_resolution_context: staticPathResolutionContext(moduleNode), + module_resolution_context: staticPathResolutionContext( + moduleNode, + context.nodePathResolve, + ), module_expression: null, }; const finding: ElectronUtilityProcessFinding = { diff --git a/src/domain/javascript/electronStaticAnalysisExportedAppPath.test.ts b/src/domain/javascript/electronStaticAnalysisExportedAppPath.test.ts new file mode 100644 index 0000000000..dbb0806a98 --- /dev/null +++ b/src/domain/javascript/electronStaticAnalysisExportedAppPath.test.ts @@ -0,0 +1,136 @@ +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +const cases = [ + ["native setter", "const change=app.setAppPath;", true, [1]], + [ + "local replacement", + "app.setAppPath=()=>{};const change=app.setAppPath;", + false, + [0], + ], + [ + "conditional replacement", + "if(flag)app.setAppPath=()=>{};const change=app.setAppPath;", + true, + [1, 0], + ], + [ + "capture before replacement", + "const change=app.setAppPath;app.setAppPath=()=>{};", + true, + [1], + ], + [ + "capture after replacement", + "app.setAppPath=()=>{};const change=app.setAppPath;", + false, + [0], + ], +] as const; + +it.each(cases)( + "retains the possible exported app-path effect after %s", + (_name, body, possible, calls) => { + const defining = `const {app}=require('electron');${body}`; + for (const [index, expected] of calls.entries()) { + let changed = 0; + // Authored module-namespace values model capture and invocation in two + // separate VM contexts; the fixture does not execute Electron itself. + const namespace = runInNewContext(`${defining}({change,app});`, { + flag: index === 1, + require: () => ({ app: { setAppPath: () => changed++ } }), + }); + expect(changed).toBe(0); + runInNewContext("change.call(app,'./lib');", namespace); + expect(changed).toBe(expected); + } + const exported = analyzeJavaScriptStaticSource( + `${defining}export{change,app};`, + ); + expect(exported.parse_error_count).toBe(0); + expect((exported.electron.app_path_changes ?? []).length > 0).toBe( + possible, + ); + const caller = analyzeJavaScriptStaticSource( + "import{change,app}from'./helper.mjs';change.call(app,'./lib');", + ); + expect(caller.parse_error_count).toBe(0); + expect(caller.electron.app_path_changes ?? []).toEqual([]); + }, +); + +it("does not treat a type-only exported setter as a runtime escape", () => { + const analysis = analyzeJavaScriptStaticSource( + "const{app}=require('electron');const change=app.setAppPath;export type{change};", + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.electron.app_path_changes ?? []).toEqual([]); +}); + +it.each([ + ["export const change=app.setAppPath;", true], + ["export default app.setAppPath;", true], + ["export const change=()=>{};", false], + ["export default ()=>{};", false], +] as const)( + "selects the published runtime value in %s", + (declaration, possible) => { + const analysis = analyzeJavaScriptStaticSource( + `const{app}=require('electron');${declaration}`, + ); + expect(analysis.parse_error_count).toBe(0); + expect((analysis.electron.app_path_changes ?? []).length > 0).toBe( + possible, + ); + }, +); + +it.each(cases)( + "retains the captured CommonJS namespace setter after %s", + (_name, body, possible, calls) => { + const defining = `const {app}=require('electron');${body}module.exports={change,app};`; + for (const [index, expected] of calls.entries()) { + let changed = 0; + const module = { exports: {} }; + runInNewContext(defining, { + module, + exports: module.exports, + flag: index === 1, + require: () => ({ app: { setAppPath: () => changed++ } }), + }); + expect(changed).toBe(0); + runInNewContext("change.call(app,'./lib');", module.exports); + expect(changed).toBe(expected); + } + const analysis = analyzeJavaScriptStaticSource(defining); + expect(analysis.parse_error_count).toBe(0); + expect((analysis.electron.app_path_changes ?? []).length > 0).toBe( + possible, + ); + }, +); + +it.each([ + ["module.exports=change;", true], + ["exports.change=change;", true], + ["module.exports.change=change;", true], + ["Object.defineProperty(exports,'change',{value:change});", true], + ["exports.change=change;module.exports={};", false], + ["const module={exports:{}};module.exports={change};", false], + ["const exports={};exports.change=change;", false], + ["export const namespace={change,app};", true], +] as const)( + "follows shared runtime export values and receivers in %s", + (published, possible) => { + const analysis = analyzeJavaScriptStaticSource( + `const {app}=require('electron');const change=app.setAppPath;${published}`, + ); + expect(analysis.parse_error_count).toBe(0); + expect((analysis.electron.app_path_changes ?? []).length > 0).toBe( + possible, + ); + }, +); diff --git a/src/domain/javascript/electronStaticAnalysisIdentity.test.ts b/src/domain/javascript/electronStaticAnalysisIdentity.test.ts index 527b0b4631..4ed2377e78 100644 --- a/src/domain/javascript/electronStaticAnalysisIdentity.test.ts +++ b/src/domain/javascript/electronStaticAnalysisIdentity.test.ts @@ -2,6 +2,130 @@ import { expect, it } from "vitest"; import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; +it.each([ + "module.require = replacement;", + "const holder = { loader: module }; holder.loader.require = replacement;", + "const holder = {}; holder.loader = module; holder.loader.require = replacement;", +])( + "does not prove Electron exports after a delegate replacement: %s", + (write) => { + const analysis = analyzeJavaScriptStaticSource( + `${write} const { BrowserWindow } = require("electron"); new BrowserWindow();`, + ); + expect(analysis.electron.browser_windows).toEqual([]); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier: null }), + ]); + }, +); + +it.each([ + ['const { app } = require("electron"); app.setAppPath("./other");', true], + [ + 'import { app as runtime } from "electron/main"; runtime.setAppPath(path);', + true, + ], + [ + 'import * as electron from "electron"; electron.app.setAppPath("./other");', + true, + ], + [ + 'const { app } = require("electron"); const runtime = app; runtime.setAppPath("./other");', + true, + ], + [ + 'const { app } = require("electron"); const change = app.setAppPath; change("./other");', + true, + ], + ['const app = { setAppPath() {} }; app.setAppPath("./other");', false], + [ + 'import { app } from "electron"; function local(app) { app.setAppPath("./other"); }', + false, + ], + [ + 'const electron = { app: { setAppPath() {} } }; electron.app.setAppPath("./other");', + false, + ], +])( + "retains Electron application path changes by binding identity: %s", + (source, expected) => { + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.electron.app_path_changes ?? []).toEqual( + expected + ? [ + expect.objectContaining({ + module_key: null, + location: expect.any(Object), + }), + ] + : [], + ); + }, +); + +it.each([ + ['app.setAppPath.call(app, "./other");', true], + ['app.setAppPath.apply(app, ["./other"]);', true], + ['Reflect.apply(app.setAppPath, app, ["./other"]);', true], + [ + 'const invoke = Reflect.apply; invoke(app.setAppPath, app, ["./other"]);', + true, + ], + [ + 'const invoke = Reflect.apply.bind(Reflect); invoke(app.setAppPath, app, ["./other"]);', + true, + ], + [ + 'const invoke = Reflect.apply; Reflect.apply = () => {}; invoke(app.setAppPath, app, ["./other"]);', + true, + ], + ['const change = app.setAppPath.bind(app); change("./other");', true], + ['app["setAppPath"][`call`](app, "./other");', true], + [ + 'const runtime = app; runtime.setAppPath.apply(runtime, ["./other"]);', + true, + ], + ['const change = app.setAppPath; change.call(app, "./other");', true], + ['function local(app) { app.setAppPath.call(app, "./other"); }', false], + [ + 'const local = { setAppPath() {} }; local.setAppPath.bind(local)("./other");', + false, + ], + ['app.setAppPath.other(app, "./other");', false], + [ + 'const Reflect = { apply() {} }; Reflect.apply(app.setAppPath, app, ["./other"]);', + false, + ], + [ + 'Reflect.apply = () => {}; Reflect.apply(app.setAppPath, app, ["./other"]);', + false, + ], + [ + 'const local = { setAppPath() {} }; Reflect.apply(local.setAppPath, local, ["./other"]);', + false, + ], +])( + "retains indirect application path changes by Electron identity: %s", + (call, expected) => { + const analysis = analyzeJavaScriptStaticSource(` + const { app } = require("electron"); + ${call} + `); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.electron.app_path_changes ?? []).toEqual( + expected + ? [ + expect.objectContaining({ + module_key: null, + location: expect.any(Object), + }), + ] + : [], + ); + }, +); + it("recovers APIs through an ES module default import of Electron", () => { const analysis = analyzeJavaScriptStaticSource(` import electron from "electron"; diff --git a/src/domain/javascript/electronStaticAnalysisPublishedApp.test.ts b/src/domain/javascript/electronStaticAnalysisPublishedApp.test.ts new file mode 100644 index 0000000000..1145736c12 --- /dev/null +++ b/src/domain/javascript/electronStaticAnalysisPublishedApp.test.ts @@ -0,0 +1,112 @@ +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; +import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; + +it.each([ + ["export {app};", true], + ["export default app;", true], + ["export const exposed={app};", true], + ["export const exposed=[app];", true], + ["app.setAppPath=()=>{};export{app};", false], + ["const local={setAppPath(){}};export{local};", false], +] as const)( + "retains whole exported application capabilities in %s", + (published, possible) => { + const result = analyzeJavaScriptStaticSource( + `import {app} from 'electron';${published}`, + ); + expect(result.parse_error_count).toBe(0); + expect((result.electron.app_path_changes ?? []).length > 0).toBe(possible); + }, +); + +it.each([ + [ + "later local", + "Object.assign(exports,{change:app.setAppPath},{change(){}});", + false, + ], + [ + "later native", + "Object.assign(exports,{change(){}},{change:app.setAppPath});", + true, + ], + [ + "inherited", + "const source={__proto__:{change:app.setAppPath}};Object.assign(exports,source);", + false, + ], + [ + "own", + "const source={__proto__:{change(){}},change:app.setAppPath};Object.assign(exports,source);", + true, + ], + [ + "later local alias", + "const local={change(){}};Object.assign(exports,{change:app.setAppPath},local);", + false, + ], + [ + "unrelated later field", + "Object.assign(exports,{change:app.setAppPath},{other(){}});", + true, + ], + ["array own", "Object.assign(exports,[app.setAppPath]);", true], + [ + "nullish sources", + "Object.assign(exports,{change:app.setAppPath},null,undefined);", + true, + ], + [ + "getter deletes a later source slot", + "Object.assign(exports,{change:app.setAppPath},{get first(){delete this.change},change(){}});", + true, + ], +] as const)("publishes only copied fields for %s", (_name, body, possible) => { + let changed = 0; + const app = { + setAppPath() { + changed++; + }, + }; + const module = { exports: {} }; + runInNewContext(body, { app, module, exports: module.exports }); + const fields = Object.values(module.exports) as unknown[]; + for (const field of fields) + if (typeof field === "function") field.call(app, "./lib"); + expect(changed > 0).toBe(possible); + const result = analyzeJavaScriptStaticSource( + `const{app}=require('electron');${body}`, + ); + expect(result.parse_error_count).toBe(0); + expect((result.electron.app_path_changes ?? []).length > 0).toBe(possible); +}); + +it("retains the final named own exports without publishing overwritten fields", () => { + const result = analyzeJavaScriptSemantics( + "const local=()=>{};Object.assign(exports,{change:require('electron').app.setAppPath,retained:local},{change:local});", + ); + expect( + result.moduleLinks + .filter(({ kind }) => kind === "commonjs-export") + .map(({ exportedName }) => exportedName) + .sort(), + ).toEqual(["change", "retained"]); +}); + +it("retains copied module provenance without publishing the namespace object", () => { + const result = analyzeJavaScriptSemantics( + "Object.assign(exports,require('./factory.cjs'));", + ); + expect(result.moduleLinks).toContainEqual( + expect.objectContaining({ + kind: "commonjs-export", + exportedName: "*", + specifier: "./factory.cjs", + importedName: null, + localName: null, + callableId: null, + }), + ); +}); diff --git a/src/domain/javascript/electronStaticAnalysisReturnedApp.test.ts b/src/domain/javascript/electronStaticAnalysisReturnedApp.test.ts new file mode 100644 index 0000000000..2169bf18ba --- /dev/null +++ b/src/domain/javascript/electronStaticAnalysisReturnedApp.test.ts @@ -0,0 +1,231 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it.each([ + [ + "direct factory", + "const getApp=()=>app;getApp().setAppPath('./lib');", + true, + ], + [ + "returned alias", + "const getApp=()=>app;const runtime=getApp();runtime.setAppPath('./lib');", + true, + ], + [ + "method factory", + "const api={get(){return app}};api.get().setAppPath('./lib');", + true, + ], + [ + "projected return", + "const getApp=()=>({runtime:app});getApp().runtime.setAppPath('./lib');", + true, + ], + ["method return", "const change=()=>app.setAppPath;change()('./lib');", true], + [ + "awaited factory", + "const getApp=async()=>app;(await getApp()).setAppPath('./lib');", + true, + ], + [ + "unawaited promise", + "const getApp=async()=>app;getApp().setAppPath('./lib');", + false, + ], + [ + "generator", + "function*getApp(){return app}getApp().setAppPath('./lib');", + false, + ], + [ + "awaited generator", + "function*getApp(){return app}(await getApp()).setAppPath('./lib');", + false, + ], + [ + "local factory", + "const getApp=()=>({setAppPath(){}});getApp().setAppPath('./lib');", + false, + ], + [ + "escaped local", + "const local={setAppPath(){}};escape(local);local.setAppPath('./lib');", + false, + ], + [ + "external local", + "const local=require('local-library');local.setAppPath('./lib');", + false, + ], + [ + "unknown return", + "const getApp=()=>unknown;getApp().setAppPath('./lib');", + false, + ], + [ + "unrelated unknown invocation", + "const local={make(){}};escape(local);local.make();", + false, + ], + [ + "local method", + "const api={get(){return {setAppPath(){}}}};api.get().setAppPath('./lib');", + false, + ], + [ + "shadowed app", + "function local(app){const getApp=()=>app;getApp().setAppPath('./lib');}", + false, + ], + [ + "mixed return", + "const getApp=()=>flag?app:{setAppPath(){}};getApp().setAppPath('./lib');", + true, + ], + [ + "replacement factory", + "const api={get:()=>({setAppPath(){}})};api.get=()=>app;api.get().setAppPath('./lib');", + true, + ], + [ + "local replacement", + "const api={get:()=>app};api.get=()=>({setAppPath(){}});api.get().setAppPath('./lib');", + false, + ], + [ + "later member replacement", + "const getApp=()=>app;getApp().setAppPath('./lib');app.setAppPath=()=>{};", + true, + ], + [ + "returned setter call", + "const getApp=()=>app;getApp().setAppPath.call(app,'./lib');", + true, + ], + [ + "returned setter reflect", + "const getApp=()=>app;Reflect.apply(getApp().setAppPath,app,['./lib']);", + true, + ], + [ + "factory call delegate", + "const getApp=()=>app;getApp.call(null).setAppPath('./lib');", + true, + ], + [ + "factory apply delegate", + "const getApp=()=>app;getApp.apply(null,[]).setAppPath('./lib');", + true, + ], + [ + "captured app before replacement", + "let runtime=app;const held=runtime;runtime={setAppPath(){}};const getApp=()=>held;getApp().setAppPath('./lib');", + true, + ], + [ + "captured local before replacement", + "let runtime={setAppPath(){}};const held=runtime;runtime=app;const getApp=()=>held;getApp().setAppPath('./lib');", + false, + ], + [ + "captured local after replacement", + "let runtime=app;runtime={setAppPath(){}};const held=runtime;const getApp=()=>held;getApp().setAppPath('./lib');", + false, + ], + [ + "captured app after replacement", + "let runtime={setAppPath(){}};runtime=app;const held=runtime;const getApp=()=>held;getApp().setAppPath('./lib');", + true, + ], +] as const)( + "retains only possible Electron app path changes from %s", + (_name, body, expected) => { + const analysis = analyzeJavaScriptStaticSource( + `const {app}=require('electron');${body}`, + ); + expect(analysis.parse_error_count).toBe(0); + expect((analysis.electron.app_path_changes ?? []).length > 0).toBe( + expected, + ); + // A possible return must not promote other APIs to proven Electron identity. + expect(analysis.electron.browser_windows).toEqual([]); + }, +); + +it.each([ + "const getApp=()=>require('electron').app;getApp().setAppPath('./lib');", + "const getElectron=()=>require('electron');getElectron().app.setAppPath('./lib');", + "const getApp=()=>require('other').app;getApp().setAppPath('./lib');", +])( + "follows returned require origins without an import shortcut: %s", + (source) => { + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect((analysis.electron.app_path_changes ?? []).length > 0).toBe( + !source.includes("'other'"), + ); + }, +); + +it("does not promote a returned Electron constructor to proven API identity", () => { + const analysis = analyzeJavaScriptStaticSource( + "const {BrowserWindow}=require('electron');const get=()=>BrowserWindow;new (get())();", + ); + expect(analysis.electron.browser_windows).toEqual([]); + expect(analysis.electron.app_path_changes ?? []).toEqual([]); +}); + +it.each([ + ["call", true], + ["call", false], + ["apply", true], + ["apply", false], +] as const)( + "uses the actual receiver of a captured %s wrapper (host=%s)", + (method, host) => { + const local = "({setAppPath(){}})"; + const analysis = analyzeJavaScriptStaticSource( + `const {app}=require('electron'); + const original=()=>${host ? local : "app"}; + const selected=()=>${host ? "app" : local}; + const invoke=original.${method}; + invoke.call(selected,null${method === "apply" ? ",[]" : ""}).setAppPath('./lib');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect((analysis.electron.app_path_changes ?? []).length > 0).toBe(host); + }, +); + +it.each([true, false])( + "keeps a finite source closure after an Electron app selector budget is reached (host=%s)", + (host) => { + const declarations = Array.from( + { length: 550 }, + (_, index) => + `const selected${index}=selected${index === 0 ? "Root" : index - 1};`, + ).join(""); + const analysis = analyzeJavaScriptStaticSource( + `const {app}=require('electron');const selectedRoot={runtime:${host ? "app" : "{setAppPath(){}}"}};${declarations}const getApp=()=>selected549.runtime;getApp().setAppPath('./lib');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect((analysis.electron.app_path_changes ?? []).length > 0).toBe(host); + expect(analysis.limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining("Alias selection expansion"), + ]), + ); + }, +); + +it.each([true, false])( + "closes cyclic returned app copies without inventing an Electron origin (host=%s)", + (host) => { + const analysis = analyzeJavaScriptStaticSource( + `const {app}=require('electron');let box={runtime:${host ? "app" : "{setAppPath(){}}"}};while(flag)box={...box};const getApp=()=>box.runtime;getApp().setAppPath('./lib');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect((analysis.electron.app_path_changes ?? []).length > 0).toBe(host); + }, +); diff --git a/src/domain/javascript/electronStaticAnalysisTypes.ts b/src/domain/javascript/electronStaticAnalysisTypes.ts index 5b8ebf4aa8..b0e4aca045 100644 --- a/src/domain/javascript/electronStaticAnalysisTypes.ts +++ b/src/domain/javascript/electronStaticAnalysisTypes.ts @@ -193,6 +193,8 @@ export interface ElectronNativeAddonBindingFinding { /** All Electron-specific facts collected during one shared AST traversal. */ export interface ElectronStaticFindings { + /** Calls that may replace the application path used by Electron loadFile. */ + readonly app_path_changes?: readonly ElectronAppPathChangeFinding[]; readonly browser_windows: readonly ElectronBrowserWindowFinding[]; readonly context_bridge_apis: readonly ElectronContextBridgeFinding[]; readonly ipc: readonly ElectronIpcFinding[]; @@ -203,6 +205,13 @@ export interface ElectronStaticFindings { readonly call_paths?: readonly ElectronCallPath[]; } +/** An Electron application path setter invocation or bound delegate capture. */ +export interface ElectronAppPathChangeFinding { + readonly module_key: string | null; + readonly module_runtime?: string; + readonly location: JavaScriptSourceRange; +} + export interface ElectronCallPathStep { readonly kind: | "exposure" diff --git a/src/domain/javascript/javascriptApplicationProjectionResources.ts b/src/domain/javascript/javascriptApplicationProjectionResources.ts index 3e5280c9e8..678a8ec3d3 100644 --- a/src/domain/javascript/javascriptApplicationProjectionResources.ts +++ b/src/domain/javascript/javascriptApplicationProjectionResources.ts @@ -10,6 +10,7 @@ export const estimateJavaScriptStaticApplicationProjection = ( // Allow for identifiers, locations, authority, coverage and schema copies, // independently of the producer's compact checkpoint representation. const findings = + (facts.electron.app_path_changes?.length ?? 0) + facts.references.length + facts.endpoints.length + facts.storage.length + diff --git a/src/domain/javascript/javascriptCommonJsAssignAppPath.test.ts b/src/domain/javascript/javascriptCommonJsAssignAppPath.test.ts new file mode 100644 index 0000000000..409151ec87 --- /dev/null +++ b/src/domain/javascript/javascriptCommonJsAssignAppPath.test.ts @@ -0,0 +1,141 @@ +import { runInNewContext } from "node:vm"; + +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +const cases = [ + [ + "native source spread", + "Object.assign(exports,...[{change:app.setAppPath,app}]);", + true, + ], + [ + "native target spread", + "Object.assign(...[exports,{change:app.setAppPath,app}]);", + true, + ], + [ + "native call", + "Object.assign.call(null,exports,{change:app.setAppPath,app});", + true, + ], + [ + "native apply", + "Object.assign.apply(null,[exports,{change:app.setAppPath,app}]);", + true, + ], + [ + "captured intrinsic before replacement", + "const publish=Object.assign;Object.assign=()=>{};publish(exports,{change:app.setAppPath,app});", + true, + ], + [ + "replaced intrinsic", + "Object.assign=()=>{};Object.assign(exports,{change:app.setAppPath,app});", + false, + ], + [ + "replaced call", + "Object.assign.call=()=>{};Object.assign.call(null,exports,{change:app.setAppPath,app});", + false, + ], + [ + "replaced apply", + "Object.assign.apply=()=>{};Object.assign.apply(null,[exports,{change:app.setAppPath,app}]);", + false, + ], + [ + "replaced Function call", + "Function.prototype.call=()=>{};Object.assign.call(null,exports,{change:app.setAppPath,app});", + false, + ], + [ + "replaced Function apply", + "Function.prototype.apply=()=>{};Object.assign.apply(null,[exports,{change:app.setAppPath,app}]);", + false, + ], + [ + "native exports", + "Object.assign(exports,{change:app.setAppPath,app});", + true, + ], + [ + "native module exports", + "Object.assign(module.exports,{change:app.setAppPath,app});", + true, + ], + [ + "detached before", + "module.exports={};Object.assign(exports,{change:app.setAppPath,app});", + false, + ], + [ + "detached after", + "Object.assign(exports,{change:app.setAppPath,app});module.exports={};", + false, + ], + [ + "reconnected", + "exports=module.exports={};Object.assign(exports,{change:app.setAppPath,app});", + true, + ], + [ + "local exports", + "const exports={};Object.assign(exports,{change:app.setAppPath,app});", + false, + ], + [ + "local Object", + "const Object={assign(){}};Object.assign(exports,{change:app.setAppPath,app});", + false, + ], + [ + "local setter", + "app.setAppPath=()=>{};Object.assign(exports,{change:app.setAppPath,app});", + false, + ], +] as const; + +it.each(cases)( + "retains an Object.assign-published setter for %s", + (_name, body, possible) => { + const defining = `const{app}=require('electron');${body}`; + let changed = 0; + const module = { exports: {} }; + runInNewContext(defining, { + module, + exports: module.exports, + require: () => ({ app: { setAppPath: () => changed++ } }), + }); + expect(changed).toBe(0); + runInNewContext( + "if(typeof change==='function')change.call(app,'./lib');", + module.exports, + ); + expect(changed).toBe(possible ? 1 : 0); + const analysis = analyzeJavaScriptStaticSource(defining); + expect(analysis.parse_error_count).toBe(0); + expect((analysis.electron.app_path_changes ?? []).length > 0).toBe( + possible, + ); + }, +); + +it("retains native source uncertainty when an array iterator is overwritten", () => { + const defining = + "const{app}=require('electron');const sources=[{change:app.setAppPath,app}];sources[Symbol.iterator]=function*(){yield{change(){},app}};Object.assign(exports,...sources);"; + let changed = 0; + const module = { exports: {} }; + runInNewContext(defining, { + module, + exports: module.exports, + require: () => ({ app: { setAppPath: () => changed++ } }), + }); + runInNewContext("change.call(app,'./lib');", module.exports); + expect(changed).toBe(0); + // The static owner does not execute this custom iterator; it must retain the + // native origin still present in the source rather than silently drop it. + const analysis = analyzeJavaScriptStaticSource(defining); + expect((analysis.electron.app_path_changes ?? []).length).toBeGreaterThan(0); +}); diff --git a/src/domain/javascript/javascriptCommonJsExportReceivers.ts b/src/domain/javascript/javascriptCommonJsExportReceivers.ts index 848f5a8d6a..68cc826ff3 100644 --- a/src/domain/javascript/javascriptCommonJsExportReceivers.ts +++ b/src/domain/javascript/javascriptCommonJsExportReceivers.ts @@ -2,16 +2,54 @@ import * as t from "@babel/types"; import { semanticStaticPropertyKey, + semanticFinalObjectProperty, semanticWriteTargets, unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; import { isUnshadowedGlobal, + resolveSemanticBindingState, type JavaScriptSemanticAnalysisState, } from "./javascriptSemanticState.js"; import { isUnmodifiedGlobalReference } from "./javascriptSemanticAmbientWrites.js"; import { traverseJavaScriptAstSteps } from "./javascriptSemanticTraversal.js"; import { semanticPropertyPathKeyMatches } from "./javascriptSemanticPropertyPaths.js"; +import { createSemanticModuleInterop } from "./javascriptModuleInterop.js"; +import { evaluateSemanticExpression } from "./javascriptSemanticValues.js"; +import type { JavaScriptSemanticPropertyPath } from "./javascriptSemanticPropertyPaths.js"; +import { semanticProjectedSourcesAtRead } from "./javascriptSemanticProjectedSources.js"; +import { semanticEffectTiming } from "./javascriptSemanticMutationInitializers.js"; +import { SEMANTIC_ALIAS_SELECTION_LIMIT } from "./javascriptSemanticResourceLimits.js"; + +const exportInvocations = new WeakMap< + JavaScriptSemanticAnalysisState, + ReturnType +>(); + +/** Share the intrinsic owner's positional and capture proofs across CJS consumers. */ +const commonJsExportInvocation = ( + node: t.CallExpression, + state: JavaScriptSemanticAnalysisState, +) => { + let interop = exportInvocations.get(state); + if (interop === undefined) { + interop = createSemanticModuleInterop(state); + for (const { node: write, path } of state.ambientWriteEffects) + if (write !== undefined && path[0] === "global") + interop.retainWrite(write, [ + "intrinsic", + ...path + .slice(1) + .map((key) => + typeof key === "string" || typeof key === "number" + ? String(key) + : null, + ), + ]); + exportInvocations.set(state, interop); + } + return interop.intrinsicInvocation(node); +}; export type CommonJsExportReceiverEffect = | "connected" @@ -21,6 +59,111 @@ export type CommonJsExportReceiverEffect = /** A CommonJS export assignment or `Object.defineProperty` export call. */ export type CommonJsExportSite = t.AssignmentExpression | t.CallExpression; +interface CopiedOwnFields { + readonly definite: ReadonlySet; + readonly complete: boolean; +} + +/** Definite own fields survive aliases, but not a possible prior shape change. */ +const copiedOwnFields = ( + source: t.Node, + state: JavaScriptSemanticAnalysisState, +): CopiedOwnFields => { + const unknown = { definite: new Set(), complete: false }; + const pending = [source]; + const seen = new Set(); + let common: Set | undefined; + let complete = true; + while (pending.length > 0) { + const node = unwrapJavaScriptExpression(pending.pop()!).node; + if (seen.has(node)) continue; + if (seen.size >= SEMANTIC_ALIAS_SELECTION_LIMIT) { + state.valueResourceLimits.add("alias-selections"); + return unknown; + } + seen.add(node); + if (t.isIdentifier(node)) { + const binding = resolveSemanticBindingState(state, node, node.name); + if (binding === undefined) return unknown; + if ( + binding.memberWrites.some( + (effect) => + semanticEffectTiming(source, effect.node, state) !== "after", + ) || + binding.escapedPaths.some( + (effect) => + effect.node !== undefined && + !( + (effect.node.start ?? -1) <= (source.start ?? -1) && + (effect.node.end ?? -1) >= (source.end ?? -1) + ) && + semanticEffectTiming(source, effect.node, state) !== "after", + ) + ) + return unknown; + const selected = semanticProjectedSourcesAtRead( + binding, + node, + [], + state, + false, + source, + ); + if ( + selected === undefined || + selected.length === 0 || + selected.some((value) => value.path.length !== 0) + ) + return unknown; + if ( + seen.size + pending.length + selected.length > + SEMANTIC_ALIAS_SELECTION_LIMIT + ) { + state.valueResourceLimits.add("alias-selections"); + return unknown; + } + for (const value of selected) pending.push(value.node); + continue; + } + // A getter can remove a later own slot while Object.assign is copying. + // Its initial presence cannot prove an earlier source was overwritten. + if ( + t.isObjectExpression(node) && + node.properties.some( + (property) => t.isObjectMethod(property) && property.kind !== "method", + ) + ) + return unknown; + const selected = evaluateSemanticExpression(node, state, source); + const fields = + selected.status === "object" + ? selected.properties + : selected.status === "array" + ? selected.items + : []; + const own = new Set( + fields + .filter((field) => field.presence === "present") + .map((field) => field.name), + ); + const exact = + selected.status === "object" + ? !selected.unknownProperties + : selected.status === "array" + ? !selected.unknownItems + : selected.status === "literal" && selected.value === null; + complete &&= + exact && + (common === undefined || + (common.size === own.size && [...common].every((key) => own.has(key)))); + common = + common === undefined + ? own + : new Set([...common].filter((key) => own.has(key))); + } + return { definite: common ?? new Set(), complete }; +}; + const isCommonJsModuleExports = ( node: t.Node, state: JavaScriptSemanticAnalysisState, @@ -90,6 +233,103 @@ export const commonJsDefinedExport = ( }; }; +/** Native Object.assign publishes source values only through a CJS receiver. */ +const commonJsAssignedExports = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, +): + | { readonly receiver: t.Node; readonly sources: readonly t.Node[] } + | undefined => { + if (!t.isCallExpression(node)) return undefined; + const invocation = commonJsExportInvocation(node, state); + if ( + invocation?.kind !== "known" || + invocation.path.length !== 3 || + invocation.path[0] !== "intrinsic" || + invocation.path[1] !== "Object" || + invocation.path[2] !== "assign" + ) + return undefined; + const [target, ...sources] = invocation.arguments ?? []; + if (!t.isExpression(target)) return undefined; + const receiver = unwrapJavaScriptExpression(target).node; + if ( + !isUnshadowedGlobal(receiver, state, "exports") && + !isCommonJsModuleExports(receiver, state) + ) + return undefined; + return { + receiver, + sources: sources.flatMap((source) => + t.isSpreadElement(source) + ? [source.argument] + : t.isExpression(source) + ? [source] + : [], + ), + }; +}; + +/** Runtime values published by the CommonJS syntax owner, before receiver applicability. */ +export const commonJsRuntimeExportValues = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, +): readonly { + readonly exportedName: string; + readonly value: t.Node | null; + /** Own enumerable fields copied at this source, after later-source overwrites. */ + readonly projection?: JavaScriptSemanticPropertyPath; + readonly ownEnumerable?: true; +}[] => { + if (t.isAssignmentExpression(node)) { + const exportedName = commonJsExportName(node.left, state); + return exportedName === undefined + ? [] + : [{ exportedName, value: node.operator === "=" ? node.right : null }]; + } + const assigned = commonJsAssignedExports(node, state); + if (assigned !== undefined) { + const exported: ReturnType[number][] = + []; + const overwritten = new Set(); + for (let index = assigned.sources.length - 1; index >= 0; index--) { + const value = assigned.sources[index]!; + const fields = copiedOwnFields(value, state); + if (fields.complete) { + for (const name of fields.definite) + if (!overwritten.has(name)) + exported.push({ + exportedName: name, + value, + projection: [name], + ownEnumerable: true, + }); + } else + exported.push({ + exportedName: "*", + value, + projection: [{ excludedKeys: [...overwritten] }], + ownEnumerable: true, + }); + // Copy applicability belongs to the shared own-slot inventory. Unknown + // later fields cannot prove that an earlier native field was replaced. + for (const field of fields.definite) overwritten.add(field); + } + return exported; + } + const defined = commonJsDefinedExport(node, state); + if (defined === undefined) return []; + const descriptor = unwrapJavaScriptExpression(defined.descriptor).node; + return [ + { + exportedName: defined.exportedName, + value: t.isObjectExpression(descriptor) + ? (semanticFinalObjectProperty(descriptor, "value") ?? null) + : null, + }, + ]; +}; + type Identity = t.Node | null; type Execution = "ordered" | "conditional" | "deferred"; @@ -217,7 +457,9 @@ export function* indexCommonJsExportReceiversSteps( : publicIdentity, ); } - const defined = commonJsDefinedExport(node, state); + const defined = + commonJsDefinedExport(node, state) ?? + commonJsAssignedExports(node, state); if (defined !== undefined && t.isCallExpression(node)) captures.set( node, diff --git a/src/domain/javascript/javascriptElectronAppPathChanges.ts b/src/domain/javascript/javascriptElectronAppPathChanges.ts new file mode 100644 index 0000000000..ae18db9aec --- /dev/null +++ b/src/domain/javascript/javascriptElectronAppPathChanges.ts @@ -0,0 +1,390 @@ +import * as t from "@babel/types"; + +import { + semanticStaticPropertyKey, + unwrapJavaScriptExpression, +} from "./javascriptAstValues.js"; +import { + semanticAliasSelectionChoices, + type SemanticAliasSelectionPath, +} from "./javascriptSemanticAliasSelections.js"; +import { + semanticArrayPropertyReferences, + semanticObjectPropertyReferences, +} from "./javascriptSemanticObjectReferences.js"; +import { + semanticNormalizeCapturedSelections, + semanticProjectedSourceFactsAtRead, + semanticShiftCapturedSelections, + type SemanticCapturedSelection, +} from "./javascriptSemanticProjectedSources.js"; +import { createSemanticReturnedReferences } from "./javascriptSemanticReturnedReferences.js"; +import { SEMANTIC_ALIAS_SELECTION_LIMIT } from "./javascriptSemanticResourceLimits.js"; +import { + resolveSemanticBindingState, + type JavaScriptSemanticAnalysisState, +} from "./javascriptSemanticState.js"; +import type { createSemanticModuleInterop } from "./javascriptModuleInterop.js"; +import { semanticReferenceSourcesInactive } from "./javascriptSemanticReferenceSources.js"; +import { assignmentValueSources } from "./javascriptSemanticRequireOrigin.js"; +import { semanticRuntimeExportValueSources } from "./javascriptModuleDeclaration.js"; +import { + commonJsRuntimeExportValues, + indexCommonJsExportReceiversSteps, + type CommonJsExportReceiverEffect, + type CommonJsExportSite, +} from "./javascriptCommonJsExportReceivers.js"; +import { completeSemanticSteps } from "./javascriptSemanticTraversal.js"; + +interface Candidate { + readonly node: t.Node; + readonly path: SemanticAliasSelectionPath | undefined; + readonly readAt?: t.Node | undefined; + readonly capturedSelections?: + | readonly SemanticCapturedSelection[] + | undefined; + readonly awaitDepths: readonly number[]; + readonly escaping?: true; +} + +const isSetterPath = (path: SemanticAliasSelectionPath): boolean => + path.length >= 2 && + path.every((segment, index) => + semanticAliasSelectionChoices(segment).some( + (key) => + key === null || + (index < 2 + ? key === ["app", "setAppPath"][index] + : ["call", "apply", "bind"].includes(String(key))), + ), + ); + +/** + * A source-local return may carry the shared Electron app even when it cannot + * establish a concrete API identity. Retain only the possibility of changing + * its root; callers must not use this query to prove other Electron APIs. + */ +export const createSemanticElectronAppPathChangeQuery = ( + state: JavaScriptSemanticAnalysisState, + exportPath: (node: t.Node) => readonly string[] | undefined, + interop: Pick< + ReturnType, + "intrinsicInvocation" | "intrinsicPath" + >, +): ((node: t.Node) => boolean) => { + const directReturns = createSemanticReturnedReferences(state, "direct-call"); + const awaitedReturns = createSemanticReturnedReferences( + state, + "awaited-call", + ); + let commonJsReceivers: + | ReadonlyMap + | undefined; + const possiblePath = ( + root: t.Node, + escaping = false, + projection: SemanticAliasSelectionPath = [], + ownEnumerable = false, + ): boolean => { + const pending: Candidate[] = []; + const seen = new WeakMap>(); + let scheduled = 0; + let widened = false; + const enqueue = (candidate: Candidate): void => { + const node = unwrapJavaScriptExpression(candidate.node).node; + // Close the finite source graph after the precise context budget is + // exhausted. Widening alone is never evidence of an Electron origin. + if (!widened && scheduled >= SEMANTIC_ALIAS_SELECTION_LIMIT) { + state.valueResourceLimits.add("alias-selections"); + widened = true; + } + if (widened) + candidate = { + node, + path: undefined, + awaitDepths: candidate.awaitDepths.length > 0 ? [0] : [], + ...(candidate.escaping ? { escaping: true as const } : {}), + }; + const capturedSelections = semanticNormalizeCapturedSelections( + candidate.capturedSelections, + candidate.path?.length, + ); + const key = JSON.stringify([ + candidate.path, + candidate.readAt?.start, + candidate.readAt?.end, + capturedSelections?.map(({ at, depth, ownEnumerable }) => [ + at.start, + at.end, + depth, + ownEnumerable, + ]), + candidate.awaitDepths, + candidate.escaping, + ]); + const contexts = seen.get(node) ?? new Set(); + if (contexts.has(key)) return; + scheduled++; + contexts.add(key); + seen.set(node, contexts); + pending.push({ ...candidate, node, capturedSelections }); + }; + enqueue({ + node: root, + path: projection, + awaitDepths: [], + ...(escaping ? { escaping: true as const } : {}), + ...(ownEnumerable + ? { + capturedSelections: [ + { at: root, depth: 1, ownEnumerable: true as const }, + ], + } + : {}), + }); + while (pending.length > 0) { + const current = pending.pop()!; + if (widened && current.path !== undefined) { + enqueue(current); + continue; + } + const { node, path } = current; + const binding = t.isIdentifier(node) + ? resolveSemanticBindingState(state, node, node.name) + : undefined; + const selected = + binding === undefined || path === undefined + ? undefined + : semanticProjectedSourceFactsAtRead( + binding, + node, + path, + state, + false, + current.readAt ?? node, + current.capturedSelections, + ); + // Ordinary aliases are followed below with shared source selections. + // Re-evaluating their whole-binding provenance at every DAG node would + // repeat that graph merely to obtain a concrete identity we do not need. + const exported = + !t.isIdentifier(node) || + binding?.directOrigins.length || + binding?.initializers.some((source) => + t.isCallExpression(unwrapJavaScriptExpression(source.node).node), + ) + ? exportPath(node) + : undefined; + if ( + current.escaping && + path?.length === 0 && + exported !== undefined && + (exported.length === 0 || + (exported.length === 1 && exported[0] === "app")) + ) { + // Publishing the app (or its namespace) lets an importer select the + // setter. Re-enter the same projected-source proof so a definite local + // method replacement still suppresses that native alternative. + enqueue({ + ...current, + path: exported.length === 0 ? ["app", "setAppPath"] : ["setAppPath"], + }); + } + if ( + exported !== undefined && + // The retained app identity alone cannot prove its selected method + // survived an own write. Conditional or unknown writes keep the native + // alternative; a definite replacement follows its selected RHS below. + selected?.retainsOrigin !== false && + (path === undefined + ? exported.length === 0 || + (exported.length === 1 && exported[0] === "app") || + isSetterPath(exported) + : isSetterPath([...exported, ...path])) + ) + return true; + if (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) { + enqueue({ + ...current, + node: node.object, + path: + path === undefined + ? undefined + : [ + semanticStaticPropertyKey(node.property, node.computed), + ...path, + ], + readAt: current.readAt ?? node, + capturedSelections: semanticShiftCapturedSelections( + current.capturedSelections, + 1, + ), + }); + } else if (t.isIdentifier(node)) { + if (binding === undefined) continue; + const sources = selected?.sources; + // Unknown iteration cannot prove an Electron origin, but must not drop + // retained source references that can still carry the shared app. + if (sources === undefined) { + for (const sources of [ + binding.initializers, + binding.referenceInitializers, + ]) + for (const source of sources) { + if (semanticReferenceSourcesInactive(source, state)) continue; + enqueue({ + ...current, + node: source.node, + path: + path === undefined + ? undefined + : [...source.projection, ...path], + }); + } + for (const effect of binding.memberWrites) + for (const source of effect.assignedSources ?? []) { + if (semanticReferenceSourcesInactive(source, state)) continue; + enqueue({ ...current, node: source.node, path: undefined }); + } + continue; + } + for (const source of sources) + enqueue({ + ...current, + ...source, + readAt: path?.length === 0 ? undefined : (current.readAt ?? node), + capturedSelections: [ + ...(semanticShiftCapturedSelections( + current.capturedSelections, + source.path.length - (path?.length ?? 0), + ) ?? []), + ...(source.capturedSelections ?? []), + ], + }); + } else if (t.isAwaitExpression(node)) { + enqueue({ + ...current, + node: node.argument, + awaitDepths: [ + ...new Set([...current.awaitDepths, path?.length ?? 0]), + ].sort((left, right) => left - right), + }); + } else if (t.isCallExpression(node) || t.isOptionalCallExpression(node)) { + const returned = current.awaitDepths.includes(path?.length ?? 0) + ? awaitedReturns + : directReturns; + for (const source of returned(node.callee, node).references) + enqueue({ + ...current, + node: source.node, + path: + path === undefined ? undefined : [...source.projection, ...path], + capturedSelections: semanticShiftCapturedSelections( + current.capturedSelections, + source.projection.length, + ), + }); + } else if ( + t.isConditionalExpression(node) || + t.isLogicalExpression(node) + ) { + enqueue({ + ...current, + node: t.isConditionalExpression(node) ? node.consequent : node.left, + }); + enqueue({ + ...current, + node: t.isConditionalExpression(node) ? node.alternate : node.right, + }); + } else if (t.isSequenceExpression(node)) { + const last = node.expressions.at(-1); + if (last !== undefined) enqueue({ ...current, node: last }); + } else if (t.isAssignmentExpression(node)) { + for (const source of assignmentValueSources(node)) + enqueue({ ...current, node: source }); + } else if ( + (t.isObjectExpression(node) || t.isArrayExpression(node)) && + (path === undefined || path.length > 0 || current.escaping) + ) { + const [key, ...remaining] = + path?.length === 0 ? [null] : (path ?? [null]); + for (const choice of semanticAliasSelectionChoices(key!)) { + const sources = t.isObjectExpression(node) + ? semanticObjectPropertyReferences(node, [choice], state) + : semanticArrayPropertyReferences(node, [choice]); + for (const source of sources) + if ( + !source.inherited || + !current.capturedSelections?.some( + (capture) => capture.depth === 1 && capture.ownEnumerable, + ) + ) + enqueue({ + ...current, + node: source.node, + path: + path === undefined + ? undefined + : [...source.path, ...remaining], + awaitDepths: current.awaitDepths.filter( + (depth) => depth !== path?.length, + ), + capturedSelections: [ + ...(semanticShiftCapturedSelections( + current.capturedSelections, + source.path.length - 1, + ) ?? []), + ...(source.capturedSelectionAt === undefined + ? [] + : [ + { + at: source.capturedSelectionAt, + depth: source.path.length, + ...(source.ownEnumerable + ? { ownEnumerable: true as const } + : {}), + }, + ]), + ], + }); + } + } + } + return false; + }; + return (node) => { + if ( + (t.isCallExpression(node) || t.isOptionalCallExpression(node)) && + (possiblePath(node.callee) || + interop.intrinsicInvocation(node, (callee) => + possiblePath(callee) + ? ["electron-app-path-change"] + : interop.intrinsicPath(callee), + )?.path[0] === "electron-app-path-change") + ) + return true; + if ( + semanticRuntimeExportValueSources(node).some((value) => + possiblePath(value, true), + ) + ) + return true; + const exported = commonJsRuntimeExportValues(node, state); + if ( + !exported.some(({ value }) => value !== null) || + (!t.isAssignmentExpression(node) && !t.isCallExpression(node)) + ) + return false; + commonJsReceivers ??= completeSemanticSteps( + indexCommonJsExportReceiversSteps(state), + ); + return ( + commonJsReceivers.get(node) !== "detached" && + exported.some( + ({ value, projection, ownEnumerable }) => + value !== null && + possiblePath(value, true, projection, ownEnumerable), + ) + ); + }; +}; diff --git a/src/domain/javascript/javascriptIntrinsicInvocation.test.ts b/src/domain/javascript/javascriptIntrinsicInvocation.test.ts index 66c58bff6f..b3858d8837 100644 --- a/src/domain/javascript/javascriptIntrinsicInvocation.test.ts +++ b/src/domain/javascript/javascriptIntrinsicInvocation.test.ts @@ -2,6 +2,11 @@ import { expect, it } from "vitest"; import * as t from "@babel/types"; import { createSemanticModuleInterop } from "./javascriptModuleInterop.js"; +import { + javaScriptSourceInvocationContext, + javaScriptGenericInvocationMethod, + resolveJavaScriptGenericInvocation, +} from "./javascriptIntrinsicInvocation.js"; import { parseJavaScriptSource } from "./javascriptSourceParser.js"; import { indexJavaScriptBindingsSteps } from "./javascriptSemanticAnalysis.js"; import { @@ -52,3 +57,44 @@ it("keeps a comma-expression intrinsic call unknown when its receiver is erased" }), ]); }); + +it.each([ + ["get.call(null)", "get"], + ["get.apply(null,[])", "get"], + ["get.call.call(other,null)", "get.call"], + ["get.apply.call(other,null,[])", "get.apply"], + ["(0,get.call)(null)", undefined], +])( + "retains one generic callable receiver transition through %s", + (source, name) => { + const parsed = parseJavaScriptSource(source); + const statement = parsed?.program.body[0]; + if ( + !t.isExpressionStatement(statement) || + !t.isCallExpression(statement.expression) + ) + throw new Error("Expected source invocation fixture"); + const call = statement.expression; + const method = javaScriptGenericInvocationMethod(call.callee); + const context = javaScriptSourceInvocationContext(call); + const result = + method === undefined + ? undefined + : resolveJavaScriptGenericInvocation(method, context); + expect(result?.callable).toEqual( + name === undefined + ? undefined + : name.includes(".") + ? expect.objectContaining({ + type: "MemberExpression", + object: expect.objectContaining({ name: "get" }), + property: expect.objectContaining({ name: name.split(".")[1] }), + }) + : expect.objectContaining({ type: "Identifier", name }), + ); + if (name?.includes(".")) + expect(result?.context.receiver).toEqual( + expect.objectContaining({ type: "Identifier", name: "other" }), + ); + }, +); diff --git a/src/domain/javascript/javascriptIntrinsicInvocation.ts b/src/domain/javascript/javascriptIntrinsicInvocation.ts index 6aeb5ab785..082fe5006d 100644 --- a/src/domain/javascript/javascriptIntrinsicInvocation.ts +++ b/src/domain/javascript/javascriptIntrinsicInvocation.ts @@ -6,9 +6,16 @@ import { unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; -/** Expand only literal array spreads whose positional arguments are all known. */ +export type JavaScriptArrayIterationProof = ( + array: t.ArrayExpression, +) => boolean; + +const unknownArrayIteration: JavaScriptArrayIterationProof = () => false; + +/** Expand literal array spreads only when their native iterator is proven. */ export const expandJavaScriptInvocationArguments = ( arguments_: readonly t.CallExpression["arguments"][number][], + nativeArrayIteration: JavaScriptArrayIterationProof = unknownArrayIteration, ): t.CallExpression["arguments"] => { const expanded: t.CallExpression["arguments"] = []; const pending = [...arguments_].reverse(); @@ -20,6 +27,7 @@ export const expandJavaScriptInvocationArguments = ( : undefined; if ( t.isArrayExpression(array) && + nativeArrayIteration(array) && array.elements.every( (element) => t.isExpression(element) || t.isSpreadElement(element), ) @@ -33,7 +41,10 @@ export const expandJavaScriptInvocationArguments = ( return expanded; }; -const arrayArguments = (node: t.Node | undefined) => { +const arrayArguments = ( + node: t.Node | undefined, + nativeArrayIteration: JavaScriptArrayIterationProof, +) => { const array = node === undefined ? undefined : unwrapJavaScriptExpression(node).node; if ( @@ -45,6 +56,7 @@ const arrayArguments = (node: t.Node | undefined) => { array.elements.filter( (element): element is t.Expression | t.SpreadElement => element !== null, ), + nativeArrayIteration, ); return expanded.every((element) => t.isExpression(element)) ? expanded.filter((element): element is t.Expression => @@ -56,13 +68,80 @@ const arrayArguments = (node: t.Node | undefined) => { const invocationArguments = ( method: "call" | "apply", arguments_: JavaScriptIntrinsicInvocation["arguments"], + nativeArrayIteration: JavaScriptArrayIterationProof, ) => t.isSpreadElement(arguments_?.[0]) ? undefined : method === "apply" - ? arrayArguments(arguments_?.[1]) + ? arrayArguments(arguments_?.[1], nativeArrayIteration) : arguments_?.slice(1); +/** Positional state needed only by generic Function.call/apply invocation. */ +export interface JavaScriptSourceInvocationContext { + readonly receiver: t.Node | undefined; + readonly receiverUnknown?: boolean; + readonly arguments: JavaScriptIntrinsicInvocation["arguments"]; +} + +/** Only these generic function members consume a callable receiver. */ +export const javaScriptGenericInvocationMethod = ( + callee: t.Node, +): "call" | "apply" | undefined => { + const expression = unwrapJavaScriptExpression(callee).node; + const method = + t.isMemberExpression(expression) || t.isOptionalMemberExpression(expression) + ? semanticStaticPropertyKey(expression.property, expression.computed) + : undefined; + return method === "call" || method === "apply" ? method : undefined; +}; + +/** A member call supplies its receiver; a captured/comma callable does not. */ +export const javaScriptSourceInvocationContext = ( + node: t.CallExpression | t.OptionalCallExpression, + nativeArrayIteration: JavaScriptArrayIterationProof = unknownArrayIteration, +): JavaScriptSourceInvocationContext => { + const callee = unwrapJavaScriptExpression(node.callee).node; + return { + receiver: + t.isMemberExpression(callee) || t.isOptionalMemberExpression(callee) + ? callee.object + : undefined, + arguments: expandJavaScriptInvocationArguments( + node.arguments, + nativeArrayIteration, + ), + }; +}; + +/** Select one generic function receiver without evaluating function arguments. */ +export const resolveJavaScriptGenericInvocation = ( + method: "call" | "apply", + context: JavaScriptSourceInvocationContext, + nativeArrayIteration: JavaScriptArrayIterationProof = unknownArrayIteration, +): + | { + readonly callable: t.Expression; + readonly context: JavaScriptSourceInvocationContext; + } + | undefined => + !t.isExpression(context.receiver) + ? undefined + : { + callable: context.receiver, + context: { + receiver: context.arguments?.[0], + ...(context.arguments === undefined || + t.isSpreadElement(context.arguments[0]) + ? { receiverUnknown: true } + : {}), + arguments: invocationArguments( + method, + context.arguments, + nativeArrayIteration, + ), + }, + }; + /** * Normalize known invocation syntax once, retaining bound argument prefixes. * Binding a function alone is not a write; only its eventual call is examined. @@ -72,11 +151,12 @@ export const resolveJavaScriptIntrinsicInvocation = ( intrinsicPath: (node: t.Node) => readonly string[] | undefined, boundIntrinsic: (node: t.Node) => JavaScriptIntrinsicInvocation | undefined, intrinsicReadChanged: (node: t.Node, path: readonly string[]) => boolean, + nativeArrayIteration: JavaScriptArrayIterationProof = unknownArrayIteration, ): JavaScriptIntrinsicInvocation | undefined => { const visited = new Set(); let callee: t.Node = node.callee; let arguments_: JavaScriptIntrinsicInvocation["arguments"] = - expandJavaScriptInvocationArguments(node.arguments); + expandJavaScriptInvocationArguments(node.arguments, nativeArrayIteration); let receiver: t.Node | undefined; let explicitReceiver = false; for (;;) { @@ -106,15 +186,21 @@ export const resolveJavaScriptIntrinsicInvocation = ( ); if (method === "call" || method === "apply") { const bound = boundIntrinsic(callee.object); + // Caller-selected native loaders share Function.prototype authority. + const callablePath = bound?.path ?? intrinsicPath(callee.object); if ( - bound !== undefined && - intrinsicReadChanged(callee, [...bound.path, method]) + callablePath !== undefined && + intrinsicReadChanged(callee, [...callablePath, method]) ) - return { kind: "opaque", path: bound.path, arguments: undefined }; + return { kind: "opaque", path: callablePath, arguments: undefined }; receiver = arguments_?.[0]; explicitReceiver = true; callee = callee.object; - arguments_ = invocationArguments(method, arguments_); + arguments_ = invocationArguments( + method, + arguments_, + nativeArrayIteration, + ); continue; } } @@ -138,12 +224,20 @@ export const resolveJavaScriptIntrinsicInvocation = ( : t.isMemberExpression(callee) || t.isOptionalMemberExpression(callee) ? callee.object : undefined; - if (!t.isExpression(callable)) + const selected = resolveJavaScriptGenericInvocation( + method, + { + receiver: callable, + arguments: arguments_, + }, + nativeArrayIteration, + ); + if (selected === undefined) return { kind: "opaque", path, arguments: undefined }; - callee = callable; - receiver = arguments_?.[0]; + callee = selected.callable; + receiver = selected.context.receiver; explicitReceiver = true; - arguments_ = invocationArguments(method, arguments_); + arguments_ = selected.context.arguments; continue; } if (path.join(".") === "intrinsic.Reflect.apply") { @@ -152,7 +246,7 @@ export const resolveJavaScriptIntrinsicInvocation = ( callee = callable; receiver = arguments_?.[1]; explicitReceiver = true; - arguments_ = arrayArguments(arguments_?.[2]); + arguments_ = arrayArguments(arguments_?.[2], nativeArrayIteration); continue; } return { diff --git a/src/domain/javascript/javascriptModuleDeclaration.ts b/src/domain/javascript/javascriptModuleDeclaration.ts index 48a8d4b9e5..2e29ba9324 100644 --- a/src/domain/javascript/javascriptModuleDeclaration.ts +++ b/src/domain/javascript/javascriptModuleDeclaration.ts @@ -1,4 +1,5 @@ import * as t from "@babel/types"; +import { semanticWriteTargets } from "./javascriptAstValues.js"; /** Declaration syntax that establishes no runtime value, regardless of export form. */ export const isErasedJavaScriptDeclaration = (node: t.Node): boolean => @@ -52,6 +53,36 @@ export const isTypeOnlyImportSpecifier = (node: t.ImportSpecifier): boolean => export const isTypeOnlyExportSpecifier = (node: t.ExportSpecifier): boolean => node.exportKind === "type"; +/** Runtime values published by a local ESM export; source re-exports have no local value. */ +export const semanticRuntimeExportValueSources = ( + node: t.Node, +): readonly t.Node[] => { + if (t.isExportDefaultDeclaration(node)) + return isErasedJavaScriptDeclaration(node.declaration) + ? [] + : [node.declaration]; + if ( + !t.isExportNamedDeclaration(node) || + node.source != null || + isTypeOnlyExportDeclaration(node) + ) + return []; + const values: t.Node[] = []; + if (t.isVariableDeclaration(node.declaration)) + for (const declaration of node.declaration.declarations) + values.push(...semanticWriteTargets(declaration.id)); + else if ( + (t.isFunctionDeclaration(node.declaration) || + t.isClassDeclaration(node.declaration)) && + node.declaration.id != null + ) + values.push(node.declaration.id); + for (const specifier of node.specifiers) + if (t.isExportSpecifier(specifier) && !isTypeOnlyExportSpecifier(specifier)) + values.push(specifier.local); + return values; +}; + /** A regular import whose bindings are all erased but may retain side effects. */ export const hasOnlyInlineTypeImportSpecifiers = ( node: t.ImportDeclaration, diff --git a/src/domain/javascript/javascriptModuleInterop.ts b/src/domain/javascript/javascriptModuleInterop.ts index 5df0252f81..e6326d8d30 100644 --- a/src/domain/javascript/javascriptModuleInterop.ts +++ b/src/domain/javascript/javascriptModuleInterop.ts @@ -29,6 +29,7 @@ import { runtimeFunctionParameters, unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; +import { semanticKnownArrayIterationAt } from "./javascriptSemanticReferenceSources.js"; // Match the complete Rolldown runtime operations, including its live getters. // Names and formatting are irrelevant; lexical roles, operators and keys are not. @@ -82,6 +83,8 @@ export const createSemanticModuleInterop = ( readonly intrinsicPath: (node: t.Node) => readonly string[] | undefined; }, ) => { + const nativeArrayIteration = (array: t.ArrayExpression) => + semanticKnownArrayIterationAt(array, state); const helpers = new Map(); const origins = new Map(); const calls = new WeakMap(); @@ -272,7 +275,10 @@ export const createSemanticModuleInterop = ( const { root, members } = electronMemberPath(current.callee); let bindTarget = root; let bindingArguments: JavaScriptIntrinsicInvocation["arguments"] = - expandJavaScriptInvocationArguments(current.arguments); + expandJavaScriptInvocationArguments( + current.arguments, + nativeArrayIteration, + ); let captured = capturedIntrinsic(current.callee); let terminal = captured?.path.at(-1) === "bind"; let path: readonly string[] | undefined = captured?.path.slice(0, -1); @@ -287,6 +293,7 @@ export const createSemanticModuleInterop = ( intrinsicPath, boundIntrinsic, intrinsicReadChanged, + nativeArrayIteration, ); if ( invocation?.kind !== "known" || @@ -618,12 +625,16 @@ export const createSemanticModuleInterop = ( origin, callOrigin, intrinsicPath, - intrinsicInvocation: (node: t.CallExpression | t.OptionalCallExpression) => + intrinsicInvocation: ( + node: t.CallExpression | t.OptionalCallExpression, + callablePath = intrinsicPath, + ) => resolveJavaScriptIntrinsicInvocation( node, - intrinsicPath, + callablePath, boundIntrinsic, intrinsicReadChanged, + nativeArrayIteration, ), boundIntrinsic, staticString, diff --git a/src/domain/javascript/javascriptModuleResolutionMode.ts b/src/domain/javascript/javascriptModuleResolutionMode.ts index ee2f6edaaf..129c572271 100644 --- a/src/domain/javascript/javascriptModuleResolutionMode.ts +++ b/src/domain/javascript/javascriptModuleResolutionMode.ts @@ -1,4 +1,12 @@ import { posix } from "node:path"; +import type { JavaScriptArtifactFileKind } from "./javascriptArtifactFiles.js"; + +/** JSON dispatch uses the exact loader filename, not the inventory's folded kind. */ +export const isJsonModuleTarget = ( + path: string, + kind: JavaScriptArtifactFileKind | undefined, +): boolean => + path.endsWith(".json") && (kind === "json" || kind === "package-json"); /** Loader evidence retained for a source-level module reference. */ export type JavaScriptModuleResolutionMode = diff --git a/src/domain/javascript/javascriptPackageImportAliases.test.ts b/src/domain/javascript/javascriptPackageImportAliases.test.ts index 6f64c1d650..6794324470 100644 --- a/src/domain/javascript/javascriptPackageImportAliases.test.ts +++ b/src/domain/javascript/javascriptPackageImportAliases.test.ts @@ -75,6 +75,48 @@ describe("exact package import lookup", () => { }); }); + it.each([ + ["#bar/*", "#foo", "missing-key"], + ["#internal/*.js", "#internal/file.cjs", "missing-key"], + ["#internal/*.js", "#internal/.js", "missing-key"], + ["#internal/*end", "#internal/end", "missing-key"], + ["#internal/*/*", "#internal/a/b", "missing-key"], + ["#internal/", "#internal/a", "missing-key"], + ["#internal/*.js", "#internal/a.js", "pattern-unresolved"], + ["#internal/*.js", "#internal/deep/a.js", "pattern-unresolved"], + ["#internal/*end", "#internal/aend", "pattern-unresolved"], + ] as const)( + "matches only a single nonempty wildcard between its prefix and suffix: %s, %s", + (pattern, specifier, kind) => { + expect( + lookupExactPackageImport( + { [pattern]: "./unsupported/*.js" }, + specifier, + ), + ).toEqual({ kind, declared: [pattern] }); + }, + ); + + it.each( + [null, { require: "./config.cjs" }, ["./config.cjs"]].map((target) => ({ + target, + })), + )( + "preserves an exact target $target ahead of a matching wildcard", + ({ target }) => { + expect( + lookupExactPackageImport( + { "#config": target, "#*": "./unsupported/*.js" }, + "#config", + ), + ).toEqual({ + kind: "found", + target, + declared: ["#config", "#*"], + }); + }, + ); + it("still selects an exact key when the same map also has patterns", () => { expect( lookupExactPackageImport( diff --git a/src/domain/javascript/javascriptPackageImportAliases.ts b/src/domain/javascript/javascriptPackageImportAliases.ts index 9eab77e100..b2fe877140 100644 --- a/src/domain/javascript/javascriptPackageImportAliases.ts +++ b/src/domain/javascript/javascriptPackageImportAliases.ts @@ -1,3 +1,5 @@ +import { packageSubpathPatternMatches } from "./javascriptPackageSubpathPatterns.js"; + /** * Exact package-private import-alias lookup. Node resolves `#` specifiers * against the importer's enclosing package `imports` map; this module owns the @@ -55,9 +57,8 @@ export const isPackageImportPatternKey = (key: string): boolean => /** * Look up one exact `#` key in a package `imports` value. - * A missing exact key is reported as missing when every declared key is also - * exact; if the map contains pattern keys, the miss stays pattern-unresolved - * rather than claiming the specifier is undefined. + * Only a matching wildcard keeps an exact miss unresolved. Unrelated patterns + * cannot select a target and therefore do not hide an undefined specifier. */ export const lookupExactPackageImport = ( imports: unknown, @@ -72,7 +73,7 @@ export const lookupExactPackageImport = ( target: Reflect.get(imports, key), declared, }; - return declared.some(isPackageImportPatternKey) + return declared.some((pattern) => packageSubpathPatternMatches(pattern, key)) ? { kind: "pattern-unresolved", declared } : { kind: "missing-key", declared }; }; diff --git a/src/domain/javascript/javascriptPackageSubpathPatterns.test.ts b/src/domain/javascript/javascriptPackageSubpathPatterns.test.ts new file mode 100644 index 0000000000..24a983c329 --- /dev/null +++ b/src/domain/javascript/javascriptPackageSubpathPatterns.test.ts @@ -0,0 +1,30 @@ +import { expect, it } from "vitest"; +import { + packageSubpathPatternMatches, + selectPackageSubpathPattern, +} from "./javascriptPackageSubpathPatterns.js"; + +it.each([ + ["./*", "./window", true], + ["./*end", "./end", false], + ["./*.js", "./nested/a.js", true], + ["./*.js", "./a.cjs", false], + ["./*/*", "./nested/a", false], + ["./other/*", "./window", false], +] as const)( + "matches only the selected package pattern %s against %s", + (pattern, subpath, expected) => { + expect(packageSubpathPatternMatches(pattern, subpath)).toBe(expected); + }, +); + +it.each([ + [["./*", "./private/*"], "./private/window", "./private/*"], + [["./*", "./*.hidden"], "./window.hidden", "./*.hidden"], + [["./private/*", "./*"], "./private/window", "./private/*"], +] as const)( + "selects the most specific pattern independently of declaration order: %j", + (patterns, subpath, expected) => { + expect(selectPackageSubpathPattern(patterns, subpath)).toBe(expected); + }, +); diff --git a/src/domain/javascript/javascriptPackageSubpathPatterns.ts b/src/domain/javascript/javascriptPackageSubpathPatterns.ts new file mode 100644 index 0000000000..1c8684b34e --- /dev/null +++ b/src/domain/javascript/javascriptPackageSubpathPatterns.ts @@ -0,0 +1,33 @@ +/** Node package patterns contain one star matching at least one character. */ +export const packageSubpathPatternMatches = ( + pattern: string, + subpath: string, +): boolean => { + const wildcard = pattern.indexOf("*"); + return ( + wildcard !== -1 && + pattern.lastIndexOf("*") === wildcard && + subpath.length >= pattern.length && + subpath.startsWith(pattern.slice(0, wildcard)) && + subpath.endsWith(pattern.slice(wildcard + 1)) + ); +}; + +/** Node chooses the longest prefix, then the longest complete pattern key. */ +export const selectPackageSubpathPattern = ( + patterns: readonly string[], + subpath: string, +): string | undefined => { + let selected: string | undefined; + for (const pattern of patterns) { + if (!packageSubpathPatternMatches(pattern, subpath)) continue; + if ( + selected === undefined || + pattern.indexOf("*") > selected.indexOf("*") || + (pattern.indexOf("*") === selected.indexOf("*") && + pattern.length > selected.length) + ) + selected = pattern; + } + return selected; +}; diff --git a/src/domain/javascript/javascriptRuntimeStaticCandidates.ts b/src/domain/javascript/javascriptRuntimeStaticCandidates.ts index a7e1afa736..5c3ef04f69 100644 --- a/src/domain/javascript/javascriptRuntimeStaticCandidates.ts +++ b/src/domain/javascript/javascriptRuntimeStaticCandidates.ts @@ -199,6 +199,8 @@ const nodePaths = (node: ApplicationNode): string[] => { isRawApplicationFileObservation(observation); if (typeof observation.properties.path === "string") paths.push(observation.properties.path); + const applicationPath = ownedApplicationPath(observation); + if (applicationPath !== undefined) paths.push(applicationPath); if (rawApplicationFile) continue; if (typeof observation.properties.resolved_path === "string") paths.push(observation.properties.resolved_path); @@ -231,20 +233,44 @@ const hasOwnedArtifactPath = (node: ApplicationNode): boolean => ) || node.observations.some( (observation) => - !isRawApplicationFileObservation(observation) && - typeof observation.properties.resolved_path === "string", + ownedApplicationPath(observation) !== undefined || + (!isRawApplicationFileObservation(observation) && + typeof observation.properties.resolved_path === "string"), ); -// loadFile receives a raw literal here: the syntax producer does not retain -// computed __dirname expressions. Electron resolves this relative to the app -// root, so its source-relative inference is not an owned runtime target. +// Ownership of the application base survives missing target bytes. A confined +// candidate is a path identity, not proof that a static file exists. +const ownedApplicationPath = ( + observation: ApplicationNode["observations"][number], +): string | undefined => { + const properties = observation.properties; + if (properties.runtime_path_base !== "application-root") return undefined; + if ( + properties.resolution_status === "resolved" && + typeof properties.resolved_path === "string" + ) + return properties.resolved_path; + if ( + properties.runtime_path_base_proven === true && + (properties.resolution_status === "not-found" || + properties.resolution_status === "unavailable") && + typeof properties.candidate_path === "string" + ) + return properties.candidate_path; + return undefined; +}; + +// Native declaration matching is needed only without an owned logical path. const isRawApplicationFileObservation = ( observation: ApplicationNode["observations"][number], ): boolean => - observation.properties.resolution_context === "filesystem-expression" && - typeof observation.properties.mechanism === "string" && - observation.properties.mechanism.startsWith("call:") && - observation.properties.mechanism.endsWith("loadFile"); + observation.properties.runtime_path_base === "application-root" + ? ownedApplicationPath(observation) === undefined + : // Compatibility for captures reconstructed before the producer retained its base. + observation.properties.resolution_context === "filesystem-expression" && + typeof observation.properties.mechanism === "string" && + observation.properties.mechanism.startsWith("call:") && + observation.properties.mechanism.endsWith("loadFile"); const usesApplicationFileDeclaration = (node: ApplicationNode): boolean => node.kind === "electron-renderer" && diff --git a/src/domain/javascript/javascriptSemanticAliasSelections.ts b/src/domain/javascript/javascriptSemanticAliasSelections.ts new file mode 100644 index 0000000000..6a0b409433 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticAliasSelections.ts @@ -0,0 +1,126 @@ +import type { JavaScriptSemanticPropertyPath } from "./javascriptSemanticPropertyPaths.js"; +import { SEMANTIC_ALIAS_SELECTION_LIMIT } from "./javascriptSemanticResourceLimits.js"; + +type PropertyKey = JavaScriptSemanticPropertyPath[number]; + +/** Finite alternatives retain selected keys without admitting unrelated slots. */ +export type SemanticAliasSelection = + | PropertyKey + | { readonly alternatives: readonly PropertyKey[] }; + +export type SemanticAliasSelectionPath = readonly SemanticAliasSelection[]; + +export const semanticAliasSelectionChoices = ( + key: SemanticAliasSelection, +): readonly PropertyKey[] => + key !== null && typeof key === "object" && "alternatives" in key + ? key.alternatives + : [key]; + +const selection = (choices: readonly PropertyKey[]): SemanticAliasSelection => { + if (choices.includes(null)) return null; + const alternatives = [ + ...new Map(choices.map((choice) => [JSON.stringify(choice), choice])), + ] + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) + .map(([, choice]) => choice); + return alternatives.length === 1 ? alternatives[0]! : { alternatives }; +}; + +const selectionCovers = ( + prior: SemanticAliasSelection, + next: SemanticAliasSelection, +): boolean => { + if (prior === null) return true; + const choices = new Set( + semanticAliasSelectionChoices(prior).map((key) => JSON.stringify(key)), + ); + return semanticAliasSelectionChoices(next).every((key) => + choices.has(JSON.stringify(key)), + ); +}; + +const pathCovers = ( + prior: SemanticAliasSelectionPath, + next: SemanticAliasSelectionPath, +): boolean => + prior.every( + (key, index) => + next[index] !== undefined && selectionCovers(key, next[index]!), + ); + +/** Only one differing selector can be joined without inventing crossed paths. */ +const joinSelections = ( + prior: SemanticAliasSelectionPath, + next: SemanticAliasSelectionPath, +): SemanticAliasSelectionPath | undefined => { + let changed = -1; + for (let index = 0; index < prior.length; index++) { + if (JSON.stringify(prior[index]) === JSON.stringify(next[index])) continue; + if (changed !== -1) return undefined; + changed = index; + } + return prior.map((key, index) => + index === changed + ? selection([ + ...semanticAliasSelectionChoices(key), + ...semanticAliasSelectionChoices(next[index]!), + ]) + : key, + ); +}; + +/** Preserve correlated selector alternatives while coalescing complete grids. */ +export class SemanticAliasSelections { + #paths = new WeakMap>(); + #calls = 0; + #limited = false; + + constructor(readonly onLimit: () => void) {} + + get limited(): boolean { + return this.#limited; + } + + normalize( + node: object, + next: SemanticAliasSelectionPath, + ): SemanticAliasSelectionPath { + if (this.#limited) return next.map(() => null); + const existing = this.#paths.get(node); + const byDepth = existing ?? new Map(); + const paths = [...(byDepth.get(next.length) ?? [])]; + let candidate = next.map((key) => + selection(semanticAliasSelectionChoices(key)), + ); + const covered = paths.find((prior) => pathCovers(prior, candidate)); + if (covered !== undefined) return covered; + if ( + existing !== undefined && + ++this.#calls > SEMANTIC_ALIAS_SELECTION_LIMIT + ) { + this.#limited = true; + this.#paths = new WeakMap(); + this.onLimit(); + return next.map(() => null); + } + for (let index = 0; index < paths.length;) { + const prior = paths[index]!; + if (pathCovers(prior, candidate)) return prior; + const joined = pathCovers(candidate, prior) + ? candidate + : joinSelections(prior, candidate); + if (joined === undefined) { + index++; + continue; + } + candidate = [...joined]; + paths.splice(index, 1); + index = 0; + } + paths.push(candidate); + byDepth.set(candidate.length, paths); + this.#paths.set(node, byDepth); + return candidate; + } +} diff --git a/src/domain/javascript/javascriptSemanticAmbientWrites.test.ts b/src/domain/javascript/javascriptSemanticAmbientWrites.test.ts index a9832ac959..0d785dbef1 100644 --- a/src/domain/javascript/javascriptSemanticAmbientWrites.test.ts +++ b/src/domain/javascript/javascriptSemanticAmbientWrites.test.ts @@ -24,6 +24,32 @@ it("keeps wrapper module aliases distinct from global-object module properties", ]); }); +it.each([ + ["const copy={...module};consume(copy);", false], + ["const copy={...{child:module}};consume(copy);", true], + [ + "const original={child:module};const copy={...original};consume(copy);", + true, + ], +])( + "exposes copied children without escaping the source container: %s", + (body, moduleEscaped) => { + const parsed = parseJavaScriptSource(`function copy(){${body}}`); + if (parsed === null) throw new Error("Expected copied escape fixture"); + const state = completeSemanticSteps( + indexJavaScriptBindingsSteps(parsed.program), + ); + expect( + state.ambientEscapeEffects.some( + ({ path }) => + path.length === 2 && + path[0] === "commonjs-module" && + path[1] === null, + ), + ).toBe(moduleEscaped); + }, +); + it.each([ { name: "reconverging projections with different alias-cycle histories", diff --git a/src/domain/javascript/javascriptSemanticAmbientWrites.ts b/src/domain/javascript/javascriptSemanticAmbientWrites.ts index 54bccbfc26..a3e7c6d8ff 100644 --- a/src/domain/javascript/javascriptSemanticAmbientWrites.ts +++ b/src/domain/javascript/javascriptSemanticAmbientWrites.ts @@ -22,6 +22,11 @@ import { type SemanticIterationSource, } from "./javascriptSemanticIterationSources.js"; import { createSemanticReturnedReferences } from "./javascriptSemanticReturnedReferences.js"; +import { + semanticCopiedReferencePath, + semanticReferenceIterationUnknown, + semanticReferenceSourcesInactive, +} from "./javascriptSemanticReferenceSources.js"; const GLOBAL_OBJECT_NAMES = ["globalThis", "global", "window", "self"]; @@ -236,30 +241,44 @@ export const createAmbientPathResolver = ( } const binding = resolveSemanticBindingState(state, root, root.name); if (binding === undefined) continue; - const stable = semanticStableInitializer(binding, state); + const stable = possibleWrites + ? undefined + : semanticStableInitializer(binding, state); const readAt = current.readsAtSelf ? root : capture; - const initializers = possibleWrites - ? readAt === undefined - ? binding.initializers + const selected = + !possibleWrites || readAt === undefined + ? binding : semanticBindingAtRead(binding, readAt, state.parentsByNode) - .binding.initializers + .binding; + const initializers = possibleWrites + ? [...selected.initializers, ...selected.referenceInitializers] : stable === undefined ? [] : [stable]; - for (const initializer of initializers) + for (const initializer of initializers) { + if (semanticReferenceSourcesInactive(initializer, state)) continue; + // Pattern assignments share the source selection owner with member + // effects: copying a container preserves its children, not itself. + const path = semanticCopiedReferencePath( + initializer, + suffix ?? [null], + escapedContents ? "escape" : "identity", + ); + if (path === null) continue; pending.push({ node: initializer.node, readsAtSelf: true, suffix: - suffix === undefined + suffix === undefined || + semanticReferenceIterationUnknown(initializer, state) ? undefined - : [ - ...initializer.projection.map((key) => - key === null ? null : String(key), - ), - ...suffix, - ], + : path.map((key) => + key === null || typeof key === "object" + ? null + : String(key), + ), }); + } continue; } if ( @@ -332,7 +351,8 @@ export const createAmbientPathResolver = ( if (t.isObjectExpression(root)) { for (const property of root.properties) if (t.isSpreadElement(property)) - continueWith(property.argument, []); + // A fresh copy exposes selected children, not its source container. + continueWith(property.argument, [null]); else if ( t.isObjectProperty(property) && t.isExpression(property.value) @@ -433,6 +453,7 @@ export function* indexJavaScriptAmbientWritesSteps( program: t.Program, state: JavaScriptSemanticAnalysisState, observeOpaqueInvocation?: (node: t.Node) => void, + observeWriteSite?: (node: t.Node) => void, ): Generator { const ambientReads = createAmbientPathResolver(state, false); const browserReceiverMethods = new Set(); @@ -482,6 +503,7 @@ export function* indexJavaScriptAmbientWritesSteps( program, (root, kind, writeSite) => { if (kind === "opaque-call") return []; + observeWriteSite?.(writeSite); // Reassigning a local alias changes the binding, not the referenced object. if (kind === "binding") return t.isIdentifier(root) && @@ -626,8 +648,11 @@ export function* indexJavaScriptAmbientWritesSteps( for (const { node, target, browserReceiver, projection } of escapes) for (const path of ambientWrites(target, node, true, projection)) // Only proven browser APIs keep their global receiver from user code. - if (!(browserReceiver && path.length === 1 && path[0] === "global")) - state.ambientEscapes.retain([...path, null]); + if (!(browserReceiver && path.length === 1 && path[0] === "global")) { + const escaped = [...path, null]; + state.ambientEscapes.retain(escaped); + state.ambientEscapeEffects.push({ path: escaped, node }); + } // A function may call a browser method replaced later in source order. if ( [...browserReceiverMethods].some((method) => diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index 239c0a79a6..c999c0cec9 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -58,7 +58,11 @@ import { completeSemanticSteps, traverseJavaScriptAstSteps, } from "./javascriptSemanticTraversal.js"; -import { semanticRequireOrigin } from "./javascriptSemanticRequireOrigin.js"; +import { + semanticRequireOrigin, + observePossibleCommonJsDelegateWrite, + invalidateSemanticRequireDelegateWrites, +} from "./javascriptSemanticRequireOrigin.js"; import { semanticBundledRequireOrigin } from "./javascriptBundledRequireOrigin.js"; import { collectSemanticMemberMutationsSteps } from "./javascriptSemanticMemberMutations.js"; import { semanticEffectTiming } from "./javascriptSemanticMutationInitializers.js"; @@ -104,9 +108,11 @@ import { type JavaScriptSemanticPropertyPath, } from "./javascriptSemanticPropertyPaths.js"; import { indexCommonJsExportReceiversSteps } from "./javascriptCommonJsExportReceivers.js"; +import { createSemanticElectronAppPathChangeQuery } from "./javascriptElectronAppPathChanges.js"; import type { JavaScriptModuleResolutionMode } from "./javascriptModuleResolutionMode.js"; interface BindPatternInput { + readonly iterationSources?: JavaScriptSemanticBindingState["initializers"][number]["iterationSources"]; readonly pattern: t.Node; readonly initializer: t.Node | null; readonly scope: JavaScriptSemanticScopeState; @@ -116,6 +122,7 @@ interface BindPatternInput { readonly kind?: JavaScriptSemanticDefinition["kind"]; readonly projection?: readonly (string | number | null)[]; readonly referenceOnly?: boolean; + readonly assignmentReference?: true; readonly copyKind?: "object-rest" | "array-rest"; readonly copyProjectionOffset?: number; readonly copyExcludedKeys?: readonly string[]; @@ -123,14 +130,21 @@ interface BindPatternInput { readonly fallbackSources?: readonly { readonly node: t.Node; readonly projection: readonly (string | number | null)[]; + readonly iterationSources?: NonNullable< + JavaScriptSemanticBindingState["initializers"][number]["iterationSources"] + >; }[]; readonly requiredSources?: readonly { readonly node: t.Node; readonly projection: readonly (string | number | null)[]; + readonly iterationSources?: NonNullable< + JavaScriptSemanticBindingState["initializers"][number]["iterationSources"] + >; }[]; } interface AddBindingInput { + readonly iterationSources?: JavaScriptSemanticBindingState["initializers"][number]["iterationSources"]; readonly state: JavaScriptSemanticAnalysisState; readonly scope: JavaScriptSemanticScopeState; readonly name: string; @@ -177,6 +191,7 @@ function* indexSemanticAmbientWritesSteps( state.program, state, createSemanticOpaqueGlobalObserver(state), + (node) => observePossibleCommonJsDelegateWrite(node, state), ); if (state.opaqueEffects.globalSites.length > 0) clearSemanticValueCaches(state); @@ -196,6 +211,7 @@ function* indexSemanticMemberMutationFactsSteps( referenceEffectsMayHaveChanged(state, site, reference, path), ); indexedMemberMutationStates.add(state); + invalidateSemanticRequireDelegateWrites(state); } /** One owner projects source-site effects for complete reports or early exit. */ @@ -420,6 +436,9 @@ export function* analyzeParsedJavaScriptSemanticsSteps( [...state.returnedReferenceWidenings].map(range), ), limitations: [ + ...(state.valueResourceLimits.has("alias-selections") + ? [semanticResourceLimitReason("alias-selections")] + : []), ...(state.returnedReferenceWidenings.size > 0 ? [RETURNED_REFERENCE_WIDENING_LIMITATION] : []), @@ -801,6 +820,10 @@ export interface JavaScriptElectronIdentity { readonly bindings: ReadonlyMap; readonly intactExport: (path: readonly (string | null)[]) => boolean; readonly exportPath: (root: t.Node) => readonly string[] | undefined; + readonly invocationPath: ( + node: t.CallExpression | t.OptionalCallExpression, + ) => readonly string[] | undefined; + readonly mayChangeAppPath: (node: t.Node) => boolean; } export function* indexJavaScriptElectronIdentitySteps( @@ -881,7 +904,35 @@ export function* indexJavaScriptElectronIdentitySteps( facts.set(root.start ?? -1, exported); }, }); - return { bindings: facts, intactExport, exportPath: rootExport }; + const invocationPath = ( + node: t.CallExpression | t.OptionalCallExpression, + ): readonly string[] | undefined => { + const invocation = interop.intrinsicInvocation(node, (callee) => { + const { root, members } = electronMemberPath(callee); + const exported = rootExport(root); + if ( + exported !== undefined && + members.every((member): member is string => member !== null) && + intactExport(["electron", ...exported, ...members]) + ) + return ["electron", ...exported, ...members]; + return interop.intrinsicPath(callee); + }); + return invocation?.path[0] === "electron" + ? invocation.path.slice(1) + : undefined; + }; + return { + bindings: facts, + intactExport, + exportPath: rootExport, + invocationPath, + mayChangeAppPath: createSemanticElectronAppPathChangeQuery( + state, + rootExport, + interop, + ), + }; } // Read import and require origins directly; general value evaluation is @@ -951,6 +1002,7 @@ const createState = ( bindings: new Map(), }; return { + arrayIterationUnknown: false, program, opaqueEffects: { mayReferenceGlobalEval: false, @@ -983,6 +1035,7 @@ const createState = ( ambientBindingWrites: new Map(), ambientWrites: new SemanticPropertyPathCoverage(), ambientEscapes: new SemanticPropertyPathCoverage(), + ambientEscapeEffects: [], ambientWriteEffects: [], }; }; @@ -1433,6 +1486,9 @@ const bindAssignment = ( ): void => { if (t.isAssignmentExpression(node)) { const left = unwrapJavaScriptExpression(node.left).node; + const selectedPattern = + node.operator === "=" && + (t.isObjectPattern(left) || t.isArrayPattern(left)); if (t.isIdentifier(left)) addAssignment( left, @@ -1446,17 +1502,28 @@ const bindAssignment = ( conditionallySkipped, node, ); - else + else { for (const identifier of semanticWriteTargets(node.left)) if (t.isIdentifier(identifier)) addAssignment( identifier, - node, + selectedPattern ? null : node, scope, state, conditionallySkipped, node, ); + if (selectedPattern) + bindPattern({ + pattern: left, + initializer: node.right, + scope, + state, + mutable: true, + referenceOnly: true, + assignmentReference: true, + }); + } } else if (t.isUpdateExpression(node)) { const argument = unwrapJavaScriptExpression(node.argument).node; if (t.isIdentifier(argument)) @@ -1677,12 +1744,14 @@ const bindPattern = (input: BindPatternInput): void => { kind = "variable", projection = [], referenceOnly = false, + assignmentReference, copyKind, copyProjectionOffset, copyExcludedKeys, copyStartIndex, fallbackSources, requiredSources, + iterationSources, } = input; if (t.isTSParameterProperty(pattern)) { bindPattern({ ...input, pattern: pattern.parameter }); @@ -1699,6 +1768,7 @@ const bindPattern = (input: BindPatternInput): void => { binding.referenceInitializers.push({ node: initializer, projection, + ...(assignmentReference === undefined ? {} : { assignment: true }), ...(copyKind === undefined ? {} : { copyKind }), ...(copyProjectionOffset === undefined ? {} @@ -1707,6 +1777,7 @@ const bindPattern = (input: BindPatternInput): void => { ...(copyStartIndex === undefined ? {} : { copyStartIndex }), ...(fallbackSources === undefined ? {} : { fallbackSources }), ...(requiredSources === undefined ? {} : { requiredSources }), + ...(iterationSources === undefined ? {} : { iterationSources }), }); return; } @@ -1720,6 +1791,7 @@ const bindPattern = (input: BindPatternInput): void => { definitionNode: pattern, initializer, projection, + ...(iterationSources === undefined ? {} : { iterationSources }), }); return; } @@ -1733,7 +1805,15 @@ const bindPattern = (input: BindPatternInput): void => { : (initializer ?? pattern.right), requiredSources: [ ...(requiredSources ?? []), - ...(initializer === null ? [] : [{ node: initializer, projection }]), + ...(initializer === null + ? [] + : [ + { + node: initializer, + projection, + ...(iterationSources === undefined ? {} : { iterationSources }), + }, + ]), ], }); bindPattern({ @@ -1745,9 +1825,18 @@ const bindPattern = (input: BindPatternInput): void => { initializer: pattern.right, projection: [], referenceOnly: true, + ...(assignmentReference === undefined ? {} : { assignmentReference }), fallbackSources: [ ...(fallbackSources ?? []), - ...(initializer === null ? [] : [{ node: initializer, projection }]), + ...(initializer === null + ? [] + : [ + { + node: initializer, + projection, + ...(iterationSources === undefined ? {} : { iterationSources }), + }, + ]), ], ...(requiredSources === undefined ? {} : { requiredSources }), }); @@ -1790,17 +1879,28 @@ const bindPattern = (input: BindPatternInput): void => { }); } } - else if (t.isArrayPattern(pattern)) + else if (t.isArrayPattern(pattern)) { + const arrayInput = { + ...input, + ...(initializer === null + ? {} + : { + iterationSources: [ + ...(iterationSources ?? []), + { node: initializer, projection }, + ], + }), + }; pattern.elements.forEach((element, index) => { if (element !== null) { bindPattern({ - ...input, + ...arrayInput, pattern: element, projection: [...projection, index], }); if (t.isRestElement(element)) bindPattern({ - ...input, + ...arrayInput, pattern: element.argument, projection: [...projection, null], referenceOnly: true, @@ -1810,6 +1910,7 @@ const bindPattern = (input: BindPatternInput): void => { }); } }); + } }; const addBinding = (input: AddBindingInput): void => { @@ -1825,6 +1926,7 @@ const addBinding = (input: AddBindingInput): void => { directOrigin, runtimeImport, projection = [], + iterationSources, } = input; let binding = scope.bindings.get(name); if (binding === undefined) { @@ -1838,7 +1940,11 @@ const addBinding = (input: AddBindingInput): void => { binding.declarationKinds.add(declarationKind); binding.definitions.push({ kind, location: range(definitionNode) }); if (initializer !== null) - binding.initializers.push({ node: initializer, projection }); + binding.initializers.push({ + node: initializer, + projection, + ...(iterationSources === undefined ? {} : { iterationSources }), + }); if (directOrigin !== undefined) binding.directOrigins.push( directOrigin.resolutionMode === undefined && @@ -1875,12 +1981,14 @@ const createBinding = ( const addAssignment = ( identifier: t.Identifier, - initializer: t.Node, + initializer: t.Node | null, scope: JavaScriptSemanticScopeState, state: JavaScriptSemanticAnalysisState, conditionallySkipped: boolean, writeSite?: t.Node, ): void => { + const mutation = writeSite ?? initializer; + if (mutation === null) return; const lookup = semanticBindingLookupFromScope(scope, identifier.name, state); if (lookup.kind === "captured") return; const binding = "binding" in lookup ? lookup.binding : undefined; @@ -1890,16 +1998,17 @@ const addAssignment = ( // Unbound identifiers can denote host wrapper bindings (such as require), // which are separate from properties written through the global object. const writes = state.ambientBindingWrites.get(identifier.name) ?? []; - writes.push(writeSite ?? initializer); + writes.push(mutation); state.ambientBindingWrites.set(identifier.name, writes); return; } const writes = state.bindingWrites.get(binding.bindingId) ?? []; - writes.push(writeSite ?? initializer); + writes.push(mutation); state.bindingWrites.set(binding.bindingId, writes); binding.mutable = true; binding.definitions.push({ kind: "assignment", location: range(identifier) }); - binding.initializers.push({ node: initializer, projection: [] }); + if (initializer !== null) + binding.initializers.push({ node: initializer, projection: [] }); if (conditionallySkipped) - state.conditionallySkippedExpressions.add(initializer); + state.conditionallySkippedExpressions.add(initializer ?? mutation); }; diff --git a/src/domain/javascript/javascriptSemanticCallResolution.test.ts b/src/domain/javascript/javascriptSemanticCallResolution.test.ts index 54e1dc0ef7..876bf4cde2 100644 --- a/src/domain/javascript/javascriptSemanticCallResolution.test.ts +++ b/src/domain/javascript/javascriptSemanticCallResolution.test.ts @@ -1,7 +1,10 @@ import * as t from "@babel/types"; import { expect, it, vi } from "vitest"; -import { indexJavaScriptBindingsSteps } from "./javascriptSemanticAnalysis.js"; +import { + analyzeJavaScriptSemantics, + indexJavaScriptBindingsSteps, +} from "./javascriptSemanticAnalysis.js"; import { resolveLocalCallables, type LocalCallableResolution, @@ -37,6 +40,88 @@ const indexed = (source: string) => { }; }; +it.each([ + "if(flag)({make}={make:replacement});", + "if(flag)[make]=[replacement];", + "({make}={make:replacement});", + "[make]=[replacement];", +])("does not prove the old callable after a projected write: %s", (write) => { + const analysis = analyzeJavaScriptSemantics( + `function original(){}function replacement(){}let make=original;make();${write}make();`, + ); + const original = analysis.callables.find(({ name }) => name === "original"); + expect(original).toBeDefined(); + const [before, after] = analysis.callSites; + expect(before).toMatchObject({ + resolution: "exact", + calleeCallableIds: [original?.callableId], + }); + expect(after).toBeDefined(); + expect(after?.resolution).not.toBe("exact"); + expect(analysis.frontiers).toContainEqual( + expect.objectContaining({ + kind: "dynamic-call", + location: after?.calleeLocation, + }), + ); +}); + +it.each([ + "if(flag)({make}={make:replacement});", + "if(flag)[make]=[replacement];", +])( + "retains known callable candidates without claiming completeness: %s", + (write) => { + const context = indexed( + `function original(){}function replacement(){}let make=original;${write}make();`, + ); + const last = context.state.program.body.at(-1); + if (!t.isExpressionStatement(last) || !t.isCallExpression(last.expression)) + throw new Error("Missing projected call"); + const original = context.state.callables.find( + ({ name }) => name === "original", + ); + expect(original).toBeDefined(); + expect(context.resolve(last.expression.callee)).toMatchObject({ + complete: false, + callableIds: [original?.callableId], + }); + }, +); + +it("retains selected callable keys without admitting an unrelated return", () => { + const context = indexed(` + const table = { x() { return {}; }, y() { return {}; }, z() { return module; } }; + const make = flag ? table.x : table.y; + make(); + `); + const last = context.state.program.body.at(-1); + if (!t.isExpressionStatement(last) || !t.isCallExpression(last.expression)) + throw new Error("Missing selected call"); + const returns = createSemanticReturnedReferences(context.state)( + last.expression.callee, + ).references; + expect(returns).toHaveLength(2); + expect(returns.every(({ node }) => t.isObjectExpression(node))).toBe(true); +}); + +it("retains correlations between selected callable path segments", () => { + const context = indexed(` + const table = { a: { b() { return {}; }, d() { return module; } }, + c: { d() { return {}; }, b() { return module; } } }; + const make = flag ? table.a.b : table.c.d; + make(); + `); + const last = context.state.program.body.at(-1); + if (!t.isExpressionStatement(last) || !t.isCallExpression(last.expression)) + throw new Error("Missing selected call"); + const returns = createSemanticReturnedReferences(context.state)( + last.expression.callee, + ).references; + expect(returns).toHaveLength(2); + expect(returns.every(({ node }) => t.isObjectExpression(node))).toBe(true); +}); + it("bounds shared partial callback alias expansion by AST size", () => { const measure = (depth: number) => { const context = diff --git a/src/domain/javascript/javascriptSemanticCallResolution.ts b/src/domain/javascript/javascriptSemanticCallResolution.ts index 4ca40cc9f6..25a9b9ecda 100644 --- a/src/domain/javascript/javascriptSemanticCallResolution.ts +++ b/src/domain/javascript/javascriptSemanticCallResolution.ts @@ -6,7 +6,10 @@ import type { } from "./javascriptSemanticIr.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; import { semanticCallableIdForNode } from "./javascriptSemanticProjection.js"; -import { semanticBindingAtRead } from "./javascriptSemanticMutationInitializers.js"; +import { + semanticBindingAtRead, + semanticEffectTiming, +} from "./javascriptSemanticMutationInitializers.js"; import { unwrapJavaScriptExpression, runtimeFunctionParameters, @@ -119,6 +122,7 @@ const resolveIdentifier = ( }; const candidates = resolveBindingCallables( read.binding, + node, selectsAll ? { ...context, @@ -168,6 +172,7 @@ const resolveAlternatives = ( const resolveBindingCallables = ( binding: JavaScriptSemanticBindingState, + site: t.Identifier, context: CallResolutionContext, ): LocalCallableResolution => { if (binding.initializers.length === 0) @@ -199,11 +204,21 @@ const resolveBindingCallables = ( const callableIds = uniqueCallableIds( candidates.flatMap((candidate) => candidate.callableIds), ); + // Pattern writes are owned reference facts, not ordinary initializers. A + // read may retain both an earlier callable and a conditional projected + // replacement. Until projected callable resolution is supported here, + // preserve known candidates without claiming they cover every target. + const complete = + !binding.referenceInitializers.some( + (initializer) => + initializer.assignment && + semanticEffectTiming(site, initializer.node, context.state) !== "after", + ) && candidates.every(({ complete }) => complete); const resolution = { callableIds, - complete: candidates.every(({ complete }) => complete), + complete, reason: - callableIds.length === 1 && candidates.every(({ complete }) => complete) + callableIds.length === 1 && complete ? "Unique local callable target." : `Call target ${binding.name} has ambiguous or unsupported assignments.`, ...(candidates.some((candidate) => candidate.cycleDependent) diff --git a/src/domain/javascript/javascriptSemanticCoverage.ts b/src/domain/javascript/javascriptSemanticCoverage.ts index cd1d55fb9d..6d73d3c5f3 100644 --- a/src/domain/javascript/javascriptSemanticCoverage.ts +++ b/src/domain/javascript/javascriptSemanticCoverage.ts @@ -4,6 +4,7 @@ import type { ApplicationCoverage } from "./javascriptApplicationEvidenceSchemas import { SEMANTIC_EXPRESSION_DEPTH_LIMIT, SEMANTIC_GETTER_EXPANSION_LIMIT, + SEMANTIC_ALIAS_SELECTION_LIMIT, SEMANTIC_PRIMITIVE_CANDIDATE_LIMIT, SEMANTIC_PRIMITIVE_JSON_BYTES_LIMIT, } from "./javascriptSemanticResourceLimits.js"; @@ -91,6 +92,10 @@ const semanticResourceBounds: Record< value: SEMANTIC_GETTER_EXPANSION_LIMIT, unit: "items", }, + "alias-selections": { + value: SEMANTIC_ALIAS_SELECTION_LIMIT, + unit: "items", + }, }; /** Map semantic evaluator bounds to their caller-visible application coverage. */ diff --git a/src/domain/javascript/javascriptSemanticDataEffectHelpers.ts b/src/domain/javascript/javascriptSemanticDataEffectHelpers.ts index fcc1d0b99d..6c0f3c210d 100644 --- a/src/domain/javascript/javascriptSemanticDataEffectHelpers.ts +++ b/src/domain/javascript/javascriptSemanticDataEffectHelpers.ts @@ -161,22 +161,31 @@ export const builtinDataEffectMethod = ( state: JavaScriptSemanticAnalysisState, specifiers: readonly string[], ): string | null => { + const path = builtinDataEffectCallablePath(callee, state, specifiers); + return path?.length === 1 ? (path[0] ?? null) : null; +}; + +/** Recover the complete callable export path using the shared mutation proof. */ +export const builtinDataEffectCallablePath = ( + callee: t.Node, + state: JavaScriptSemanticAnalysisState, + specifiers: readonly string[], +): readonly string[] | null => { const expression = unwrapJavaScriptExpression(callee).node; if (expression !== callee) - return builtinDataEffectMethod(expression, state, specifiers); + return builtinDataEffectCallablePath(expression, state, specifiers); if (t.isSequenceExpression(expression)) { const final = expression.expressions.at(-1); return final === undefined ? null - : builtinDataEffectMethod(final, state, specifiers); + : builtinDataEffectCallablePath(final, state, specifiers); } const required = semanticRequireOrigin(callee, state); if (required !== undefined) { const origin = builtinNamespaceOrigin(required.origin); - return origin.importedPath.length === 1 && - specifiers.some( - (specifier) => specifier.replace(/^node:/u, "") === origin.specifier, - ) && + return specifiers.some( + (specifier) => specifier.replace(/^node:/u, "") === origin.specifier, + ) && !requiredModuleChanged( required.origin.specifier, required.origin.importedPath, @@ -184,7 +193,7 @@ export const builtinDataEffectMethod = ( state, true, ) - ? (origin.importedPath[0] ?? null) + ? origin.importedPath : null; } if (t.isIdentifier(callee)) { @@ -197,7 +206,7 @@ export const builtinDataEffectMethod = ( [], true, ); - return path?.length === 1 ? (path[0] ?? null) : null; + return path; } if (!t.isMemberExpression(callee) && !t.isOptionalMemberExpression(callee)) return null; @@ -227,27 +236,26 @@ export const builtinDataEffectMethod = ( callee, [...receiver.path, method], true, - )?.length === 0 - ? method + ) !== null + ? [...origin.importedPath, method] : null; } } const source = semanticExpressionModuleOrigin(callee.object, state); if (method === null || source === undefined) return null; const origin = builtinNamespaceOrigin(source); - return origin.importedPath.length === 0 && - specifiers.some( - (specifier) => specifier.replace(/^node:/u, "") === origin.specifier, - ) && + return specifiers.some( + (specifier) => specifier.replace(/^node:/u, "") === origin.specifier, + ) && !requiredModuleChanged( origin.specifier, - [...origin.importedPath, method], + [...source.importedPath, method], callee, state, true, callee.object, ) - ? method + ? [...origin.importedPath, method] : null; } const binding = resolveSemanticBindingState( @@ -265,7 +273,7 @@ export const builtinDataEffectMethod = ( [method], true, ); - if (resolvedPath?.length === 0) return method; + if (resolvedPath !== null) return [...resolvedPath, method]; const initializer = binding?.initializers[0]; if ( initializer === undefined || @@ -298,7 +306,7 @@ export const builtinDataEffectMethod = ( true, initializer.node, ) - ? method + ? [method] : null; }; @@ -588,6 +596,9 @@ const affectsBuiltinPath = ( const BUILTIN_RECEIVER_METHODS: Readonly>> = { + path: new Set(["resolve", "join", "dirname"]), + "path/posix": new Set(["resolve", "join", "dirname"]), + url: new Set(["fileURLToPath"]), fs: new Set([ "readFile", "readFileSync", @@ -762,7 +773,7 @@ const computeOriginalBuiltinReceiver = ( ): boolean => { if (!t.isCallExpression(node) && !t.isOptionalCallExpression(node)) return false; - const callee = node.callee; + const callee = unwrapJavaScriptExpression(node.callee).node; if (!t.isMemberExpression(callee) && !t.isOptionalMemberExpression(callee)) return false; const receiver = t.isIdentifier(callee.object) @@ -780,14 +791,21 @@ const computeOriginalBuiltinReceiver = ( semanticGlobalProcessOrigin(callee.object, state)); if (sourceOrigin === undefined) return false; const origin = builtinNamespaceOrigin(sourceOrigin); - if (origin.importedPath.length !== 0) return false; + const posixReceiver = + origin.specifier === "path" && + origin.importedPath.length === 1 && + origin.importedPath[0] === "posix"; + if (origin.importedPath.length !== 0 && !posixReceiver) return false; const method = semanticStaticPropertyKey(callee.property, callee.computed); const identity = semanticRequireExportsIdentity(origin.specifier, [ + ...origin.importedPath, method ?? "", ]); if ( method === null || - !BUILTIN_RECEIVER_METHODS[origin.specifier]?.has(method) || + !BUILTIN_RECEIVER_METHODS[ + posixReceiver ? "path/posix" : origin.specifier + ]?.has(method) || receiver?.memberWrites.some( ({ path, node: write }) => affectsBuiltinPath(path, [method]) && diff --git a/src/domain/javascript/javascriptSemanticDataEffects.ts b/src/domain/javascript/javascriptSemanticDataEffects.ts index a98ed5d729..69deae89cc 100644 --- a/src/domain/javascript/javascriptSemanticDataEffects.ts +++ b/src/domain/javascript/javascriptSemanticDataEffects.ts @@ -21,7 +21,10 @@ import { type JavaScriptSemanticAnalysisState, type JavaScriptSemanticBindingState, } from "./javascriptSemanticState.js"; -import { semanticBindingAtRead } from "./javascriptSemanticMutationInitializers.js"; +import { + semanticBindingAtRead, + semanticEffectTiming, +} from "./javascriptSemanticMutationInitializers.js"; import { builtinDataEffectMethod as builtinMethod, dataEffectArgumentBindingId as argumentBindingId, @@ -254,11 +257,35 @@ const soleWriteReachesRead = ( binding: JavaScriptSemanticBindingState | undefined, read: t.Node, context: TraversalContext, -): boolean => - binding?.initializers.length === 1 && - (semanticStableInitializer(binding, context.state) !== undefined || - semanticBindingAtRead(binding, read, context.state.parentsByNode) - .hasDefiniteInitializer); + request: t.Node | undefined, +): boolean => { + if (binding === undefined || request === undefined) return false; + const snapshot = semanticBindingAtRead( + binding, + read, + context.state.parentsByNode, + ); + const initializers = snapshot.binding.initializers.filter( + ({ node }) => semanticEffectTiming(read, node, context.state) !== "after", + ); + const [initializer] = initializers; + if (initializers.length !== 1 || initializer === undefined) return false; + // The result itself must be this request. Containing a request in a branch + // or child allocation does not prove the entire initializer is its response. + let source = unwrapJavaScriptExpression(initializer.node).node; + while (t.isAwaitExpression(source)) + source = unwrapJavaScriptExpression(source.argument).node; + return ( + source === request && + !snapshot.binding.referenceInitializers.some( + ({ assignment, node }) => + assignment && + semanticEffectTiming(read, node, context.state) !== "after", + ) && + (semanticStableInitializer(binding, context.state) !== undefined || + snapshot.hasDefiniteInitializer) + ); +}; const collectResponseConsumers = ( program: t.Program, @@ -308,7 +335,12 @@ const collectResponseConsumers = ( ) ? linked[0] : undefined; - const soleWrite = soleWriteReachesRead(binding, member.object, context); + const soleWrite = soleWriteReachesRead( + binding, + member.object, + context, + uniqueRequest?.node, + ); output.push({ requestId: `response:${String(node.start ?? -1)}:${String(node.end ?? -1)}`, kind: "response-consumer", diff --git a/src/domain/javascript/javascriptSemanticFunctionEscapes.test.ts b/src/domain/javascript/javascriptSemanticFunctionEscapes.test.ts new file mode 100644 index 0000000000..f203bff5a8 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticFunctionEscapes.test.ts @@ -0,0 +1,47 @@ +import { expect, it } from "vitest"; + +import { indexJavaScriptBindingsWithReceiversSteps } from "./javascriptSemanticAnalysis.js"; +import { completeSemanticSteps } from "./javascriptSemanticTraversal.js"; +import { parseJavaScriptSource } from "./javascriptSourceParser.js"; + +const indexedBindings = (body: string) => { + const parsed = parseJavaScriptSource( + `const shared={};const get=()=>shared;${body}`, + ); + if (parsed === null) throw new Error("Expected callback escape fixture"); + const index = completeSemanticSteps( + indexJavaScriptBindingsWithReceiversSteps(parsed.program), + ); + completeSemanticSteps(index.indexAmbientWritesSteps()); + completeSemanticSteps(index.indexMemberMutationFactsSteps()); + return [...index.bindings.bindingsById.values()]; +}; + +it.each([ + ["consume(get);", true], + ["consume(()=>shared);", true], + ["const alias=get;consume(alias);", true], + ["consume(()=>({}));", false], + ["const local=()=>({});consume(local);", false], + ["const local=()=>({});const alias=local;consume(alias);", false], + ["get.own=1;", false], + ["const local=()=>({});get.call=local;consume(get);", true], + ["const local=()=>({});get.call=local;consume(get.call);", false], +])("preserves callback result escape sources through %s", (body, escaped) => { + const bindings = indexedBindings(body); + const shared = bindings.find(({ name }) => name === "shared"); + expect(shared).toBeDefined(); + expect((shared?.escapedPaths.length ?? 0) > 0).toBe(escaped); +}); + +it.each(["call", "apply", "other"])( + "keeps a function's own %s write separate from its returned object", + (member) => { + const bindings = indexedBindings(`get.${member}=()=>({});`); + const get = bindings.find(({ name }) => name === "get"); + const shared = bindings.find(({ name }) => name === "shared"); + expect(get?.memberWrites.map(({ path }) => path)).toContainEqual([member]); + expect(shared?.memberWrites).toEqual([]); + expect(shared?.escapedPaths).toEqual([]); + }, +); diff --git a/src/domain/javascript/javascriptSemanticIterationSources.ts b/src/domain/javascript/javascriptSemanticIterationSources.ts index 0fc3962b26..debe56235e 100644 --- a/src/domain/javascript/javascriptSemanticIterationSources.ts +++ b/src/domain/javascript/javascriptSemanticIterationSources.ts @@ -4,20 +4,24 @@ import { semanticObjectPatternKeys, runtimeFunctionParameters, semanticStaticPropertyKey, + unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; import { semanticArrayIndex, type JavaScriptSemanticPropertyPath, } from "./javascriptSemanticPropertyPaths.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; +import { semanticCopiedReferencePath } from "./javascriptSemanticReferenceSources.js"; interface IterationSourceValue { readonly node: t.Node; readonly projection: JavaScriptSemanticPropertyPath; + readonly iterationSources?: readonly IterationSourceValue[]; } /** An iterator receiver or property reference store, with default guards. */ export interface SemanticIterationSource extends IterationSourceValue { + readonly iterationSources?: readonly IterationSourceValue[]; readonly kind: "iteration" | "reference-store"; readonly mutation: t.Node; readonly target?: t.MemberExpression; @@ -30,7 +34,44 @@ interface PatternSource extends IterationSourceValue { readonly restStartIndex?: number; } +/** Select an assigned slot while retaining fresh rest-copy identity and offsets. */ +export const semanticStoredSourceSelection = ( + source: SemanticIterationSource, + selected: JavaScriptSemanticPropertyPath, +): { + readonly projection: JavaScriptSemanticPropertyPath; + readonly capturedSelectionDepth?: number; + readonly ownEnumerable?: true; +} | null => { + const copy = source.projection.at(-1); + if (copy === null || typeof copy !== "object") + return { projection: [...source.projection, ...selected] }; + const projection = semanticCopiedReferencePath( + { + projection: source.projection, + copyProjectionOffset: source.projection.length - 1, + copyKind: copy.startIndex === undefined ? "object-rest" : "array-rest", + copyExcludedKeys: copy.excludedKeys, + ...(copy.startIndex === undefined + ? {} + : { copyStartIndex: copy.startIndex }), + }, + selected, + "identity", + ); + return projection === null + ? null + : { + projection, + capturedSelectionDepth: source.projection.length, + ...(copy.startIndex === undefined + ? { ownEnumerable: true as const } + : {}), + }; +}; + interface SourceGuards { + readonly iterationSources?: readonly IterationSourceValue[]; readonly fallbackSources?: readonly IterationSourceValue[]; readonly requiredSources?: readonly IterationSourceValue[]; } @@ -79,6 +120,7 @@ const visit = ( source: PatternSource | undefined, guards: SourceGuards = {}, ): void => { + pattern = unwrapJavaScriptExpression(pattern).node; const { sources, mutation } = context; if (t.isMemberExpression(pattern)) { // Property stores have no lexical alias binding. Preserve the selected @@ -111,7 +153,16 @@ const visit = ( ...guards, requiredSources: [ ...(guards.requiredSources ?? []), - ...(source === undefined ? [] : [source]), + ...(source === undefined + ? [] + : [ + { + ...source, + ...(guards.iterationSources === undefined + ? {} + : { iterationSources: guards.iterationSources }), + }, + ]), ], }); visit( @@ -122,7 +173,16 @@ const visit = ( ...guards, fallbackSources: [ ...(guards.fallbackSources ?? []), - ...(source === undefined ? [] : [source]), + ...(source === undefined + ? [] + : [ + { + ...source, + ...(guards.iterationSources === undefined + ? {} + : { iterationSources: guards.iterationSources }), + }, + ]), ], }, ); @@ -168,6 +228,13 @@ const visit = ( if (t.isArrayPattern(pattern)) { if (source !== undefined && source.restStartIndex === undefined) add(sources, source, mutation, guards); + const selectedGuards = { + ...guards, + iterationSources: [ + ...(guards.iterationSources ?? []), + ...(source === undefined ? [] : [source]), + ], + }; pattern.elements.forEach((element, index) => { if (element === null) return; if (t.isRestElement(element)) @@ -181,9 +248,9 @@ const visit = ( projection: source.projection, restStartIndex: (source.restStartIndex ?? 0) + index, }, - guards, + selectedGuards, ); - else visit(context, element, project(source, index), guards); + else visit(context, element, project(source, index), selectedGuards); }); return; } diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts index 872d8bb46c..fc26d58a63 100644 --- a/src/domain/javascript/javascriptSemanticMemberMutations.ts +++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts @@ -8,6 +8,7 @@ import { semanticMutableBuiltinOrigin, semanticRequireExportsIdentity, semanticRequireOrigin, + hasSemanticCreateRequireFactoryOrigin, } from "./javascriptSemanticRequireOrigin.js"; import { @@ -23,16 +24,22 @@ import { } from "./javascriptSemanticValues.js"; import { semanticContainer } from "./javascriptSemanticSlots.js"; import { + semanticArrayPropertyReferences, semanticClassCandidates, semanticClassPropertyReferences, semanticObjectPropertyReferences, type JavaScriptSemanticObjectReference, } from "./javascriptSemanticObjectReferences.js"; -import { semanticIterationSources } from "./javascriptSemanticIterationSources.js"; +import { + semanticIterationSources, + type SemanticIterationSource, +} from "./javascriptSemanticIterationSources.js"; import { semanticCallableResultExpressions } from "./javascriptSemanticCallableResults.js"; import { createSemanticReturnedReferences } from "./javascriptSemanticReturnedReferences.js"; import { semanticMutationInitializers } from "./javascriptSemanticMutationInitializers.js"; import { + semanticCopiedReferencePath, + semanticKnownArrayIteration, semanticReferenceSourcesInactive, semanticSelectedReferenceSlot, } from "./javascriptSemanticReferenceSources.js"; @@ -54,6 +61,23 @@ import { type PropertyPath = JavaScriptSemanticPropertyPath; type ValueEffect = "write" | "escape"; +/** Only replacing or possibly selecting an iterator-bearing prototype slot changes iteration. */ +const arrayPrototypeIterationMayChange = (path: PropertyPath): boolean => { + // Array.prototype and Object.prototype are on native arrays' prototype + // chain. Their default constructor.prototype pairs return to the same owner. + let offset = 0; + while (path[offset] === "constructor" && path[offset + 1] === "prototype") + offset += 2; + const key = path[offset]; + return ( + key === undefined || + key === null || + typeof key === "object" || + semanticArrayIndex(key) !== null || + key === "__proto__" + ); +}; + /** Preserve uncertainty from property writes and references exposed to calls, yielding during traversal and fixpoint iteration. */ /** Setter keys of one object literal; a computed setter key may match any key. */ @@ -198,7 +222,7 @@ export function* collectSemanticMemberMutationsSteps( recordRequiredEffect("node:process", path.slice(2), "write", node); const selections = new Map(); const coveringPaths = new Map(); - let arrayIterationUnknown = false; + state.arrayIterationUnknown = false; // Definition collection finishes before this phase, so a scope's bindings no // longer change and one node resolves to one binding. Repeated visits of the // same node otherwise repeat the same scope walk. @@ -278,11 +302,21 @@ export function* collectSemanticMemberMutationsSteps( source.node.start, source.node.end, source.projection, + source.iterationSources?.map((iterable) => [ + iterable.node.start, + iterable.node.end, + iterable.projection, + ]), ]), reference.fallbackSources?.map((source) => [ source.node.start, source.node.end, source.projection, + source.iterationSources?.map((iterable) => [ + iterable.node.start, + iterable.node.end, + iterable.projection, + ]), ]), [...reference.bindings].sort(compareUnicodeCodePoints), ]); @@ -300,6 +334,7 @@ export function* collectSemanticMemberMutationsSteps( originAt: t.Node | undefined = mutation, ownerInitializer?: t.Node, widenedGetterResult = false, + assignedSources?: readonly SemanticIterationSource[], ): void => { const pending: (ReferencedValue & { readonly bindings: ReadonlySet; @@ -351,15 +386,18 @@ export function* collectSemanticMemberMutationsSteps( mutation, ); if ( - (expression.name === "Array" && - (current.path[0] === "prototype" || current.path[0] === null)) || + (["Array", "Object"].includes(expression.name) && + (current.path[0] === "prototype" || current.path[0] === null) && + arrayPrototypeIterationMayChange(current.path.slice(1))) || (["globalThis", "global", "window", "self"].includes( expression.name, ) && - (current.path[0] === "Array" || current.path[0] === null) && - (current.path[1] === "prototype" || current.path[1] === null)) + (["Array", "Object"].includes(String(current.path[0])) || + current.path[0] === null) && + (current.path[1] === "prototype" || current.path[1] === null) && + arrayPrototypeIterationMayChange(current.path.slice(2))) ) - arrayIterationUnknown = true; + state.arrayIterationUnknown = true; continue; } // An alias restored from a saved reference can revisit the same binding @@ -487,7 +525,7 @@ export function* collectSemanticMemberMutationsSteps( current.originAt?.start ?? -1 }\u0001${current.originAt?.end ?? -1}\u0001${ current.ownerInitializer?.start ?? -1 - }\u0001${current.ownerInitializer?.end ?? -1}\u0001${widenedGetterResult}`; + }\u0001${current.ownerInitializer?.end ?? -1}\u0001${widenedGetterResult}\u0001${assignedSources?.[0]?.target?.start ?? -1}`; let covering = coveringPaths.get(pathState); if (covering === undefined) { covering = new SemanticPropertyPathCoverage(); @@ -521,9 +559,29 @@ export function* collectSemanticMemberMutationsSteps( mutation, ); } + const receiverInitializers = [ + ...origins.initializers, + ...origins.referenceInitializers, + ]; if (effect === "escape") - binding.escapedPaths.push({ path, node: mutation }); - else binding.memberWrites.push({ path, node: mutation }); + binding.escapedPaths.push({ + path, + node: mutation, + receiverInitializers, + ...(current.originAt === undefined + ? {} + : { receiverAt: current.originAt }), + }); + else + binding.memberWrites.push({ + path, + node: mutation, + receiverInitializers, + ...(current.originAt === undefined + ? {} + : { receiverAt: current.originAt }), + ...(assignedSources === undefined ? {} : { assignedSources }), + }); clearSemanticValueCaches(state); for (const initializer of origins.initializers) pending.push({ @@ -534,7 +592,11 @@ export function* collectSemanticMemberMutationsSteps( }); } for (const initializer of origins.referenceInitializers) { - const copiedPath = copiedReferencePath(initializer, path, effect); + const copiedPath = semanticCopiedReferencePath( + initializer, + path, + effect, + ); if (copiedPath === null) continue; pendingReferences.push({ initializer, @@ -617,10 +679,14 @@ export function* collectSemanticMemberMutationsSteps( ): readonly ReferencedValue[] { if ( t.isArrayExpression(expression) && - ((path[0] === "__proto__" && (effect === "escape" || path.length > 1)) || - (path[0] === "constructor" && path[1] === "prototype")) + ((path[0] === "__proto__" && + (effect === "escape" || path.length > 1) && + arrayPrototypeIterationMayChange(path.slice(1))) || + (path[0] === "constructor" && + path[1] === "prototype" && + arrayPrototypeIterationMayChange(path.slice(2)))) ) - arrayIterationUnknown = true; + state.arrayIterationUnknown = true; const references = referencedValues( expression, path, @@ -732,6 +798,21 @@ export function* collectSemanticMemberMutationsSteps( markEscaped(expression.object, mutation); }; const markTarget = (node: t.Node, mutation: t.Node): void => { + const sources = semanticIterationSources( + mutation, + state.parentsByNode.get(mutation) ?? null, + ); + const sourcesByTarget = new Map< + t.MemberExpression | t.OptionalMemberExpression, + SemanticIterationSource[] + >(); + for (const source of sources) { + if (source.kind !== "reference-store" || source.target === undefined) + continue; + const selected = sourcesByTarget.get(source.target) ?? []; + selected.push(source); + sourcesByTarget.set(source.target, selected); + } for (const target of semanticWriteTargets(node)) if (!t.isIdentifier(target)) markValue( @@ -740,6 +821,10 @@ export function* collectSemanticMemberMutationsSteps( new Set(), "write", mutation, + mutation, + undefined, + false, + sourcesByTarget.get(target) ?? [], ); }; yield* traverseJavaScriptAstSteps(program, { @@ -798,7 +883,7 @@ export function* collectSemanticMemberMutationsSteps( ) { const effectsBefore = recordedEffectCount; const iterablesBefore = iterableReferences.length; - const iterationBefore: boolean = arrayIterationUnknown; + const iterationBefore: boolean = state.arrayIterationUnknown; for (const [index, reference] of pendingReferences.splice(0).entries()) { if (index % 64 === 0) yield; const { initializer } = reference; @@ -866,12 +951,7 @@ export function* collectSemanticMemberMutationsSteps( const slot = semanticSelectedReferenceSlot(reference, state); // Custom iteration can yield the iterable itself as well as any child. // Treat its root as escaped, including writes through yielded references. - const knownArray = - !arrayIterationUnknown && - slot?.value.status === "array" && - slot.value.items.every( - (item) => semanticArrayIndex(item.name) !== null, - ); + const knownArray = semanticKnownArrayIteration(slot?.value, state); if (knownArray && reference.iterationOnly) continue; markValue( reference.node, @@ -901,7 +981,7 @@ export function* collectSemanticMemberMutationsSteps( if ( recordedEffectCount === effectsBefore && iterableReferences.length === iterablesBefore && - arrayIterationUnknown === iterationBefore + state.arrayIterationUnknown === iterationBefore ) break; } @@ -922,14 +1002,12 @@ interface IterableReference extends ReferencedValue { readonly mutation?: t.Node; readonly originAt: t.Node | undefined; readonly iterationOnly?: boolean; - readonly requiredSources?: readonly { - readonly node: t.Node; - readonly projection: PropertyPath; - }[]; - readonly fallbackSources?: readonly { - readonly node: t.Node; - readonly projection: PropertyPath; - }[]; + readonly requiredSources?: NonNullable< + SemanticIterationSource["requiredSources"] + >; + readonly fallbackSources?: NonNullable< + SemanticIterationSource["fallbackSources"] + >; } const expandedIterationReferences = ( @@ -1004,7 +1082,7 @@ const referencedValues = ( if (effect === "write" && path.length < 2) return []; return t.isObjectExpression(node) ? semanticObjectPropertyReferences(node, path, state) - : arrayReferencedValues(node, path); + : semanticArrayPropertyReferences(node, path); } if (t.isClass(node) || t.isNewExpression(node)) return constructedReferenceValues( @@ -1081,6 +1159,7 @@ type MutationObservableOrigin = | "ambient" | "array" | "array-prototype" + | "function" | "opaque"; interface MutationObserverNode { @@ -1172,7 +1251,9 @@ const observableMutationPaths = ( // CommonJS namespace from `require("node:fs")`). Keep those writes at their // exact member paths instead of widening the first path to a wildcard. const hasParsedRequireOrigin = binding.initializers.some( - ({ node }) => semanticRequireOrigin(node, state) !== undefined, + ({ node }) => + semanticRequireOrigin(node, state) !== undefined || + hasSemanticCreateRequireFactoryOrigin(node, state), ); if ( binding.directOrigins.length > 0 || @@ -1205,6 +1286,10 @@ const observableMutationPaths = ( const selected = pendingExpressions.pop(); if (selected === undefined) break; const { expression, observer } = selected; + if (t.isFunction(expression)) + // Keep own-member writes precise without replacing the existing + // links to references handed to a function's eventual caller. + (observer.origins ??= new Set()).add("function"); // Keep class/getter projections distinct so cycle pruning does not // discard observable escapes through a shared getter result. if ( @@ -1386,98 +1471,3 @@ const observableMutationPaths = ( }; return { normalize, repeatsUnpruned }; }; - -const arrayReferencedValues = ( - node: t.ArrayExpression, - path: PropertyPath, -): readonly ReferencedValue[] => { - const [key, ...remaining] = path; - const selected = key ?? null; - const selectedIndex = - typeof selected === "object" ? null : semanticArrayIndex(selected); - if (typeof selected !== "object" && selectedIndex === null) return []; - const references: ReferencedValue[] = []; - let minimumIndex = 0; - let uncertainIndex = false; - for (const element of node.elements) { - if (t.isSpreadElement(element)) { - if (selectedIndex === null || selectedIndex >= minimumIndex) { - let sourceKey: PropertyPath[number] = null; - if (!uncertainIndex) { - if (selectedIndex !== null) sourceKey = selectedIndex - minimumIndex; - else if (selected !== null && typeof selected === "object") - sourceKey = { - excludedKeys: selected.excludedKeys.flatMap((name) => { - const index = semanticArrayIndex(name); - return index === null || index < minimumIndex - ? [] - : [String(index - minimumIndex)]; - }), - startIndex: Math.max( - 0, - (selected.startIndex ?? 0) - minimumIndex, - ), - }; - } - references.push({ - node: element.argument, - path: [sourceKey, ...remaining], - iterableFallbackPath: [], - }); - } - uncertainIndex = true; - continue; - } - if ( - element !== null && - (uncertainIndex - ? selectedIndex === null || selectedIndex >= minimumIndex - : semanticPropertyPathKeyMatches(selected, String(minimumIndex))) - ) - references.push({ node: element, path: remaining }); - minimumIndex++; - } - return references; -}; - -const copiedReferencePath = ( - initializer: JavaScriptSemanticBindingState["referenceInitializers"][number], - path: PropertyPath, - effect: ValueEffect, -): PropertyPath | null => { - const offset = initializer.copyProjectionOffset; - if (offset === undefined) return [...initializer.projection, ...path]; - const copiedPath = [...initializer.projection.slice(offset + 1), ...path]; - if (effect === "write" && copiedPath.length < 2) return null; - let [key, ...remaining] = copiedPath; - if (initializer.copyKind === "array-rest") { - const startIndex = initializer.copyStartIndex ?? 0; - if (key === undefined || key === null) - key = { excludedKeys: [], startIndex }; - else if (typeof key === "object") - key = { - excludedKeys: key.excludedKeys.flatMap((name) => { - const index = semanticArrayIndex(name); - return index === null ? [] : [String(index + startIndex)]; - }), - startIndex: (key.startIndex ?? 0) + startIndex, - }; - else { - const index = semanticArrayIndex(key); - if (index === null) return null; - key = index + startIndex; - } - } else { - const excludedKeys = initializer.copyExcludedKeys ?? []; - if (key === undefined || key === null) key = { excludedKeys }; - else if (typeof key === "object") - key = { - ...key, - excludedKeys: [...new Set([...key.excludedKeys, ...excludedKeys])].sort( - compareUnicodeCodePoints, - ), - }; - else if (excludedKeys.includes(String(key))) return null; - } - return [...initializer.projection.slice(0, offset), key, ...remaining]; -}; diff --git a/src/domain/javascript/javascriptSemanticMutationInitializers.ts b/src/domain/javascript/javascriptSemanticMutationInitializers.ts index c297caa13d..12bf68748e 100644 --- a/src/domain/javascript/javascriptSemanticMutationInitializers.ts +++ b/src/domain/javascript/javascriptSemanticMutationInitializers.ts @@ -471,6 +471,25 @@ const directMutationPosition = ( return computed; }; +/** Prove that an unconditional member replacement precedes this read. */ +export const semanticDefiniteEffectPrecedesReference = ( + effect: t.Node, + reference: t.Node, + parents: WeakMap, +): boolean => { + const before = directMutationPosition(effect, parents); + const after = bindingReadPosition(reference, parents); + if (before === null || after.position === null || after.repeating) + return false; + if (positionPrecedes(before, after.position)) return true; + return ( + before.body === after.position.body && + before.index === after.position.index && + before.declaratorIndex === after.position.declaratorIndex && + expressionEvaluationOrder(effect, reference, parents) === "before" + ); +}; + const computeReferenceInitializerPosition = ( node: t.Node, parents: WeakMap, @@ -481,6 +500,11 @@ const computeReferenceInitializerPosition = ( const direct = directInitializerPosition(current, parents); if (direct !== null) return direct; const parent = parents.get(current); + if ( + t.isAssignmentExpression(parent, { operator: "=" }) && + (t.isObjectPattern(parent.left) || t.isArrayPattern(parent.left)) + ) + return directMutationPosition(parent, parents); if (t.isFunction(parent)) return t.isBlockStatement(parent.body) ? { body: parent.body.body, index: -1, declaratorIndex: -1 } @@ -660,8 +684,15 @@ export const semanticBindingAtRead = ( // can cover entry even when another write's order remains unknown. const hasDefiniteInitializer = position !== null && - binding.initializers.some((initializer) => { - const source = initializerPosition(initializer, parents); + [ + ...binding.initializers, + ...binding.referenceInitializers.filter(({ assignment }) => assignment), + ].some((initializer) => { + const source = initializerPosition( + initializer, + parents, + binding.referenceInitializers.includes(initializer), + ); return ( source !== null && positionPrecedes(source, position) && diff --git a/src/domain/javascript/javascriptSemanticObjectReferences.ts b/src/domain/javascript/javascriptSemanticObjectReferences.ts index 41d79ffd4a..58a0756599 100644 --- a/src/domain/javascript/javascriptSemanticObjectReferences.ts +++ b/src/domain/javascript/javascriptSemanticObjectReferences.ts @@ -8,6 +8,7 @@ import { unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; import { + semanticArrayIndex, semanticPropertyPathKeyMatches, type JavaScriptSemanticPropertyPath as PropertyPath, } from "./javascriptSemanticPropertyPaths.js"; @@ -21,6 +22,12 @@ import { expandSemanticGetterResult } from "./javascriptSemanticGetterResults.js export interface JavaScriptSemanticObjectReference { readonly node: t.Node; readonly path: PropertyPath; + /** This selection crossed the object's prototype, rather than an own slot. */ + readonly inherited?: true; + /** A shallow copy selects this prefix now while retaining its child identity. */ + readonly capturedSelectionAt?: t.Node; + readonly ownEnumerable?: true; + readonly iterableFallbackPath?: PropertyPath; readonly escapes?: true; /** A retry omitted suffixes, so escaping constructed children must cover them. */ readonly widenedGetterResult?: true; @@ -638,6 +645,8 @@ class ObjectReferenceWalk { this.#references.push({ node: property.argument, path: [this.#spreadKey(), ...this.#remaining], + capturedSelectionAt: property.argument, + ownEnumerable: true, }); if ( selected !== null && @@ -672,6 +681,7 @@ class ObjectReferenceWalk { this.#references.push({ node: property.value, path: [selected, ...this.#remaining], + inherited: true, }); return true; } @@ -777,3 +787,57 @@ const objectExpressionDefinesKey = ( } return false; }; + +export const semanticArrayPropertyReferences = ( + node: t.ArrayExpression, + path: PropertyPath, +): readonly JavaScriptSemanticObjectReference[] => { + const [key, ...remaining] = path; + const selected = key ?? null; + const selectedIndex = + typeof selected === "object" ? null : semanticArrayIndex(selected); + if (typeof selected !== "object" && selectedIndex === null) return []; + const references: JavaScriptSemanticObjectReference[] = []; + let minimumIndex = 0; + let uncertainIndex = false; + for (const element of node.elements) { + if (t.isSpreadElement(element)) { + if (selectedIndex === null || selectedIndex >= minimumIndex) { + let sourceKey: PropertyPath[number] = null; + if (!uncertainIndex) { + if (selectedIndex !== null) sourceKey = selectedIndex - minimumIndex; + else if (selected !== null && typeof selected === "object") + sourceKey = { + excludedKeys: selected.excludedKeys.flatMap((name) => { + const index = semanticArrayIndex(name); + return index === null || index < minimumIndex + ? [] + : [String(index - minimumIndex)]; + }), + startIndex: Math.max( + 0, + (selected.startIndex ?? 0) - minimumIndex, + ), + }; + } + references.push({ + node: element.argument, + path: [sourceKey, ...remaining], + capturedSelectionAt: element.argument, + iterableFallbackPath: [], + }); + } + uncertainIndex = true; + continue; + } + if ( + element !== null && + (uncertainIndex + ? selectedIndex === null || selectedIndex >= minimumIndex + : semanticPropertyPathKeyMatches(selected, String(minimumIndex))) + ) + references.push({ node: element, path: remaining }); + minimumIndex++; + } + return references; +}; diff --git a/src/domain/javascript/javascriptSemanticProjectedSources.ts b/src/domain/javascript/javascriptSemanticProjectedSources.ts new file mode 100644 index 0000000000..722f6b41bd --- /dev/null +++ b/src/domain/javascript/javascriptSemanticProjectedSources.ts @@ -0,0 +1,403 @@ +import * as t from "@babel/types"; +import type { + JavaScriptSemanticAnalysisState, + JavaScriptSemanticBindingState, +} from "./javascriptSemanticState.js"; +import { + semanticPropertyPathKeyMatches, + type JavaScriptSemanticPropertyPath, +} from "./javascriptSemanticPropertyPaths.js"; +import { unwrapJavaScriptExpression } from "./javascriptAstValues.js"; +import { + semanticObjectPropertyReferences, + semanticArrayPropertyReferences, +} from "./javascriptSemanticObjectReferences.js"; +import { SEMANTIC_ALIAS_SELECTION_LIMIT } from "./javascriptSemanticResourceLimits.js"; +import { + semanticBindingAtRead, + semanticDefiniteEffectPrecedesReference, + semanticEffectTiming, + semanticTopLevelEffectFollowsCapture, +} from "./javascriptSemanticMutationInitializers.js"; +import { + semanticCopiedReferencePath, + semanticReferenceIterationUnknown, + semanticReferenceSourcesInactive, +} from "./javascriptSemanticReferenceSources.js"; +import { semanticStoredSourceSelection } from "./javascriptSemanticIterationSources.js"; +import { + semanticAliasSelectionChoices, + type SemanticAliasSelectionPath, +} from "./javascriptSemanticAliasSelections.js"; + +export interface SemanticCapturedSelection { + readonly at: t.Node; + readonly depth: number; + /** Object spread/rest reads only an own enumerable slot at this depth. */ + readonly ownEnumerable?: true; +} + +/** The next origin crossed owns a repeated copy boundary at the same depth. */ +export const semanticNormalizeCapturedSelections = ( + selections: readonly SemanticCapturedSelection[] | undefined, + pathLength?: number, +): readonly SemanticCapturedSelection[] | undefined => { + const byDepth = new Map(); + for (const capture of selections ?? []) + if ( + capture.depth > 0 && + (pathLength === undefined || capture.depth <= pathLength) + ) + byDepth.set(capture.depth, capture); + const normalized = [...byDepth.values()].sort( + (left, right) => left.depth - right.depth, + ); + return normalized.length ? normalized : undefined; +}; + +/** Keep capture boundaries aligned as a selected prefix is added or consumed. */ +export const semanticShiftCapturedSelections = ( + selections: readonly SemanticCapturedSelection[] | undefined, + delta: number, +): readonly SemanticCapturedSelection[] | undefined => { + const shifted = selections?.flatMap((selection) => + selection.depth + delta > 0 + ? [{ ...selection, depth: selection.depth + delta }] + : [], + ); + return semanticNormalizeCapturedSelections(shifted); +}; + +interface ProjectedSource { + readonly node: t.Node; + readonly path: SemanticAliasSelectionPath; + readonly capturedSelections?: readonly SemanticCapturedSelection[]; +} + +/** A fresh parent does not prove a descendant receiver is a fresh allocation. */ +const hasFreshProjectedReceiver = ( + root: t.Node, + path: JavaScriptSemanticPropertyPath, + state: JavaScriptSemanticAnalysisState, +): boolean => { + const pending = [{ node: root, path }]; + const examined = new WeakMap>(); + let selections = 0; + while (pending.length > 0) { + const current = pending.pop()!; + const node = unwrapJavaScriptExpression(current.node).node; + const key = JSON.stringify(current.path); + const paths = examined.get(node) ?? new Set(); + if (paths.has(key)) return false; + paths.add(key); + examined.set(node, paths); + if (++selections > SEMANTIC_ALIAS_SELECTION_LIMIT) { + state.valueResourceLimits.add("alias-selections"); + return false; + } + if (current.path.length === 0) { + if ( + !t.isObjectExpression(node) && + !t.isArrayExpression(node) && + !t.isFunction(node) && + !t.isClassExpression(node) + ) + return false; + continue; + } + const [selected, ...remaining] = current.path; + const sources = t.isObjectExpression(node) + ? semanticObjectPropertyReferences(node, [selected!], state) + : t.isArrayExpression(node) + ? semanticArrayPropertyReferences(node, [selected!]) + : []; + if (sources.length === 0) return false; + if ( + selections + pending.length + sources.length > + SEMANTIC_ALIAS_SELECTION_LIMIT + ) { + state.valueResourceLimits.add("alias-selections"); + return false; + } + pending.push( + ...sources.map((source) => ({ + node: source.node, + path: [...source.path, ...remaining], + })), + ); + } + return true; +}; + +/** Select retained assignment and copy sources at one projected member read. */ +export const semanticProjectedSourcesAtRead = ( + ...arguments_: Parameters +): readonly ProjectedSource[] | undefined => + semanticProjectedSourceFactsAtRead(...arguments_)?.sources; + +/** Retain the selected receiver's possible owners after its current value becomes opaque. */ +export const semanticProjectedSourceEligibilityAtRead = ( + binding: JavaScriptSemanticBindingState, + reference: t.Node, + path: SemanticAliasSelectionPath, + state: JavaScriptSemanticAnalysisState, + memberReadAt: t.Node = reference, + capturedSelections: readonly SemanticCapturedSelection[] = [], +): readonly ProjectedSource[] | undefined => + projectedSourceFactsAtRead( + binding, + reference, + path, + state, + false, + memberReadAt, + capturedSelections, + true, + )?.sources; + +/** Keep imported origins subject to the same selected writes as other values. */ +export const semanticProjectedSourceFactsAtRead = ( + ...arguments_: Parameters +): ReturnType => + projectedSourceFactsAtRead(...arguments_); + +const projectedSourceFactsAtRead = ( + binding: JavaScriptSemanticBindingState, + reference: t.Node, + path: SemanticAliasSelectionPath, + state: JavaScriptSemanticAnalysisState, + includeUnknown: boolean, + memberReadAt: t.Node = reference, + capturedSelections: readonly SemanticCapturedSelection[] = [], + retainEligibility = false, +): + | { + readonly sources: readonly ProjectedSource[]; + readonly retainsOrigin: boolean; + } + | undefined => { + const read = semanticBindingAtRead(binding, reference, state.parentsByNode); + const captured = [ + ...read.binding.initializers, + ...read.binding.referenceInitializers, + ]; + const normalizedCaptures = semanticNormalizeCapturedSelections( + capturedSelections, + path.length, + ); + const captureForDepth = (depth: number): t.Node | undefined => + normalizedCaptures?.find((capture) => capture.depth >= depth)?.at; + const selectedAt = (depth: number): t.Node => + captureForDepth(depth) ?? memberReadAt; + const samePrefix = ( + prefix: JavaScriptSemanticBindingState["memberWrites"][number], + effect: JavaScriptSemanticBindingState["memberWrites"][number], + ): boolean => + prefix.path.length < effect.path.length && + prefix.path.every( + (key, index) => + key !== null && + typeof key !== "object" && + effect.path[index] !== null && + typeof effect.path[index] !== "object" && + String(key) === String(effect.path[index]), + ); + const targetsReplacedPrefix = ( + effect: JavaScriptSemanticBindingState["memberWrites"][number], + ): boolean => + capturedSelections.length > 0 && + binding.memberWrites.some((replacement) => { + const capture = captureForDepth(replacement.path.length); + const receiverAt = effect.receiverAt ?? effect.node; + return ( + capture !== undefined && + receiverAt !== undefined && + t.isAssignmentExpression(replacement.node, { operator: "=" }) && + samePrefix(replacement, effect) && + semanticTopLevelEffectFollowsCapture( + capture, + replacement.node, + state.parentsByNode, + ) && + semanticDefiniteEffectPrecedesReference( + replacement.node, + receiverAt, + state.parentsByNode, + ) && + hasFreshProjectedReceiver( + replacement.node.right, + effect.path.slice(replacement.path.length, -1), + state, + ) + ); + }); + const matches = ( + effect: JavaScriptSemanticBindingState["memberWrites"][number], + ) => + (effect.receiverInitializers === undefined || + effect.receiverInitializers.length === 0 || + captured.length === 0 || + effect.receiverInitializers.some((receiver) => + captured.some( + (source) => + source.node === receiver.node && + JSON.stringify(source.projection) === + JSON.stringify(receiver.projection), + ), + )) && + path.length > 0 && + effect.path.length <= path.length && + effect.path.every((key, index) => { + const selected = path[index]; + return ( + selected !== undefined && + semanticAliasSelectionChoices(selected).some( + (choice) => + choice === null || + typeof choice === "object" || + semanticPropertyPathKeyMatches(key, String(choice)), + ) + ); + }) && + semanticEffectTiming(selectedAt(effect.path.length), effect.node, state) !== + "after" && + !targetsReplacedPrefix(effect); + const writes = binding.memberWrites.filter(matches); + let replacement: + | JavaScriptSemanticBindingState["memberWrites"][number] + | undefined; + for (const effect of writes) { + if ( + t.isAssignmentExpression(effect.node, { operator: "=" }) && + effect.path.every( + (key, index) => + key !== null && + typeof key !== "object" && + path[index] !== null && + typeof path[index] !== "object" && + String(key) === String(path[index]), + ) && + semanticDefiniteEffectPrecedesReference( + effect.node, + selectedAt(effect.path.length), + state.parentsByNode, + ) && + (replacement === undefined || + semanticDefiniteEffectPrecedesReference( + replacement.node!, + effect.node, + state.parentsByNode, + ) || + (replacement.node === effect.node && + (replacement.assignedSources?.[0]?.target?.start ?? -1) < + (effect.assignedSources?.[0]?.target?.start ?? -1))) + ) + replacement = effect; + } + const superseded = ( + effect: JavaScriptSemanticBindingState["memberWrites"][number], + ) => + replacement !== undefined && + effect.node !== undefined && + effect !== replacement && + (semanticEffectTiming(replacement.node!, effect.node, state) === "before" || + (effect.node === replacement.node && + (effect.assignedSources?.[0]?.target?.start ?? -1) < + (replacement.assignedSources?.[0]?.target?.start ?? -1))); + if ( + includeUnknown && + binding.escapedPaths.some( + (effect) => matches(effect) && !superseded(effect), + ) + ) + return undefined; + const sources: ProjectedSource[] = []; + for (const effect of writes) { + if (!retainEligibility && superseded(effect)) continue; + if ( + !t.isAssignmentExpression(effect.node) || + !["=", "||=", "&&=", "??="].includes(effect.node.operator) + ) { + if (includeUnknown) return undefined; + continue; + } + if (effect.assignedSources === undefined) { + if (includeUnknown) return undefined; + continue; + } + const remaining = path.slice(effect.path.length); + for (const source of effect.assignedSources) { + if (semanticReferenceIterationUnknown(source, state)) return undefined; + if (semanticReferenceSourcesInactive(source, state)) continue; + const [key, ...suffix] = remaining; + for (const choice of key === undefined + ? [undefined] + : semanticAliasSelectionChoices(key)) { + const selection = semanticStoredSourceSelection( + source, + choice === undefined ? [] : [choice], + ); + if (selection !== null) + sources.push({ + node: source.node, + path: [...selection.projection, ...suffix], + ...(selection.capturedSelectionDepth === undefined + ? {} + : { + capturedSelections: [ + { + at: source.node, + depth: selection.capturedSelectionDepth, + ...(selection.ownEnumerable + ? { ownEnumerable: true as const } + : {}), + }, + ], + }), + }); + } + } + } + if (!retainEligibility && replacement !== undefined) + return { sources, retainsOrigin: false }; + for (const initializer of read.binding.initializers) { + if (semanticReferenceIterationUnknown(initializer, state)) return undefined; + if (semanticReferenceSourcesInactive(initializer, state)) continue; + sources.push({ + node: initializer.node, + path: [...initializer.projection, ...path], + }); + } + for (const initializer of read.binding.referenceInitializers) { + if (semanticReferenceIterationUnknown(initializer, state)) return undefined; + if (semanticReferenceSourcesInactive(initializer, state)) continue; + const [key, ...suffix] = path; + for (const choice of key === undefined + ? [undefined] + : semanticAliasSelectionChoices(key)) { + const projection = semanticCopiedReferencePath( + initializer, + choice === undefined ? [] : [choice], + "identity", + ); + if (projection !== null) + sources.push({ + node: initializer.node, + path: [...projection, ...suffix], + capturedSelections: + initializer.copyProjectionOffset === undefined + ? [] + : [ + { + at: initializer.node, + depth: projection.length + suffix.length - path.length + 1, + ...(initializer.copyKind === "object-rest" + ? { ownEnumerable: true as const } + : {}), + }, + ], + }); + } + } + return { sources, retainsOrigin: true }; +}; diff --git a/src/domain/javascript/javascriptSemanticProjection.ts b/src/domain/javascript/javascriptSemanticProjection.ts index 25a065e57f..b7a723fea7 100644 --- a/src/domain/javascript/javascriptSemanticProjection.ts +++ b/src/domain/javascript/javascriptSemanticProjection.ts @@ -25,7 +25,6 @@ import { import { range } from "./javascriptStaticAnalysisHelpers.js"; import { propertyName, - semanticFinalObjectProperty, semanticStaticPropertyKey, semanticWrappedWriteSite, semanticWriteSiteReadsFirst, @@ -40,11 +39,11 @@ import { semanticImportEqualsSpecifier, } from "./javascriptModuleDeclaration.js"; import { - commonJsDefinedExport, - commonJsExportName, + commonJsRuntimeExportValues, type CommonJsExportReceiverEffect, type CommonJsExportSite, } from "./javascriptCommonJsExportReceivers.js"; +import type { JavaScriptSemanticPropertyPath } from "./javascriptSemanticPropertyPaths.js"; interface CollectCallableInput { readonly node: t.Node; @@ -124,7 +123,7 @@ export const collectSemanticModuleLink = ( else if (t.isAssignmentExpression(node)) collectCommonJsExport(node, state, commonJsReceivers.get(node)); else if (t.isCallExpression(node)) { - collectDefinedCommonJsExport(node, state, commonJsReceivers.get(node)); + collectCommonJsExport(node, state, commonJsReceivers.get(node)); collectUnboundRequireLink(node, state); } }; @@ -472,34 +471,21 @@ const requirePatternBindings = ( }; const collectCommonJsExport = ( - node: t.AssignmentExpression, + node: CommonJsExportSite, state: JavaScriptSemanticAnalysisState, receiverEffect: CommonJsExportReceiverEffect | undefined, ): void => { - const exportedName = commonJsExportName(node.left, state); - if (exportedName === undefined) return; if (receiverEffect === "detached") return; // Compound and logical writes also depend on the previous value, so remain // unknown. - const value = - node.operator === "=" && receiverEffect !== "uncertain" ? node.right : null; - addCommonJsExport(node, state, exportedName, value); -}; - -const collectDefinedCommonJsExport = ( - node: t.CallExpression, - state: JavaScriptSemanticAnalysisState, - receiverEffect: CommonJsExportReceiverEffect | undefined, -): void => { - const defined = commonJsDefinedExport(node, state); - if (defined === undefined || receiverEffect === "detached") return; - // A data descriptor exports its value; an accessor's result stays unknown. - const descriptor = unwrapJavaScriptExpression(defined.descriptor).node; - const value = - receiverEffect !== "uncertain" && t.isObjectExpression(descriptor) - ? (semanticFinalObjectProperty(descriptor, "value") ?? null) - : null; - addCommonJsExport(node, state, defined.exportedName, value); + for (const exported of commonJsRuntimeExportValues(node, state)) + addCommonJsExport( + node, + state, + exported.exportedName, + receiverEffect === "uncertain" ? null : exported.value, + exported.projection, + ); }; const addCommonJsExport = ( @@ -507,6 +493,7 @@ const addCommonJsExport = ( state: JavaScriptSemanticAnalysisState, exportedName: string, exported: t.Node | null, + projection?: JavaScriptSemanticPropertyPath, ): void => { // Simple assignment chains yield their rightmost assigned value. let value = exported; @@ -514,21 +501,41 @@ const addCommonJsExport = ( value = value.right; const required = value === null ? undefined : semanticRequireOrigin(value, state); - const origin = required?.origin; + // Copies retain their module provenance, but do not export the source object + // itself. A wildcard with exclusions cannot express one module link. + const origin = projection?.some( + (key) => + typeof key === "object" && key !== null && key.excludedKeys.length > 0, + ) + ? undefined + : required?.origin; + const selectedName = projection?.at(-1); addModuleLink( state, { kind: "commonjs-export", specifier: origin?.specifier ?? null, - importedName: origin?.importedPath.at(-1) ?? null, - localName: value !== null && t.isIdentifier(value) ? value.name : null, + importedName: + origin === undefined + ? null + : typeof selectedName === "string" || typeof selectedName === "number" + ? String(selectedName) + : (origin.importedPath.at(-1) ?? null), + localName: + projection === undefined && value !== null && t.isIdentifier(value) + ? value.name + : null, exportedName, - callableId: value === null ? null : semanticCallableIdForNode(value), + callableId: + projection !== undefined || value === null + ? null + : semanticCallableIdForNode(value), location: range(node), }, node, ); - if (required !== undefined) state.linkedRequireCalls.add(required.call); + if (origin !== undefined && required !== undefined) + state.linkedRequireCalls.add(required.call); }; const addModuleLink = ( diff --git a/src/domain/javascript/javascriptSemanticProvenanceAssignments.test.ts b/src/domain/javascript/javascriptSemanticProvenanceAssignments.test.ts new file mode 100644 index 0000000000..6a26af7bd5 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticProvenanceAssignments.test.ts @@ -0,0 +1,87 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +const provenance = (source: string, name: string) => { + const result = analyzeJavaScriptSemantics(source); + const binding = result.bindings.find((candidate) => candidate.name === name); + if (binding === undefined) throw new Error(`Missing binding ${name}`); + return binding.provenance; +}; + +it.each([ + "let api;({api}={api:require('node:fs')});", + "let api;[api]=[require('node:fs')];", + "let api;({api}={api:{readFileSync(){}}});", + "let api={};({api}={api:require('node:fs')});", + "let api;if(flag)({api}={api:require('node:fs')});", +])("keeps projected replacement provenance unresolved: %s", (source) => { + expect(provenance(source, "api")).toMatchObject({ + status: "unknown", + origins: [], + }); +}); + +it.each([ + "let api=require('node:fs');({api}={api:{}});", + "let api=require('node:fs');if(flag)({api}={api:{}});", + "let api=require('node:fs');[api]=[{}];", +])( + "retains possible earlier module origins without proving them current: %s", + (source) => { + expect(provenance(source, "api")).toMatchObject({ + status: "ambiguous", + origins: [{ specifier: "node:fs", importedPath: [] }], + }); + }, +); + +it.each([ + ["let api=require('node:fs');const saved=api;({api}={api:{}});", "module"], + ["let api=require('node:fs');({api}={api:{}});const saved=api;", "unknown"], + [ + "let api=require('node:fs');if(flag)({api}={api:{}});const saved=api;", + "ambiguous", + ], +])("selects provenance at the alias capture: %s", (source, status) => { + expect(provenance(source, "saved").status).toBe(status); + if (status === "module") + expect(provenance(source, "saved").origins).toEqual([ + { specifier: "node:fs", importedPath: [], resolutionMode: "require" }, + ]); +}); + +it.each([ + ["const {api=fallback}={};", "local"], + ["const {...api}=require('node:fs');", "local"], + ["const api=require('node:fs');", "module"], +])( + "preserves declaration and ordinary initializer policy: %s", + (source, status) => { + expect(provenance(source, "api").status).toBe(status); + }, +); + +it("retains an uncertain sqlite candidate instead of asserting a local constructor", () => { + const source = + "let Database;({Database}={Database:require('better-sqlite3')});new Database('x.db');"; + expect(provenance(source, "Database").status).toBe("unknown"); + expect(analyzeJavaScriptStaticSource(source).storage).toEqual([ + expect.objectContaining({ + kind: "sqlite", + interpretation: "candidate", + name: "x.db", + }), + ]); +}); + +it("preserves captured Electron origin and rejects certainty after replacement", () => { + const source = + "let electron=require('electron');const saved=electron;({electron}={electron:{}});const later=electron;"; + expect(provenance(source, "saved")).toMatchObject({ + status: "module", + origins: [{ specifier: "electron" }], + }); + expect(provenance(source, "later").status).toBe("unknown"); +}); diff --git a/src/domain/javascript/javascriptSemanticReferenceSources.ts b/src/domain/javascript/javascriptSemanticReferenceSources.ts index 54e1d3ba98..58b1307c00 100644 --- a/src/domain/javascript/javascriptSemanticReferenceSources.ts +++ b/src/domain/javascript/javascriptSemanticReferenceSources.ts @@ -1,17 +1,114 @@ import type * as t from "@babel/types"; import type { JavaScriptSemanticAnalysisState } from "./javascriptSemanticState.js"; -import type { JavaScriptSemanticPropertyPath as PropertyPath } from "./javascriptSemanticPropertyPaths.js"; -import { evaluateSemanticExpression } from "./javascriptSemanticValues.js"; +import { + semanticArrayIndex, + semanticPropertyPathKeyMatches, + type JavaScriptSemanticPropertyPath as PropertyPath, +} from "./javascriptSemanticPropertyPaths.js"; +import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; +import { + evaluateSemanticExpression, + semanticValueDefinitelyDefined, +} from "./javascriptSemanticValues.js"; import { semanticSlotAtPath } from "./javascriptSemanticSlots.js"; +import type { JavaScriptSemanticValue } from "./javascriptSemanticValueTypes.js"; +import { semanticEffectTiming } from "./javascriptSemanticMutationInitializers.js"; + +/** + * Node's wrapper module inherits require. A copy cannot read that delegate + * unless an earlier own-slot write or opaque module escape may have exposed it. + */ +export const semanticCommonJsRequireOwnEnumerableAt = ( + capture: t.Node, + state: JavaScriptSemanticAnalysisState, +): boolean => { + const selected = ["commonjs-module", "require"]; + const mayAffect = ( + effect: JavaScriptSemanticAnalysisState["ambientWriteEffects"][number], + ) => + effect.path.length <= selected.length && + effect.path.every((key, index) => + semanticPropertyPathKeyMatches(key, selected[index] ?? ""), + ) && + semanticEffectTiming(capture, effect.node, state) !== "after"; + return ( + state.ambientWriteEffects.some(mayAffect) || + state.ambientEscapeEffects.some(mayAffect) + ); +}; interface ReferenceSource { readonly node: t.Node; readonly projection: PropertyPath; + readonly iterationSources?: readonly ReferenceSource[]; } +/** Native array iteration preserves numeric slots only while its iterator is intact. */ +export const semanticKnownArrayIteration = ( + value: JavaScriptSemanticValue | undefined, + state: JavaScriptSemanticAnalysisState, +): boolean => + !state.arrayIterationUnknown && + value?.status === "array" && + value.items.every((item) => semanticArrayIndex(item.name) !== null); + +/** Invocation spreads use the same native-iteration proof at their source read. */ +export const semanticKnownArrayIterationAt = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, +): boolean => + semanticKnownArrayIteration( + evaluateSemanticExpression(node, state, node), + state, + ); + +/** Numeric destructuring projections do not select slots of custom iterators. */ +export const semanticReferenceIterationUnknown = ( + source: { readonly iterationSources?: readonly ReferenceSource[] }, + state: JavaScriptSemanticAnalysisState, +): boolean => + source.iterationSources?.some( + (iterable) => !selectedReferenceIterationKnown(iterable, state), + ) === true; + +const selectedReferenceIterationKnown = ( + source: ReferenceSource, + state: JavaScriptSemanticAnalysisState, +): boolean => { + let values = [evaluateSemanticExpression(source.node, state, source.node)]; + for (const key of source.projection) { + if (typeof key === "object" && key !== null) return false; + const selected = new Set(); + for (const value of values) { + if (key === null) { + // for-of projects any position of a native array. Its finite slot + // inventory can prove every possible nested iterable without treating + // a computed object property or custom iterator as an own-index read. + if ( + !semanticKnownArrayIteration(value, state) || + value.status !== "array" || + value.unknownItems + ) + return false; + for (const slot of value.items) { + if (slot.presence !== "present") return false; + selected.add(slot.value); + } + } else { + const slot = semanticSlotAtPath(value, [String(key)]); + if (slot.presence !== "present") return false; + selected.add(slot.value); + } + } + values = [...selected]; + } + return values.every((value) => semanticKnownArrayIteration(value, state)); +}; + /** Exclude references whose source slot or destructuring fallback is absent. */ export const semanticReferenceSourcesInactive = ( reference: { + readonly iterationSources?: readonly ReferenceSource[]; readonly requiredSources?: readonly ReferenceSource[]; readonly fallbackSources?: readonly ReferenceSource[]; }, @@ -19,19 +116,18 @@ export const semanticReferenceSourcesInactive = ( ): boolean => reference.requiredSources?.some( (source) => + !semanticReferenceIterationUnknown(source, state) && semanticSelectedReferenceSlot(source, state, source.node)?.presence === - "absent", + "absent", ) === true || reference.fallbackSources?.some((source) => { // Selection precedes effects through the chosen reference. A later escape // cannot make a fallback that was excluded at destructuring reachable. const slot = semanticSelectedReferenceSlot(source, state, source.node); return ( + !semanticReferenceIterationUnknown(source, state) && slot?.presence === "present" && - (slot.value.status === "literal" || - slot.value.status === "union" || - slot.value.status === "object" || - slot.value.status === "array") + semanticValueDefinitelyDefined(slot.value) ); }) === true; @@ -51,3 +147,55 @@ export const semanticSelectedReferenceSlot = ( path, ); }; + +interface CopiedReferenceSource { + readonly projection: PropertyPath; + readonly copyKind?: "object-rest" | "array-rest"; + readonly copyProjectionOffset?: number; + readonly copyExcludedKeys?: readonly string[]; + readonly copyStartIndex?: number; +} + +/** Map copied child selections once; the fresh copy itself keeps its own identity. */ +export const semanticCopiedReferencePath = ( + initializer: CopiedReferenceSource, + path: PropertyPath, + effect: "write" | "escape" | "identity", +): PropertyPath | null => { + const offset = initializer.copyProjectionOffset; + if (offset === undefined) return [...initializer.projection, ...path]; + const copiedPath = [...initializer.projection.slice(offset + 1), ...path]; + if (effect === "identity" && copiedPath.length === 0) return null; + if (effect === "write" && copiedPath.length < 2) return null; + let [key, ...remaining] = copiedPath; + if (initializer.copyKind === "array-rest") { + const startIndex = initializer.copyStartIndex ?? 0; + if (key === undefined || key === null) + key = { excludedKeys: [], startIndex }; + else if (typeof key === "object") + key = { + excludedKeys: key.excludedKeys.flatMap((name) => { + const index = semanticArrayIndex(name); + return index === null ? [] : [String(index + startIndex)]; + }), + startIndex: (key.startIndex ?? 0) + startIndex, + }; + else { + const index = semanticArrayIndex(key); + if (index === null) return null; + key = index + startIndex; + } + } else { + const excludedKeys = initializer.copyExcludedKeys ?? []; + if (key === undefined || key === null) key = { excludedKeys }; + else if (typeof key === "object") + key = { + ...key, + excludedKeys: [...new Set([...key.excludedKeys, ...excludedKeys])].sort( + compareUnicodeCodePoints, + ), + }; + else if (excludedKeys.includes(String(key))) return null; + } + return [...initializer.projection.slice(0, offset), key, ...remaining]; +}; diff --git a/src/domain/javascript/javascriptSemanticRequireCaptures.test.ts b/src/domain/javascript/javascriptSemanticRequireCaptures.test.ts new file mode 100644 index 0000000000..ec7e04bc26 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticRequireCaptures.test.ts @@ -0,0 +1,151 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it.each([ + ['const h={require:module.require};h.require("./shared.cjs");', true], + [ + 'const h={require:module.require};h.require.call(module,"./shared.cjs");', + true, + ], + [ + 'const h={require:module.require};h.require.apply(module,["./shared.cjs"]);', + true, + ], + [ + 'const h={inner:{require:module.require}};h.inner.require("./shared.cjs");', + true, + ], + ['const h={require(){}};h.require("./shared.cjs");', false], + ['const h={require(){}};h.require.call(module,"./shared.cjs");', false], + [ + 'const h={require:module.require};h.require=()=>{};h.require("./shared.cjs");', + false, + ], + [ + 'const h={require(){}};h.require=module.require;h.require("./shared.cjs");', + true, + ], +])( + "follows the actual named delegate as well as its receiver: %s", + (source, expected) => { + expect(analyzeJavaScriptStaticSource(source).references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); + +it.each([ + ['(0,module).require("./shared.cjs");', true], + ['(module,{}).require("./shared.cjs");', false], + ['const get=()=>module;(0,get()).require("./shared.cjs");', true], + ['const loader=(0,module);loader.require("./shared.cjs");', true], + ['const load=(0,module.require);load("./shared.cjs");', true], + ['const load=(module.require,()=>{});load("./shared.cjs");', false], +])( + "selects only the final sequence value as a loader: %s", + (source, expected) => { + const result = analyzeJavaScriptStaticSource(source); + expect(result.parse_error_count).toBe(0); + expect(result.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); + +it.each([ + [ + 'const old={loader:module};const middle={...old};old.loader={};const box={...middle};middle.loader={};box.loader.require("./x");', + true, + ], + [ + 'const old={loader:{}};const middle={...old};old.loader=module;const box={...middle};middle.loader=module;box.loader.require("./x");', + false, + ], + [ + 'const old={loader:module};const middle={...old};old.loader={};const {...box}=middle;middle.loader={};box.loader.require("./x");', + true, + ], + [ + 'const old={loader:{}};const middle={...old};old.loader=module;const {...box}=middle;middle.loader=module;box.loader.require("./x");', + false, + ], + [ + 'const inner={loader:{}};const box={inner};inner.loader=module;box.inner.loader.require("./x");', + true, + ], + [ + 'const inner={loader:module};const box={inner};inner.loader={};box.inner.loader.require("./x");', + false, + ], + [ + 'let inner={loader:module};const box={inner};inner={loader:{}};inner.loader={};box.inner.loader.require("./x");', + true, + ], + [ + 'let inner={loader:{}};const box={inner};inner={loader:{}};inner.loader=module;box.inner.loader.require("./x");', + false, + ], + [ + 'const old={loader:module};const box={...old};old.loader={};box.loader.require("./x");', + true, + ], + [ + 'const old={loader:{}};const box={...old};old.loader=module;box.loader.require("./x");', + false, + ], + [ + 'const old={child:{loader:module}};const box={...old};old.child={loader:{}};box.child.loader.require("./x");', + true, + ], + [ + 'const old={child:{loader:module}};const box={...old};old.child.loader={};box.child.loader.require("./x");', + false, + ], + [ + 'const old={child:{loader:{}}};const box={...old};old.child.loader=module;box.child.loader.require("./x");', + true, + ], + [ + 'const old={child:{loader:{}}};const box={...old};old.child={loader:module};old.child.loader=module;box.child.loader.require("./x");', + false, + ], + [ + 'const old=[module];const box=[...old];old[0]={};box[0].require("./x");', + true, + ], + [ + 'const old=[{}];const box=[...old];old[0]=module;box[0].require("./x");', + false, + ], + [ + 'const old={loader:module};const {...box}=old;old.loader={};box.loader.require("./x");', + true, + ], + [ + 'const old={loader:{}};const {...box}=old;old.loader=module;box.loader.require("./x");', + false, + ], + [ + 'const old=[module];const [...box]=old;old[0]={};box[0].require("./x");', + true, + ], + [ + 'const old=[{}];const [...box]=old;old[0]=module;box[0].require("./x");', + false, + ], +])( + "keeps module receiver capture and later member selection separate: %s", + (source, expected) => { + const result = analyzeJavaScriptStaticSource(source); + expect( + result.references + .filter(({ kind }) => kind === "require") + .map(({ specifier }) => specifier), + ).toEqual(expected ? [null] : []); + }, +); diff --git a/src/domain/javascript/javascriptSemanticRequireOrigin.ts b/src/domain/javascript/javascriptSemanticRequireOrigin.ts index 825eefef1a..2c4477849c 100644 --- a/src/domain/javascript/javascriptSemanticRequireOrigin.ts +++ b/src/domain/javascript/javascriptSemanticRequireOrigin.ts @@ -1,3 +1,10 @@ +import { + semanticProjectedSourceFactsAtRead, + semanticProjectedSourceEligibilityAtRead, + semanticNormalizeCapturedSelections, + semanticShiftCapturedSelections, + type SemanticCapturedSelection, +} from "./javascriptSemanticProjectedSources.js"; import * as t from "@babel/types"; import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; @@ -9,6 +16,7 @@ import { commonJsWrapperBindingDeclaration, semanticResolutionBlocked, semanticStableInitializer, + semanticBindingHasPriorValueWrite, type JavaScriptSemanticAnalysisState, } from "./javascriptSemanticState.js"; import { @@ -19,9 +27,25 @@ import { semanticPropertyPathKeyMatches } from "./javascriptSemanticPropertyPath import { stringValue } from "./javascriptStaticAnalysisHelpers.js"; import { semanticEffectTiming, + semanticBindingAtRead, semanticTopLevelEffectFollowsCapture, } from "./javascriptSemanticMutationInitializers.js"; import { createSemanticModuleInterop } from "./javascriptModuleInterop.js"; +import { createSemanticReturnedReferences } from "./javascriptSemanticReturnedReferences.js"; +import { semanticIterationSources } from "./javascriptSemanticIterationSources.js"; +import { semanticCommonJsRequireOwnEnumerableAt } from "./javascriptSemanticReferenceSources.js"; +import { + semanticArrayPropertyReferences, + semanticObjectPropertyReferences, +} from "./javascriptSemanticObjectReferences.js"; +import { + SemanticAliasSelections, + semanticAliasSelectionChoices, + type SemanticAliasSelectionPath, +} from "./javascriptSemanticAliasSelections.js"; +import { SEMANTIC_ALIAS_SELECTION_LIMIT } from "./javascriptSemanticResourceLimits.js"; +import { isUnmodifiedImportMetaUrl } from "./javascriptSemanticAmbientWrites.js"; +import { semanticReferenceSourcesInactive } from "./javascriptSemanticReferenceSources.js"; export interface JavaScriptRequireOrigin { readonly origin: JavaScriptModuleOrigin; @@ -35,46 +59,124 @@ export type JavaScriptRequireLoaderFact = | "local" | "unknown"; -/** Classify a bare `require(...)` using the shared lexical binding owner. */ +const possibleCommonJsDelegateWriteStates = + new WeakSet(); + +/** Source eligibility only; the shared write owner still proves receiver identity. */ +export const observePossibleCommonJsDelegateWrite = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, +): void => { + if (possibleCommonJsDelegateWriteStates.has(state)) return; + const target = t.isAssignmentExpression(node) + ? node.left + : t.isUpdateExpression(node) || + t.isUnaryExpression(node, { operator: "delete" }) + ? node.argument + : undefined; + if ( + semanticIterationSources(node, state.parentsByNode.get(node) ?? null).some( + (source) => source.kind === "reference-store", + ) + ) + possibleCommonJsDelegateWriteStates.add(state); + if ( + (t.isMemberExpression(target) || t.isOptionalMemberExpression(target)) && + ["require", null].includes( + semanticStaticPropertyKey(target.property, target.computed), + ) + ) + possibleCommonJsDelegateWriteStates.add(state); + else if (t.isCallExpression(node) || t.isOptionalCallExpression(node)) + possibleCommonJsDelegateWriteStates.add(state); +}; + +export const hasPossibleCommonJsDelegateWrite = ( + state: JavaScriptSemanticAnalysisState, +): boolean => possibleCommonJsDelegateWriteStates.has(state); + +/** Member effects settle at the shared indexing boundary, invalidating derived delegates. */ +export const invalidateSemanticRequireDelegateWrites = ( + state: JavaScriptSemanticAnalysisState, +): void => { + moduleRequireWrites.delete(state); + moduleReturnedReferences.delete(state); +}; + +const commonJsDelegateChangedAt = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, +): boolean => + writesModuleRequire(state).some( + (write) => semanticEffectTiming(node, write, state) !== "after", + ); + +const commonJsLoaderAt = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, +): JavaScriptRequireLoaderFact => + commonJsDelegateChangedAt(node, state) ? "unknown" : "commonjs-loader"; + +/** Associate wrapper lookups with their loader; this does not claim execution. */ export const semanticRequireLoaderAt = ( node: t.CallExpression | t.OptionalCallExpression | t.NewExpression, state: JavaScriptSemanticAnalysisState, factoryBindingIds: ReadonlySet, ): JavaScriptRequireLoaderFact => { const callee = unwrapJavaScriptExpression(node.callee).node; - const loader = t.isIdentifier(callee, { name: "require" }) - ? callee - : (t.isMemberExpression(callee) || t.isOptionalMemberExpression(callee)) && - ["resolve", "main"].includes( - semanticStaticPropertyKey(callee.property, callee.computed) ?? "", - ) && - t.isIdentifier(callee.object, { name: "require" }) + const loader = + (t.isMemberExpression(callee) || t.isOptionalMemberExpression(callee)) && + ["resolve", "main"].includes( + semanticStaticPropertyKey(callee.property, callee.computed) ?? "", + ) ? callee.object - : undefined; - if (loader === undefined) return "local"; - const binding = resolveSemanticBindingState(state, loader, "require"); + : callee; + return requireLoaderValueAt(loader, node, state, factoryBindingIds); +}; + +/** Classify Node's wrapper and exact factory bindings by their lexical owner. */ +const requireLoaderValueAt = ( + value: t.Node, + node: t.Node, + state: JavaScriptSemanticAnalysisState, + factoryBindingIds: ReadonlySet, +): JavaScriptRequireLoaderFact => { + const callee = unwrapJavaScriptExpression(value).node; + // Only the selected require function is callable as the host loader. + // Its resolve/main members remain association facts in the wrapper above. + if (!t.isIdentifier(callee)) return "local"; + const loader = callee; + const binding = resolveSemanticBindingState(state, loader, loader.name); + if (binding !== undefined && factoryBindingIds.has(binding.bindingId)) + return semanticBindingHasPriorValueWrite(node, binding, state) + ? "unknown" + : "factory-loader"; + if (loader.name !== "require") return "local"; if (binding === undefined) { if (semanticResolutionBlocked(state, loader, "require")) return "unknown"; return (state.ambientBindingWrites.get("require") ?? []).some( (write) => semanticEffectTiming(node, write, state) !== "after", ) ? "unknown" - : "commonjs-loader"; + : commonJsLoaderAt(node, state); } - if (factoryBindingIds.has(binding.bindingId)) - return binding.initializers.length === 0 ? "factory-loader" : "unknown"; if ( binding.hostOrigin === "commonjs-wrapper" && [...binding.declarationKinds].every((kind) => kind === "var") ) { if (binding.definitions.some(({ kind }) => kind === "function")) return "local"; - return commonJsWrapperBindingHasPriorWrite(node, binding, state) + return semanticBindingHasPriorValueWrite(node, binding, state) ? "unknown" - : "commonjs-loader"; + : commonJsLoaderAt(node, state); } - if (isWrapperRequireInitializerCall(binding, node, state)) - return "commonjs-loader"; + if ( + (t.isCallExpression(node) || + t.isOptionalCallExpression(node) || + t.isNewExpression(node)) && + isWrapperRequireInitializerCall(binding, node, state) + ) + return commonJsLoaderAt(node, state); const wrapperBinding = commonJsWrapperBindingDeclaration( loader, "require", @@ -82,9 +184,9 @@ export const semanticRequireLoaderAt = ( ); if ( wrapperBinding !== undefined && - !commonJsWrapperBindingHasPriorWrite(node, wrapperBinding, state) + !semanticBindingHasPriorValueWrite(node, wrapperBinding, state) ) - return "commonjs-loader"; + return commonJsLoaderAt(node, state); if (isSemanticCreateRequireBinding(binding, state)) return "create-require"; const initializer = semanticStableInitializer(binding, state)?.node; if ( @@ -102,42 +204,6 @@ export const semanticRequireLoaderAt = ( return "unknown"; }; -const commonJsWrapperBindingHasPriorWrite = ( - site: t.Node, - binding: NonNullable>, - state: JavaScriptSemanticAnalysisState, -): boolean => { - const writes = state.bindingWrites.get(binding.bindingId) ?? []; - const declarationInitializers = binding.initializers - .map(({ node }) => node) - .filter( - (initializer) => - !writes.some((write) => semanticNodeWithin(initializer, write, state)), - ); - return ( - writes.some( - (write) => semanticEffectTiming(site, write, state) !== "after", - ) || - declarationInitializers.some( - (initializer) => - semanticEffectTiming(site, initializer, state) !== "after", - ) - ); -}; - -const semanticNodeWithin = ( - node: t.Node, - ancestor: t.Node, - state: JavaScriptSemanticAnalysisState, -): boolean => { - let current: t.Node | undefined = node; - while (current !== undefined) { - if (current === ancestor) return true; - current = state.parentsByNode.get(current); - } - return false; -}; - /** Classify a call through an exact recovered factory-loader binding. */ export const semanticFactoryRequireLoaderAt = ( node: t.CallExpression | t.OptionalCallExpression | t.NewExpression, @@ -177,7 +243,9 @@ export const semanticFactoryLoaderMemberAt = ( const binding = resolveSemanticBindingState(state, loader, loader.name); if (binding === undefined || !bindingIds.has(binding.bindingId)) return undefined; - return binding.initializers.length === 0 ? "factory-loader" : "unknown"; + return semanticBindingHasPriorValueWrite(node, binding, state) + ? "unknown" + : "factory-loader"; }; const isSemanticCreateRequireBinding = ( @@ -186,40 +254,414 @@ const isSemanticCreateRequireBinding = ( ): boolean => { if (binding.initializers.length !== 1) return false; const initializer = semanticStableInitializer(binding, state)?.node; - if (!t.isCallExpression(initializer)) return false; - const callee = unwrapJavaScriptExpression(initializer.callee).node; - if (t.isIdentifier(callee)) { - const origins = resolveSemanticBindingState( - state, - callee, - callee.name, - )?.directOrigins; - if (origins?.length !== 1) return false; - const [origin] = origins; - return ( - origin?.importedPath.at(-1) === "createRequire" && - ["module", "node:module"].includes(origin.specifier) - ); - } - const chain = semanticMemberChain(callee); - if (chain === undefined || chain.members.at(-1) !== "createRequire") - return false; - if (!t.isNode(chain.root)) return false; - const root = unwrapJavaScriptExpression(chain.root).node; - if (!t.isIdentifier(root)) return false; - const origins = resolveSemanticBindingState( - state, - root, - root.name, - )?.directOrigins; - if (origins?.length !== 1) return false; - const [origin] = origins; return ( - origin?.importedPath.length === 0 && - ["module", "node:module"].includes(origin.specifier) + createRequireFactoryCallAt(initializer, state)?.loader === "create-require" ); }; +/** Retain the factory's actual certainty for both proof and eligibility consumers. */ +const createRequireFactoryCallAt = ( + initializer: t.Node | undefined, + state: JavaScriptSemanticAnalysisState, + factoryBindingIds: ReadonlySet = new Set(), +): Pick | undefined => + !t.isCallExpression(initializer) + ? undefined + : selectedRequireSourcesAt( + initializer.callee, + initializer, + state, + factoryBindingIds, + "factory", + ); + +/** Eligibility for observable loader slots; an unknown factory is not invocation proof. */ +export const hasSemanticCreateRequireFactoryOrigin = ( + initializer: t.Node | undefined, + state: JavaScriptSemanticAnalysisState, +): initializer is t.CallExpression => { + const factory = createRequireFactoryCallAt(initializer, state); + return factory !== undefined && factory.loader !== "local"; +}; + +const isCreateRequireOrigin = ( + origin: JavaScriptModuleOrigin, + selected: SemanticAliasSelectionPath, +): boolean => { + if (!["module", "node:module"].includes(origin.specifier)) return false; + const path = [...origin.importedPath, ...selected]; + return ( + (path.length === 1 && path[0] === "createRequire") || + ((origin.resolutionMode ?? "import") === "import" && + path.length === 2 && + path[0] === "default" && + path[1] === "createRequire") + ); +}; + +/** Assignment expressions expose their RHS; short-circuit stores may keep the old value. */ +export const assignmentValueSources = ( + node: t.AssignmentExpression, +): readonly t.Node[] => + node.operator === "=" + ? [node.right] + : ["||=", "&&=", "??="].includes(node.operator) + ? [node.left, node.right] + : []; + +/** Follow selected values to Node's captured loader or its creating factory. */ +const selectedRequireSourcesAt = ( + root: t.Node, + site: t.Node, + state: JavaScriptSemanticAnalysisState, + factoryBindingIds: ReadonlySet, + target: "loader" | "factory" = "loader", +): Pick | undefined => { + const pending: ModuleObjectSource[] = [{ node: root, path: [] }]; + const seen = new WeakMap>(); + let count = 0; + let found = false; + let loaderBinding = false; + let selectedLoader: JavaScriptRequireLoaderFact | undefined; + let unknown = false; + let local = false; + let nativeMemberAlternative = false; + let sourceRelative = true; + while (pending.length > 0) { + const current = pending.pop()!; + const node = unwrapJavaScriptExpression(current.node).node; + const genericMember = + current.path.length === 1 && + semanticAliasSelectionChoices(current.path[0]!).every( + (key) => key === "call" || key === "apply", + ); + const key = JSON.stringify([current.path, loaderSourceContextKey(current)]); + const paths = seen.get(node) ?? new Set(); + if (paths.has(key)) continue; + paths.add(key); + seen.set(node, paths); + if (++count > SEMANTIC_ALIAS_SELECTION_LIMIT) { + state.valueResourceLimits.add("alias-selections"); + return { loader: "unknown", sourceRelative: false }; + } + if (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) { + pending.push( + ...inheritLoaderSourceContext(current, [ + loaderMemberSource(current, node), + ]), + ); + } else if (t.isIdentifier(node)) { + if (target === "loader" && genericMember) + nativeMemberAlternative ||= + requireLoaderValueAt(node, node, state, factoryBindingIds) !== + "local"; + if (target === "loader" && current.path.length === 0) { + const captured = requireLoaderValueAt( + node, + node, + state, + factoryBindingIds, + ); + if (captured !== "local" && captured !== "create-require") { + const invoked = + captured === "commonjs-loader" + ? commonJsLoaderAt(site, state) + : captured; + found = true; + unknown ||= + invoked === "unknown" || + (selectedLoader !== undefined && selectedLoader !== invoked); + selectedLoader ??= invoked; + continue; + } + } + const binding = resolveSemanticBindingState(state, node, node.name); + if (binding === undefined) { + unknown = true; + continue; + } + // Eligibility is separate from the selected value: replacing a known + // loader with an opaque value keeps a gap, while a proven local function + // no longer claims the previous factory's base or identity. + loaderBinding ||= + target === "loader" && + binding.initializers.some( + (source) => + hasSemanticCreateRequireFactoryOrigin(source.node, state) || + (t.isIdentifier(source.node) && + requireLoaderValueAt( + source.node, + source.node, + state, + factoryBindingIds, + ) !== "local"), + ); + const facts = semanticProjectedSourceFactsAtRead( + binding, + node, + current.path, + state, + true, + current.readAt ?? node, + current.capturedSelections, + ); + if (target === "factory" && binding.directOrigins.length > 0) { + const native = binding.directOrigins.some((origin) => + isCreateRequireOrigin(origin, current.path), + ); + loaderBinding ||= native; + if (facts?.retainsOrigin) { + found ||= native; + local ||= !native; + if (native) selectedLoader = "create-require"; + } + } + if (facts === undefined) { + unknown = true; + const retained = semanticBindingAtRead( + binding, + node, + state.parentsByNode, + ).binding; + pending.push( + ...inheritLoaderSourceContext( + current, + projectedLoaderSources( + current, + [...retained.initializers, ...retained.referenceInitializers] + .filter( + (source) => !semanticReferenceSourcesInactive(source, state), + ) + .map((source) => ({ + node: source.node, + path: [...source.projection, ...current.path], + })), + ), + ), + ); + } else { + unknown ||= + facts.sources.length === 0 && + !( + target === "factory" && + facts.retainsOrigin && + binding.directOrigins.length > 0 + ); + pending.push( + ...inheritLoaderSourceContext( + current, + projectedLoaderSources(current, facts.sources), + ), + ); + } + } else if (t.isObjectExpression(node) || t.isArrayExpression(node)) { + if (current.path.length === 0) { + local = true; + continue; + } + const sources = loaderObjectPropertySources(current, node, state); + unknown ||= sources.length === 0; + pending.push(...inheritLoaderSourceContext(current, sources)); + } else if (t.isSequenceExpression(node)) { + const last = node.expressions.at(-1); + if (last !== undefined) pending.push({ ...current, node: last }); + } else if (t.isAssignmentExpression(node)) { + for (const source of assignmentValueSources(node)) + pending.push({ ...current, node: source }); + } else if (t.isConditionalExpression(node) || t.isLogicalExpression(node)) { + pending.push( + { + ...current, + node: t.isConditionalExpression(node) ? node.consequent : node.left, + }, + { + ...current, + node: t.isConditionalExpression(node) ? node.alternate : node.right, + }, + ); + } else if (t.isCallExpression(node)) { + if (target === "factory") { + const origin = semanticRequireOrigin(node, state)?.origin; + if ( + origin !== undefined && + isCreateRequireOrigin(origin, current.path) + ) { + found = true; + selectedLoader = "create-require"; + } else unknown = true; + } else if (current.path.length === 0 || genericMember) { + const factory = createRequireFactoryCallAt( + node, + state, + factoryBindingIds, + ); + if (factory !== undefined && factory.loader !== "local") { + if (genericMember) { + // A conditional own write retains this inherited native method. + // Alone it still belongs to the invocation owner's fallback. + nativeMemberAlternative = true; + if (factory.loader === "unknown") { + loaderBinding = true; + unknown = true; + } + } else { + found = true; + unknown ||= + factory.loader === "unknown" || + (selectedLoader !== undefined && + selectedLoader !== "create-require"); + selectedLoader ??= "create-require"; + unknown ||= state.conditionallySkippedExpressions.has(node); + sourceRelative &&= isUnmodifiedImportMetaUrl( + node.arguments[0], + state, + ); + } + } else if (factory?.loader === "local") local = true; + else unknown = true; + } else unknown = true; + } else if ( + current.path.length === 0 && + (t.isFunction(node) || t.isClassExpression(node) || t.isLiteral(node)) + ) { + local = true; + } else { + // Every selected expression contributes a possible value. An opaque + // alternative cannot leave another factory as the sole proven origin. + unknown = true; + } + } + return found || local || (loaderBinding && unknown) + ? { + loader: + (unknown && (found || loaderBinding)) || + (local && (found || nativeMemberAlternative)) + ? "unknown" + : found + ? (selectedLoader ?? "unknown") + : "local", + sourceRelative: found && sourceRelative && !unknown, + } + : undefined; +}; + +const loaderInvocationInterops = new WeakMap< + JavaScriptSemanticAnalysisState, + { + readonly writes: number; + readonly interop: ReturnType; + } +>(); + +const loaderInvocationInterop = (state: JavaScriptSemanticAnalysisState) => { + const prior = loaderInvocationInterops.get(state); + if (prior?.writes === state.ambientWriteEffects.length) return prior.interop; + const interop = createSemanticModuleInterop(state); + for (const effect of state.ambientWriteEffects) + if (effect.path[0] === "global") + interop.retainWrite(effect.node ?? state.program, [ + "intrinsic", + ...effect.path + .slice(1) + .map((key) => + key === null || typeof key === "object" ? null : String(key), + ), + ]); + loaderInvocationInterops.set(state, { + writes: state.ambientWriteEffects.length, + interop, + }); + return interop; +}; + +export interface JavaScriptRequireCallFact { + readonly loader: JavaScriptRequireLoaderFact; + readonly arguments: + | readonly t.CallExpression["arguments"][number][] + | undefined; + readonly sourceRelative: boolean; +} + +/** Classify the selected loader and normalize invocation arguments once. */ +export const semanticRequireCallAt = ( + node: t.CallExpression | t.OptionalCallExpression | t.NewExpression, + state: JavaScriptSemanticAnalysisState, + factoryBindingIds: ReadonlySet, +): JavaScriptRequireCallFact | undefined => { + if (semanticModuleRequireCallAt(node, state)) + return { loader: "unknown", arguments: undefined, sourceRelative: false }; + if (t.isNewExpression(node)) { + const loader = requireLoaderValueAt( + node.callee, + node, + state, + factoryBindingIds, + ); + const selected = selectedRequireSourcesAt( + node.callee, + node, + state, + factoryBindingIds, + ); + if (selected !== undefined || loader !== "local") + return { + loader: selected?.loader ?? loader, + arguments: node.arguments, + sourceRelative: selected?.sourceRelative ?? loader !== "create-require", + }; + } + const interop = loaderInvocationInterop(state); + if (!t.isNewExpression(node)) { + const invocation = interop.intrinsicInvocation(node, (callee) => { + const loader = requireLoaderValueAt( + callee, + node, + state, + factoryBindingIds, + ); + if (loader !== "local" && loader !== "create-require") + return ["require-loader", loader, "relative"]; + const selected = selectedRequireSourcesAt( + callee, + node, + state, + factoryBindingIds, + ); + if (selected !== undefined) + return [ + "require-loader", + selected.loader, + selected.sourceRelative ? "relative" : "unknown", + ]; + return interop.intrinsicPath(callee); + }); + if (invocation?.path[0] === "require-loader") + return { + loader: + invocation.path[1] === "local" + ? "local" + : invocation.kind === "opaque" + ? "unknown" + : (invocation.path[1] as JavaScriptRequireLoaderFact), + arguments: invocation.arguments, + sourceRelative: + invocation.kind === "known" && invocation.path[2] === "relative", + }; + } + const factory = semanticFactoryRequireLoaderAt( + node, + state, + factoryBindingIds, + ); + return factory === undefined + ? undefined + : { + loader: factory, + arguments: node.arguments.slice(t.isIdentifier(node.callee) ? 0 : 1), + sourceRelative: true, + }; +}; + /** The RHS call runs before its assignment replaces Node's wrapper loader. */ const isWrapperRequireInitializerCall = ( binding: NonNullable>, @@ -263,6 +705,494 @@ const moduleRequireWrites = new WeakMap< JavaScriptSemanticAnalysisState, readonly t.Node[] >(); +const moduleReturnedReferences = new WeakMap< + JavaScriptSemanticAnalysisState, + Map< + "direct-call" | "awaited-call", + ReturnType + > +>(); + +type ModuleObjectSource = { + readonly node: t.Node; + readonly path: SemanticAliasSelectionPath; + readonly eligibility?: true; + readonly readAt?: t.Node | undefined; + readonly capturedSelections?: + | readonly SemanticCapturedSelection[] + | undefined; +}; + +/** Both loader queries retain the same read and shallow-copy selection owner. */ +const loaderSourceContextKey = (source: ModuleObjectSource): string => + JSON.stringify([ + source.eligibility, + source.readAt?.start, + source.readAt?.end, + source.capturedSelections?.map(({ at, depth, ownEnumerable }) => [ + at.start, + at.end, + depth, + ownEnumerable, + ]), + ]); + +const inheritLoaderSourceContext = ( + current: ModuleObjectSource, + sources: readonly ModuleObjectSource[], +): ModuleObjectSource[] => + sources.map((source) => ({ + ...(current.eligibility ? { eligibility: true as const } : {}), + ...(current.readAt === undefined ? {} : { readAt: current.readAt }), + ...source, + capturedSelections: semanticNormalizeCapturedSelections( + source.capturedSelections ?? current.capturedSelections, + source.path.length, + ), + })); + +const loaderMemberSource = ( + current: ModuleObjectSource, + member: t.MemberExpression | t.OptionalMemberExpression, +): ModuleObjectSource => ({ + node: member.object, + readAt: current.readAt ?? member, + capturedSelections: semanticShiftCapturedSelections( + current.capturedSelections, + 1, + ), + path: [ + semanticStaticPropertyKey(member.property, member.computed), + ...current.path, + ], +}); + +const projectedLoaderSources = ( + current: ModuleObjectSource, + sources: readonly ModuleObjectSource[], +): ModuleObjectSource[] => + sources.map((source) => ({ + ...source, + readAt: + current.path.length === 0 ? undefined : (current.readAt ?? current.node), + capturedSelections: [ + ...(semanticShiftCapturedSelections( + current.capturedSelections, + source.path.length - current.path.length, + ) ?? []), + ...(source.capturedSelections ?? []), + ], + })); + +const loaderObjectPropertySources = ( + current: ModuleObjectSource, + object: t.ObjectExpression | t.ArrayExpression, + state: JavaScriptSemanticAnalysisState, +): ModuleObjectSource[] => { + const [key, ...remaining] = current.path; + if (key === undefined) return []; + return semanticAliasSelectionChoices(key).flatMap((choice) => + (t.isObjectExpression(object) + ? semanticObjectPropertyReferences(object, [choice], state) + : semanticArrayPropertyReferences(object, [choice]) + ) + .filter( + (source) => + !source.inherited || + !current.capturedSelections?.some( + (capture) => capture.depth === 1 && capture.ownEnumerable, + ), + ) + .map((source) => ({ + node: source.node, + path: [...source.path, ...remaining], + capturedSelections: [ + ...(semanticShiftCapturedSelections( + current.capturedSelections, + source.path.length - 1, + ) ?? []), + ...(source.capturedSelectionAt === undefined + ? [] + : [ + { + at: source.capturedSelectionAt, + depth: source.path.length, + ...(source.ownEnumerable + ? { ownEnumerable: true as const } + : {}), + }, + ]), + ], + })), + ); +}; + +const isCommonJsModuleObject = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, + includeUnknown = true, +): boolean => hasCommonJsModuleSource(node, state, includeUnknown, "module"); + +/** A captured delegate uses the same receiver and assignment ownership facts. */ +export const isCommonJsRequireDelegate = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, + includeUnknown = false, + readAt?: t.Node, +): boolean => + hasCommonJsModuleSource(node, state, includeUnknown, "delegate", readAt); + +const hasCommonJsModuleSource = ( + node: t.Node, + state: JavaScriptSemanticAnalysisState, + includeUnknown: boolean, + target: "module" | "delegate", + readAt?: t.Node, +): boolean => { + type Work = ModuleObjectSource & { + readonly ancestors: ReadonlySet; + /** Await occurs after this many requested outer property selections. */ + readonly awaitDepths: readonly number[]; + }; + const pending: Work[] = [ + { node, path: [], readAt, ancestors: new Set(), awaitDepths: [] }, + ]; + // Keep depths separate and join finite key alternatives at each source read. + const selections = new SemanticAliasSelections(() => { + state.valueResourceLimits.add("alias-selections"); + }); + const contexts = new WeakMap< + t.Node, + Map< + string, + { + readonly identities: Map; + } + > + >(); + const examined = new WeakSet(); + let additionalStates = 0; + for (let next = pending.pop(); next !== undefined; next = pending.pop()) { + const value = unwrapJavaScriptExpression(next.node).node; + const { awaitDepths } = next; + const byMode = + contexts.get(value) ?? + new Map }>(); + const modeKey = JSON.stringify([awaitDepths, loaderSourceContextKey(next)]); + let context = byMode.get(modeKey); + if (context === undefined) { + context = { identities: new Map() }; + byMode.set(modeKey, context); + contexts.set(value, byMode); + } + const path = selections.normalize(context, next.path); + if (selections.limited) return true; + const pathKey = JSON.stringify(path); + let identity = context.identities.get(pathKey); + if (identity === undefined) { + // Consumption contexts must not provide an unbounded second path to + // expansion beyond the shared selector budget. + if ( + (byMode.size > 1 || context.identities.size > 0) && + ++additionalStates > SEMANTIC_ALIAS_SELECTION_LIMIT + ) { + state.valueResourceLimits.add("alias-selections"); + return true; + } + identity = {}; + context.identities.set(pathKey, identity); + } + if (next.ancestors.has(identity)) { + if (includeUnknown) return true; + continue; + } + if (examined.has(identity)) continue; + examined.add(identity); + const ancestors = new Set([...next.ancestors, identity]); + const enqueueAt = ( + depths: readonly number[], + ...sources: ModuleObjectSource[] + ) => { + pending.push( + ...inheritLoaderSourceContext(next, sources).map((source) => ({ + ...source, + ancestors, + awaitDepths: depths, + })), + ); + }; + const enqueue = (...sources: ModuleObjectSource[]) => { + enqueueAt(awaitDepths, ...sources); + }; + if (t.isAwaitExpression(value)) { + enqueueAt( + [...new Set([...awaitDepths, path.length])].sort( + (left, right) => left - right, + ), + { node: value.argument, path }, + ); + continue; + } + if (t.isMemberExpression(value) || t.isOptionalMemberExpression(value)) { + enqueue(loaderMemberSource({ ...next, path }, value)); + continue; + } + if ( + path.length > 0 && + (t.isObjectExpression(value) || t.isArrayExpression(value)) + ) { + enqueueAt( + // Awaiting a fresh container does not await one of its children. + awaitDepths.filter((depth) => depth !== path.length), + ...loaderObjectPropertySources({ ...next, path }, value, state), + ); + continue; + } + if (t.isConditionalExpression(value) || t.isLogicalExpression(value)) { + enqueue( + { + node: t.isConditionalExpression(value) + ? value.consequent + : value.left, + path, + }, + { + node: t.isConditionalExpression(value) + ? value.alternate + : value.right, + path, + }, + ); + continue; + } + if (t.isSequenceExpression(value)) { + const last = value.expressions.at(-1); + if (last !== undefined) enqueue({ node: last, path }); + continue; + } + if (t.isAssignmentExpression(value)) { + enqueue(...assignmentValueSources(value).map((node) => ({ node, path }))); + continue; + } + if ( + t.isCallExpression(value) || + t.isOptionalCallExpression(value) || + t.isNewExpression(value) + ) { + let factories = moduleReturnedReferences.get(state); + if (factories === undefined) { + factories = new Map(); + moduleReturnedReferences.set(state, factories); + } + const mode = awaitDepths.includes(path.length) + ? "awaited-call" + : "direct-call"; + let returned = factories.get(mode); + if (returned === undefined) { + returned = createSemanticReturnedReferences(state, mode); + factories.set(mode, returned); + } + const returnedResult = returned( + value.callee, + t.isNewExpression(value) ? undefined : value, + ); + if (returnedResult.unresolvedInvocation) return true; + enqueue( + ...returnedResult.references.map((source) => ({ + node: source.node, + path: [...source.projection, ...path], + capturedSelections: semanticShiftCapturedSelections( + next.capturedSelections, + source.projection.length, + ), + })), + ); + continue; + } + if (!t.isIdentifier(value)) continue; + const binding = resolveSemanticBindingState(state, value, value.name); + const processMainModulePath = + semanticGlobalProcessOrigin(value, state) !== undefined && + path.length === (target === "module" ? 1 : 2) && + semanticAliasSelectionChoices(path[0]!).some( + (key) => + key === null || semanticPropertyPathKeyMatches(key, "mainModule"), + ) && + (target === "module" || + semanticAliasSelectionChoices(path[1]!).some( + (key) => + key === null || semanticPropertyPathKeyMatches(key, "require"), + )); + if (processMainModulePath) return true; + const mainModulePath = + value.name === "require" && + state.program.sourceType === "script" && + path.length === (target === "module" ? 1 : 2) && + semanticAliasSelectionChoices(path[0]!).some( + (key) => key === null || semanticPropertyPathKeyMatches(key, "main"), + ) && + (target === "module" || + semanticAliasSelectionChoices(path[1]!).some( + (key) => + key === null || semanticPropertyPathKeyMatches(key, "require"), + )); + // Native wrapper require.main is a Module, with a process-relative loader. + // Its selected binding writes below can replace that source with local data. + if ( + mainModulePath && + binding === undefined && + !semanticResolutionBlocked(state, value, "require") + ) + return true; + const copyAdmitsModuleDelegate = + value.name !== "module" || + target !== "delegate" || + path.length !== 1 || + !next.capturedSelections?.some( + (capture) => + capture.depth === 1 && + capture.ownEnumerable && + !semanticCommonJsRequireOwnEnumerableAt(capture.at, state), + ); + if (value.name === "module" && binding === undefined) { + if ( + copyAdmitsModuleDelegate && + (target === "module" + ? path.length === 0 + : path.length === 1 && + semanticAliasSelectionChoices(path[0]!).some( + (key) => + key === null || semanticPropertyPathKeyMatches(key, "require"), + )) && + !semanticResolutionBlocked(state, value, "module") + ) + return true; + continue; + } + if (binding === undefined) continue; + const read = semanticBindingAtRead(binding, value, state.parentsByNode); + const retainsMainModule = + mainModulePath && + commonJsWrapperBindingDeclaration(value, "require", state) !== + undefined && + !read.hasDefiniteInitializer; + if ( + value.name === "module" && + copyAdmitsModuleDelegate && + (target === "module" + ? path.length === 0 + : path.length === 1 && + semanticAliasSelectionChoices(path[0]!).some( + (key) => + key === null || semanticPropertyPathKeyMatches(key, "require"), + )) && + commonJsWrapperBindingDeclaration(value, "module", state) !== undefined && + !read.hasDefiniteInitializer + ) + return true; + let eligibility = next.eligibility; + const facts = eligibility + ? undefined + : semanticProjectedSourceFactsAtRead( + binding, + value, + path, + state, + includeUnknown, + next.readAt ?? value, + next.capturedSelections, + ); + if (retainsMainModule && (eligibility || facts?.retainsOrigin)) return true; + let sources = eligibility + ? semanticProjectedSourceEligibilityAtRead( + binding, + value, + path, + state, + next.readAt ?? value, + next.capturedSelections, + ) + : facts?.sources; + if (sources === undefined) { + if (target !== "delegate") return true; + // An opaque member effect can obscure a selected native delegate, but + // cannot make every escaped callable a possible Node loader. Keep its + // receiver's retained sources as the eligibility boundary for that gap. + sources = semanticProjectedSourceEligibilityAtRead( + binding, + value, + path, + state, + next.readAt ?? value, + next.capturedSelections, + ); + if (sources === undefined) return true; + eligibility = true; + } + enqueue( + ...projectedLoaderSources({ ...next, node: value, path }, sources).map( + (source) => (eligibility ? { ...source, eligibility } : source), + ), + ); + } + return false; +}; + +/** Select an unchecked member-loader receiver before its identity is examined. */ +export const semanticModuleRequireReceiver = ( + node: t.CallExpression | t.OptionalCallExpression | t.NewExpression, +): t.Node | undefined => { + let callee = unwrapJavaScriptExpression(node.callee).node; + while ( + (t.isMemberExpression(callee) || t.isOptionalMemberExpression(callee)) && + ["bind", "call", "apply"].includes( + semanticStaticPropertyKey(callee.property, callee.computed) ?? "", + ) + ) + // Capturing a bound delegate can load outside the visible module graph. + // Keep that possibility without asserting bind/call/apply is unmodified. + callee = unwrapJavaScriptExpression(callee.object).node; + return (t.isMemberExpression(callee) || + t.isOptionalMemberExpression(callee)) && + semanticStaticPropertyKey(callee.property, callee.computed) === "require" + ? callee.object + : undefined; +}; + +/** Recognize an unchecked module loader without asserting its current delegate. */ +export const semanticModuleRequireCallAt = ( + node: t.CallExpression | t.OptionalCallExpression | t.NewExpression, + state: JavaScriptSemanticAnalysisState, + includeUnknown = true, +): boolean => { + const receiver = semanticModuleRequireReceiver(node); + if ( + receiver !== undefined && + isCommonJsModuleObject(receiver, state, includeUnknown) + ) + return true; + let target = unwrapJavaScriptExpression(node.callee).node; + while ( + (t.isMemberExpression(target) || t.isOptionalMemberExpression(target)) && + ["bind", "call", "apply"].includes( + semanticStaticPropertyKey(target.property, target.computed) ?? "", + ) + ) + target = unwrapJavaScriptExpression(target.object).node; + // The callee is selected before the current invocation can escape or mutate + // its receiver. Earlier captured member reads retain their own source site. + if (isCommonJsRequireDelegate(target, state, includeUnknown, node)) + return true; + if (t.isNewExpression(node)) return false; + const interop = loaderInvocationInterop(state); + return ( + interop.intrinsicInvocation(node, (callee) => + isCommonJsRequireDelegate(callee, state, false, node) + ? ["commonjs-require-delegate"] + : interop.intrinsicPath(callee), + )?.path[0] === "commonjs-require-delegate" + ); +}; const writesModuleRequire = ( state: JavaScriptSemanticAnalysisState, @@ -274,24 +1204,10 @@ const writesModuleRequire = ( moduleRequireWrites.set(state, []); const writes: t.Node[] = []; const interop = createSemanticModuleInterop(state); - const isModuleObject = ( - node: t.Node, - visited = new Set(), - ): boolean => { - const value = unwrapJavaScriptExpression(node).node; - if (!t.isIdentifier(value)) return false; - if (value.name === "module") - return resolveSemanticBindingState(state, value, "module") === undefined - ? !semanticResolutionBlocked(state, value, "module") - : isUnmodifiedCommonJsWrapperBinding(value, "module", state); - const binding = resolveSemanticBindingState(state, value, value.name); - if (binding === undefined || visited.has(binding.bindingId)) return false; - const next = new Set(visited).add(binding.bindingId); - return ( - binding.initializers.length > 0 && - binding.initializers.every(({ node: init }) => isModuleObject(init, next)) - ); - }; + const isModuleObject = (node: t.Node): boolean => + // Only a retained module identity can revoke the host delegate. Escaping + // an unrelated local table is not evidence that it became the host module. + isCommonJsModuleObject(node, state, false); traverseJavaScriptAst(state.program, { enter: (node) => { const target = t.isAssignmentExpression(node) @@ -300,34 +1216,15 @@ const writesModuleRequire = ( t.isUnaryExpression(node, { operator: "delete" }) ? node.argument : undefined; - if (target !== undefined) { - const members: string[] = []; - let root: t.Node = target; - while ( - t.isMemberExpression(root) || - t.isOptionalMemberExpression(root) - ) { - const key = semanticStaticPropertyKey(root.property, root.computed); - if (key === null) break; - members.unshift(key); - root = root.object; - } + if ( + t.isMemberExpression(target) || + t.isOptionalMemberExpression(target) + ) { + const key = semanticStaticPropertyKey(target.property, target.computed); if ( - members.length === 1 && - members[0] === "require" && - isModuleObject(root) - ) - writes.push(node); - else if ( - members.length === 0 && - (t.isMemberExpression(target) || - t.isOptionalMemberExpression(target)) && - target.computed && - semanticStaticPropertyKey(target.property, true) === null && + (key === "require" || (key === null && target.computed)) && isModuleObject(target.object) ) - // An unknown computed key may be "require"; retain uncertainty rather - // than treating this direct delegate write as a proven unrelated slot. writes.push(node); } if (t.isCallExpression(node) || t.isOptionalCallExpression(node)) { @@ -508,17 +1405,7 @@ export const semanticRequireOrigin = ( !t.isIdentifier(current.callee, { name: "require" }) ) return undefined; - if ( - writesModuleRequire(state).some( - (write) => - !semanticTopLevelEffectFollowsCapture( - current, - write, - state.parentsByNode, - ), - ) - ) - return undefined; + if (commonJsDelegateChangedAt(current, state)) return undefined; const requireBinding = resolveSemanticBindingState( state, current.callee, @@ -553,13 +1440,25 @@ export const semanticRequireOrigin = ( }; }; -/** Node's fs.promises export and fs/promises require share one object. */ +/** Node's exported child objects retain their direct builtin-module identity. */ export const semanticRequireExportsIdentity = ( specifier: string, path: readonly Key[], ): { readonly specifier: string; readonly path: readonly (Key | string)[] } => { const name = specifier.replace(/^node:/u, ""); if (name === "fs/promises") return { specifier: name, path }; + if (["path", "path/posix", "path/win32"].includes(name)) { + let selected = name; + let remaining = path; + while ( + remaining.length > 1 && + (remaining[0] === "posix" || remaining[0] === "win32") + ) { + selected = `path/${remaining[0]}`; + remaining = remaining.slice(1); + } + return { specifier: selected, path: remaining }; + } return name === "fs" && path[0] === "promises" && path.length > 1 ? { specifier: "fs/promises", path: path.slice(1) } : { specifier: name, path }; @@ -572,6 +1471,10 @@ const MUTABLE_BUILTIN_DEFAULTS = new Set([ "child_process", "timers", "process", + "path", + "path/posix", + "path/win32", + "url", ]); /** Mutable default/object selections share CommonJS identity; namespace slots do not. */ diff --git a/src/domain/javascript/javascriptSemanticResourceLimits.ts b/src/domain/javascript/javascriptSemanticResourceLimits.ts index 52f836da24..5a94e90e80 100644 --- a/src/domain/javascript/javascriptSemanticResourceLimits.ts +++ b/src/domain/javascript/javascriptSemanticResourceLimits.ts @@ -9,6 +9,8 @@ export const SEMANTIC_PRIMITIVE_CANDIDATE_LIMIT = 256; export const SEMANTIC_EXPRESSION_DEPTH_LIMIT = 256; /** Getter jobs and local result work retained in one precise query. */ export const SEMANTIC_GETTER_EXPANSION_LIMIT = 512; +/** Additional selector states retained by one precise alias query. */ +export const SEMANTIC_ALIAS_SELECTION_LIMIT = 512; /** Bound normalized strings and transient keys independently of transport limits. */ export const SEMANTIC_PRIMITIVE_JSON_BYTES_LIMIT = 1_048_576; @@ -21,6 +23,8 @@ const RESOURCE_LIMIT_REASON: Record = { "Semantic expression depth budget exceeded (maximum 256 nested expressions).", "getter-expansion": "Getter expansion was conservatively bounded (maximum 512 scheduled states or a growing receiver cycle).", + "alias-selections": + "Alias selection expansion was conservatively bounded (maximum 512 additional selector states); selected paths remain unknown.", }; /** Check UTF-16 length against worst-case JSON escaping without flattening. */ diff --git a/src/domain/javascript/javascriptSemanticResponseAssignments.test.ts b/src/domain/javascript/javascriptSemanticResponseAssignments.test.ts new file mode 100644 index 0000000000..73b39fbdee --- /dev/null +++ b/src/domain/javascript/javascriptSemanticResponseAssignments.test.ts @@ -0,0 +1,81 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; + +it.each([ + ["response.json();", "/original"], + ["response=other;response.json();", null], + ["response.json();response=other;", "/original"], + ["if(flag)response=other;response.json();", null], + ["({response}={response:other});response.json();", null], + ["response.json();({response}={response:other});", "/original"], + ["if(flag)({response}={response:other});response.json();", null], + ["response.json();if(flag)({response}={response:other});", "/original"], + ["[response]=[other];response.json();", null], + ["response.json();[response]=[other];", "/original"], + ["if(flag)[response]=[other];response.json();", null], + ["response.json();if(flag)[response]=[other];", "/original"], +])( + "retains request endpoints only before response replacement: %s", + (body, endpoint) => { + const result = analyzeJavaScriptSemantics( + `let response=await fetch('/original');${body}`, + ); + const request = result.requestOperations.find( + (operation) => operation.kind === "request", + ); + expect(request?.endpoint).toBe("/original"); + const consumers = result.requestOperations.filter( + (operation) => operation.kind === "response-consumer", + ); + expect(consumers).toHaveLength(1); + expect(consumers[0]).toMatchObject({ endpoint, resolution: "complete" }); + expect(consumers[0]?.linkedRequestIds).toEqual([request?.requestId]); + }, +); + +it("does not prove an incoming response was initialized by a conditional request", () => { + const result = analyzeJavaScriptSemantics( + "let response;if(flag)response=await fetch('/original');response.json();", + ); + expect( + result.requestOperations.filter( + (operation) => operation.kind === "response-consumer", + ), + ).toEqual([ + expect.objectContaining({ endpoint: null, resolution: "complete" }), + ]); +}); + +it.each([ + "let response=flag?await fetch('/original'):other;response.json();", + "let response=other||(await fetch('/original'));response.json();", +])( + "does not invent a response link through conditional value combinations: %s", + (source) => { + const result = analyzeJavaScriptSemantics(source); + expect( + result.requestOperations.filter( + (operation) => operation.kind === "request", + ), + ).toEqual([expect.objectContaining({ endpoint: "/original" })]); + expect( + result.requestOperations.filter( + (operation) => operation.kind === "response-consumer", + ), + ).toEqual([]); + }, +); + +it("keeps the existing contract for a response linked to multiple requests", () => { + const result = analyzeJavaScriptSemantics( + "let response=await fetch('/original');response=await fetch('/other');response.json();", + ); + expect( + result.requestOperations.filter( + (operation) => operation.kind === "response-consumer", + ), + ).toEqual([ + expect.objectContaining({ endpoint: null, resolution: "partial" }), + ]); +}); diff --git a/src/domain/javascript/javascriptSemanticRestAssignmentCaptures.test.ts b/src/domain/javascript/javascriptSemanticRestAssignmentCaptures.test.ts new file mode 100644 index 0000000000..9e8cce3189 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticRestAssignmentCaptures.test.ts @@ -0,0 +1,110 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; +import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; +import { topLevelBinding } from "./javascriptSemanticAnalysis.fixture.js"; + +it.each([ + [ + "let box;const old={loader:module};({...box}=old);old.loader={};box.loader", + true, + ], + [ + "let box;const old={loader:{}};({...box}=old);old.loader=module;box.loader", + false, + ], + [ + "const holder={};const old={loader:module};({...holder.box}=old);old.loader={};holder.box.loader", + true, + ], + [ + "const holder={};const old={loader:{}};({...holder.box}=old);old.loader=module;holder.box.loader", + false, + ], + ["let box;const old=[module];([...box]=old);old[0]={};box[0]", true], + ["let box;const old=[{}];([...box]=old);old[0]=module;box[0]", false], + [ + "const holder={};const old=[module];([...holder.box]=old);old[0]={};holder.box[0]", + true, + ], + [ + "const holder={};const old=[{}];([...holder.box]=old);old[0]=module;holder.box[0]", + false, + ], + [ + "let box;const old={make:()=>module};({...box}=old);old.make=()=>({});box.make()", + true, + ], + [ + "let box;const old={make:()=>({})};({...box}=old);old.make=()=>module;box.make()", + false, + ], + [ + "const holder={};const old={make:()=>module};({...holder.box}=old);old.make=()=>({});holder.box.make()", + true, + ], + [ + "const holder={};const old={make:()=>({})};({...holder.box}=old);old.make=()=>module;holder.box.make()", + false, + ], + [ + "let box;const old=[()=>module];([...box]=old);old[0]=()=>({});box[0]()", + true, + ], + [ + "let box;const old=[()=>({})];([...box]=old);old[0]=()=>module;box[0]()", + false, + ], + [ + "const holder={};const old=[()=>module];([...holder.box]=old);old[0]=()=>({});holder.box[0]()", + true, + ], + [ + "const holder={};const old=[()=>({})];([...holder.box]=old);old[0]=()=>module;holder.box[0]()", + false, + ], + [ + "let box;const old={child:{loader:module}};({child:{...box}}=old);old.child={loader:{}};box.loader", + true, + ], + [ + "const holder={};const old={child:[module]};({child:[...holder.box]}=old);old.child=[{}];holder.box[0]", + true, + ], + [ + "const holder={};const old={child:{loader:module}};({...holder.box}=old);old.child.loader={};holder.box.child.loader", + false, + ], + ["let make=()=>module;({make}={make:()=>({})});make()", false], + ["let make;({make=()=>module}={});make()", true], + ["let make;({make=()=>module}={make:()=>({})});make()", false], + ["let make=()=>module;if(flag)({make}={make:()=>({})});make()", true], + [ + "let make=()=>module;const saved=make;({make}={make:()=>({})});saved()", + true, + ], +])( + "retains assigned rest-copy slots at their capture: %s", + (receiver, expected) => { + const result = analyzeJavaScriptStaticSource( + `${receiver}.require('./shared.cjs');`, + ); + expect(result.parse_error_count).toBe(0); + expect(result.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); + +it.each(["[value]=['new']", "({key:value}={key:'new'})"])( + "preserves whole-binding and read-value ambiguity after %s", + (assignment) => { + const result = analyzeJavaScriptSemantics( + `let value='old';${assignment};const observed=value;`, + ); + expect(topLevelBinding(result, "value").value.status).toBe("ambiguous"); + expect(topLevelBinding(result, "observed").value.status).toBe("ambiguous"); + }, +); diff --git a/src/domain/javascript/javascriptSemanticReturnedReferences.ts b/src/domain/javascript/javascriptSemanticReturnedReferences.ts index 403a2c01a8..47e3f4c977 100644 --- a/src/domain/javascript/javascriptSemanticReturnedReferences.ts +++ b/src/domain/javascript/javascriptSemanticReturnedReferences.ts @@ -5,14 +5,43 @@ import { unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; import { semanticCallableResultExpressions } from "./javascriptSemanticCallableResults.js"; -import { semanticBindingAtRead } from "./javascriptSemanticMutationInitializers.js"; import { - semanticArrayIndex, + javaScriptSourceInvocationContext, + javaScriptGenericInvocationMethod, + resolveJavaScriptGenericInvocation, + type JavaScriptSourceInvocationContext, +} from "./javascriptIntrinsicInvocation.js"; +import { + semanticBindingAtRead, + semanticEffectTiming, +} from "./javascriptSemanticMutationInitializers.js"; +import { + semanticReferenceSourcesInactive, + semanticKnownArrayIteration, + semanticKnownArrayIterationAt, + semanticSelectedReferenceSlot, +} from "./javascriptSemanticReferenceSources.js"; +import { semanticArrayPropertyReferences } from "./javascriptSemanticObjectReferences.js"; +import { + semanticNormalizeCapturedSelections, + semanticProjectedSourcesAtRead, + semanticShiftCapturedSelections, + type SemanticCapturedSelection, +} from "./javascriptSemanticProjectedSources.js"; +import { SEMANTIC_ALIAS_SELECTION_LIMIT } from "./javascriptSemanticResourceLimits.js"; +import { + semanticPropertyPathKeyMatches, type JavaScriptSemanticPropertyPath, } from "./javascriptSemanticPropertyPaths.js"; +import { + SemanticAliasSelections, + semanticAliasSelectionChoices, + type SemanticAliasSelectionPath, +} from "./javascriptSemanticAliasSelections.js"; import { resolveSemanticBindingState, type JavaScriptSemanticAnalysisState, + type JavaScriptSemanticBindingState, } from "./javascriptSemanticState.js"; /** A reference potentially handed to a caller or generator consumer. */ @@ -24,34 +53,23 @@ export interface SemanticReturnedReference { interface Candidate { readonly node: t.Node; /** `undefined` means any projection depth after different projection depths merge. */ - readonly path: readonly (string | number | null)[] | undefined; + readonly path: SemanticAliasSelectionPath | undefined; + /** Member selection happens at its use, while a captured function keeps its read. */ + readonly readAt?: t.Node; + readonly capturedSelections?: + | readonly SemanticCapturedSelection[] + | undefined; + readonly invocation?: JavaScriptSourceInvocationContext; } export interface SemanticReturnedReferenceResult { readonly references: readonly SemanticReturnedReference[]; + /** A visible invocation-method mutation prevents a source-local result proof. */ + readonly unresolvedInvocation?: boolean; /** Both states close the supported source-local may-target graph. Widened joins projections without dropping represented targets; it does not claim runtime completeness. */ readonly precision: "exact" | "widened"; } -const joinProjection = ( - left: Candidate["path"], - right: Candidate["path"], -): Candidate["path"] => { - if (left === undefined || right === undefined || left.length !== right.length) - return undefined; - return left.map((key, index) => (key === right[index] ? key : null)); -}; - -const sameProjection = ( - left: Candidate["path"], - right: Candidate["path"], -): boolean => - left === undefined - ? right === undefined - : right !== undefined && - left.length === right.length && - left.every((key, index) => key === right[index]); - interface ObjectTargetIndex { readonly propertyByName: ReadonlyMap< string, @@ -71,49 +89,301 @@ interface ObjectTargetUnknown { /** Resolve local result references conservatively, without claiming call execution. */ export const createSemanticReturnedReferences = ( state: JavaScriptSemanticAnalysisState, -): ((callee: t.Node) => SemanticReturnedReferenceResult) => { - const results = new WeakMap(); + consumption: + | "potential-reference" + | "direct-call" + | "awaited-call" = "potential-reference", +): (( + callee: t.Node, + invocation?: t.CallExpression | t.OptionalCallExpression, +) => SemanticReturnedReferenceResult) => { + const results = new WeakMap< + t.Node, + { + readonly result: SemanticReturnedReferenceResult; + readonly invocation: t.Node | undefined; + readonly iterationUnknown: boolean; + readonly ambientWrites: number; + readonly dependencies: readonly { + readonly binding: JavaScriptSemanticBindingState; + readonly writes: number; + readonly escapes: number; + }[]; + } + >(); const objectTargets = new WeakMap(); - return (callee) => { + return (callee, invocation) => { const existing = results.get(callee); - if (existing !== undefined) return existing; + if ( + existing !== undefined && + existing.invocation === invocation && + existing.iterationUnknown === state.arrayIterationUnknown && + existing.ambientWrites === state.ambientWriteEffects.length && + existing.dependencies.every( + ({ binding, writes, escapes }) => + binding.memberWrites.length === writes && + binding.escapedPaths.length === escapes, + ) + ) + return existing.result; + const dependencies = new Map< + JavaScriptSemanticBindingState, + { readonly writes: number; readonly escapes: number } + >(); const functions = new Set(); - const work: t.Node[] = []; + const work: Candidate[] = []; // A transition depends on the source read and current projection, not the // initializer history used to reach it. Keeping history out of the key lets // shared diamonds and reassignment cycles converge to the graph fixed point. - const projections = new WeakMap(); - const queued = new WeakSet(); + const projectionDepths = new WeakMap< + t.Node, + Map + >(); + const expanded = new WeakMap>(); + let additionalContexts = 0; + let contextsWidened = false; const result: { references: readonly SemanticReturnedReference[]; precision: SemanticReturnedReferenceResult["precision"]; + unresolvedInvocation?: boolean; } = { references: [], precision: "exact" }; - const enqueue = (node: t.Node, path: Candidate["path"]): void => { - const source = unwrapJavaScriptExpression(node).node; - if (!projections.has(source)) { - projections.set(source, path); - } else { - const previous = projections.get(source); - const joined = joinProjection(previous, path); - if (sameProjection(previous, joined)) return; - projections.set(source, joined); - result.precision = "widened"; - } - if (!queued.has(source)) { - work.push(source); - queued.add(source); - } + // The invocation owner retains the actual callable receiver. A bare alias + // of fn.call has lost that receiver and cannot execute fn by itself. + const nativeArrayIteration = (array: t.ArrayExpression) => + semanticKnownArrayIterationAt(array, state); + const sourceInvocation = + invocation === undefined + ? undefined + : javaScriptSourceInvocationContext(invocation, nativeArrayIteration); + const sourceGenericInvocation = + invocation !== undefined && + javaScriptGenericInvocationMethod(invocation.callee) !== undefined; + const inheritedInvocationChanged = (method: "call" | "apply"): boolean => { + if (invocation === undefined) return false; + const inheritedPath = ["global", "Function", "prototype", method]; + return state.ambientWriteEffects.some( + (effect) => + effect.path.length <= inheritedPath.length && + effect.path.every( + (key, index) => + key === null || + semanticPropertyPathKeyMatches(key, inheritedPath[index] ?? ""), + ) && + semanticEffectTiming(invocation, effect.node, state) !== "after", + ); }; - enqueue(callee, []); + // Exact finite alternatives retain unrelated keys and cross-path + // correlations. Growing/shrinking cycles still converge at any depth; + // selector-budget widening retains the complete may-target closure. + const selections = new SemanticAliasSelections(() => { + state.valueResourceLimits.add("alias-selections"); + result.precision = "widened"; + }); + const enqueue = ( + node: t.Node, + path: Candidate["path"], + readAt?: t.Node, + capturedSelections?: readonly SemanticCapturedSelection[], + invocationContext?: JavaScriptSourceInvocationContext, + ): void => { + const source = unwrapJavaScriptExpression(node).node; + if (contextsWidened) { + path = undefined; + capturedSelections = undefined; + invocationContext = undefined; + } + const invocationKey = () => + invocationContext === undefined + ? null + : [ + invocationContext.receiver?.start, + invocationContext.receiver?.end, + invocationContext.receiverUnknown, + invocationContext.arguments?.map((argument) => [ + argument.start, + argument.end, + ]), + ]; + let readKey = JSON.stringify([ + readAt?.start ?? -1, + readAt?.end ?? -1, + invocationKey(), + ]); + const depths = projectionDepths.get(source) ?? new Map(); + projectionDepths.set(source, depths); + if (!depths.has(readKey)) { + depths.set(readKey, path?.length); + } else { + const previous = depths.get(readKey); + if (previous === undefined || previous !== path?.length) { + if (previous !== undefined) result.precision = "widened"; + depths.set(readKey, undefined); + path = undefined; + } + } + if (path !== undefined) path = selections.normalize(source, path); + capturedSelections = + path === undefined + ? undefined + : semanticNormalizeCapturedSelections( + capturedSelections, + path.length, + ); + const expansionKey = () => + JSON.stringify([ + readKey, + path ?? null, + capturedSelections?.map(({ at, depth, ownEnumerable }) => [ + at.start, + at.end, + depth, + ownEnumerable, + ]), + ]); + let key = expansionKey(); + const paths = expanded.get(source) ?? new Set(); + if (paths.has(key)) return; + // Copy/invocation contexts add selector dimensions. Bound them by the + // same query budget, then visit the complete unknown-projection closure. + if ( + !contextsWidened && + (capturedSelections !== undefined || invocationContext !== undefined) && + paths.size > 0 && + ++additionalContexts > SEMANTIC_ALIAS_SELECTION_LIMIT + ) { + contextsWidened = true; + state.valueResourceLimits.add("alias-selections"); + result.precision = "widened"; + // Generic transitions select only the source receiver and authored + // argument values. Keep that complete finite source closure when their + // precise positional contexts merge, including later callable receivers. + if (sourceInvocation !== undefined) { + if (sourceInvocation.receiver !== undefined) + enqueue(sourceInvocation.receiver, undefined); + for (const argument of sourceInvocation.arguments ?? []) + if (t.isExpression(argument)) enqueue(argument, undefined); + else if (t.isSpreadElement(argument)) + enqueue(argument.argument, undefined); + } + } + if (contextsWidened) { + path = undefined; + capturedSelections = undefined; + invocationContext = undefined; + readKey = JSON.stringify([ + readAt?.start ?? -1, + readAt?.end ?? -1, + null, + ]); + if (sourceGenericInvocation) result.unresolvedInvocation = true; + depths.set(readKey, undefined); + key = expansionKey(); + if (paths.has(key)) return; + } + paths.add(key); + expanded.set(source, paths); + work.push({ + node: source, + path, + ...(readAt === undefined ? {} : { readAt }), + ...(capturedSelections === undefined ? {} : { capturedSelections }), + ...(invocationContext === undefined + ? {} + : { invocation: invocationContext }), + }); + }; + enqueue(callee, [], undefined, undefined, sourceInvocation); while (work.length > 0) { - const node = work.pop(); - if (node === undefined) break; - queued.delete(node); - const current: Candidate = { node, path: projections.get(node) }; + const current = work.pop(); + if (current === undefined) break; + const { node } = current; + // An exact empty selection with an invocation is the value being called, + // not a container whose callable children are potential escape targets. + // Ordinary objects and arrays cannot supply that callable receiver. An + // unknown projection still visits the full closure after budget widening. + if ( + current.invocation !== undefined && + current.path?.length === 0 && + (t.isObjectExpression(node) || t.isArrayExpression(node)) + ) + continue; const pending: Candidate[] = []; if (t.isIdentifier(node)) { const binding = resolveSemanticBindingState(state, node, node.name); - if (binding === undefined) continue; + if (binding === undefined) { + if (sourceGenericInvocation && current.path?.length === 0) + result.unresolvedInvocation = true; + continue; + } + dependencies.set(binding, { + writes: binding.memberWrites.length, + escapes: binding.escapedPaths.length, + }); + if (current.path !== undefined) { + if ( + sourceGenericInvocation && + current.path.length > 0 && + semanticProjectedSourcesAtRead( + binding, + node, + current.path, + state, + true, + // The current invocation receives its callee before its own + // opaque effects; earlier captured member reads keep their site. + invocation !== undefined && + current.readAt === + unwrapJavaScriptExpression(invocation.callee).node + ? invocation + : (current.readAt ?? node), + current.capturedSelections, + ) === undefined + ) + result.unresolvedInvocation = true; + const selectedSources = semanticProjectedSourcesAtRead( + binding, + node, + current.path, + state, + false, + current.readAt ?? node, + current.capturedSelections, + ); + if (selectedSources !== undefined) { + for (const source of selectedSources) { + const capturedSelections = [ + ...(semanticShiftCapturedSelections( + current.capturedSelections, + source.path.length - current.path.length, + ) ?? []), + ...(source.capturedSelections ?? []), + ]; + pending.push({ + ...source, + ...(current.path.length === 0 + ? {} + : { readAt: current.readAt ?? node }), + ...(capturedSelections.length === 0 + ? {} + : { capturedSelections }), + }); + } + for (let index = pending.length - 1; index >= 0; index--) { + const child = pending[index]; + if (child !== undefined) + enqueue( + child.node, + child.path, + child.readAt, + child.capturedSelections, + child.invocation ?? current.invocation, + ); + } + continue; + } + result.precision = "widened"; + if (sourceGenericInvocation) result.unresolvedInvocation = true; + } const selected = semanticBindingAtRead( binding, node, @@ -123,14 +393,31 @@ export const createSemanticReturnedReferences = ( ...selected.initializers, ...selected.referenceInitializers, ]) { + if (semanticReferenceSourcesInactive(initializer, state)) continue; pending.push({ node: initializer.node, path: current.path === undefined ? undefined : [...initializer.projection, ...current.path], + ...(current.path?.length === 0 + ? {} + : { readAt: current.readAt ?? node }), + ...(current.capturedSelections === undefined + ? {} + : { + capturedSelections: semanticShiftCapturedSelections( + current.capturedSelections, + initializer.projection.length, + ), + }), }); } + if (current.path === undefined) + for (const effect of binding.memberWrites) + for (const source of effect.assignedSources ?? []) + if (!semanticReferenceSourcesInactive(source, state)) + pending.push({ node: source.node, path: undefined }); } else if ( t.isMemberExpression(node) || t.isOptionalMemberExpression(node) @@ -149,6 +436,15 @@ export const createSemanticReturnedReferences = ( pending.push({ ...current, node: receiver, + readAt: current.readAt ?? node, + ...(current.capturedSelections === undefined + ? {} + : { + capturedSelections: semanticShiftCapturedSelections( + current.capturedSelections, + path.length, + ), + }), path: current.path === undefined ? undefined @@ -171,19 +467,94 @@ export const createSemanticReturnedReferences = ( } else if (t.isSequenceExpression(node)) { const last = node.expressions.at(-1); if (last !== undefined) pending.push({ ...current, node: last }); - } else if ( - t.isFunction(node) && - (current.path === undefined || current.path.length === 0) - ) - functions.add(node); - else if (t.isObjectExpression(node)) + } else if (t.isFunction(node)) { + if (current.path === undefined || current.path.length === 0) + functions.add(node); + else if ( + current.invocation !== undefined && + current.path.every((key) => + semanticAliasSelectionChoices(key).every( + (choice) => choice === "call" || choice === "apply", + ), + ) + ) { + // Own member writes have already selected their actual RHS above. + // Only a remaining inherited invocation member reaches this branch. + const methods = semanticAliasSelectionChoices( + current.path.at(-1)!, + ).filter( + (method): method is "call" | "apply" => + method === "call" || method === "apply", + ); + for (const method of methods) { + const selected = resolveJavaScriptGenericInvocation( + method, + current.invocation, + nativeArrayIteration, + ); + if (selected !== undefined) + pending.push({ + node: selected.callable, + path: [], + invocation: selected.context, + }); + else if (current.invocation.receiverUnknown) + result.unresolvedInvocation = true; + if (inheritedInvocationChanged(method)) + result.unresolvedInvocation = true; + } + } + } else if (t.isObjectExpression(node)) selectObjectTargets(node, current, pending, objectTargets); else if (t.isArrayExpression(node)) { const [key, ...path] = current.path ?? []; - const index = - current.path === undefined || key === undefined || key === null - ? null - : semanticArrayIndex(key); + if (current.path !== undefined && key !== undefined) { + for (const choice of semanticAliasSelectionChoices(key)) + for (const source of semanticArrayPropertyReferences(node, [ + choice, + ])) { + let projection = source.path; + if ( + source.capturedSelectionAt !== undefined && + !semanticKnownArrayIteration( + semanticSelectedReferenceSlot( + { node: source.node, projection: [] }, + state, + source.capturedSelectionAt, + )?.value, + state, + ) + ) + projection = [null, ...source.path.slice(1)]; + const selectedPath = [...projection, ...path]; + pending.push({ + ...current, + node: source.node, + path: selectedPath, + capturedSelections: [ + ...(semanticShiftCapturedSelections( + current.capturedSelections, + selectedPath.length - current.path.length, + ) ?? []), + ...(source.capturedSelectionAt === undefined + ? [] + : [{ at: source.capturedSelectionAt, depth: 1 }]), + ], + }); + } + for (let index = pending.length - 1; index >= 0; index--) { + const child = pending[index]; + if (child !== undefined) + enqueue( + child.node, + child.path, + child.readAt, + child.capturedSelections, + child.invocation ?? current.invocation, + ); + } + continue; + } let uncertainOffset = false; node.elements.forEach((element, offset) => { if (element === null) return; @@ -193,12 +564,28 @@ export const createSemanticReturnedReferences = ( ...current, node: element.argument, path: current.path === undefined ? undefined : [null, ...path], + capturedSelections: [ + ...(current.capturedSelections ?? []), + { at: element.argument, depth: 1 }, + ], }); - } else if (index === null || uncertainOffset || index === offset) + } else if ( + current.path === undefined || + key === undefined || + uncertainOffset || + semanticAliasSelectionChoices(key).some((choice) => + semanticPropertyPathKeyMatches(choice, String(offset)), + ) + ) pending.push({ ...current, node: element, path: current.path === undefined ? undefined : path, + capturedSelections: + semanticShiftCapturedSelections( + current.capturedSelections, + -1, + ) ?? [], }); }); } else if (t.isNewExpression(node)) @@ -212,14 +599,25 @@ export const createSemanticReturnedReferences = ( member.kind === "constructor") || current.path === undefined || (key !== undefined && - (key === null || - semanticStaticPropertyKey(member.key, member.computed) === - String(key))) + semanticAliasSelectionChoices(key).some((choice) => { + const name = semanticStaticPropertyKey( + member.key, + member.computed, + ); + return ( + name === null || + semanticPropertyPathKeyMatches(choice, name) + ); + })) ) pending.push({ ...current, node: member, path: current.path === undefined ? undefined : path, + capturedSelections: semanticShiftCapturedSelections( + current.capturedSelections, + -1, + ), }); } else if ( t.isClassProperty(member) && @@ -227,14 +625,26 @@ export const createSemanticReturnedReferences = ( member.value !== undefined && (key !== undefined || current.path === undefined) && (current.path === undefined || - key === null || - semanticStaticPropertyKey(member.key, member.computed) === - String(key)) + (key !== undefined && + semanticAliasSelectionChoices(key).some((choice) => { + const name = semanticStaticPropertyKey( + member.key, + member.computed, + ); + return ( + name === null || + semanticPropertyPathKeyMatches(choice, name) + ); + }))) ) pending.push({ ...current, node: member.value, path: current.path === undefined ? undefined : path, + capturedSelections: semanticShiftCapturedSelections( + current.capturedSelections, + -1, + ), }); } if (node.superClass !== null && node.superClass !== undefined) @@ -242,17 +652,45 @@ export const createSemanticReturnedReferences = ( } for (let index = pending.length - 1; index >= 0; index--) { const child = pending[index]; - if (child !== undefined) enqueue(child.node, child.path); + if (child !== undefined) + enqueue( + child.node, + child.path, + child.readAt, + child.capturedSelections, + child.invocation ?? current.invocation, + ); } } - const output = [...functions].flatMap((callable) => - semanticCallableResultExpressions(callable).flatMap( - ({ node, delegated }) => - node === null ? [] : [{ node, projection: delegated ? [null] : [] }], - ), - ); + const output = [...functions].flatMap((callable) => { + // A direct generator/async call produces a fresh iterator/Promise, not + // the yielded/resolved contents. Escape analysis still retains those + // potential references for their eventual consumers. + if ( + consumption !== "potential-reference" && + (callable.generator || + (consumption === "direct-call" && callable.async)) + ) + return []; + return semanticCallableResultExpressions(callable).flatMap( + ({ node, delegated, kind }) => + node === null || + (consumption !== "potential-reference" && kind !== "return") + ? [] + : [{ node, projection: delegated ? [null] : [] }], + ); + }); result.references = output; - results.set(callee, result); + results.set(callee, { + result, + invocation, + iterationUnknown: state.arrayIterationUnknown, + ambientWrites: state.ambientWriteEffects.length, + dependencies: [...dependencies].map(([binding, counts]) => ({ + binding, + ...counts, + })), + }); if (result.precision === "widened") state.returnedReferenceWidenings.add(callee); return result; @@ -267,7 +705,17 @@ const selectObjectTargets = ( ): void => { const [key, ...path] = item.path ?? []; const childPath = item.path === undefined ? undefined : path; - if (key !== null && key !== undefined) { + if (key !== null && typeof key === "object" && "alternatives" in key) { + for (const choice of semanticAliasSelectionChoices(key)) + selectObjectTargets( + node, + { ...item, path: [choice, ...path] }, + pending, + indexes, + ); + return; + } + if (key !== null && key !== undefined && typeof key !== "object") { const index = objectTargetIndex(node, indexes); const property = index.propertyByName.get(String(key)); if (property !== undefined) { @@ -287,11 +735,20 @@ const selectObjectTargets = ( ? unknown.value : unknown, ...(t.isSpreadElement(unknown) ? {} : { path: childPath }), + capturedSelections: t.isSpreadElement(unknown) + ? [ + ...(item.capturedSelections ?? []), + { at: unknown.argument, depth: 1, ownEnumerable: true }, + ] + : (semanticShiftCapturedSelections(item.capturedSelections, -1) ?? + []), }); pending.push({ ...item, node: t.isObjectProperty(property) ? property.value : property, path: childPath, + capturedSelections: + semanticShiftCapturedSelections(item.capturedSelections, -1) ?? [], }); return; } @@ -300,7 +757,14 @@ const selectObjectTargets = ( for (const property of [...node.properties].reverse()) { if (replaced) break; if (t.isSpreadElement(property)) { - pending.push({ ...item, node: property.argument }); + pending.push({ + ...item, + node: property.argument, + capturedSelections: [ + ...(item.capturedSelections ?? []), + { at: property.argument, depth: 1, ownEnumerable: true }, + ], + }); continue; } const name = semanticStaticPropertyKey(property.key, property.computed); @@ -308,13 +772,31 @@ const selectObjectTargets = ( key !== null && key !== undefined && name !== null && - name !== String(key) + !semanticPropertyPathKeyMatches(key, name) ) continue; if (t.isObjectProperty(property)) - pending.push({ ...item, node: property.value, path: childPath }); - else pending.push({ ...item, node: property, path: childPath }); - if (key !== null && key !== undefined && name === String(key)) + pending.push({ + ...item, + node: property.value, + path: childPath, + capturedSelections: + semanticShiftCapturedSelections(item.capturedSelections, -1) ?? [], + }); + else + pending.push({ + ...item, + node: property, + path: childPath, + capturedSelections: + semanticShiftCapturedSelections(item.capturedSelections, -1) ?? [], + }); + if ( + key !== null && + key !== undefined && + typeof key !== "object" && + name === String(key) + ) replaced = true; } }; diff --git a/src/domain/javascript/javascriptSemanticReturnedReferencesBounded.test.ts b/src/domain/javascript/javascriptSemanticReturnedReferencesBounded.test.ts index e9aec4049a..00a3eee4e4 100644 --- a/src/domain/javascript/javascriptSemanticReturnedReferencesBounded.test.ts +++ b/src/domain/javascript/javascriptSemanticReturnedReferencesBounded.test.ts @@ -123,3 +123,35 @@ it("bounds returned-reference expansion across a React-shaped switch mesh", () = expect(small.reads).toBeGreaterThan(0); expect(large.reads).toBeLessThan(4 * small.reads); }); + +it.each([ + ["{make:()=>module}", "{...box}", "box.make"], + ["[()=>module]", "[...box]", "box[0]"], +])( + "converges repeated copy boundaries without dropping callable results: %s", + (initial, replacement, target) => { + const state = indexed( + `let box=${initial};while(flag)box=${replacement};${target}();`, + ); + let callee: t.Node | undefined; + t.traverseFast(state.program, (node) => { + if (t.isCallExpression(node)) callee = node.callee; + }); + if (callee === undefined) throw new Error("Missing copied call"); + const lookups = state.scopes.map((scope) => + vi.spyOn(scope.bindings, "get"), + ); + try { + const result = createSemanticReturnedReferences(state)(callee); + expect(result.references).toContainEqual({ + node: expect.objectContaining({ type: "Identifier", name: "module" }), + projection: [], + }); + expect( + lookups.reduce((count, lookup) => count + lookup.mock.calls.length, 0), + ).toBeLessThan(100); + } finally { + for (const lookup of lookups) lookup.mockRestore(); + } + }, +); diff --git a/src/domain/javascript/javascriptSemanticReturnedReferencesCopies.test.ts b/src/domain/javascript/javascriptSemanticReturnedReferencesCopies.test.ts new file mode 100644 index 0000000000..f61bb7b7c3 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticReturnedReferencesCopies.test.ts @@ -0,0 +1,83 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it.each([ + [ + "const old={child:{make:()=>module}};const box={...old};old.child={make:()=>({})};box.child.make()", + true, + ], + [ + "const old={child:{make:()=>module}};const box={...old};old.child.make=()=>({});box.child.make()", + false, + ], + [ + "const old={child:{make:()=>({})}};const box={...old};old.child.make=()=>module;box.child.make()", + true, + ], + [ + "const old={child:{make:()=>({})}};const box={...old};old.child={make:()=>module};old.child.make=()=>module;box.child.make()", + false, + ], + [ + "const old={child:{make:()=>module}};const {...box}=old;old.child={make:()=>({})};box.child.make()", + true, + ], + [ + "const old={child:{make:()=>module}};const {...box}=old;old.child.make=()=>({});box.child.make()", + false, + ], + [ + "let old={child:{make:()=>module}};const {...box}=old;old={child:{make:()=>({})}};box.child.make()", + true, + ], + [ + "const old=[{make:()=>module}];const box=[...old];old[0]={make:()=>({})};box[0].make()", + true, + ], + [ + "const old=[{make:()=>module}];const box=[...old];old[0].make=()=>({});box[0].make()", + false, + ], + [ + "const old=[{make:()=>module}];const [...box]=old;old[0]={make:()=>({})};box[0].make()", + true, + ], + [ + "const old=[{}, {make:()=>module}];const [skip,...box]=old;old[1]={make:()=>({})};box[0].make()", + true, + ], + [ + "const old=[{}, {make:()=>module}];const [skip,...box]=old;old[1].make=()=>({});box[0].make()", + false, + ], + [ + "const old={make:()=>module};const middle={...old};old.make=()=>({});const box={...middle};middle.make=()=>({});box.make()", + true, + ], + [ + "const old={make:()=>({})};const middle={...old};old.make=()=>module;const box={...middle};middle.make=()=>module;box.make()", + false, + ], + [ + "const old={child:{make:()=>module}};const middle={...old};old.child={make:()=>({})};const box={...middle};box.child.make()", + true, + ], + [ + "const old={make:()=>module};const middle={...old};old.make=()=>({});const {...box}=middle;box.make()", + true, + ], +])( + "preserves copied child identity and later writes through it: %s", + (receiver, expected) => { + const result = analyzeJavaScriptStaticSource( + `${receiver}.require('./shared.cjs');`, + ); + expect(result.parse_error_count).toBe(0); + expect(result.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); diff --git a/src/domain/javascript/javascriptSemanticReturnedReferencesWrites.test.ts b/src/domain/javascript/javascriptSemanticReturnedReferencesWrites.test.ts new file mode 100644 index 0000000000..87e35c6dcb --- /dev/null +++ b/src/domain/javascript/javascriptSemanticReturnedReferencesWrites.test.ts @@ -0,0 +1,203 @@ +import * as t from "@babel/types"; +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; +import { indexJavaScriptBindingsWithReceiversSteps } from "./javascriptSemanticAnalysis.js"; +import { createSemanticReturnedReferences } from "./javascriptSemanticReturnedReferences.js"; +import { parseJavaScriptSource } from "./javascriptSourceParser.js"; +import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; + +it.each([ + [ + 'const box={make(){return {};}};box.make=()=>module;box.make().require("./shared.cjs");', + true, + ], + [ + 'const box={make(){return module;}};box.make=()=>({});box.make().require("./shared.cjs");', + false, + ], + [ + 'const box={make(){return {};}};if(flag)box.make=()=>module;box.make().require("./shared.cjs");', + true, + ], + [ + 'const box={make(){return module;}};if(flag)box.make=()=>({});box.make().require("./shared.cjs");', + true, + ], + [ + 'const box={make(){return {};}};box.make().require("./shared.cjs");box.make=()=>module;', + false, + ], + [ + 'const box={make(){return {};}};const alias=box;box.make=()=>module;alias.make().require("./shared.cjs");', + true, + ], + [ + 'const box={make(){return module;}};const alias=box;box.make=()=>({});alias.make().require("./shared.cjs");', + false, + ], + [ + 'const box={make(){return {};}};const make=box.make;box.make=()=>module;make().require("./shared.cjs");', + false, + ], + [ + 'const box={make(){return module;}};const make=box.make;box.make=()=>({});make().require("./shared.cjs");', + true, + ], + [ + 'const box={inner:{make(){return {};}}};box.inner.make=()=>module;box.inner.make().require("./shared.cjs");', + true, + ], + [ + 'const box={inner:{make(){return module;}}};box.inner={make(){return {};}};box.inner.make().require("./shared.cjs");', + false, + ], + [ + 'const box={};({make:box.make}={make:()=>module});box.make().require("./shared.cjs");', + true, + ], + [ + 'const box=[()=>({})];box[0]=()=>module;box[0]().require("./shared.cjs");', + true, + ], + [ + 'const box={make(){return {};}};box.other=()=>module;box.make().require("./shared.cjs");', + false, + ], + [ + 'const box={make(){return {};}};box.make=async()=>module;box.make().require("./shared.cjs");', + false, + ], + [ + 'const box={make(){return {};}};box.make=async()=>module;(await box.make()).require("./shared.cjs");', + true, + ], + [ + 'const box={make(){return {};}};box.make=function*(){return module;};(await box.make()).require("./shared.cjs");', + false, + ], +] as const)( + "selects returned references at the member read: %s", + (source, expected) => { + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); + +it.each([ + ["let make=()=>module;const box={make};make=()=>({});box.make()", true], + ["let make=()=>({});const box={make};make=()=>module;box.make()", false], + ["let make=()=>module;const box={make:make};make=()=>({});box.make()", true], + ["let make=()=>module;const box=[make];make=()=>({});box[0]()", true], + ["let make=()=>({});const box=[make];make=()=>module;box[0]()", false], + [ + "let make=()=>module;const box={make};const saved=box.make;make=()=>({});saved()", + true, + ], + [ + "let make=()=>module;const box={};box.make=make;make=()=>({});box.make()", + true, + ], + [ + "let make=()=>({});const box={};box.make=make;make=()=>module;box.make()", + false, + ], + [ + "let make=()=>module;const box={};[box.make]=[make];make=()=>({});box.make()", + true, + ], + [ + "let make=()=>module;const old={make};const box={...old};make=()=>({});box.make()", + true, + ], + [ + "const old={make:()=>module};const box={...old};old.make=()=>({});box.make()", + true, + ], + [ + "const old={make:()=>({})};const box={...old};old.make=()=>module;box.make()", + false, + ], + [ + "const old={make:()=>module};const {...box}=old;old.make=()=>({});box.make()", + true, + ], + [ + "const old={make:()=>({})};const {...box}=old;old.make=()=>module;box.make()", + false, + ], + ["const old=[()=>module];const [...box]=old;old[0]=()=>({});box[0]()", true], + ["const old=[()=>({})];const [...box]=old;old[0]=()=>module;box[0]()", false], + [ + "let make=()=>module;const {make:saved=make}={};make=()=>({});saved()", + true, + ], + [ + "let inner={make:()=>module};const box={inner};inner={make:()=>({})};inner.make=()=>({});box.inner.make()", + true, + ], + [ + "let inner={make:()=>({})};const box={inner};inner={make:()=>module};inner.make=()=>module;box.inner.make()", + false, + ], + [ + "let inner={make:()=>({})};const box={inner};inner.make=()=>module;box.inner.make()", + true, + ], + [ + "let inner={make:()=>({})};const saved=inner;const box={inner};inner={make:()=>({})};saved.make=()=>module;box.inner.make()", + true, + ], +] as const)( + "retains captured values independently of later member reads: %s", + (expression, expected) => { + const analysis = analyzeJavaScriptStaticSource( + `${expression}.require('./shared.cjs');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); + +it("refreshes cached returned targets after the shared member index settles", () => { + const parsed = parseJavaScriptSource( + "const box={make(){return {};}};box.make=()=>module;box.make();", + "member-writes.cjs", + ); + if (parsed === null) throw new Error("Expected parsed member writes"); + const steps = indexJavaScriptBindingsWithReceiversSteps(parsed.program); + let step = steps.next(); + while (!step.done) step = steps.next(); + const index = step.value; + let callee: t.Node | undefined; + traverseJavaScriptAst(parsed.program, { + enter(node) { + if (t.isCallExpression(node)) callee = node.callee; + }, + }); + if (callee === undefined) throw new Error("Expected member call"); + const returned = createSemanticReturnedReferences( + index.bindings, + "direct-call", + ); + expect( + returned(callee).references.some(({ node }) => t.isIdentifier(node)), + ).toBe(false); + const effects = index.indexMemberMutationFactsSteps(); + let effect = effects.next(); + while (!effect.done) effect = effects.next(); + expect(returned(callee).references).toEqual([ + expect.objectContaining({ + node: expect.objectContaining({ name: "module" }), + }), + ]); +}); diff --git a/src/domain/javascript/javascriptSemanticSharedChildWrites.test.ts b/src/domain/javascript/javascriptSemanticSharedChildWrites.test.ts new file mode 100644 index 0000000000..48b7a405b0 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticSharedChildWrites.test.ts @@ -0,0 +1,223 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; +import * as t from "@babel/types"; +import { parseJavaScriptSource } from "./javascriptSourceParser.js"; +import { indexJavaScriptBindingsWithReceiversSteps } from "./javascriptSemanticAnalysis.js"; +import { semanticProjectedSourcesAtRead } from "./javascriptSemanticProjectedSources.js"; + +const shapes = [ + { + before: "const old={child:{inner:INITIAL}};const box={...old};", + shared: "old.child={inner:box.child.inner};", + fresh: "old.child={inner:INITIAL};", + write: "old.child.inner.SLOT=NEXT;", + read: "box.child.inner.READ", + }, + { + before: "const old={child:{inner:INITIAL}};const {...box}=old;", + shared: "old.child={inner:box.child.inner};", + fresh: "old.child={inner:INITIAL};", + write: "old.child.inner.SLOT=NEXT;", + read: "box.child.inner.READ", + }, + { + before: "const old=[{inner:INITIAL}];const box=[...old];", + shared: "old[0]={inner:box[0].inner};", + fresh: "old[0]={inner:INITIAL};", + write: "old[0].inner.SLOT=NEXT;", + read: "box[0].inner.READ", + }, + { + before: "const old=[{inner:INITIAL}];const [...box]=old;", + shared: "old[0]={inner:box[0].inner};", + fresh: "old[0]={inner:INITIAL};", + write: "old[0].inner.SLOT=NEXT;", + read: "box[0].inner.READ", + }, + { + before: "const old={child:[INITIAL]};const box={...old};", + shared: "old.child=[box.child[0]];", + fresh: "old.child=[INITIAL];", + write: "old.child[0].SLOT=NEXT;", + read: "box.child[0].READ", + }, +] as const; +const values = [ + { + initial: "{make:()=>({})}", + next: "()=>module", + slot: "make", + read: "make().require('./shared.cjs')", + host: false, + }, + { + initial: "{make:()=>module}", + next: "()=>({})", + slot: "make", + read: "make().require('./shared.cjs')", + host: true, + }, + { + initial: "{loader:{}}", + next: "module", + slot: "loader", + read: "loader.require('./shared.cjs')", + host: false, + }, + { + initial: "{loader:module}", + next: "{}", + slot: "loader", + read: "loader.require('./shared.cjs')", + host: true, + }, + { + initial: "{load:()=>({})}", + next: "module.require", + slot: "load", + read: "load('./shared.cjs')", + host: false, + }, + { + initial: "{load:module.require}", + next: "()=>({})", + slot: "load", + read: "load('./shared.cjs')", + host: true, + }, +] as const; + +it.each( + shapes.flatMap((shape) => + values.flatMap((value) => + [true, false].map((shared) => ({ + source: ( + shape.before + + (shared ? shape.shared : shape.fresh) + + shape.write + + shape.read + + ";" + ) + .replaceAll("INITIAL", value.initial) + .replaceAll("NEXT", value.next) + .replaceAll("SLOT", value.slot) + .replaceAll("READ", value.read), + expression: shape.read.replaceAll("READ", value.read), + // The public module/delegate owner retains unknown after the copy's child + // escapes into the new parent. Factory result selection can exclude it. + expected: shared ? !value.host || value.slot !== "make" : value.host, + })), + ), + ), +)( + "distinguishes shared and newly allocated descendant receivers: $source", + ({ source, expression, expected }) => { + const result = analyzeJavaScriptStaticSource(source); + expect(result.parse_error_count).toBe(0); + expect( + result.references.filter( + (reference) => reference.expression === expression, + ), + ).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); + +it.each(["module", "{}"])( + "selects the later write to a shared projected receiver: %s", + (next) => { + const parsed = parseJavaScriptSource( + `const old={child:{inner:{loader:module}}};const box={...old};old.child={inner:box.child.inner};old.child.inner.loader=${next};box.child.inner.loader.require('./shared.cjs');`, + "shared-child.cjs", + ); + if (parsed === null) throw new Error("Expected parsed fixture"); + const steps = indexJavaScriptBindingsWithReceiversSteps(parsed.program); + let step = steps.next(); + while (!step.done) step = steps.next(); + const index = step.value; + const effects = index.indexMemberMutationFactsSteps(); + let effect = effects.next(); + while (!effect.done) effect = effects.next(); + const binding = [...index.bindings.bindingsById.values()].find( + (candidate) => candidate.name === "old", + ); + let capture: t.Node | undefined; + let assignment: t.AssignmentExpression | undefined; + t.traverseFast(parsed.program, (node) => { + if (t.isSpreadElement(node)) capture = node.argument; + if (t.isAssignmentExpression(node)) assignment = node; + }); + if ( + binding === undefined || + capture === undefined || + assignment === undefined + ) + throw new Error("Missing selected receiver"); + const statement = parsed.program.body.at(-1)!; + if (!t.isExpressionStatement(statement)) + throw new Error("Missing receiver use"); + const use = statement.expression; + expect( + semanticProjectedSourcesAtRead( + binding, + capture, + ["child", "inner", "loader"], + index.bindings, + true, + use, + [{ at: capture, depth: 1 }], + ), + ).toEqual([{ node: assignment.right, path: [] }]); + }, +); + +it.each([ + "old.child={inner:flag?box.child.inner:{make:()=>({})}};", + "old.child={get inner(){return box.child.inner}};", + "old.child={...box.child};", +])( + "retains a possible shared receiver when identity is incomplete: %s", + (replacement) => { + const expression = "box.child.inner.make().require('./shared.cjs')"; + const result = analyzeJavaScriptStaticSource( + `const old={child:{inner:{make:()=>({})}}};const box={...old};${replacement}old.child.inner.make=()=>module;${expression};`, + ); + expect(result.parse_error_count).toBe(0); + expect(result.references).toContainEqual( + expect.objectContaining({ kind: "require", specifier: null, expression }), + ); + }, +); + +it.each([16, 600])( + "bounds descendant receiver alternatives before expanding %i selections", + (count) => { + const properties = Array.from( + { length: count }, + (_, index) => `p${index}:{make:()=>({})}`, + ).join(","); + const expression = "box.child.inner.make().require('./shared.cjs')"; + const result = analyzeJavaScriptStaticSource( + `const old={child:{inner:{make:()=>({})}}};const box={...old};old.child={${properties}};old.child[key].make=()=>module;${expression};`, + ); + expect(result.parse_error_count).toBe(0); + const limited = result.limitations.some((limitation) => + limitation.includes( + "Alias selection expansion was conservatively bounded", + ), + ); + expect(limited).toBe(count === 600); + const selected = result.references.filter( + (reference) => reference.expression === expression, + ); + expect(selected).toEqual( + count === 600 + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); diff --git a/src/domain/javascript/javascriptSemanticState.ts b/src/domain/javascript/javascriptSemanticState.ts index e6e388280a..11fd41e3b4 100644 --- a/src/domain/javascript/javascriptSemanticState.ts +++ b/src/domain/javascript/javascriptSemanticState.ts @@ -1,9 +1,14 @@ import type { JavaScriptOpaqueScopeEffects } from "./javascriptSemanticOpaqueEffects.js"; +import type { SemanticIterationSource } from "./javascriptSemanticIterationSources.js"; import * as t from "@babel/types"; import { semanticStaticPropertyKey, unwrapJavaScriptExpression, } from "./javascriptAstValues.js"; +import { + semanticBindingAtRead, + semanticEffectTiming, +} from "./javascriptSemanticMutationInitializers.js"; import type { JavaScriptSemanticPropertyPath, @@ -29,10 +34,16 @@ export interface JavaScriptSemanticInitializer { readonly node: t.Node; readonly projection: readonly (string | number | null)[]; readonly entryBody?: t.BlockStatement; + readonly iterationSources?: readonly { + readonly node: t.Node; + readonly projection: JavaScriptSemanticPropertyPath; + }[]; } /** Additional references from destructuring copies and possible defaults. */ interface JavaScriptSemanticReferenceInitializer extends JavaScriptSemanticInitializer { + /** This source is a value replacement, rather than a declaration fallback. */ + readonly assignment?: true; readonly copyKind?: "object-rest" | "array-rest"; readonly copyProjectionOffset?: number; readonly copyExcludedKeys?: readonly string[]; @@ -45,6 +56,12 @@ interface JavaScriptSemanticReferenceInitializer extends JavaScriptSemanticIniti export interface JavaScriptSemanticMemberEffect { readonly path: JavaScriptSemanticPropertyPath; readonly node: t.Node | undefined; + /** Selected RHS values retained by the shared pattern-store owner. */ + readonly assignedSources?: readonly SemanticIterationSource[]; + /** Binding values whose receiver identities were selected for this effect. */ + readonly receiverInitializers?: readonly JavaScriptSemanticInitializer[]; + /** Capture of this binding used while following an aliased effect receiver. */ + readonly receiverAt?: t.Node; } export type JavaScriptSemanticEffectTiming = "before" | "after" | "unknown"; @@ -101,6 +118,7 @@ export interface JavaScriptSemanticScopeState { /** Shared state for semantic collection and evaluation. */ export interface JavaScriptSemanticAnalysisState { + arrayIterationUnknown: boolean; readonly program: t.Program; readonly opaqueEffects: JavaScriptOpaqueScopeEffects; /** Effective loader for static imports in this source artifact. */ @@ -150,6 +168,8 @@ export interface JavaScriptSemanticAnalysisState { readonly ambientWrites: SemanticPropertyPathCoverage; /** Ambient object identities passed to calls whose effects are not evaluated. */ readonly ambientEscapes: SemanticPropertyPathCoverage; + /** Capture sites keep later opaque effects from changing earlier copies. */ + readonly ambientEscapeEffects: JavaScriptSemanticMemberEffect[]; /** Source sites distinguish replacement before an ambient identity capture from later writes. */ readonly ambientWriteEffects: JavaScriptSemanticMemberEffect[]; } @@ -537,7 +557,52 @@ export const isUnmodifiedCommonJsWrapperBinding = ( state: JavaScriptSemanticAnalysisState, ): boolean => { const binding = commonJsWrapperBindingDeclaration(node, name, state); - return binding !== undefined && binding.initializers.length === 0; + return ( + binding !== undefined && + !semanticBindingHasPriorValueWrite(node, binding, state) + ); +}; + +/** Retained value and pattern stores share one source-ordered write boundary. */ +export const semanticBindingHasPriorValueWrite = ( + site: t.Node, + binding: JavaScriptSemanticBindingState, + state: JavaScriptSemanticAnalysisState, +): boolean => { + if (binding.definitions.some(({ kind }) => kind === "function")) return true; + const writes = state.bindingWrites.get(binding.bindingId) ?? []; + const read = semanticBindingAtRead(binding, site, state.parentsByNode); + const retained = + read.status === "ordered" + ? new Set([ + ...read.binding.initializers, + ...read.binding.referenceInitializers, + ]) + : undefined; + const representedWrites = new Set(); + const retainedWrites = new Set(); + for (const source of [ + ...binding.initializers, + ...binding.referenceInitializers, + ]) { + let owner: t.Node | undefined = source.node; + while (owner !== undefined && !writes.includes(owner)) + owner = state.parentsByNode.get(owner); + if (owner !== undefined) { + representedWrites.add(owner); + if (retained === undefined || retained.has(source)) + retainedWrites.add(owner); + } else if ( + (retained === undefined || retained.has(source)) && + semanticEffectTiming(site, source.node, state) !== "after" + ) + return true; + } + return writes.some( + (write) => + (!representedWrites.has(write) || retainedWrites.has(write)) && + semanticEffectTiming(site, write, state) !== "after", + ); }; /** Program `var` redeclarations retain wrapper identity until a source write. */ @@ -550,7 +615,9 @@ export const commonJsWrapperBindingDeclaration = ( // specifier resolution mode does not establish that distinction. if ( state.program.sourceType !== "script" || - !["module", "exports", "require"].includes(name) || + !["module", "exports", "require", "__dirname", "__filename"].includes( + name, + ) || !t.isIdentifier(node, { name }) ) return undefined; diff --git a/src/domain/javascript/javascriptSemanticValueTypes.ts b/src/domain/javascript/javascriptSemanticValueTypes.ts index bb545d8662..41a698d4c6 100644 --- a/src/domain/javascript/javascriptSemanticValueTypes.ts +++ b/src/domain/javascript/javascriptSemanticValueTypes.ts @@ -6,7 +6,8 @@ export type JavaScriptSemanticResourceLimit = | "primitive-candidates" | "primitive-bytes" | "expression-depth" - | "getter-expansion"; + | "getter-expansion" + | "alias-selections"; type JavaScriptSemanticObjectValue = { readonly status: "object"; diff --git a/src/domain/javascript/javascriptSemanticValues.ts b/src/domain/javascript/javascriptSemanticValues.ts index 966c11a13e..50d0f8714a 100644 --- a/src/domain/javascript/javascriptSemanticValues.ts +++ b/src/domain/javascript/javascriptSemanticValues.ts @@ -63,6 +63,27 @@ import { semanticResourceLimitReason, } from "./javascriptSemanticResourceLimits.js"; +// Function identity is outside the public constant lattice. Retain its +// non-undefined proof with the immutable value rather than interpreting an +// "unknown" reason or recovering callable syntax in default consumers. +const definitelyDefinedValues = new WeakSet(); + +const definedNonPrimitiveValue = (reason: string): JavaScriptSemanticValue => { + const value: JavaScriptSemanticValue = { status: "unknown", reason }; + definitelyDefinedValues.add(value); + return value; +}; + +/** Definedness is independent of whether the constant lattice retains a value. */ +export const semanticValueDefinitelyDefined = ( + value: JavaScriptSemanticValue, +): boolean => + value.status === "literal" || + value.status === "union" || + value.status === "object" || + value.status === "array" || + definitelyDefinedValues.has(value); + interface MemoizedEvaluation { readonly value: T; readonly dependencies: SemanticDependencyProof; @@ -433,6 +454,16 @@ const evaluateBinding = ( const reusable = readMemoizedEvaluation(retained, context); if (reusable !== undefined) return reusable; } + if ( + binding.referenceInitializers.some(({ assignment }) => assignment) && + (binding.initializers.length > 0 || + (context.state.bindingsById.get(binding.bindingId)?.initializers.length ?? + 0) > 0) + ) + return { + status: "ambiguous", + reason: `Binding ${binding.name} has value and projected-reference assignments.`, + }; if (binding.initializers.length === 0) return { status: "unknown", @@ -615,6 +646,14 @@ const evaluateExpressionValue = ( } const literal = readExactJavaScriptLiteral(node); if (literal.found) return primitiveSet([literal.value]); + if ( + t.isFunctionExpression(node) || + t.isFunctionDeclaration(node) || + t.isArrowFunctionExpression(node) || + t.isClassExpression(node) || + t.isClassDeclaration(node) + ) + return definedNonPrimitiveValue(`Unsupported ${node.type} value.`); if (t.isIdentifier(node)) { const binding = resolveSemanticBindingState(context.state, node, node.name); if (binding === undefined) @@ -759,10 +798,14 @@ const evaluateObject = ( presence: "present", value: t.isObjectProperty(property) ? evaluateCapturedExpression(property.value, nestedContext(context)) - : { - status: "unknown", - reason: "Object method or accessor value is not a primitive.", - }, + : property.kind === "method" + ? definedNonPrimitiveValue( + "Object method or accessor value is not a primitive.", + ) + : { + status: "unknown", + reason: "Object method or accessor value is not a primitive.", + }, }); } const properties = [...propertiesByName.values()].sort((left, right) => @@ -1011,6 +1054,29 @@ const provenanceForBindingUncached = ( const provenanceForBindingValue = ( binding: JavaScriptSemanticBindingState, context: EvaluationContext, +): JavaScriptBindingProvenance => { + const provenance = provenanceForBindingInitializers(binding, context); + if (!binding.referenceInitializers.some(({ assignment }) => assignment)) + return provenance; + // Projected replacements are retained reference sources, not evidence of a + // local value or an unchanged earlier module. Apply this only to the selected + // read snapshot so a later replacement does not erase captured origins. + if (provenance.status === "local") + return semanticUnresolvedProvenance( + "unknown", + `Binding ${binding.name} has projected reference assignments whose module provenance is unresolved.`, + ); + return provenance.status === "module" + ? semanticAmbiguousProvenance( + provenance.origins, + `Binding ${binding.name} has projected reference assignments that may replace its module origin.`, + ) + : provenance; +}; + +const provenanceForBindingInitializers = ( + binding: JavaScriptSemanticBindingState, + context: EvaluationContext, ): JavaScriptBindingProvenance => { if (binding.kind === "parameter" || binding.kind === "catch") return semanticUnresolvedProvenance( @@ -1239,11 +1305,14 @@ const mergeValues = ( const primitives = values.flatMap( (value) => primitiveCandidates(value) ?? [], ); - return values.every( + const merged: JavaScriptSemanticValue = values.every( (value) => value.status === "union" || value.status === "literal", ) ? primitiveSet(primitives) : { status: "ambiguous", reason: "Branches have incompatible values." }; + if (values.length > 0 && values.every(semanticValueDefinitelyDefined)) + definitelyDefinedValues.add(merged); + return merged; }; const nestedContext = (context: EvaluationContext): EvaluationContext => ({ diff --git a/src/domain/javascript/javascriptSemanticWrapperAssignments.test.ts b/src/domain/javascript/javascriptSemanticWrapperAssignments.test.ts new file mode 100644 index 0000000000..d44bcb4a03 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticWrapperAssignments.test.ts @@ -0,0 +1,119 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; +import { topLevelBinding } from "./javascriptSemanticAnalysis.fixture.js"; +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it.each([ + 'var require;require("./before.cjs");', + 'var require;require("./before.cjs");({require}={require:()=>({})});', + 'var require;require("./before.cjs");[require]=[()=>({})];', + 'var require;({require}={require:require("./before.cjs")});', + 'var require;[require]=[require("./before.cjs")];', +])("preserves the host loader at its earlier read: %s", (source) => { + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toContainEqual( + expect.objectContaining({ kind: "require", specifier: "./before.cjs" }), + ); +}); + +it.each([ + "({require}={require:()=>({BrowserWindow:class{}})});", + "[require]=[()=>({BrowserWindow:class{}})];", + "({nested:{require}}={nested:{require:()=>({BrowserWindow:class{}})}});", + "({require=()=>({BrowserWindow:class{}})}={});", + "if(flag)({require}={require:()=>({BrowserWindow:class{}})});", +])( + "keeps projected loader writes out of proven Electron identity: %s", + (write) => { + const analysis = analyzeJavaScriptStaticSource( + `var require;${write}const {BrowserWindow}=require("electron");new BrowserWindow({webPreferences:{preload:"./unused.cjs"}});`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toContainEqual( + expect.objectContaining({ kind: "require", specifier: null }), + ); + expect(analysis.references).not.toContainEqual( + expect.objectContaining({ kind: "require", specifier: "electron" }), + ); + expect(analysis.electron.browser_windows).toEqual([]); + }, +); + +it.each([ + "({require}={require:()=>({BrowserWindow:class{}})});", + "[require]=[()=>({BrowserWindow:class{}})];", +])("preserves an Electron capture before the projected write: %s", (write) => { + const analysis = analyzeJavaScriptStaticSource( + `var require;const {BrowserWindow}=require("electron");${write}new BrowserWindow({webPreferences:{preload:"./before.cjs"}});`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toContainEqual( + expect.objectContaining({ kind: "require", specifier: "electron" }), + ); + expect(analysis.electron.browser_windows).toMatchObject([ + { preload_path: "./before.cjs" }, + ]); +}); + +it.each([ + ["({module}={module:{require(){}}});", false], + ["[module]=[{require(){}}];", false], + ["if(flag)({module}={module:{require(){}}});", true], + ["if(flag)[module]=[{require(){}}];", true], +])( + "distinguishes definite and conditional wrapper replacement: %s", + (write, unknown) => { + const analysis = analyzeJavaScriptStaticSource( + `var module;${write}module.require("./unused.cjs");`, + ); + expect(analysis.parse_error_count).toBe(0); + expect( + analysis.references.filter(({ kind }) => kind === "require"), + ).toEqual(unknown ? [expect.objectContaining({ specifier: null })] : []); + }, +); + +it.each([ + ["({loader}={loader:()=>({})});loader('./unused.cjs');", null], + ["[loader]=[()=>({})];loader('./unused.cjs');", null], + ["if(flag)({loader}={loader:()=>({})});loader('./unused.cjs');", null], + ["loader('./before.cjs');({loader}={loader:()=>({})});", "./before.cjs"], + ["loader('./before.cjs');[loader]=[()=>({})];", "./before.cjs"], +])( + "uses the shared write facts for a factory loader: %s", + (body, specifier) => { + const analysis = analyzeJavaScriptStaticSource( + `globalThis.webpackChunkApp.push([[1],{entry:function(module,exports,loader){${body}}}]);`, + ); + expect(analysis.parse_error_count).toBe(0); + expect( + analysis.references.filter(({ kind }) => kind === "require"), + ).toEqual([expect.objectContaining({ specifier })]); + }, +); + +it.each([ + "({require}={require:()=>({BrowserWindow:class{}})});", + "if(flag)({require}={require:()=>({BrowserWindow:class{}})});", +])("retains ambient projected writes as unknown: %s", (write) => { + const analysis = analyzeJavaScriptStaticSource( + `${write}const {BrowserWindow}=require("electron");new BrowserWindow({});`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toContainEqual( + expect.objectContaining({ kind: "require", specifier: null }), + ); + expect(analysis.electron.browser_windows).toEqual([]); +}); + +it("keeps projected reference provenance unknown until its sources are resolved", () => { + const analysis = analyzeJavaScriptSemantics( + 'let selected;({selected}={selected:require("electron")});', + ); + expect(topLevelBinding(analysis, "selected").provenance).toMatchObject({ + status: "unknown", + origins: [], + }); +}); diff --git a/src/domain/javascript/javascriptStaticAnalysis.ts b/src/domain/javascript/javascriptStaticAnalysis.ts index ebf69a2f0e..0c84220e0b 100644 --- a/src/domain/javascript/javascriptStaticAnalysis.ts +++ b/src/domain/javascript/javascriptStaticAnalysis.ts @@ -12,6 +12,9 @@ import { } from "./javascriptSemanticTraversal.js"; import { inspectElectronStaticNode } from "./electronStaticAnalysis.js"; +import { hasPossibleCommonJsDelegateWrite } from "./javascriptSemanticRequireOrigin.js"; +import { semanticResourceLimitReason } from "./javascriptSemanticResourceLimits.js"; +import { createStaticNodePathResolveQuery } from "./javascriptStaticPathIntrinsics.js"; import { finishElectronBrowserWindowsSteps } from "./electronStaticAnalysisBrowser.js"; import { detectVendors, @@ -24,6 +27,7 @@ import { addReference, addSourceMapDirectives, finishPartialNetworkEndpoints, + finishModuleRequireCalls, finishSqliteStorageCalls, inspectCall, inspectRouteProperty, @@ -89,20 +93,29 @@ export function* analyzeParsedJavaScriptStaticSourceSteps( const index = yield* indexJavaScriptBindingsWithReceiversSteps(file.program); const { bindings, receivers } = index; yield* index.indexAmbientWritesSteps(); + if (hasPossibleCommonJsDelegateWrite(bindings)) + yield* index.indexMemberMutationFactsSteps(); const electronIdentity = yield* indexJavaScriptElectronIdentitySteps( file.program, bindings, ); const findings: JavaScriptFindingContext = { + nodePathResolve: createStaticNodePathResolveQuery(index), source, accumulator, vitePreloadDependencyTable: indexVitePreloadDependencyTable(file.program), openReceiverFacts: receivers.openReceivers, electronBindings: electronIdentity.bindings, + electronInvocationPath: electronIdentity.invocationPath, + electronAppPathChangePossible: electronIdentity.mayChangeAppPath, rpcScan: createJavaScriptRpcScan(source, receivers.rpcClientRoots, index), }; yield* traverseStaticSourceSteps(file, findings, bindings); yield* finishElectronBrowserWindowsSteps(findings, index); + if (accumulator.pendingModuleRequireCalls.length > 0) { + yield* index.indexMemberMutationFactsSteps(); + finishModuleRequireCalls(findings, bindings); + } if ( accumulator.pendingNetworkCalls.length > 0 || accumulator.pendingSqliteCalls.length > 0 @@ -145,6 +158,9 @@ const finalizeStaticAnalysis = ( ): JavaScriptStaticAnalysis => { const parserErrors = file.errors.length; const limitations = [ + ...(bindings.valueResourceLimits.has("alias-selections") + ? [semanticResourceLimitReason("alias-selections")] + : []), ...(bindings.returnedReferenceWidenings.size > 0 ? [RETURNED_REFERENCE_WIDENING_LIMITATION] : []), @@ -183,7 +199,9 @@ const finalizeStaticAnalysis = ( ]; return { parse_status: - parserErrors > 0 || accumulator.unknownFindings > 0 + parserErrors > 0 || + accumulator.unknownFindings > 0 || + bindings.valueResourceLimits.has("alias-selections") ? "partial" : "complete", parse_error_count: parserErrors, @@ -211,6 +229,14 @@ const finalizeStaticAnalysis = ( source_map_urls: accumulator.sourceMaps, vendors: detectVendors(source), electron: { + ...(accumulator.appPathChanges.length > 0 + ? { + app_path_changes: finalizeLocatedFindings( + accumulator.appPathChanges, + accumulator.moduleRangeIndex, + ), + } + : {}), ...(accumulator.electronCallPaths?.length ? { call_paths: accumulator.electronCallPaths } : {}), diff --git a/src/domain/javascript/javascriptStaticAnalysisAssignedRequire.test.ts b/src/domain/javascript/javascriptStaticAnalysisAssignedRequire.test.ts new file mode 100644 index 0000000000..2f6c21f168 --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisAssignedRequire.test.ts @@ -0,0 +1,158 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it.each([ + [ + "const values={*[Symbol.iterator](){yield module;}};const h={};[h.loader]=values;h.loader", + true, + ], + [ + "const values=[{}];values[Symbol.iterator]=function*(){yield module;};const h={};[h.loader]=values;h.loader", + true, + ], + [ + "const values={*[Symbol.iterator](){yield module;}};const [api]=values;api", + true, + ], + [ + "const values={*[Symbol.iterator](){yield module;}};const [...api]=values;api[0]", + true, + ], + [ + "Array.prototype[Symbol.iterator]=function*(){yield module;};const [api]=[{}];api", + true, + ], + ["const h={};[h.loader]=[{}];h.loader", false], + ["const [api]=[{}];api", false], + ["const [api]=module;api", true], +])("keeps custom iterable selections unknown: %s", (source, expected) => { + const analysis = analyzeJavaScriptStaticSource( + `${source}.require('./shared.cjs');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); +}); + +it.each(["??=", "||=", "&&="])( + "retains possible logical assignment module sources for %s", + (operator) => { + const analysis = analyzeJavaScriptStaticSource(` + const h={ loader: choose() }; + h.loader ${operator} module; + h.loader.require=replacement; + require('./shared.cjs'); + `); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier: null }), + ]); + }, +); + +it.each(["??=", "||=", "&&="])( + "keeps logical assignment local sources separate for %s", + (operator) => { + const analysis = analyzeJavaScriptStaticSource(` + const h={ loader: {} }; + h.loader ${operator} { require() {} }; + h.loader.require=replacement; + require('./shared.cjs'); + `); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier: "./shared.cjs" }), + ]); + }, +); + +it.each(["??=", "||=", "&&="])( + "keeps a load before a later logical module assignment for %s", + (operator) => { + const analysis = analyzeJavaScriptStaticSource(` + require('./shared.cjs'); + const h={ loader: choose() }; + h.loader ${operator} module; + h.loader.require=replacement; + `); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier: "./shared.cjs" }), + ]); + }, +); + +it.each([ + ["const {...h}={loader:module};h.loader", true], + ["const [...h]=[module];h[0]", true], + ["const [ignored,...h]=[{},module];h[0]", true], + ["const {ignored,...h}={ignored:module,loader:module};h.loader", true], + ["const {loader:ignored,...h}={loader:module};h.loader", false], + ["const {...h}=module;h", false], + ["const [...h]=[module];h", false], + ["const {...h}={loader:module.exports};h.loader", false], + ["const [...h]=[{}];h[0]", false], +])( + "preserves child identities of declaration rest copies: %s", + (source, expected) => { + const analysis = analyzeJavaScriptStaticSource( + `${source}.require('./shared.cjs');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); + +it.each([ + ["({x:h.loader}={x:module});h.loader", true], + ["[h.loader]=[module];h.loader", true], + ["({x:[{y:h.loader}]}={x:[{y:module}]});h.loader", true], + ["({x:h.loader=module}={});h.loader", true], + ["({x:{y:h.loader=module}={}}={});h.loader", true], + ["({[key]:h.loader}={x:module});h.loader", true], + ["({['x']:h.loader}={x:module});h.loader", true], + ["[ignore,...h.loader]=[{},module];h.loader[0]", true], + ["({...h.loader}={loader:module});h.loader.loader", true], + ["({exports:h.loader}=module);h.loader", false], + ["({...h.loader}=module);h.loader", false], + ["({x:ignore,...h.loader}={x:module});h.loader.x", false], + ["({x:h.loader=module}={x:{}});h.loader", false], + ["h.loader=module;({x:h.loader}={x:{}});h.loader", false], + ["({a:h.loader,b:h.loader}={a:module,b:{}});h.loader", false], + ["({a:h.loader,b:h.loader}={a:{},b:module});h.loader", true], + ["({x:h.a,y:h.b}={x:{},y:module});h.a", false], + ["({x:h.a,y:h.b}={x:{},y:module});h.b", true], +])("preserves selected assigned module sources: %s", (assignment, expected) => { + const analysis = analyzeJavaScriptStaticSource( + `const h={};${assignment}.require('./shared.cjs');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); +}); + +it("retains a destructured member delegate call", () => { + const analysis = analyzeJavaScriptStaticSource( + `const h={};({load:h.load}={load:module.require});h.load.call(module,'./shared.cjs');`, + ); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier: null }), + ]); +}); + +it("revokes the host delegate after a destructured module receiver write", () => { + const analysis = analyzeJavaScriptStaticSource( + `const h={};({x:h.loader}={x:module});h.loader.require=replacement;require('./shared.cjs');`, + ); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier: null }), + ]); +}); diff --git a/src/domain/javascript/javascriptStaticAnalysisAssignmentLoaders.test.ts b/src/domain/javascript/javascriptStaticAnalysisAssignmentLoaders.test.ts new file mode 100644 index 0000000000..95435f7ac0 --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisAssignmentLoaders.test.ts @@ -0,0 +1,88 @@ +import { Module } from "node:module"; +import * as path from "node:path"; +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +const cases = [ + { + name: "assigned host receiver", + source: "let holder;(holder=module).require('node:path');", + outcomes: ["host"], + possible: true, + }, + { + name: "assigned local receiver", + source: + "let holder;(holder={require:()=>({kind:'local'})}).require('node:path');", + outcomes: ["local"], + possible: false, + }, + { + name: "assigned delegate", + source: "let load;(load=module.require).call(module,'node:path');", + outcomes: ["host"], + possible: true, + }, + { + name: "conditional assignment value", + source: + "let holder;(holder=flag?module:{require:()=>({kind:'local'})}).require('node:path');", + outcomes: ["local", "host"], + possible: true, + }, + { + name: "logical or assignment", + source: + "let holder=flag?{require:()=>({kind:'local'})}:null;(holder||=module).require('node:path');", + outcomes: ["host", "local"], + possible: true, + }, + { + name: "logical and assignment", + source: "let holder=flag?{}:null;(holder&&=module).require('node:path');", + outcomes: ["throw", "host"], + possible: true, + }, + { + name: "nullish assignment", + source: + "let holder=flag?{require:()=>({kind:'local'})}:null;(holder??=module).require('node:path');", + outcomes: ["host", "local"], + possible: true, + }, + { + name: "logical assignment of local receivers", + source: + "let holder=flag?{require:()=>({kind:'local'})}:null;(holder||={require:()=>({kind:'local'})}).require('node:path');", + outcomes: ["local", "local"], + possible: false, + }, +]; + +it.each(cases)( + "retains the expression result for $name", + ({ source, outcomes, possible }) => { + for (const [index, expected] of outcomes.entries()) { + const execute = () => + runInNewContext(source, { + module: new Module("assignment-loader.cjs"), + flag: index === 1, + }); + if (expected === "throw") expect(execute).toThrow(); + else { + const result = execute(); + expect(result.join === path.join).toBe(expected === "host"); + if (expected === "local") expect(result.kind).toBe("local"); + } + } + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + possible + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); diff --git a/src/domain/javascript/javascriptStaticAnalysisCallableReceivers.test.ts b/src/domain/javascript/javascriptStaticAnalysisCallableReceivers.test.ts new file mode 100644 index 0000000000..25d8f3a8bc --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisCallableReceivers.test.ts @@ -0,0 +1,99 @@ +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +const generic = "const get=()=>({});const invoke=get.call;"; +const cases = [ + { + name: "plain object receiver", + source: `${generic}const holder={invoke,other:()=>module};holder.invoke(null)`, + callable: false, + }, + { + name: "sequence object receiver", + source: `${generic}const holder={invoke,other:()=>module};(0,holder).invoke(null)`, + callable: false, + }, + { + name: "spread object receiver", + source: `${generic}({...{invoke,other:()=>module}}).invoke(null)`, + callable: false, + }, + { + name: "array receiver", + source: `${generic}const holder=[invoke,()=>module];holder[0](null)`, + callable: false, + }, + { + name: "explicit object receiver", + source: `${generic}invoke.call({other:()=>module},null)`, + callable: false, + }, + { + name: "conditional object receiver", + source: `${generic}const holder=flag?{invoke,other:()=>module}:{invoke,other:()=>module};holder.invoke(null)`, + callable: false, + }, + { + name: "direct object callee", + source: "const holder={other:()=>module};holder(null)", + callable: false, + }, + { + name: "direct array callee", + source: "const holder=[()=>module];holder(null)", + callable: false, + }, + { + name: "callable receiver with an own invocation alias", + source: `${generic}const holder=()=>module;holder.invoke=invoke;holder.other=()=>({});holder.invoke(null)`, + callable: true, + }, + { + name: "explicit callable member receiver", + source: `${generic}const holder={invoke,other:()=>module};holder.invoke.call(holder.other,null)`, + callable: true, + }, + { + name: "selected object callable", + source: `${generic}const holder={other:()=>module};invoke.call(holder.other,null)`, + callable: true, + }, + { + name: "selected array callable", + source: `${generic}const holder=[()=>module];invoke.call(holder[0],null)`, + callable: true, + }, + { + name: "selected spread callable", + source: `${generic}const holder={...{other:()=>module}};invoke.call(holder.other,null)`, + callable: true, + }, +]; + +it.each(cases)( + "distinguishes the callable value from children for $name", + ({ source, callable }) => { + const invocation = `${source}.require('./shared.cjs');`; + expect( + runInNewContext( + `try{${invocation}}catch(error){error.name;}`, + { module: { require: () => "host" }, flag: true }, + { timeout: 1_000 }, + ), + ).toBe(callable ? "host" : "TypeError"); + const analysis = analyzeJavaScriptStaticSource(invocation); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + callable + ? [ + expect.objectContaining({ + kind: "require", + specifier: null, + }), + ] + : [], + ); + }, +); diff --git a/src/domain/javascript/javascriptStaticAnalysisCalleeBoundary.test.ts b/src/domain/javascript/javascriptStaticAnalysisCalleeBoundary.test.ts index dde8b84dbe..38c1859043 100644 --- a/src/domain/javascript/javascriptStaticAnalysisCalleeBoundary.test.ts +++ b/src/domain/javascript/javascriptStaticAnalysisCalleeBoundary.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; +import { aliasSelectionBudgetSource } from "../../../tests/fixtures/javascriptAliasSelectionBudget.js"; const expressionWrappers = [ ["plain", (value: string) => value], @@ -9,6 +10,207 @@ const expressionWrappers = [ ["satisfies", (value: string) => `(${value} satisfies any)`], ] as const; +it.each([ + ["module.require = replacement;", true], + [ + "const holder = { loader: module }; holder.loader.require = replacement;", + true, + ], + [ + "const holder = {}; holder.loader = module; holder.loader.require = replacement;", + true, + ], + ["const holder = [module]; holder[0].require = replacement;", true], + ["Object.defineProperty(module, 'require', { value: replacement });", true], + ["const api = module; Object.assign(api, { require: replacement });", true], + ["const module = { require() {} }; module.require = replacement;", false], + ["const api = module.exports; api.require = replacement;", false], + [ + "const holder = { a: module }; const target = flag ? holder.a : holder; target.require = replacement;", + true, + ], + [ + "const table = { x: {}, y: {}, z: module }; const target = flag ? table.x : table.y; target.require = replacement;", + false, + ], +])("keeps CommonJS delegate replacement unknown: %s", (write, changed) => { + const analysis = analyzeJavaScriptStaticSource( + `${write} require("./shared.cjs");`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual([ + expect.objectContaining({ + kind: "require", + specifier: changed ? null : "./shared.cjs", + }), + ]); +}); + +it("keeps a require capture before a later delegate replacement", () => { + const analysis = analyzeJavaScriptStaticSource( + 'require("./shared.cjs"); const holder = { loader: module }; holder.loader.require = replacement;', + ); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier: "./shared.cjs" }), + ]); +}); + +it.each([ + ["const holder = {}; holder.loader = { require() {} }; holder.loader", false], + [ + "const holder = { loader: module }; holder.loader = {}; holder.loader", + false, + ], + [ + "const holder = {}; if (flag) holder.loader = { require() {} }; holder.loader", + false, + ], + ["const holder = { loader: module }; holder.loader", true], + ["const holder = { inner: { loader: module } }; holder.inner.loader", true], + ["const holder = [module]; holder[0]", true], + [ + "const holder = { loader: module }; const { loader } = holder; loader", + true, + ], + [ + "const holder = { loader: module }; const loader = holder.loader; loader", + true, + ], + [ + "const holder = { loader: module }; const alias = { ...holder }; alias.loader", + true, + ], + ["const holder = { loader: module, loader: {} }; holder.loader", false], + ["const holder = { loader: {} }; holder.loader", false], + ["const holder = { loader: module.exports }; holder.loader", false], + ["const holder = { loader: module }; holder.missing", false], + [ + "const table = { a: { b: {}, d: module }, c: { d: {}, b: {} } }; const selected = flag ? table.a.b : table.c.d; selected", + false, + ], + ["let holder = {}; while (flag) holder = holder.slot; holder", true], + ["const holder = {}; holder.loader = module; holder.loader", true], + ["const holder = {}; if (flag) holder.loader = module; holder.loader", true], + [ + "const holder = {}; const alias = holder; alias.loader = module; holder.loader", + true, + ], +])( + "tracks the selected module-object container slot: %s", + (receiver, expected) => { + const analysis = analyzeJavaScriptStaticSource( + `${receiver}.require("./shared.cjs");`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); + +it("bounds branching selector work and reports unknown ownership explicitly", () => { + const analysis = analyzeJavaScriptStaticSource(aliasSelectionBudgetSource()); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.parse_status).toBe("partial"); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier: null }), + ]); + expect(analysis.limitations).toContainEqual( + expect.stringMatching(/Alias selection expansion.*512/u), + ); +}); + +it.each([ + ['const load = module.require.bind(module); load("./shared.cjs");', true], + [ + 'const api = module; const load = api.require.bind(api); load("./shared.cjs");', + true, + ], + [ + 'const load = module["require"][`bind`](module); load("./shared.cjs");', + true, + ], + ['module.require.call(module, "./shared.cjs");', true], + ['module.require.apply(module, ["./shared.cjs"]);', true], + [ + 'const load = module.require.bind.call(module.require, module); load("./shared.cjs");', + true, + ], + [ + 'const module = { require() {} }; const load = module.require.bind(module); load("./shared.cjs");', + false, + ], + [ + 'const api = module.exports; const load = api.require.bind(api); load("./shared.cjs");', + false, + ], +])( + "retains unchecked module delegate invocation and capture: %s", + (source, possibleModule) => { + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + possibleModule + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); + +it.each([ + ["const { api = module } = options;", true], + ["const { api = module } = {};", true], + ["const { api = module } = { api: {} };", false], + ["const { api = module.exports } = options;", false], + ["const { ...api } = module;", false], +])( + "tracks active direct module defaults: %s", + (declaration, possibleModule) => { + const analysis = analyzeJavaScriptStaticSource( + `${declaration} api.require("./shared.cjs");`, + ); + expect(analysis.references).toEqual( + possibleModule + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); + +it.each([ + "const { exports: api } = module;", + "const { exports: { api } } = module;", + "const [api] = [module.exports];", +])( + "keeps projected module members separate from the loader: %s", + (declaration) => { + const analysis = analyzeJavaScriptStaticSource(`${declaration} + api.require = () => {}; api.require("./shared.cjs");`); + expect(analysis.references).toEqual([]); + }, +); + +it.each(["module", "local"])( + "examines a repeated conditional alias graph once per source read for %s", + (origin) => { + const source = [`const a0 = ${origin === "module" ? "module" : "{}"};`]; + for (let index = 1; index <= 128; index += 1) + source.push( + `let a${index} = a${index - 1}; if (flag) a${index} = a${index - 1};`, + ); + source.push('a128.require("./shared.cjs");'); + const analysis = analyzeJavaScriptStaticSource(source.join("\n")); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + origin === "module" + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); + it("shares erased call-target names with endpoint consumers without inventing dynamic keys", () => { const analysis = analyzeJavaScriptStaticSource(` (client.get as typeof client.get)("/api/items"); diff --git a/src/domain/javascript/javascriptStaticAnalysisCalls.ts b/src/domain/javascript/javascriptStaticAnalysisCalls.ts index 3655e71bb0..8abf7a9354 100644 --- a/src/domain/javascript/javascriptStaticAnalysisCalls.ts +++ b/src/domain/javascript/javascriptStaticAnalysisCalls.ts @@ -50,8 +50,9 @@ import { import { semanticTopLevelEffectFollowsCapture } from "./javascriptSemanticMutationInitializers.js"; import { semanticFactoryLoaderMemberAt, - semanticFactoryRequireLoaderAt, - semanticRequireLoaderAt, + semanticRequireCallAt, + semanticModuleRequireCallAt, + semanticModuleRequireReceiver, } from "./javascriptSemanticRequireOrigin.js"; type JavaScriptCallLike = @@ -69,7 +70,13 @@ export const inspectCall = ( const { accumulator } = context; const name = calleeName(node.callee); const first = stringValue(node.arguments[0]); - const requireCall = requireCallSpecifier(node, bindings, context); + const deferredModuleRequire = + semanticModuleRequireReceiver(node) !== undefined || + semanticModuleRequireCallAt(node, bindings, false); + if (deferredModuleRequire) accumulator.pendingModuleRequireCalls.push(node); + const requireCall = deferredModuleRequire + ? null + : requireCallSpecifier(node, bindings, context); if (requireCall !== null) addReference(context, { node, @@ -249,43 +256,37 @@ const requireCallSpecifier = ( bindings: JavaScriptSemanticAnalysisState, context: FindingContext, ): { readonly specifier: string | undefined } | null => { - const bareRequire = t.isIdentifier(node.callee, { name: "require" }); - if (bareRequire) { - const loader = semanticRequireLoaderAt( - node, - bindings, - context.accumulator.factoryRequireBindingIds, - ); - if (loader === "local") return null; - if (loader === "unknown") context.accumulator.unknownFindings += 1; - return { - specifier: - loader === "unknown" ? undefined : argumentValue(node.arguments[0]), - }; - } - const factoryLoader = semanticFactoryRequireLoaderAt( + const call = semanticRequireCallAt( node, bindings, context.accumulator.factoryRequireBindingIds, ); - if (factoryLoader === "factory-loader") - return { - specifier: argumentValue( - node.arguments[ - t.isMemberExpression(unwrapJavaScriptExpression(node.callee).node) || - t.isOptionalMemberExpression( - unwrapJavaScriptExpression(node.callee).node, - ) - ? 1 - : 0 - ], - ), - }; - if (factoryLoader === "unknown") { + if (call === undefined || call.loader === "local") return null; + if (call.loader === "unknown" || !call.sourceRelative) { context.accumulator.unknownFindings += 1; return { specifier: undefined }; } - return null; + return { specifier: argumentValue(call.arguments?.[0]) }; +}; + +/** Examine member-loader receivers only after shared alias mutation facts settle. */ +export const finishModuleRequireCalls = ( + context: FindingContext, + bindings: JavaScriptSemanticAnalysisState, +): void => { + for (const node of context.accumulator.pendingModuleRequireCalls) { + const call = requireCallSpecifier(node, bindings, context); + if (call !== null) + addReference(context, { + node, + kind: "require", + specifier: call.specifier, + }); + } + context.accumulator.pendingModuleRequireCalls.length = 0; + context.accumulator.references.sort( + (left, right) => left.offset - right.offset, + ); }; const chunkLoadSpecifier = ( @@ -760,7 +761,7 @@ export const inspectRoleProperty = ( ): void => { if (semanticStaticPropertyName(node.key, node.computed) !== "preload") return; if (context.accumulator.mediaElementPreloads.has(node)) return; - const path = staticPath(node.value); + const path = staticPath(node.value, context.nodePathResolve); if (path === undefined) return; addLocatedFinding(context, { collection: context.accumulator.roles, @@ -769,7 +770,10 @@ export const inspectRoleProperty = ( value: { role: "preload", path, - resolution_context: staticPathResolutionContext(node.value), + resolution_context: staticPathResolutionContext( + node.value, + context.nodePathResolve, + ), mechanism: "property:preload", module_key: null, location: range(node), diff --git a/src/domain/javascript/javascriptStaticAnalysisCapturedGeneric.test.ts b/src/domain/javascript/javascriptStaticAnalysisCapturedGeneric.test.ts new file mode 100644 index 0000000000..b9f67c7c0c --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisCapturedGeneric.test.ts @@ -0,0 +1,98 @@ +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; +import * as t from "@babel/types"; + +import { capturedGenericInvocations } from "../../../tests/fixtures/javascriptCapturedGenericInvocations.js"; +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; +import { parseJavaScriptSource } from "./javascriptSourceParser.js"; +import { indexJavaScriptBindingsWithReceiversSteps } from "./javascriptSemanticAnalysis.js"; +import { completeSemanticSteps } from "./javascriptSemanticTraversal.js"; +import { createSemanticReturnedReferences } from "./javascriptSemanticReturnedReferences.js"; + +it.each(capturedGenericInvocations)( + "retains the authored runtime receiver for $name", + ({ source, host, expression }) => { + expect( + runInNewContext( + source, + { + module: { require: () => "host" }, + }, + { timeout: 1_000 }, + ), + ).toBe(host ? "host" : "local"); + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + host + ? [ + expect.objectContaining({ + kind: "require", + specifier: null, + expression, + }), + ] + : [], + ); + }, +); + +it.each([ + "const table={x:()=>({}),y:()=>({}),z:()=>module};const selected=flag?table.x:table.y;const get=()=>module;const invoke=get.call;invoke.call.call(selected,null).require('./unused.cjs');", + "const get=()=>({});const invoke=get.call;const other=function*(){return module;};invoke.call.call(other,null).require('./unused.cjs');", + "const get=()=>({});const invoke=get.call;const other=async()=>module;invoke.call.call(other,null).require('./unused.cjs');", +])("preserves unrelated, generator and promise receivers: %s", (source) => { + expect(analyzeJavaScriptStaticSource(source).references).toEqual([]); +}); + +it("does not reclassify an ordinary Electron construction as an unknown loader", () => { + const analysis = analyzeJavaScriptStaticSource( + "const {BrowserWindow}=require('electron');new BrowserWindow().loadFile('index.html');", + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier: "electron" }), + ]); +}); + +it.each( + [32, 550].flatMap((depth) => [true, false].map((host) => ({ depth, host }))), +)( + "retains generic receiver sources when invocation contexts converge at depth $depth (host=$host)", + ({ depth, host }) => { + const source = `const get=()=>({kind:'local'});const invoke=get.call;const other=()=>${host ? "module" : "({kind:'local'})"};invoke.call(${Array(depth).fill("invoke").join(",")},other,null);`; + expect( + runInNewContext(source, { module: { kind: "host" } }, { timeout: 1_000 }) + .kind, + ).toBe(host ? "host" : "local"); + const parsed = parseJavaScriptSource(source); + if (parsed === null) + throw new Error("Expected bounded generic invocation fixture"); + const index = completeSemanticSteps( + indexJavaScriptBindingsWithReceiversSteps(parsed.program), + ); + completeSemanticSteps(index.indexAmbientWritesSteps()); + completeSemanticSteps(index.indexMemberMutationFactsSteps()); + const statement = parsed.program.body.at(-1); + if ( + !t.isExpressionStatement(statement) || + !t.isCallExpression(statement.expression) + ) + throw new Error("Expected final generic invocation"); + const call = statement.expression; + const returned = createSemanticReturnedReferences( + index.bindings, + "direct-call", + )(call.callee, call); + expect( + returned.references.some(({ node }) => + t.isIdentifier(node, { name: "module" }), + ), + ).toBe(host); + expect(returned.precision).toBe(depth > 512 ? "widened" : "exact"); + expect(index.bindings.valueResourceLimits.has("alias-selections")).toBe( + depth > 512, + ); + if (depth > 512) expect(returned.unresolvedInvocation).toBe(true); + }, +); diff --git a/src/domain/javascript/javascriptStaticAnalysisCapturedRequire.test.ts b/src/domain/javascript/javascriptStaticAnalysisCapturedRequire.test.ts new file mode 100644 index 0000000000..3065ba811b --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisCapturedRequire.test.ts @@ -0,0 +1,49 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it.each([ + ['const load = module.require; load.call(module, "../shared.cjs");', true], + ['const load = module.require; load.apply(module, ["../shared.cjs"]);', true], + ['const load = module.require; load("../shared.cjs");', true], + [ + 'const load = module.require; const alias = load; alias.call(module, "../shared.cjs");', + true, + ], + [ + 'const { require: load } = module; load.call(module, "../shared.cjs");', + true, + ], + [ + 'const holder = { load: module.require }; holder.load.call(module, "../shared.cjs");', + true, + ], + ['const holder = [module.require]; holder[0]("../shared.cjs");', true], + [ + 'let load = () => {}; if (flag) load = module.require; load("../shared.cjs");', + true, + ], + [ + 'let load = module.require; load = () => {}; load.call(module, "../shared.cjs");', + false, + ], + [ + 'const api = { require() {} }; const load = api.require; load.call(api, "../shared.cjs");', + false, + ], + [ + 'const load = module.exports.require; load.call(module.exports, "../shared.cjs");', + false, + ], +])( + "retains captured CommonJS delegates by source identity: %s", + (source, expected) => { + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); diff --git a/src/domain/javascript/javascriptStaticAnalysisCopiedRequire.test.ts b/src/domain/javascript/javascriptStaticAnalysisCopiedRequire.test.ts new file mode 100644 index 0000000000..e879964473 --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisCopiedRequire.test.ts @@ -0,0 +1,75 @@ +import { Module } from "node:module"; +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it("checks inherited and own enumerable require against a real Node Module", () => { + const host = new Module("copied-require-oracle.cjs"); + expect(Object.hasOwn(host, "require")).toBe(false); + expect(typeof host.require).toBe("function"); + const { ...rest } = host; + expect(Object.hasOwn(rest, "require")).toBe(false); + expect(Object.hasOwn({ ...host }, "require")).toBe(false); + + host.require = host.require; + expect(Object.getOwnPropertyDescriptor(host, "require")?.enumerable).toBe( + true, + ); + const { ...ownRest } = host; + expect(ownRest.require).toBe(host.require); + expect({ ...host }.require).toBe(host.require); +}); + +it("keeps copied child module identity without copying its prototype", () => { + const host = new Module("copied-child-oracle.cjs"); + const { ...copy } = { loader: host }; + expect(Object.hasOwn(copy, "loader")).toBe(true); + expect(copy.loader).toBe(host); + expect(Object.hasOwn(copy.loader, "require")).toBe(false); + expect(copy.loader.require).toBe(host.require); +}); + +it.each(["const {...h}=module;", "const h={...module};"])( + "keeps a copied inherited delegate absent inside a function: %s", + (declaration) => { + const result = analyzeJavaScriptStaticSource( + `function copy(){${declaration}h.require('./shared.cjs');}`, + ); + expect(result.references).toEqual([]); + }, +); + +it.each([ + ["const {...h}=module;h", false], + ["const holder={};({...holder.h}=module);holder.h", false], + ["const h={...module};h", false], + ["const source=module;const h={...source};h", false], + ["const source=module;const {...h}=source;h", false], + ["const h={...{loader:module}};h.loader", true], + ["const {...h}={loader:module};h.loader", true], + ["const h={...[module]};h[0]", true], + ["const source={require:module.require};const h={...source};h", true], + ["const source={require:module.require};const {...h}=source;h", true], + ["module.require=module.require;const h={...module};h", true], + ["module.require=module.require;const {...h}=module;h", true], + ["const h={...module};module.require=module.require;h", false], + ["const {...h}=module;module.require=module.require;h", false], + ["if(flag)module.require=module.require;const h={...module};h", true], + ["globalThis.module.require=unknown;const h={...module};h", false], + ["expose(module);const h={...module};h", true], + ["const h={...module};expose(module);h", false], + ["const module={require(){}};const h={...module};h", false], +])( + "selects only possibly own enumerable copied delegates: %s", + (receiver, expected) => { + const result = analyzeJavaScriptStaticSource( + `${receiver}.require('./shared.cjs');`, + ); + expect(result.parse_error_count).toBe(0); + expect(result.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); diff --git a/src/domain/javascript/javascriptStaticAnalysisDelegateEligibility.test.ts b/src/domain/javascript/javascriptStaticAnalysisDelegateEligibility.test.ts new file mode 100644 index 0000000000..e022014567 --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisDelegateEligibility.test.ts @@ -0,0 +1,60 @@ +import { Module } from "node:module"; +import * as path from "node:path"; +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it("keeps repeated Node resolver calls separate from loader delegate eligibility", () => { + const expression = "path.resolve(runtimePath,__dirname,'preload.js')"; + const source = ` + const path=require('node:path'); + const {BrowserWindow,utilityProcess}=require('electron'); + new BrowserWindow({webPreferences:{preload:${expression}}}); + utilityProcess.fork(${expression}); + `; + const observed: string[] = []; + runInNewContext(source, { + runtimePath: "/runtime", + __dirname: "/fixture/app", + require: (specifier: string) => + specifier === "node:path" + ? path.posix + : { + BrowserWindow: class { + constructor(options: { webPreferences: { preload: string } }) { + observed.push(options.webPreferences.preload); + } + }, + utilityProcess: { fork: (file: string) => observed.push(file) }, + }, + }); + expect(observed).toEqual([ + "/fixture/app/preload.js", + "/fixture/app/preload.js", + ]); + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_status).toBe("complete"); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier: "node:path" }), + expect.objectContaining({ kind: "require", specifier: "electron" }), + ]); + expect(analysis.electron.browser_windows[0]?.preload_path).toBe("preload.js"); + expect(analysis.electron.utility_processes[0]?.module_path).toBe( + "preload.js", + ); +}); + +it("retains an escaped selected native delegate as a possible loader", () => { + const source = + "const box={load:module.require};opaque(box);box.load.call(module,'node:path');"; + const module = new Module("delegate-eligibility.cjs"); + expect(runInNewContext(source, { module, opaque: () => {} }).join).toBe( + path.join, + ); + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier: null }), + ]); +}); diff --git a/src/domain/javascript/javascriptStaticAnalysisFactoryAlternatives.test.ts b/src/domain/javascript/javascriptStaticAnalysisFactoryAlternatives.test.ts new file mode 100644 index 0000000000..dabe4c424b --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisFactoryAlternatives.test.ts @@ -0,0 +1,50 @@ +import { createRequire } from "node:module"; +import { dirname, resolve } from "node:path"; +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it("observes an authored factory returning a loader from another directory", () => { + const source = resolve("factory-oracle-source/main.cjs"); + const alternate = resolve("factory-oracle-other/main.cjs"); + for (const flag of [false, true]) { + const actual = runInNewContext( + "const make=flag?native:getFactory();const load=make(source);load.resolve.paths('./window.cjs')[0];", + { + native: createRequire, + getFactory: () => () => createRequire(alternate), + source, + flag, + }, + ); + expect(actual).toBe(dirname(flag ? source : alternate)); + } +}); + +it.each([ + ["conditional opaque factory", "flag?native:getFactory()", null], + ["logical opaque factory", "native||getFactory()", null], + ["optional opaque factory", "flag?native:getFactory?.()", null], + ["constructed opaque factory", "flag?native:new Factory()", null], + ["opaque member factory", "flag?native:getContainer().make", null], + ["unbound member factory", "flag?native:holder.make", null], + ["known native choices", "flag?native:native", "./window.cjs"], + ["only opaque factory", "getFactory()", undefined], + ["only local factory", "()=>()=>({})", undefined], +] as const)( + "retains every selected origin for %s", + (_name, choice, specifier) => { + const analysis = analyzeJavaScriptStaticSource( + `import{createRequire as native}from'node:module';const make=${choice};const load=make(import.meta.url);load('./window.cjs');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect( + analysis.references.filter(({ kind }) => kind === "require"), + ).toEqual( + specifier === undefined + ? [] + : [expect.objectContaining({ kind: "require", specifier })], + ); + }, +); diff --git a/src/domain/javascript/javascriptStaticAnalysisHelpers.ts b/src/domain/javascript/javascriptStaticAnalysisHelpers.ts index d1ee8b4cdb..8a5b2fab17 100644 --- a/src/domain/javascript/javascriptStaticAnalysisHelpers.ts +++ b/src/domain/javascript/javascriptStaticAnalysisHelpers.ts @@ -171,22 +171,31 @@ export const staticArrayValues = ( }; /** Classify whether inert path syntax is a module specifier or file expression. */ +export type JavaScriptStaticPathIntrinsicQuery = ( + node: t.Node, + fact?: "source-directory", +) => boolean; + export const staticPathResolutionContext = ( node: t.Node, + nodePathResolve?: JavaScriptStaticPathIntrinsicQuery, ): JavaScriptStaticPathContext => - isFilesystemPathExpression(node) + isFilesystemPathExpression(node, nodePathResolve) ? "filesystem-expression" : "module-specifier"; -/** Resolve a path composed only from inert literal syntax. */ -export const staticPath = (node: t.Node): string | undefined => { +/** Resolve a path string composed only from inert literal syntax. */ +export const staticPath = ( + node: t.Node, + nodePathResolve?: JavaScriptStaticPathIntrinsicQuery, +): string | undefined => { node = unwrapJavaScriptExpression(node).node; const value = stringValue(node); if (value !== undefined) return value; if (t.isBinaryExpression(node, { operator: "+" })) - return staticConcatenation(node); + return staticConcatenation(node, nodePathResolve); if (t.isCallExpression(node) || t.isNewExpression(node)) - return staticCallPath(node); + return staticCallPath(node, nodePathResolve); return undefined; }; @@ -195,18 +204,21 @@ export const staticPath = (node: t.Node): string | undefined => { * `__dirname + "/preload.js"` names a file inside it, while text without a * leading separator extends the directory's own name and stays unknown. */ -const staticConcatenation = (node: t.BinaryExpression): string | undefined => { +const staticConcatenation = ( + node: t.BinaryExpression, + nodePathResolve?: JavaScriptStaticPathIntrinsicQuery, +): string | undefined => { const operands: t.Node[] = []; let current: t.Node = node; while (t.isBinaryExpression(current, { operator: "+" })) { operands.push(current.right); current = unwrapJavaScriptExpression(current.left).node; } - const anchored = isDirectoryPath(current); + const anchored = isDirectoryPath(current, nodePathResolve); if (!anchored) operands.push(current); let text = ""; for (const operand of operands.reverse()) { - const value = staticPath(operand); + const value = staticPath(operand, nodePathResolve); if (value === undefined) return undefined; text += value; } @@ -216,29 +228,27 @@ const staticConcatenation = (node: t.BinaryExpression): string | undefined => { const staticCallPath = ( node: t.CallExpression | t.NewExpression, + nodePathResolve?: JavaScriptStaticPathIntrinsicQuery, ): string | undefined => { + if (t.isCallExpression(node) && nodePathResolve?.(node.callee)) + return staticResolvedPath(node, nodePathResolve); const name = calleeName(node.callee); - if (name === "URL" || name.endsWith(".URL")) - return isFileIdentity(argumentNode(node.arguments[1])) - ? stringValue(node.arguments[0]) - : undefined; if (name === "fileURLToPath" || name.endsWith(".fileURLToPath")) { - return staticFileUrlPath(node); + return staticFileUrlPath(node, nodePathResolve); } - if (name.endsWith(".resolve")) - return name === "win32.resolve" || name.endsWith(".win32.resolve") - ? undefined - : staticResolvedPath(node); + if (name.endsWith(".resolve")) return undefined; if (!name.endsWith(".join") && name !== "join") return undefined; const parts: string[] = []; for (const argument of node.arguments) { - if (isDirectoryIdentity(argumentNode(argument))) { + if (isDirectoryPath(argumentNode(argument), nodePathResolve)) { // join never resets at an absolute segment, so `/preload.js` after the // source directory stays inside it. parts.push("."); continue; } - const value = t.isNode(argument) ? staticPath(argument) : undefined; + const value = t.isNode(argument) + ? staticPath(argument, nodePathResolve) + : undefined; if (value === undefined) return undefined; parts.push(value); } @@ -247,13 +257,14 @@ const staticCallPath = ( const staticFileUrlPath = ( node: t.CallExpression | t.NewExpression, + nodePathResolve?: JavaScriptStaticPathIntrinsicQuery, ): string | undefined => { if (node.arguments.length !== 1) return undefined; const rawArgument = argumentNode(node.arguments[0]); if (rawArgument === undefined) return undefined; const argument = unwrapJavaScriptExpression(rawArgument).node; if (isSourceFileUrl(argument)) return sourceRelativeFileUrlPath(argument); - const value = staticPath(argument); + const value = staticPath(argument, nodePathResolve); const rawPath = value?.split(/[?#]/u, 1)[0] ?? ""; if ( value === undefined || @@ -324,82 +335,86 @@ const isSourceFileUrl = ( const staticResolvedPath = ( node: t.CallExpression | t.NewExpression, + nodePathResolve?: JavaScriptStaticPathIntrinsicQuery, ): string | undefined => { - const parts: (string | null)[] = []; - for (const argument of node.arguments) { - if (isDirectoryIdentity(argumentNode(argument))) { - parts.push(null); - continue; + const parts: string[] = []; + let anchored = false; + // Node walks right-to-left and never inspects arguments before its anchor. + for (let index = node.arguments.length - 1; index >= 0; index -= 1) { + const argument = node.arguments[index]; + if (isDirectoryPath(argumentNode(argument), nodePathResolve)) { + anchored = true; + break; } - const value = t.isNode(argument) ? staticPath(argument) : undefined; - // Validate every argument, including those before the last anchor. + const value = t.isNode(argument) + ? staticPath(argument, nodePathResolve) + : undefined; if (value === undefined) return undefined; parts.push(value); + if (posix.isAbsolute(value)) { + anchored = true; + break; + } } - const anchor = parts.findLastIndex( - (part) => part === null || posix.isAbsolute(part), - ); // Relative-only calls depend on the target process cwd, which is unknown. - if (anchor === -1) return undefined; - // A null anchor is the source directory; preserve its relative projection. - const path = posix.join( - ...parts.slice(anchor).filter((part) => part !== null), - ); + if (!anchored) return undefined; + const path = posix.join(...parts.reverse()); return path === "/" ? path : path.replace(/\/$/u, ""); }; -const isFilesystemPathExpression = (node: t.Node): boolean => { +const isFilesystemPathExpression = ( + node: t.Node, + nodePathResolve?: JavaScriptStaticPathIntrinsicQuery, +): boolean => { node = unwrapJavaScriptExpression(node).node; const value = stringValue(node); if (value !== undefined) return value.startsWith("/"); - if (isFileIdentity(node) || isDirectoryIdentity(node)) return true; + if (isFileIdentity(node) || isDirectoryPath(node, nodePathResolve)) + return true; if (t.isBinaryExpression(node, { operator: "+" })) return ( - isFilesystemPathExpression(node.left) || - isFilesystemPathExpression(node.right) + isFilesystemPathExpression(node.left, nodePathResolve) || + isFilesystemPathExpression(node.right, nodePathResolve) ); if (t.isCallExpression(node) || t.isNewExpression(node)) { const name = calleeName(node.callee); - if (name.endsWith(".resolve") && staticCallPath(node) !== undefined) + if ( + nodePathResolve?.(node.callee) && + staticCallPath(node, nodePathResolve) !== undefined + ) return true; - if (name === "URL" || name.endsWith(".URL")) - return isFileIdentity(argumentNode(node.arguments[1])); + // A URL object is not a path string until fileURLToPath converts it. + if (name === "URL" || name.endsWith(".URL")) return false; if (name === "fileURLToPath" || name.endsWith(".fileURLToPath")) return ( - staticFileUrlPath(node) !== undefined || + staticFileUrlPath(node, nodePathResolve) !== undefined || node.arguments.some((argument) => { const value = argumentNode(argument); - return value !== undefined && isFilesystemPathExpression(value); + return ( + value !== undefined && + isFilesystemPathExpression(value, nodePathResolve) + ); }) ); return node.arguments.some((argument) => { const value = argumentNode(argument); - return value !== undefined && isFilesystemPathExpression(value); + return ( + value !== undefined && + isFilesystemPathExpression(value, nodePathResolve) + ); }); } return false; }; -const isDirectoryIdentity = (node: t.Node | undefined): boolean => { - if (node !== undefined) node = unwrapJavaScriptExpression(node).node; - if (isDirectoryPath(node)) return true; - if (!t.isCallExpression(node) && !t.isNewExpression(node)) return false; - const name = calleeName(node.callee); - return ( - (name === "URL" || name.endsWith(".URL")) && - isFileIdentity(argumentNode(node.arguments[1])) - ); -}; - /** `__dirname` or the dirname of the source file, as a string. */ -const isDirectoryPath = (node: t.Node | undefined): boolean => { +const isDirectoryPath = ( + node: t.Node | undefined, + nodePathResolve?: JavaScriptStaticPathIntrinsicQuery, +): boolean => { if (node !== undefined) node = unwrapJavaScriptExpression(node).node; - if (t.isIdentifier(node, { name: "__dirname" })) return true; - if (!t.isCallExpression(node)) return false; - const name = calleeName(node.callee); return ( - (name === "dirname" || name.endsWith(".dirname")) && - isFileIdentity(argumentNode(node.arguments[0])) + node !== undefined && nodePathResolve?.(node, "source-directory") === true ); }; diff --git a/src/domain/javascript/javascriptStaticAnalysisInvocationIteration.test.ts b/src/domain/javascript/javascriptStaticAnalysisInvocationIteration.test.ts new file mode 100644 index 0000000000..b09fa2a1b0 --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisInvocationIteration.test.ts @@ -0,0 +1,122 @@ +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +const changedIterator = + "Array.prototype[Symbol.iterator]=function*(){yield '../shared/window.cjs'};"; +const cases = [ + [ + "ordinary spread", + "", + "require(...['./inert.cjs'])", + "./inert.cjs", + "./inert.cjs", + ], + [ + "ordinary nested spread", + "", + "require(...[...['./inert.cjs']])", + "./inert.cjs", + "./inert.cjs", + ], + [ + "changed direct spread", + changedIterator, + "require(...['./inert.cjs'])", + "../shared/window.cjs", + null, + ], + [ + "changed nested spread", + changedIterator, + "require(...[...['./inert.cjs']])", + "../shared/window.cjs", + null, + ], + [ + "changed generic call spread", + changedIterator, + "require.call(null,...['./inert.cjs'])", + "../shared/window.cjs", + null, + ], + [ + "changed apply argument spread", + changedIterator, + "require.apply(null,[...['./inert.cjs']])", + "../shared/window.cjs", + null, + ], + [ + "changed reflected argument spread", + changedIterator, + "Reflect.apply(require,null,[...['./inert.cjs']])", + "../shared/window.cjs", + null, + ], + [ + "apply arraylike without spread", + changedIterator, + "require.apply(null,['./inert.cjs'])", + "./inert.cjs", + "./inert.cjs", + ], + [ + "reflected arraylike without spread", + changedIterator, + "Reflect.apply(require,null,['./inert.cjs'])", + "./inert.cjs", + "./inert.cjs", + ], + [ + "known argument before custom spread", + changedIterator, + "require('./inert.cjs',...[])", + "./inert.cjs", + "./inert.cjs", + ], + [ + "shadowed Array mutation", + "const Array={prototype:{}};Array.prototype[Symbol.iterator]=function*(){yield '../shared/window.cjs'};", + "require(...['./inert.cjs'])", + "./inert.cjs", + "./inert.cjs", + ], + [ + "ordinary captured generic spread", + "const get=()=>module;const invoke=get.call;", + "invoke.call(...[get,null]).require('./inert.cjs')", + "./inert.cjs", + null, + ], + [ + "changed captured generic spread receiver", + "const get=()=>({});const invoke=get.call;Array.prototype[Symbol.iterator]=function*(){yield ()=>module;yield null;};", + "invoke.call(...[get,null]).require('./inert.cjs')", + "./inert.cjs", + null, + ], +] as const; + +it.each(cases)( + "retains invocation iterator authority for %s", + (_name, setup, call, actual, specifier) => { + const source = `${setup}${call};`; + expect( + runInNewContext( + source, + { + require: (request: string) => request, + module: { require: (request: string) => request }, + }, + { timeout: 1_000 }, + ), + ).toBe(actual); + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier }), + ]); + }, +); diff --git a/src/domain/javascript/javascriptStaticAnalysisInvocationReadSite.test.ts b/src/domain/javascript/javascriptStaticAnalysisInvocationReadSite.test.ts new file mode 100644 index 0000000000..abbf4637b7 --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisInvocationReadSite.test.ts @@ -0,0 +1,56 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it.each([ + "handlers[key]()", + "handlers[key].call(null)", + "handlers[key].apply(null,[])", +])("selects a local table callee before its own opaque effects: %s", (call) => { + const result = analyzeJavaScriptStaticSource( + `const handlers={first(){},second(){}};${call};`, + ); + expect(result.references).toEqual([]); + expect(result.parse_status).toBe("complete"); +}); + +it.each([ + 'handlers[key]=module.require;handlers[key].call(module,"./shared.cjs");', + 'handlers.first=module.require;handlers.first.call(module,"./shared.cjs");', +])("retains earlier uncertainty or a real delegate: %s", (use) => { + const result = analyzeJavaScriptStaticSource( + `const handlers={first(){},second(){}};${use}`, + ); + expect(result.references).toContainEqual( + expect.objectContaining({ kind: "require", specifier: null }), + ); + expect(result.parse_status).toBe("partial"); +}); + +it("retains complete coverage for an inline Electron factory export", () => { + const result = analyzeJavaScriptStaticSource(`(() => { + const modules={ + 5(module,exports){ + exports.self=exports; + exports.events=require("electron").ipcMain; + }, + 9(module,exports,load){ + const helper=load(5); + helper.self.events.addListener("test:copy",()=>{}); + } + }; + function load(id){ + const m={exports:{}}; + modules[id].call(m.exports,m,m.exports,load); + return m.exports; + } + load(9); + })();`); + expect(result.parse_status).toBe("complete"); + expect(result.references.every(({ specifier }) => specifier !== null)).toBe( + true, + ); + expect(result.electron.ipc).toContainEqual( + expect.objectContaining({ channel: "test:copy", side: "main" }), + ); +}); diff --git a/src/domain/javascript/javascriptStaticAnalysisLoaderInvocations.test.ts b/src/domain/javascript/javascriptStaticAnalysisLoaderInvocations.test.ts new file mode 100644 index 0000000000..8841b2a1c9 --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisLoaderInvocations.test.ts @@ -0,0 +1,290 @@ +import { Module, createRequire } from "node:module"; +import * as path from "node:path"; +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; +import { parseJavaScriptSource } from "./javascriptSourceParser.js"; +import { unshadowedReferenceSourceRequireCalls } from "../referenceSourceRequireBindings.js"; + +it.each(["require", "require.resolve", "require.main"])( + "distinguishes invoked loading from reference association for %s", + (callee) => { + const source = `${callee}('node:path');`; + const execute = () => + runInNewContext(source, { require: createRequire(import.meta.url) }); + if (callee === "require") expect(execute().join).toBe(path.join); + else if (callee === "require.resolve") expect(execute()).toBe("node:path"); + else expect(execute).toThrow(); + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + callee === "require" + ? [expect.objectContaining({ kind: "require", specifier: "node:path" })] + : [], + ); + const parsed = parseJavaScriptSource(source); + if (parsed === null) throw new Error("Expected loader association fixture"); + expect( + unshadowedReferenceSourceRequireCalls(parsed, "entry.cjs").size, + ).toBe(1); + }, +); + +const imported = "import {createRequire as make} from 'node:module';"; +it.each([ + [ + "proven factory", + `${imported}const require=make(import.meta.url);require('node:path');`, + 1, + "node:path", + ], + [ + "opaque replacement factory", + "import*as mod from'node:module';mod.default.createRequire=replacement;const require=mod.default.createRequire(import.meta.url);require('node:path');", + 0, + null, + ], +] as const)( + "uses invocation proof for external source association after a %s", + (_name, source, associated, specifier) => { + const parsed = parseJavaScriptSource(source); + if (parsed === null) throw new Error("Expected factory proof fixture"); + expect( + unshadowedReferenceSourceRequireCalls(parsed, "entry.mjs").size, + ).toBe(associated); + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect( + analysis.references.filter(({ kind }) => kind === "require"), + ).toEqual([expect.objectContaining({ kind: "require", specifier })]); + }, +); + +it.each([ + [ + "captured wrapper", + "const load=require;load('node:path')", + "node:path", + true, + ], + [ + "wrapper member", + "const box={load:require};box.load('node:path')", + "node:path", + true, + ], + [ + "wrapper array", + "const box=[require];box[0]('node:path')", + "node:path", + true, + ], + [ + "projected wrapper", + "const {load}={load:require};load('node:path')", + "node:path", + true, + ], + [ + "captured before wrapper replacement", + "const load=require;require=()=>({});load('node:path')", + "node:path", + true, + ], + [ + "shadowed wrapper", + "const require=()=>({});const load=require;load('node:path')", + undefined, + false, + ], + [ + "delegate replaced after capture", + "const load=require;module.require=()=>({});load('node:path')", + null, + false, + ], +] as const)( + "retains the selected native %s value", + (_name, source, specifier, host) => { + const module = new Module("wrapper-alias.cjs"); + const result = runInNewContext(source, { + module, + require: (request: string) => module.require(request), + }); + expect(result.join === path.join).toBe(host); + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + specifier === undefined + ? [] + : [expect.objectContaining({ kind: "require", specifier })], + ); + }, +); + +it.each([ + [ + "renamed loader", + "const load=make(import.meta.url);load('./shared.cjs')", + "./shared.cjs", + ], + [ + "captured renamed loader", + "const load=make(import.meta.url);const saved=load;saved('./shared.cjs')", + "./shared.cjs", + ], + [ + "object member loader", + "const box={load:make(import.meta.url)};box.load('./shared.cjs')", + "./shared.cjs", + ], + [ + "destructured loader", + "const {load}={load:make(import.meta.url)};load('./shared.cjs')", + "./shared.cjs", + ], + [ + "array member loader", + "const box=[make(import.meta.url)];box[0]('./shared.cjs')", + "./shared.cjs", + ], + [ + "generic loader invocation", + "const load=make(import.meta.url);load.call(null,'./shared.cjs')", + "./shared.cjs", + ], + [ + "reflected created loader", + "const load=make(import.meta.url);Reflect.apply(load,null,['./shared.cjs'])", + "./shared.cjs", + ], + [ + "unknown create base", + "const load=make('/elsewhere/entry.cjs');load('./shared.cjs')", + null, + ], + [ + "conditional create bases", + "const load=flag?make(import.meta.url):make('/elsewhere/entry.cjs');load('./shared.cjs')", + null, + ], + [ + "opaque loader replacement", + "let load=make(import.meta.url);load=replacement;load('./shared.cjs')", + null, + ], + [ + "copied child with a later leaf replacement", + "const box={child:{load:make(import.meta.url)}};const copy={...box};box.child.load=make('/elsewhere/entry.cjs');copy.child.load('./shared.cjs')", + null, + ], + [ + "copied child before a parent replacement", + "const box={child:{load:make(import.meta.url)}};const copy={...box};box.child={load:make('/elsewhere/entry.cjs')};copy.child.load('./shared.cjs')", + "./shared.cjs", + ], + [ + "spread excludes an inherited loader", + "const box={__proto__:{load:make(import.meta.url)}};const copy={...box};copy.load('./shared.cjs')", + undefined, + ], + [ + "local loader replacement", + "let load=make(import.meta.url);load=()=>({});load('./shared.cjs')", + undefined, + ], + ["local loader", "const load=()=>({});load('./shared.cjs')", undefined], + [ + "shadowed create factory", + "function f(make){const load=make(import.meta.url);load('./shared.cjs')}", + undefined, + ], +] as const)( + "classifies %s from the selected loader value", + (_name, body, specifier) => { + const result = analyzeJavaScriptStaticSource(`${imported}${body};`); + expect(result.parse_error_count).toBe(0); + const calls = result.references.filter(({ kind }) => kind === "require"); + expect(calls).toEqual( + specifier === undefined ? [] : [expect.objectContaining({ specifier })], + ); + }, +); + +it("preserves a renamed native createRequire callable", () => { + const load = createRequire(import.meta.url); + const saved = load; + expect(saved("node:path").join).toBe(path.join); +}); + +it.each(["leaf", "parent"])( + "keeps the native created loader identity after a copied child %s write", + (write) => { + const selected = runInNewContext( + `const original=make(base);const outside=make(outsideBase); + const box={child:{load:original}};const copy={...box}; + ${write === "leaf" ? "box.child.load=outside" : "box.child={load:outside}"}; + ({actual:copy.child.load,expected:${write === "leaf" ? "outside" : "original"}});`, + { + make: createRequire, + base: import.meta.url, + outsideBase: new URL("./elsewhere/entry.cjs", import.meta.url).href, + }, + ); + expect(selected.actual).toBe(selected.expected); + }, +); + +it.each([ + [ + "direct Reflect.apply", + "Reflect.apply(module.require,module,['node:path'])", + true, + ], + [ + "captured Reflect.apply", + "const invoke=Reflect.apply;invoke(module.require,module,['node:path'])", + true, + ], + [ + "generic Reflect.apply", + "Reflect.apply.call(null,module.require,module,['node:path'])", + true, + ], + [ + "Reflect.apply captured before replacement", + "const invoke=Reflect.apply;Reflect.apply=()=>({join:false});invoke(module.require,module,['node:path'])", + true, + ], + [ + "local Reflect", + "const Reflect={apply(){return {join:false}}};Reflect.apply(module.require,module,['node:path'])", + false, + ], + [ + "local callable", + "Reflect.apply(()=>({join:false}),module,['node:path'])", + false, + ], + [ + "local module", + "const module={require(){return {join:false}}};Reflect.apply(module.require,module,['node:path'])", + false, + ], +] as const)( + "preserves module delegate provenance through %s", + (_name, source, host) => { + const result = runInNewContext(source, { + module: new Module("reflected-loader.cjs"), + }); + expect(result.join === path.join).toBe(host); + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + host + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); diff --git a/src/domain/javascript/javascriptStaticAnalysisLoaderMayTargets.test.ts b/src/domain/javascript/javascriptStaticAnalysisLoaderMayTargets.test.ts new file mode 100644 index 0000000000..aa5c9bc080 --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisLoaderMayTargets.test.ts @@ -0,0 +1,70 @@ +import { createRequire } from "node:module"; +import * as path from "node:path"; +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it("keeps the actual CommonJS module export separate from an ESM synthetic default", () => { + const require = createRequire(import.meta.url); + const exported = require("node:module"); + expect(Object.hasOwn(exported, "default")).toBe(false); + expect(exported.default).toBeUndefined(); + expect(() => + runInNewContext( + "const load=require('node:module').default.createRequire(base);load('node:path');", + { require, base: import.meta.url }, + ), + ).toThrow(); + const analysis = analyzeJavaScriptStaticSource( + "const load=require('node:module').default.createRequire(import.meta.url);load('node:path');", + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier: "node:module" }), + ]); +}); + +it.each(["call", "apply"] as const)( + "retains both native and local created-loader %s alternatives", + (method) => { + const body = (base: string) => + `const load=make(${base});if(flag)load.${method}=()=>({kind:'local'});load.${method}(null,${method === "call" ? "'node:path'" : "['node:path']"});`; + for (const flag of [false, true]) { + const result = runInNewContext(body("base"), { + make: createRequire, + base: import.meta.url, + flag, + }); + expect(result.join === path.join).toBe(!flag); + if (flag) expect(result.kind).toBe("local"); + } + const analysis = analyzeJavaScriptStaticSource( + `import {createRequire as make} from 'node:module';${body("import.meta.url")}`, + ); + expect(analysis.parse_error_count).toBe(0); + expect( + analysis.references.filter(({ kind }) => kind === "require"), + ).toEqual([expect.objectContaining({ kind: "require", specifier: null })]); + }, +); + +it.each(["call", "apply"] as const)( + "retains both native and local captured-wrapper %s alternatives", + (method) => { + const body = `const load=require;if(flag)load.${method}=()=>({kind:'local'});load.${method}(null,${method === "call" ? "'node:path'" : "['node:path']"});`; + for (const flag of [false, true]) { + const result = runInNewContext(body, { + require: createRequire(import.meta.url), + flag, + }); + expect(result.join === path.join).toBe(!flag); + if (flag) expect(result.kind).toBe("local"); + } + const analysis = analyzeJavaScriptStaticSource(body); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier: null }), + ]); + }, +); diff --git a/src/domain/javascript/javascriptStaticAnalysisLoaderMutationControls.test.ts b/src/domain/javascript/javascriptStaticAnalysisLoaderMutationControls.test.ts new file mode 100644 index 0000000000..90388435c2 --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisLoaderMutationControls.test.ts @@ -0,0 +1,79 @@ +import { createRequire } from "node:module"; +import * as path from "node:path"; +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +const named = "import {createRequire as make} from 'node:module';"; +const namespace = "import * as mod from 'node:module';"; +const cases = [ + { + name: "own call replacement", + prefix: named, + body: (base: string) => + `const load=make(${base});load.call=()=>({kind:'local'});load.call(null,'node:path');`, + host: false, + }, + { + name: "own apply replacement", + prefix: named, + body: (base: string) => + `const load=make(${base});load.apply=()=>({kind:'local'});load.apply(null,['node:path']);`, + host: false, + }, + { + name: "callable captured before method replacement", + prefix: named, + body: (base: string) => + `const load=make(${base});const saved=load;load.call=()=>({kind:'local'});saved('node:path');`, + host: true, + }, + { + name: "mutable default factory replacement", + prefix: namespace, + body: (base: string) => + `mod.default.createRequire=()=>()=>({kind:'local'});const load=mod.default.createRequire(${base});load('node:path');`, + host: false, + }, + { + name: "factory captured before its default slot replacement", + prefix: namespace, + body: (base: string) => + `const make=mod.default.createRequire;mod.default.createRequire=()=>()=>({kind:'local'});const load=make(${base});load('node:path');`, + host: true, + }, + { + name: "loader created before its default factory replacement", + prefix: namespace, + body: (base: string) => + `const load=mod.default.createRequire(${base});mod.default.createRequire=()=>()=>({kind:'local'});load('node:path');`, + host: true, + }, +]; + +it.each(cases)( + "keeps loader and selected member identity for $name", + ({ prefix, body, host }) => { + // The mutable namespace default belongs to this VM fixture. Native created + // require functions retain their real Node behavior without patching builtins. + const result = runInNewContext(body("base"), { + make: createRequire, + mod: { default: { createRequire } }, + base: import.meta.url, + }); + expect(result.join === path.join).toBe(host); + if (!host) expect(result.kind).toBe("local"); + const analysis = analyzeJavaScriptStaticSource( + `${prefix}${body("import.meta.url")}`, + ); + expect(analysis.parse_error_count).toBe(0); + expect( + analysis.references.filter(({ kind }) => kind === "require"), + ).toEqual( + host + ? [expect.objectContaining({ kind: "require", specifier: "node:path" })] + : [], + ); + }, +); diff --git a/src/domain/javascript/javascriptStaticAnalysisMainModule.test.ts b/src/domain/javascript/javascriptStaticAnalysisMainModule.test.ts new file mode 100644 index 0000000000..e1c405ca38 --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisMainModule.test.ts @@ -0,0 +1,90 @@ +import { Module } from "node:module"; +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +const cases = [ + ["native main", "require.main.require('node:path');", true, true], + [ + "captured main", + "const main=require.main;main.require('node:path');", + true, + true, + ], + [ + "captured delegate", + "const load=require.main.require;load.call(require.main,'node:path');", + true, + true, + ], + ["wrapper var", "var require;require.main.require('node:path');", true, true], + [ + "local require", + "const require={main:{require:()=>({kind:'local'})}};require.main.require('node:path');", + false, + false, + ], + [ + "local alias", + "const load={main:{require:()=>({kind:'local'})}};load.main.require('node:path');", + false, + false, + ], + [ + "wrapper main replacement stays conservative", + "var require;require.main={require:()=>({kind:'local'})};require.main.require('node:path');", + false, + true, + ], + [ + "conditional main replacement", + "var require;if(flag)require.main={require:()=>({kind:'local'})};require.main.require('node:path');", + false, + true, + ], + [ + "main captured before replacement", + "const main=require.main;require.main={require:()=>({kind:'local'})};main.require('node:path');", + true, + true, + ], + [ + "ambient main replacement stays conservative", + "require.main={require:()=>({kind:'local'})};require.main.require('node:path');", + false, + true, + ], + [ + "shared deep child local write", + "const old={child:{inner:{make:()=>module}}};const box={...old};old.child={inner:box.child.inner};old.child.inner.make=()=>({require(){}});box.child.inner.make().require('node:path');", + false, + false, + ], + [ + "shared deep child host write", + "const old={child:{inner:{make:()=>({})}}};const box={...old};old.child={inner:box.child.inner};old.child.inner.make=()=>module;box.child.inner.make().require('node:path');", + true, + true, + ], +] as const; + +it.each(cases)( + "retains main-module source for %s", + (_name, source, host, unknown) => { + const module = new Module("main-module-source.cjs"); + const require = Object.assign( + (request: string) => module.require(request), + { main: module }, + ); + const observed = runInNewContext(source, { module, require, flag: true }); + expect(typeof observed?.join === "function").toBe(host); + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + unknown + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); diff --git a/src/domain/javascript/javascriptStaticAnalysisProcessMainModule.test.ts b/src/domain/javascript/javascriptStaticAnalysisProcessMainModule.test.ts new file mode 100644 index 0000000000..fb31d16c03 --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisProcessMainModule.test.ts @@ -0,0 +1,49 @@ +import { Module } from "node:module"; +import { runInNewContext } from "node:vm"; + +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +const cases = [ + ["native process", "process.mainModule.require('node:path');", true], + [ + "captured main", + "const main=process.mainModule;main.require('node:path');", + true, + ], + [ + "captured delegate", + "const main=process.mainModule;const load=main.require;load.call(main,'node:path');", + true, + ], + [ + "local process", + "const process={mainModule:{require:()=>({})}};process.mainModule.require('node:path');", + false, + ], + [ + "replaced process", + "process={mainModule:{require:()=>({})}};process.mainModule.require('node:path');", + false, + ], + ["unrelated process member", "process.other.require('node:path');", false], +] as const; + +it.each(cases)( + "retains the native Module loader for %s", + (_name, source, possible) => { + const main = new Module("process-main-source.cjs"); + const observed = runInNewContext(source, { + process: { mainModule: main, other: { require: () => ({}) } }, + }); + expect(typeof observed?.join === "function").toBe(possible); + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + possible + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); diff --git a/src/domain/javascript/javascriptStaticAnalysisProjectedModuleWrites.test.ts b/src/domain/javascript/javascriptStaticAnalysisProjectedModuleWrites.test.ts new file mode 100644 index 0000000000..e9f8f3e962 --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisProjectedModuleWrites.test.ts @@ -0,0 +1,144 @@ +import { Module } from "node:module"; +import * as path from "node:path"; +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +const projectedModuleWrites = [ + [ + "object assignment", + "let selected;({selected}={selected:module});selected.require=module.require;", + true, + ], + [ + "object declaration", + "const {selected}={selected:module};selected.require=module.require;", + true, + ], + [ + "array assignment", + "let selected;[selected]=[module];selected.require=module.require;", + true, + ], + [ + "array declaration", + "const [selected]=[module];selected.require=module.require;", + true, + ], + [ + "nested assignment", + "let selected;({nested:{selected}}={nested:{selected:module}});selected.require=module.require;", + true, + ], + [ + "assignment fallback", + "let selected;({selected=module}={});selected.require=module.require;", + true, + ], + [ + "nested fallback", + "let selected;({nested:{selected=module}}={nested:{}});selected.require=module.require;", + true, + ], + [ + "object rest child", + "let selected;({...selected}={loader:module});selected.loader.require=module.require;", + true, + ], + [ + "array rest child", + "let selected;[,...selected]=[0,module];selected[0].require=module.require;", + true, + ], + [ + "captured alias", + "let selected;({selected}={selected:module});const saved=selected;selected={};saved.require=module.require;", + true, + ], + [ + "captured source container", + "let source={selected:module};let selected;({selected}=source);source={selected:{}};selected.require=module.require;", + true, + ], + [ + "projected escape", + "let selected;({selected}={selected:module});expose(selected);", + true, + ], + [ + "copied child escape", + "let selected;({...selected}={loader:module});expose(selected.loader);", + true, + ], + [ + "inactive fallback", + "const {selected=module}={selected:{}};selected.require=module.require;", + false, + ], + [ + "local alias", + "let selected;({selected}={selected:{require(){}}});selected.require=module.require;", + false, + ], + [ + "object rest container", + "let selected;({...selected}=module);selected.require=module.require;", + false, + ], + [ + "rebound alias", + "let selected;({selected}={selected:module});selected={};selected.require=module.require;", + false, + ], + [ + "rebound fallback", + "let selected;({selected=module}={});selected={};selected.require=module.require;", + false, + ], + [ + "copied container escape", + "let selected;({...selected}=module);expose(selected);", + false, + ], +] as const; + +it.each(projectedModuleWrites)( + "tracks copied require after a projected module write: %s", + (_name, setup, expected) => { + const body = `${setup}const copy={...module};const load=copy.require;load.call(module,'node:path');`; + const host = new Module("projected-module-write-oracle.cjs"); + const context = { + module: host, + expose: (value: Module) => { + value.require = host.require; + }, + }; + if (expected) expect(runInNewContext(body, context).join).toBe(path.join); + else expect(() => runInNewContext(body, context)).toThrow(); + const result = analyzeJavaScriptStaticSource(body); + expect(result.parse_error_count).toBe(0); + expect(result.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); + +it.each(["selected.require=module.require;", "expose(selected);"])( + "captures copied require before a later projected module effect: %s", + (effect) => { + const body = `let selected;({selected}={selected:module});const copy={...module};${effect}const load=copy.require;load.call(module,'node:path');`; + const host = new Module("late-module-effect.cjs"); + expect(() => + runInNewContext(body, { + module: host, + expose: (value: Module) => { + value.require = host.require; + }, + }), + ).toThrow(); + expect(analyzeJavaScriptStaticSource(body).references).toEqual([]); + }, +); diff --git a/src/domain/javascript/javascriptStaticAnalysisReturnedInvocation.test.ts b/src/domain/javascript/javascriptStaticAnalysisReturnedInvocation.test.ts new file mode 100644 index 0000000000..cdbd2b82cb --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisReturnedInvocation.test.ts @@ -0,0 +1,180 @@ +import { expect, it } from "vitest"; +import * as t from "@babel/types"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; +import { parseJavaScriptSource } from "./javascriptSourceParser.js"; +import { indexJavaScriptBindingsWithReceiversSteps } from "./javascriptSemanticAnalysis.js"; +import { completeSemanticSteps } from "./javascriptSemanticTraversal.js"; +import { createSemanticReturnedReferences } from "./javascriptSemanticReturnedReferences.js"; + +it.each([ + ["const get=()=>module;get.call(null)", true], + ["const get=()=>module;get.apply(null,[])", true], + ["const get=()=>module;get['call'](null)", true], + ["const get=()=>module;const alias=get;alias.apply(null,[])", true], + ["const box={get(){return module;}};box.get.call(null)", true], + ["const get=()=>module;get.call.call(get,null)", true], + ["const get=()=>module;get.apply.call(get,null,[])", true], + ["const get=()=>module;const invoke=get.call;invoke.call(get,null)", true], + [ + "const get=()=>module;const invoke=get.call;const other=()=>({});invoke.call(other,null)", + false, + ], + [ + "const get=()=>({});const invoke=get.call;const other=()=>module;invoke.call(other,null)", + true, + ], + [ + "const get=()=>module;const invoke=get.apply;invoke.call(get,null,[])", + true, + ], + ["const get=()=>module;const invoke=get.call;invoke.apply(get,[null])", true], + ["const get=()=>({});get.call(null)", false], + ["const get=()=>({});get.apply(null,[])", false], + ["const get=()=>module.exports;get.call(null)", false], + ["const get=()=>module;get.call=()=>({require(){}});get.call(null)", false], + [ + "const get=()=>module;get.apply=()=>({require(){}});get.apply(null,[])", + false, + ], + ["const get=()=>({});get.call=()=>module;get.call(null)", true], + ["const get=()=>({});get.apply=()=>module;get.apply(null,[])", true], + ["const get=()=>module;const invoke=get.call;invoke(get)", false], + ["const get=()=>module;(0,get.call)(null)", false], + ["const get=()=>module;(0,get.apply)(null,[])", false], + [ + "const get=()=>module;const other=()=>({});get.call.call(other,null)", + false, + ], + ["const get=()=>({});const other=()=>module;get.call.call(other,null)", true], + ["function* get(){yield module;}get.call(null)", false], + ["async function get(){return module;}get.apply(null,[])", false], + ["async function get(){return module;}(await get.call(null))", true], + ["const get=()=>module;get.call=unknown;get.call(null)", true], + ["const get=()=>module;if(flag)get.call=()=>({});get.call(null)", true], + [ + "const get=()=>({});Object.setPrototypeOf(get,{call:()=>module});get.call(null)", + true, + ], + [ + "const get=()=>module;Object.setPrototypeOf(get,{call:()=>({})});get.call(null)", + true, + ], + [ + "const get=()=>1;Object.setPrototypeOf(get,{call:()=>module});get.call(null)", + true, + ], + [ + "const get=()=>({});Object.setPrototypeOf(get,{call:()=>module});get.call=()=>({});get.call(null)", + false, + ], + [ + "const get=()=>({});Object.setPrototypeOf(get,{call:()=>({})});get.call=()=>module;get.call(null)", + true, + ], +])( + "retains invocation results after selecting source-local member identity: %s", + (receiver, expected) => { + const analysis = analyzeJavaScriptStaticSource( + `${receiver}.require('./shared.cjs');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); + +it.each([ + ["Function.prototype.call=unknown;const get=()=>({});get.call(null)", true], + [ + "Function.prototype.apply=unknown;const get=()=>({});get.apply(null,[])", + true, + ], + [ + "Function.prototype.call=unknown;const get=()=>module;get.call=()=>({});get.call(null)", + false, + ], + ["Function.prototype.extra=unknown;const get=()=>({});get.call(null)", false], +])( + "retains inherited invocation uncertainty without revoking own slots: %s", + (receiver, expected) => { + const analysis = analyzeJavaScriptStaticSource( + `${receiver}.require('./shared.cjs');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + expected ? [expect.objectContaining({ specifier: null })] : [], + ); + }, +); + +it("keeps source reads before later function-method replacement", () => { + const analysis = analyzeJavaScriptStaticSource(` + const get=()=>module; + get.call(null).require('./shared.cjs'); + get.call=()=>({}); + get.call(null).require('./unused.cjs'); + `); + expect(analysis.references).toEqual([ + expect.objectContaining({ specifier: null }), + ]); +}); + +it("keeps an earlier native invocation before a later prototype write", () => { + const analysis = analyzeJavaScriptStaticSource(` + const get=()=>({}); + get.call(null).require('./unused.cjs'); + Function.prototype.call=unknown; + `); + expect(analysis.references).toEqual([]); +}); + +it("does not treat a shadowed Function namespace as native prototype mutation", () => { + const analysis = analyzeJavaScriptStaticSource(` + function local(Function){ + Function.prototype.call=unknown; + const get=()=>({}); + get.call(null).require('./unused.cjs'); + } + `); + expect(analysis.references).toEqual([]); +}); + +it("retains a finite shared callable alias graph", () => { + const aliases = Array.from( + { length: 64 }, + (_, index) => `const get${index + 1}=flag?get${index}:get${index};`, + ).join(""); + const parsed = parseJavaScriptSource( + `const get0=()=>module;${aliases}get64.apply(null,[]);`, + ); + if (parsed === null) throw new Error("Expected valid shared alias fixture"); + const index = completeSemanticSteps( + indexJavaScriptBindingsWithReceiversSteps(parsed.program), + ); + completeSemanticSteps(index.indexAmbientWritesSteps()); + completeSemanticSteps(index.indexMemberMutationFactsSteps()); + const statement = parsed.program.body.at(-1); + if ( + !t.isExpressionStatement(statement) || + !t.isCallExpression(statement.expression) + ) + throw new Error("Expected final invocation"); + const call = statement.expression; + const result = createSemanticReturnedReferences( + index.bindings, + "direct-call", + )(call.callee, call); + expect(result).toEqual({ + references: [ + { + node: expect.objectContaining({ type: "Identifier", name: "module" }), + projection: [], + }, + ], + precision: "exact", + }); +}); diff --git a/src/domain/javascript/javascriptStaticAnalysisReturnedRequire.test.ts b/src/domain/javascript/javascriptStaticAnalysisReturnedRequire.test.ts new file mode 100644 index 0000000000..eed35ede1a --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisReturnedRequire.test.ts @@ -0,0 +1,285 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it.each([ + ["async function get(){return module;}(await get())", true], + ["async function get(){return module;}const api=await get();api", true], + [ + "async function get(){return module;}const pending=get();const api=await pending;api", + true, + ], + ["function get(){return module;}const api=await get();api", true], + ["function* get(){yield module;}const api=await get();api", false], + ["function* get(){return module;}const api=await get();api", false], + ["async function* get(){yield module;}const api=await get();api", false], + [ + "async function get(){return {require(){}};}const api=await get();api", + false, + ], + ["async function get(){return module;}const api=get();api", false], + [ + "async function child(){return module;}async function get(){return {child:child()};}(await get()).child", + false, + ], + [ + "async function child(){return module;}function get(){return {child:child()};}(await get().child)", + true, + ], + [ + "async function child(){return module;}async function get(){return {child:child()};}const {child:pending}=await get();const api=await pending;api", + true, + ], + [ + "async function get(){return module;}const pending=get();const api=flag?pending:await pending;api", + true, + ], + [ + "async function get(){return module;}const pending=get();const api=flag?await pending:pending;api", + true, + ], + [ + "async function inner(){return module;}function get(){return inner();}(await get())", + true, + ], +])( + "preserves awaited result consumption through aliases and projections: %s", + (receiver, expected) => { + const analysis = analyzeJavaScriptStaticSource( + `${receiver}.require('./shared.cjs');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); + +it.each([ + "const {make=host}={make:class{}};", + "class Local{}const {make=host}={make:Local};", + "const [make=host]=[class{}];", +])( + "excludes inactive defaults for known class constructors: %s", + (declaration) => { + const analysis = analyzeJavaScriptStaticSource( + `function host(){return module;}${declaration}new make().require=()=>{};require('./shared.cjs');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier: "./shared.cjs" }), + ]); + }, +); + +it.each([ + ["const {make=host}={make(){return {};}};", false], + ["const {make=host}={make:()=>({})};", false], + ["const [make=host]=[()=>({})];", false], + ["const {box:{make=host}={}}={box:{make(){return {};}}};", false], + ["const {make=host}={};", true], + ["const [make=host]=[];", true], + ["const {box:{make=host}={}}={};", true], + ["const {make=host}=unknown;", true], + ["const [make=host]=unknown;", true], + ["const {make=host}={make:unknown};", true], + ["const {make=host}={make:undefined};", true], + ["const [make=host]=[undefined];", true], + ["const {make=host}={get make(){return undefined;}};", true], + ["const {make=host}={make:flag?()=>({}):()=>({})};", false], + [ + "const values=[()=>({})];values[Symbol.iterator]=function*(){yield undefined;};const [make=host]=values;", + true, + ], + [ + "Array.prototype[Symbol.iterator]=function*(){yield undefined;};const [make=host]=[()=>({})];", + true, + ], + ["const {make=host}={make:host};", true], + ["const [make=host]=[host];", true], + ["const make=host;", true], + ["const make=()=>({});", false], +])( + "retains only applicable default callable sources: %s", + (declaration, expected) => { + const analysis = analyzeJavaScriptStaticSource( + `function host(){return module;}${declaration}make().require=()=>{};require('./shared.cjs');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references.at(-1)).toEqual( + expect.objectContaining({ + kind: "require", + specifier: expected ? null : "./shared.cjs", + }), + ); + }, +); + +it.each([ + "function* get(){yield module;}", + "function* get(){return module;}", + "async function get(){return module;}", + "async function* get(){yield module;}", + "const get=async()=>module;", +])( + "distinguishes direct wrapper objects from their eventual contents: %s", + (declaration) => { + const analysis = analyzeJavaScriptStaticSource( + `${declaration}const api=get();api.require=()=>{};api.require('./unused.cjs');require('./shared.cjs');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier: "./shared.cjs" }), + ]); + }, +); + +it.each([ + "const table={x(){return{};},y(){return{};},z(){return module;}};const get=flag?table.x:table.y;", + "const table={a:{b(){return{};},d(){return module;}},c:{d(){return{};},b(){return module;}}};const get=flag?table.a.b:table.c.d;", +])("retains precise selected direct-return receivers: %s", (declaration) => { + const analysis = analyzeJavaScriptStaticSource( + `${declaration}get().require('./unused.cjs');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual([]); +}); + +it.each([ + ["Object.prototype[0]=module;", true], + ["Object.prototype['0']=module;", true], + ["Object.prototype.constructor.prototype[0]=module;", true], + ["globalThis.Object.prototype[0]=module;", true], + ["global.Object.prototype[0]=module;", true], + ["window.Object.prototype[0]=module;", true], + ["self.Object.prototype[0]=module;", true], + ["Object.prototype['01']=module;", false], + ["Object.prototype.extra=module;", false], + ["Object.prototype.require=replacement;", false], + ["Array.prototype[0]=module;", true], + ["Array.prototype['0']=module;", true], + ["Array.prototype.constructor.prototype[0]=module;", true], + ["Array.prototype['01']=module;", false], + ["Array.prototype.extra=module;", false], + ["Array.prototype.require=replacement;", false], +])("retains inherited numeric array iteration slots: %s", (write, expected) => { + const analysis = analyzeJavaScriptStaticSource( + `${write}const [api]=[,];api.require('./shared.cjs');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); +}); + +it.each([ + "function local(Object){Object.prototype[0]=module;const[api]=[,];api.require('./shared.cjs');}", + "function local(globalThis){globalThis.Object.prototype[0]=module;const[api]=[,];api.require('./shared.cjs');}", +])("retains shadowed prototype controls: %s", (source) => { + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual([]); +}); + +it.each([ + [ + "Array.prototype.constructor.prototype[Symbol.iterator]=function*(){yield module;};", + true, + ], + [ + "Array.prototype.constructor.prototype.constructor.prototype[Symbol.iterator]=custom;", + true, + ], + ["Array.prototype.constructor.prototype.extra=1;", false], + ["Array.prototype.constructor.prototype.require=replacement;", false], + ["Array.prototype.require=replacement;", false], +])( + "tracks iterator changes through prototype constructor cycles: %s", + (write, expected) => { + const analysis = analyzeJavaScriptStaticSource( + `${write}const [selected]=[];selected.require('./shared.cjs');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); + +it.each([ + ["const get=()=>module;get()", true], + ["function get(){return module;}get()", true], + ["const get=()=>module;const alias=get;alias()", true], + ["const get=()=>({loader:module});get().loader", true], + ["const get=()=>flag?module:{};get()", true], + ["const inner=()=>module;const get=()=>inner();get()", true], + ["(()=>module)()", true], + ["const get=()=>({require(){}});get()", false], + ["const get=()=>module.exports;get()", false], + ["function get(module){return module;}get({require(){}})", false], + ["let get=()=>module;get=()=>({require(){}});get()", false], +])("retains possible returned module receivers: %s", (receiver, expected) => { + const analysis = analyzeJavaScriptStaticSource( + `${receiver}.require('./shared.cjs');`, + ); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + expected + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); +}); + +it.each([ + [ + "const [api]=[module];Array.prototype.extra=1;api.require=replacement;", + true, + ], + [ + "const [api]=[module];Array.prototype[Symbol.iterator]=custom;api.require=replacement;", + true, + ], + ["const [api]=[{}];Array.prototype.extra=1;api.require=replacement;", false], + [ + "const values={*[Symbol.iterator](){yield module;}};const [api]=values;api.require=replacement;", + true, + ], + [ + "const values=[{}];values[Symbol.iterator]=function*(){yield module;};const [api]=values;api.require=replacement;", + true, + ], + [ + "Array.prototype[Symbol.iterator]=custom;const [api]=[{}];api.require=replacement;", + true, + ], + ["const get=()=>module;get().require=replacement;", true], + ["const get=()=>({require(){}});get().require=replacement;", false], +])( + "retains host delegate uncertainty after possible receiver writes: %s", + (write, expected) => { + const analysis = analyzeJavaScriptStaticSource( + `${write}require('./shared.cjs');`, + ); + expect(analysis.references).toEqual([ + expect.objectContaining({ + kind: "require", + specifier: expected ? null : "./shared.cjs", + }), + ]); + }, +); + +it("preserves a require before later returned-receiver delegate writes", () => { + const analysis = analyzeJavaScriptStaticSource( + `require('./shared.cjs');const get=()=>module;get().require=replacement;`, + ); + expect(analysis.references).toEqual([ + expect.objectContaining({ kind: "require", specifier: "./shared.cjs" }), + ]); +}); diff --git a/src/domain/javascript/javascriptStaticAnalysisSharedHelperWrites.test.ts b/src/domain/javascript/javascriptStaticAnalysisSharedHelperWrites.test.ts new file mode 100644 index 0000000000..5a07f99fe3 --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisSharedHelperWrites.test.ts @@ -0,0 +1,83 @@ +import changedPath from "node:path"; +import path from "node:path"; +import posix from "node:path/posix"; +import changedUrl from "node:url"; +import url from "node:url"; +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it("observes that separate Node default imports share mutable helper slots", () => { + expect(changedPath).toBe(path); + expect(changedUrl).toBe(url); + expect(path.posix).toBe(posix); + const resolve = changedPath.resolve; + const decode = changedUrl.fileURLToPath; + try { + changedPath.resolve = (first) => first; + changedUrl.fileURLToPath = () => "/changed/main.mjs"; + expect(path.resolve("/runtime/actual.cjs", "/source", "decoy.cjs")).toBe( + "/runtime/actual.cjs", + ); + expect(url.fileURLToPath("file:///source/main.mjs")).toBe( + "/changed/main.mjs", + ); + } finally { + changedPath.resolve = resolve; + changedUrl.fileURLToPath = decode; + } +}); + +it.each([ + [ + "path resolve", + "import changed from 'node:path';import path from 'path';changed.resolve=first=>first;", + "path.resolve(runtimePath,__dirname,'decoy.cjs')", + ], + [ + "POSIX path resolve", + "import changed from 'node:path/posix';import path from 'path/posix';changed.resolve=first=>first;", + "path.resolve(runtimePath,__dirname,'decoy.cjs')", + ], + [ + "decoded URL source anchor", + "import changed from 'node:url';import url from 'url';import path from 'node:path';changed.fileURLToPath=()=>'/changed/main.mjs';", + "path.resolve(runtimePath,path.dirname(url.fileURLToPath(import.meta.url)),'decoy.cjs')", + ], + [ + "POSIX exported child identity", + "import changed from 'node:path/posix';import path from 'node:path';changed.resolve=first=>first;", + "path.posix.resolve(runtimePath,__dirname,'decoy.cjs')", + ], + [ + "direct POSIX module identity", + "import changed from 'node:path';import path from 'node:path/posix';changed.posix.resolve=first=>first;", + "path.resolve(runtimePath,__dirname,'decoy.cjs')", + ], +] as const)( + "retains shared default writes before proving %s", + (_name, setup, expression) => { + const analysis = analyzeJavaScriptStaticSource(`${setup} + import {BrowserWindow,utilityProcess} from 'electron'; + new BrowserWindow({webPreferences:{preload:${expression}}}); + utilityProcess.fork(${expression});`); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.electron.browser_windows).toMatchObject([ + { preload_path: null }, + ]); + expect(analysis.electron.utility_processes).toMatchObject([ + { module_path: null }, + ]); + }, +); + +it("distinguishes replacing a path child slot from changing its exported object", () => { + const analysis = analyzeJavaScriptStaticSource(` + import changed from 'node:path';import path from 'node:path/posix'; + changed.posix={resolve(first){return first}}; + import{BrowserWindow}from'electron'; + new BrowserWindow({webPreferences:{preload:path.resolve(runtimePath,__dirname,'decoy.cjs')}});`); + expect(analysis.electron.browser_windows).toMatchObject([ + { preload_path: "decoy.cjs" }, + ]); +}); diff --git a/src/domain/javascript/javascriptStaticAnalysisState.ts b/src/domain/javascript/javascriptStaticAnalysisState.ts index 25d06463ed..79438ec3ed 100644 --- a/src/domain/javascript/javascriptStaticAnalysisState.ts +++ b/src/domain/javascript/javascriptStaticAnalysisState.ts @@ -1,6 +1,7 @@ import type * as t from "@babel/types"; import type { JavaScriptOpenReceiverFact } from "./javascriptSemanticAnalysis.js"; import type { JavaScriptRpcScan } from "./javascriptRpcEndpoints.js"; +import type { JavaScriptStaticPathIntrinsicQuery } from "./javascriptStaticAnalysisHelpers.js"; import type { JavaScriptBundlerRegistration, @@ -11,6 +12,7 @@ import type { JavaScriptStaticStorage, } from "./javascriptStaticAnalysisTypes.js"; import type { + ElectronAppPathChangeFinding, ElectronCallPath, ElectronBrowserWindowFinding, ElectronContextBridgeFinding, @@ -38,6 +40,12 @@ export interface LocatedJavaScriptFinding { /** Mutable collections used during one inert AST traversal. */ export interface JavaScriptAnalysisAccumulator { + readonly pendingModuleRequireCalls: ( + | t.CallExpression + | t.OptionalCallExpression + | t.NewExpression + )[]; + readonly appPathChanges: LocatedJavaScriptFinding[]; electronCallPaths?: readonly ElectronCallPath[]; electronCallPathLimitReached?: boolean; readonly references: LocatedJavaScriptFinding[]; @@ -80,6 +88,8 @@ export interface JavaScriptAnalysisAccumulator { /** Shared source and accumulator for helper inspections. */ export interface JavaScriptFindingContext { + /** Node path.resolve identity established by shared callee and mutation owners. */ + readonly nodePathResolve?: JavaScriptStaticPathIntrinsicQuery; readonly rpcScan: JavaScriptRpcScan; readonly source: string; readonly accumulator: JavaScriptAnalysisAccumulator; @@ -87,6 +97,12 @@ export interface JavaScriptFindingContext { readonly openReceiverFacts?: ReadonlyMap; /** Proven Electron export path segments by root offset. */ readonly electronBindings?: ReadonlyMap; + /** Normalized invocation target from the shared Electron and intrinsic owners. */ + readonly electronInvocationPath?: ( + node: t.CallExpression | t.OptionalCallExpression, + ) => readonly string[] | undefined; + /** A possible shared-app mutation does not establish concrete API identity. */ + readonly electronAppPathChangePossible?: (node: t.Node) => boolean; } /** Candidate import, require, worker, or service-worker reference. */ @@ -120,6 +136,8 @@ export interface LocatedJavaScriptFindingInput< export const createJavaScriptAnalysisAccumulator = ( sourceLength: number, ): JavaScriptAnalysisAccumulator => ({ + pendingModuleRequireCalls: [], + appPathChanges: [], references: [], endpoints: [], pendingNetworkCalls: [], diff --git a/src/domain/javascript/javascriptStaticAnalysisTypedMemberStores.test.ts b/src/domain/javascript/javascriptStaticAnalysisTypedMemberStores.test.ts new file mode 100644 index 0000000000..333a852296 --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisTypedMemberStores.test.ts @@ -0,0 +1,55 @@ +import { Module, stripTypeScriptTypes } from "node:module"; +import { runInNewContext } from "node:vm"; +import * as t from "@babel/types"; +import { expect, it } from "vitest"; + +import { semanticWriteTargets } from "./javascriptAstValues.js"; +import { semanticIterationSources } from "./javascriptSemanticIterationSources.js"; +import { parseJavaScriptSource } from "./javascriptSourceParser.js"; +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it.each( + [false, true].flatMap((typed) => + [false, true].map((host) => ({ typed, host })), + ), +)( + "retains the selected member store with types=$typed and host=$host", + ({ typed, host }) => { + const target = typed ? "(box.make as any)" : "box.make"; + const source = `const box={make:()=>({require(){}})};${target}=()=>${host ? "module" : "({require(){}})"};box.make().require('./shared.cjs');`; + const parsed = parseJavaScriptSource(source); + if (parsed === null) throw new Error("Expected a parsed member store"); + const statement = parsed.program.body[1]; + if ( + !t.isExpressionStatement(statement) || + !t.isAssignmentExpression(statement.expression) + ) + throw new Error("Expected the member assignment"); + const assignment = statement.expression; + const stores = semanticIterationSources(assignment, statement).filter( + ({ kind }) => kind === "reference-store", + ); + expect(stores).toHaveLength(1); + expect(stores[0]?.target).toBe( + [...semanticWriteTargets(assignment.left)][0], + ); + expect(stores[0]?.node).toBe(assignment.right); + expect(stores[0]?.projection).toEqual([]); + + const requests: string[] = []; + const module = new Module("typed-member-store-oracle.cjs"); + module.require = (request) => { + requests.push(request); + return {}; + }; + runInNewContext(stripTypeScriptTypes(source), { module }); + expect(requests).toEqual(host ? ["./shared.cjs"] : []); + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.references).toEqual( + host + ? [expect.objectContaining({ kind: "require", specifier: null })] + : [], + ); + }, +); diff --git a/src/domain/javascript/javascriptStaticPathIntrinsics.ts b/src/domain/javascript/javascriptStaticPathIntrinsics.ts new file mode 100644 index 0000000000..701ca98e7e --- /dev/null +++ b/src/domain/javascript/javascriptStaticPathIntrinsics.ts @@ -0,0 +1,73 @@ +import * as t from "@babel/types"; + +import { builtinDataEffectCallablePath } from "./javascriptSemanticDataEffectHelpers.js"; +import type { JavaScriptBindingIndex } from "./javascriptSemanticState.js"; +import { completeSemanticSteps } from "./javascriptSemanticTraversal.js"; +import { evaluateSemanticExpressionProvenance } from "./javascriptSemanticValues.js"; +import { + isUnmodifiedGlobalReference, + isUnmodifiedImportMetaUrl, +} from "./javascriptSemanticAmbientWrites.js"; +import { unwrapJavaScriptExpression } from "./javascriptAstValues.js"; +import type { JavaScriptStaticPathIntrinsicQuery } from "./javascriptStaticAnalysisHelpers.js"; + +/** Prove Node's path resolver before applying its argument-selection semantics. */ +export const createStaticNodePathResolveQuery = ( + index: JavaScriptBindingIndex, +): JavaScriptStaticPathIntrinsicQuery => { + const facts = new WeakMap(); + const nativeMethod = ( + callee: t.Node, + method: string, + modules: readonly string[], + ): boolean => { + const provenance = evaluateSemanticExpressionProvenance( + callee, + index.bindings, + ); + if ( + provenance.status !== "module" || + provenance.origins[0].importedPath.at(-1) !== method || + !modules.includes(provenance.origins[0].specifier.replace(/^node:/u, "")) + ) + return false; + completeSemanticSteps(index.indexMemberMutationFactsSteps()); + const path = builtinDataEffectCallablePath(callee, index.bindings, modules); + return ( + path !== null && + ((path.length === 1 && path[0] === method) || + (modules.includes("path") && + path.length === 2 && + path[0] === "posix" && + path[1] === method)) + ); + }; + const sourceFile = (node: t.Node | undefined): boolean => { + if (node === undefined) return false; + node = unwrapJavaScriptExpression(node).node; + if (t.isIdentifier(node, { name: "__filename" })) + return isUnmodifiedGlobalReference(node, index.bindings, "__filename"); + return ( + t.isCallExpression(node) && + node.arguments.length === 1 && + nativeMethod(node.callee, "fileURLToPath", ["url"]) && + isUnmodifiedImportMetaUrl(node.arguments[0], index.bindings) + ); + }; + return (callee, fact) => { + if (fact === "source-directory") { + callee = unwrapJavaScriptExpression(callee).node; + return t.isIdentifier(callee, { name: "__dirname" }) + ? isUnmodifiedGlobalReference(callee, index.bindings, "__dirname") + : t.isCallExpression(callee) && + callee.arguments.length === 1 && + nativeMethod(callee.callee, "dirname", ["path", "path/posix"]) && + sourceFile(callee.arguments[0]); + } + const cached = facts.get(callee); + if (cached !== undefined) return cached; + const proven = nativeMethod(callee, "resolve", ["path", "path/posix"]); + facts.set(callee, proven); + return proven; + }; +}; diff --git a/src/domain/javascript/javascriptStaticPathResolution.test.ts b/src/domain/javascript/javascriptStaticPathResolution.test.ts index 83d6daba8f..56b44c7c4c 100644 --- a/src/domain/javascript/javascriptStaticPathResolution.test.ts +++ b/src/domain/javascript/javascriptStaticPathResolution.test.ts @@ -7,6 +7,7 @@ import { describe, expect, it } from "vitest"; import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; import { parserPluginsForPath } from "./javascriptSourceParser.js"; import { + calleeName, staticPath, staticPathResolutionContext, } from "./javascriptStaticAnalysisHelpers.js"; @@ -27,6 +28,22 @@ const anchoredExpressions = [ ['path[`resolve`](__dirname, "preload.js")', "preload.js"], ['path.posix.resolve("ignored", __dirname, "preload.js")', "preload.js"], ['path.resolve("/ignored", __dirname, "preload.js")', "preload.js"], + ['path.resolve(runtimeBase, "/app/preload.js")', "/app/preload.js"], + ['path.resolve(runtimeBase, __dirname, "preload.js")', "preload.js"], + ['path.resolve(...parts, "/app/preload.js")', "/app/preload.js"], + ['path.resolve(42, "/app/preload.js")', "/app/preload.js"], + [ + 'path.resolve(new URL("./", import.meta.url), __dirname, "preload.js")', + "preload.js", + ], + [ + 'path.resolve(new URL("./", import.meta.url), dirname(fileURLToPath(import.meta.url)), "preload.js")', + "preload.js", + ], + [ + 'path.posix.resolve(new URL("./", import.meta.url), "/app", "preload.js")', + "/app/preload.js", + ], [ 'path.posix.resolve("ignored", dirname(fileURLToPath(import.meta.url)), "preload.js")', "preload.js", @@ -58,13 +75,9 @@ const unknownExpressions = [ "path.posix.resolve()", 'path.resolve("sub", "../preload.js")', 'path.resolve("", "")', - 'path.resolve(runtimeBase, "/app/preload.js")', 'path.resolve("/app", runtimeFile)', - 'path.resolve(runtimeBase, __dirname, "preload.js")', "path.resolve(__dirname, runtimeFile)", - 'path.resolve(...parts, "/app/preload.js")', 'path.resolve("/app", ...parts)', - 'path.resolve(42, "/app/preload.js")', 'path.resolve("/app", null)', "path.resolve(`/app/${runtimeFile}`)", 'path.win32.resolve("/app", "/shared/preload.js")', @@ -73,6 +86,12 @@ const unknownExpressions = [ 'resolve("/app", "preload.js")', '__dirname + "preload.js"', 'path.dirname(__filename) + "preload.js"', + 'new URL("./pre%2Fload.js", import.meta.url)', + 'path.join(new URL("./", import.meta.url), "preload.js")', + 'path.resolve(new URL("./", import.meta.url), "preload.js")', + 'path.join(__dirname, new URL("./pre%2Fload.js", import.meta.url))', + 'path.resolve(__dirname, new URL("./", import.meta.url), "preload.js")', + 'path.resolve("/app", new URL("./", import.meta.url), "preload.js")', ]; const pathValueExpressions = [ @@ -324,6 +343,36 @@ it.each([ ); }); +it("decodes a source-relative file URL once before composing a filesystem path", () => { + assertPathProjection( + 'fileURLToPath(new URL("./pre%252Fload.js", import.meta.url))', + "./pre%2Fload.js", + "filesystem-expression", + ); + assertPathProjection( + 'path.join(dirname(fileURLToPath(import.meta.url)), "pre%2Fload.js")', + "pre%2Fload.js", + "filesystem-expression", + ); +}); + +it("retains Worker URL references through their separate URL-aware producer", () => { + const specifier = "./worker%252Fentry.js?version=1#worker"; + const analysis = analyzeJavaScriptStaticSource(` + import { Worker } from "node:worker_threads"; + new Worker(new URL(${JSON.stringify(specifier)}, import.meta.url)); + `); + + expect(analysis.parse_status).toBe("complete"); + expect(analysis.parse_error_count).toBe(0); + expect( + analysis.references + .filter(({ kind }) => kind === "worker") + .map(({ specifier, expression }) => ({ specifier, expression })), + ).toEqual([{ specifier, expression: null }]); + expect(analysis.role_paths).toEqual([]); +}); + it.each([ 'fileURLToPath("file:///app/%2Fpreload.js")', 'fileURLToPath("file:///app/%5Cpreload.js")', @@ -344,7 +393,8 @@ const parsePathExpression = (expression: string) => const analyzePathConsumers = (expression: string) => { const property = `preload: ${expression}`; - const genericPrefix = "const options = { "; + const genericPrefix = + 'import path, {dirname} from "node:path"; import {fileURLToPath} from "node:url"; const options = { '; const windowPrefix = "new BrowserWindow({ webPreferences: { "; const windowExpression = `${windowPrefix}${property} } })`; const utilityExpression = `utilityProcess.fork(${expression})`; @@ -366,8 +416,24 @@ const assertPathProjection = ( context: JavaScriptStaticPathContext, ) => { const node = parsePathExpression(expression); - expect(staticPath(node)).toBe(path); - expect(staticPathResolutionContext(node)).toBe(context); + // This syntax matrix supplies identity explicitly; full-source cases prove it. + const nodePathResolve = ( + callee: import("@babel/types").Node, + fact?: "source-directory", + ) => { + if (fact === "source-directory") { + const name = calleeName( + callee.type === "CallExpression" ? callee.callee : callee, + ); + return ( + name === "__dirname" || name === "dirname" || name.endsWith(".dirname") + ); + } + const name = calleeName(callee); + return name.endsWith(".resolve") && !name.includes("win32"); + }; + expect(staticPath(node, nodePathResolve)).toBe(path); + expect(staticPathResolutionContext(node, nodePathResolve)).toBe(context); const { analysis, property, diff --git a/src/domain/javascript/javascriptStaticPathResolveOwners.test.ts b/src/domain/javascript/javascriptStaticPathResolveOwners.test.ts new file mode 100644 index 0000000000..324ba9033d --- /dev/null +++ b/src/domain/javascript/javascriptStaticPathResolveOwners.test.ts @@ -0,0 +1,166 @@ +import { posix } from "node:path"; + +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it.each(["/other", "relative"])( + "keeps a shadowed source-directory parameter dynamic: %s", + (directory) => { + const actual = posix.resolve("/runtime/base", directory, "decoy.cjs"); + expect(actual).not.toBe("/source/app/decoy.cjs"); + const analysis = analyzeJavaScriptStaticSource(` + const path = require('node:path'); + const {BrowserWindow, utilityProcess} = require('electron'); + function build(__dirname) { + new BrowserWindow({webPreferences:{preload:path.resolve(runtimeBase,__dirname,'decoy.cjs')}}); + utilityProcess.fork(path.resolve(runtimeBase,__dirname,'decoy.cjs')); + } + build(${JSON.stringify(directory)}); + `); + expect(analysis.role_paths).toEqual([]); + expect(analysis.electron.browser_windows).toMatchObject([ + { preload_path: null, preload_resolution_context: null }, + ]); + expect(analysis.electron.utility_processes).toMatchObject([ + { module_path: null, module_resolution_context: null }, + ]); + }, +); + +const localResolvers = [ + [ + "local object", + "const helper = {resolve(first) {return first;}};", + "helper.resolve", + ], + [ + "familiar name", + "const path = {resolve(first) {return first;}};", + "path.resolve", + ], + [ + "shadowed require", + "const require = () => ({resolve(first) {return first;}}); const helper = require('node:path');", + "helper.resolve", + ], + [ + "replaced Node method", + "const helper = require('node:path'); helper.resolve = first => first;", + "helper.resolve", + ], + [ + "ambiguous owner", + "const helper = flag ? require('node:path') : {resolve(first) {return first;}};", + "helper.resolve", + ], +] as const; + +it.each(localResolvers)( + "keeps %s resolve calls dynamic", + (_name, setup, callee) => { + const expression = `${callee}(runtimePath, __dirname, "decoy.cjs")`; + const native = new Function( + "require", + "runtimePath", + "__dirname", + "flag", + `{ ${setup} return ${expression}; }`, + ); + expect( + native( + () => ({ ...posix }), + "/actual/runtime.cjs", + "/fixture/app", + false, + ), + ).toBe("/actual/runtime.cjs"); + const analysis = analyzeJavaScriptStaticSource(` + import {BrowserWindow, utilityProcess} from 'electron'; + ${setup} + new BrowserWindow({webPreferences:{preload:${expression}}}); + utilityProcess.fork(${expression}); + `); + expect(analysis.role_paths).toEqual([]); + expect(analysis.electron.browser_windows).toMatchObject([ + { preload_path: null, preload_resolution_context: null }, + ]); + expect(analysis.electron.utility_processes).toMatchObject([ + { module_path: null, module_resolution_context: null }, + ]); + }, +); + +const nodeResolvers = [ + ["require", "const helper = require('node:path');", "helper.resolve"], + [ + "destructured alias", + "const {resolve: select} = require('path');", + "select", + ], + [ + "POSIX namespace", + "const helper = require('node:path');", + "helper.posix.resolve", + ], + [ + "POSIX alias", + "const {posix: helper} = require('node:path');", + "helper.resolve", + ], + [ + "POSIX module", + "const helper = require('node:path/posix');", + "helper.resolve", + ], + ["default import", "import helper from 'node:path';", "helper.resolve"], + [ + "namespace import", + "import * as helper from 'node:path';", + "helper.posix.resolve", + ], + ["named import", "import {resolve as select} from 'node:path';", "select"], +] as const; + +it("does not treat a constructed Node resolver as a path call", () => { + expect(() => Reflect.construct(posix.resolve, ["/app", "decoy.cjs"])).toThrow( + TypeError, + ); + const analysis = analyzeJavaScriptStaticSource(` + const helper = require('node:path'); + const {BrowserWindow} = require('electron'); + new BrowserWindow({webPreferences:{preload:new helper.resolve(runtimePath,__dirname,'decoy.cjs')}}); + `); + expect(analysis.role_paths).toEqual([]); + expect(analysis.electron.browser_windows).toMatchObject([ + { preload_path: null }, + ]); +}); + +it.each(nodeResolvers)( + "retains proven Node %s right-to-left anchoring", + (_name, setup, callee) => { + expect( + posix.resolve("/actual/runtime.cjs", "/fixture/app", "decoy.cjs"), + ).toBe("/fixture/app/decoy.cjs"); + const expression = `${callee}(runtimePath, __dirname, "decoy.cjs")`; + const analysis = analyzeJavaScriptStaticSource(` + const {BrowserWindow, utilityProcess} = require('electron'); + ${setup} + new BrowserWindow({webPreferences:{preload:${expression}}}); + utilityProcess.fork(${expression}); + `); + expect(analysis.electron.browser_windows).toMatchObject([ + { + preload_path: "decoy.cjs", + preload_resolution_context: "filesystem-expression", + }, + ]); + expect(analysis.electron.utility_processes).toMatchObject([ + { + module_path: "decoy.cjs", + module_resolution_context: "filesystem-expression", + }, + ]); + }, +); diff --git a/src/domain/javascript/javascriptStaticPathSourceAnchors.test.ts b/src/domain/javascript/javascriptStaticPathSourceAnchors.test.ts new file mode 100644 index 0000000000..6566050d82 --- /dev/null +++ b/src/domain/javascript/javascriptStaticPathSourceAnchors.test.ts @@ -0,0 +1,121 @@ +import { createRequire, wrap } from "node:module"; +import { posix } from "node:path"; +import { fileURLToPath } from "node:url"; +import { runInThisContext } from "node:vm"; + +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it.each([false, true])( + "preserves only untouched authored CommonJS dirname vars: assigned=%s", + (assigned) => { + const setup = `const path=require('node:path/posix');var __dirname;${assigned ? '__dirname="relative";' : ""}`; + const fixture = { exports: "" }; + const run = runInThisContext( + wrap(`${setup}module.exports=path.join(__dirname,'preload.cjs');`), + ); + run( + fixture.exports, + createRequire(import.meta.url), + fixture, + "/source/app/main.cjs", + "/source/app", + ); + expect(fixture.exports).toBe( + assigned ? "relative/preload.cjs" : "/source/app/preload.cjs", + ); + const analysis = analyzeJavaScriptStaticSource( + `${setup}const{BrowserWindow}=require('electron');new BrowserWindow({webPreferences:{preload:path.join(__dirname,'preload.cjs')}});`, + ); + expect(analysis.electron.browser_windows).toMatchObject([ + { preload_path: assigned ? null : "preload.cjs" }, + ]); + }, +); + +it("keeps local dirname methods from proving a source anchor", () => { + const helper = { dirname: (_url: string) => "/other" }; + expect( + posix.resolve( + "/runtime/base", + helper.dirname("file:///source/app/main.mjs"), + "decoy.cjs", + ), + ).toBe("/other/decoy.cjs"); + const analysis = analyzeJavaScriptStaticSource( + `import path from 'node:path';const helper={dirname:()=>'/other'};const{BrowserWindow}=require('electron');new BrowserWindow({webPreferences:{preload:path.resolve(runtimeBase,helper.dirname(import.meta.url),'decoy.cjs')}});`, + ); + expect(analysis.electron.browser_windows).toMatchObject([ + { preload_path: null }, + ]); +}); + +it("retains a proven Node dirname of the decoded module URL", () => { + expect( + posix.dirname( + fileURLToPath("file:///source/app/main.mjs", { windows: false }), + ), + ).toBe("/source/app"); + const analysis = analyzeJavaScriptStaticSource( + `import path from 'node:path';import{fileURLToPath}from'node:url';const{BrowserWindow}=require('electron');new BrowserWindow({webPreferences:{preload:path.resolve(runtimeBase,path.dirname(fileURLToPath(import.meta.url)),'preload.cjs')}});`, + ); + expect(analysis.electron.browser_windows).toMatchObject([ + { preload_path: "preload.cjs" }, + ]); +}); + +it.each([false, true])( + "preserves a decoded module URL through its real dirname owner: local=%s", + (local) => { + const decoded = fileURLToPath("file:///source/app/main.mjs", { + windows: false, + }); + const helper = { dirname: (_file: string) => "/other" }; + expect(local ? helper.dirname(decoded) : posix.dirname(decoded)).toBe( + local ? "/other" : "/source/app", + ); + const analysis = analyzeJavaScriptStaticSource(` + import path from 'node:path';import*as url from 'node:url'; + const helper={dirname:()=>'/other'};const{BrowserWindow}=require('electron'); + new BrowserWindow({webPreferences:{preload:path.resolve(runtimeBase,${local ? "helper" : "path"}.dirname(url.fileURLToPath(import.meta.url)),'preload.cjs')}}); + `); + expect(analysis.electron.browser_windows).toMatchObject([ + { preload_path: local ? null : "preload.cjs" }, + ]); + }, +); + +it("does not use a URL string as a filesystem source-file anchor", () => { + expect(posix.dirname("file:///source/app/main.mjs")).not.toBe("/source/app"); + const analysis = analyzeJavaScriptStaticSource( + `import path from 'node:path';const{BrowserWindow}=require('electron');new BrowserWindow({webPreferences:{preload:path.resolve(runtimeBase,path.dirname(import.meta.url),'decoy.cjs')}});`, + ); + expect(analysis.electron.browser_windows).toMatchObject([ + { preload_path: null }, + ]); +}); + +it.each([false, true])( + "proves the CommonJS filename dirname owner: local=%s", + (local) => { + const analysis = analyzeJavaScriptStaticSource( + `const path=require('node:path');const helper={dirname:()=>'.'};const{BrowserWindow}=require('electron');new BrowserWindow({webPreferences:{preload:path.resolve(runtimeBase,${local ? "helper" : "path"}.dirname(__filename),'decoy.cjs')}});`, + ); + expect(analysis.electron.browser_windows).toMatchObject([ + { preload_path: local ? null : "decoy.cjs" }, + ]); + }, +); + +it.each([false, true])( + "preserves the utility-process dirname parameter until assigned: assigned=%s", + (assigned) => { + const analysis = analyzeJavaScriptStaticSource( + `var __dirname;${assigned ? "__dirname='relative';" : ""}const path=require('node:path');const{utilityProcess}=require('electron');utilityProcess.fork(path.join(__dirname,'worker.cjs'));`, + ); + expect(analysis.electron.utility_processes).toMatchObject([ + { module_path: assigned ? null : "worker.cjs" }, + ]); + }, +); diff --git a/src/domain/referenceSourceRequireBindings.ts b/src/domain/referenceSourceRequireBindings.ts index a7ea10a97a..773c052324 100644 --- a/src/domain/referenceSourceRequireBindings.ts +++ b/src/domain/referenceSourceRequireBindings.ts @@ -39,11 +39,13 @@ export const unshadowedReferenceSourceRequireCalls = ( if (!t.isCallExpression(node)) return; const identifier = requireCalleeIdentifier(node.callee); if (identifier === undefined) return; - const fact = semanticRequireLoaderAt( - node, - index.bindings, - factoryBindings, - ); + let fact = semanticRequireLoaderAt(node, index.bindings, factoryBindings); + if (fact === "create-require") { + // Definitions identify a possible factory. Its mutable member reads + // need the shared write index before they can prove an association. + completeSemanticSteps(index.indexMemberMutationFactsSteps()); + fact = semanticRequireLoaderAt(node, index.bindings, factoryBindings); + } if ( fact === "create-require" || (fact === "commonjs-loader" && wrapperMode !== "absent") diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.test.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.test.ts index c20ee54d93..c4601eadf6 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.test.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.test.ts @@ -8,6 +8,41 @@ import { } from "./JavaScriptAnalysisWorkerSchemas.js"; import { analyzeJavaScriptSemantics } from "../../domain/javascript/javascriptSemanticAnalysis.js"; import { getterExpansionSource } from "../../../tests/fixtures/javascriptGetterExpansion.js"; +import { aliasSelectionBudgetSource } from "../../../tests/fixtures/javascriptAliasSelectionBudget.js"; + +it("transfers bounded alias selection coverage across the worker boundary", () => { + const source = aliasSelectionBudgetSource(); + const facts = analyzeJavaScriptStaticSource(source); + const ir = analyzeJavaScriptSemantics(source); + const header = { coverage: ir.coverage, limitations: ir.limitations }; + expect(ir.coverage).toMatchObject({ + status: "partial", + resourceLimits: ["alias-selections"], + }); + expect( + javaScriptWorkerModuleHeaderSchema.parse( + JSON.parse(JSON.stringify(header)) as unknown, + ), + ).toEqual(header); + expect( + javaScriptWorkerStaticAnalysisSchema.parse( + JSON.parse(JSON.stringify(facts)) as unknown, + ), + ).toEqual(facts); +}); + +it("retains application path change identity and provenance across the worker boundary", () => { + const facts = analyzeJavaScriptStaticSource(` + const { app } = require("electron"); + app.setAppPath("./other"); + `); + expect(facts.electron.app_path_changes).toHaveLength(1); + expect( + javaScriptWorkerStaticAnalysisSchema.parse( + JSON.parse(JSON.stringify(facts)) as unknown, + ), + ).toEqual(facts); +}); it("transfers getter expansion limits and unknown bindings across the worker boundary", () => { const ir = analyzeJavaScriptSemantics(getterExpansionSource()); diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts index 2ac16ad355..30bb2c176b 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts @@ -155,6 +155,9 @@ const bridgeShape = { const utilityShape = { ...findingContext, service_name: z.string().nullable() }; const electron = z .strictObject({ + app_path_changes: z + .array(z.strictObject(findingContext).transform(withRuntime)) + .optional(), call_paths: z .array( z.strictObject({ @@ -285,8 +288,9 @@ const electron = z }), ), }) - .transform(({ call_paths, ...rest }) => ({ + .transform(({ call_paths, app_path_changes, ...rest }) => ({ ...rest, + ...(app_path_changes === undefined ? {} : { app_path_changes }), ...(call_paths === undefined ? {} : { call_paths }), })); const referenceShape = { @@ -422,6 +426,7 @@ const resourceLimit = z.enum([ "primitive-bytes", "expression-depth", "getter-expansion", + "alias-selections", ]); const primitive = z.union([z.string(), z.number(), z.boolean(), z.null()]); const property: z.ZodType = z.lazy(() => diff --git a/src/process/ProcessOwnership.baselineAbsence.test.ts b/src/process/ProcessOwnership.baselineAbsence.test.ts new file mode 100644 index 0000000000..3621fa9d91 --- /dev/null +++ b/src/process/ProcessOwnership.baselineAbsence.test.ts @@ -0,0 +1,124 @@ +import { expect, it, vi } from "vitest"; + +import { + verifyNoTokenOwnedProcesses, + type ProcessIdentityObservation, + type ProcessOwnershipHost, + type ProcessTableEntry, +} from "./ProcessOwnership.js"; +import { ProcessOwnershipEvidenceOwner } from "./ProcessOwnershipEvidence.js"; + +const row: ProcessTableEntry = { + pid: 200, + parentPid: 100, + processGroupId: 100, + state: "S", + command: "listed-before-native-exit", +}; + +const absent = (platform: "linux" | "darwin"): ProcessIdentityObservation => ({ + state: "absent", + reason: + platform === "linux" + ? "not-found-in-caller-pid-namespace" + : "not-in-snapshot", +}); +const unavailable: ProcessIdentityObservation = { + state: "unavailable", + reason: "native identity could not be read", +}; + +const fixture = ( + platform: "linux" | "darwin", + observe: () => ProcessIdentityObservation, +) => { + const evidenceOwner = new ProcessOwnershipEvidenceOwner(); + const signalGroup = vi.fn(); + let listed = true; + const host: ProcessOwnershipHost = { + platform, + listProcesses: async () => (listed ? [row] : []), + environment: async () => ({}), + runTokens: async (entries) => + new Map( + entries.map(({ pid }) => [ + pid, + { state: "readable", runId: undefined } as const, + ]), + ), + processIdentities: async (entries) => + new Map(entries.map(({ pid }) => [pid, observe()])), + signalGroup, + }; + return { + evidenceOwner, + signalGroup, + omitRow: () => { + listed = false; + }, + verify: (identity: string | null) => + verifyNoTokenOwnedProcesses( + "run-token", + host, + [{ pid: row.pid, identity }], + { leaderPid: 100, processGroupId: 100, evidenceOwner }, + ), + }; +}; + +it.each( + (["linux", "darwin"] as const).flatMap((platform) => + (["old-start", null] as const).flatMap((baseline) => + (["unavailable", "absent"] as const).map((first) => ({ + platform, + baseline, + first, + })), + ), + ), +)( + "settles baseline $baseline after $first becomes native absence on $platform", + async ({ platform, baseline, first }) => { + let reads = 0; + const owned = fixture(platform, () => + ++reads === 1 && first === "unavailable" ? unavailable : absent(platform), + ); + await expect(owned.verify(baseline)).resolves.toEqual({ + cleaned: true, + signaled: false, + }); + expect(owned.evidenceOwner.candidates).toEqual([]); + expect(owned.signalGroup).not.toHaveBeenCalled(); + }, +); + +it.each(["linux", "darwin"] as const)( + "retains an unavailable generation after process-table omission on %s", + async (platform) => { + const owned = fixture(platform, () => unavailable); + const expected = { + cleaned: false, + reason: "process identity could not be revalidated", + failures: [ + expect.objectContaining({ + pid: row.pid, + reason: "process-identity-unavailable", + }), + ], + }; + await expect(owned.verify(null)).resolves.toMatchObject(expected); + expect(owned.evidenceOwner.candidates).toEqual([ + expect.objectContaining({ + pid: row.pid, + applicability: "identity-unresolved", + }), + ]); + owned.omitRow(); + await expect(owned.verify(null)).resolves.toMatchObject(expected); + expect(owned.evidenceOwner.candidates).toEqual([ + expect.objectContaining({ pid: row.pid }), + ]); + expect(owned.evidenceOwner.coverage.retainedOnlyPids).toBe(1); + expect(owned.signalGroup).not.toHaveBeenCalled(); + }, +); diff --git a/src/process/capture/ProcessCaptureLifecycle.test.ts b/src/process/capture/ProcessCaptureLifecycle.test.ts index abad38ac2c..cd6dc19a4c 100644 --- a/src/process/capture/ProcessCaptureLifecycle.test.ts +++ b/src/process/capture/ProcessCaptureLifecycle.test.ts @@ -1,4 +1,4 @@ -import { expect, it } from "vitest"; +import { expect, it, vi } from "vitest"; import type { IPty } from "@lydell/node-pty"; import { parseProcessScenario } from "../../domain/process/processScenario.js"; @@ -93,6 +93,36 @@ const start = ( recordEvent: () => undefined, }); +it("uses the monotonic running deadline after the wall clock moves backward", async () => { + const fake = fakeTerminal(); + const controller = new AbortController(); + const wallStart = Date.now(); + let reads = 0; + const wall = vi.spyOn(Date, "now").mockImplementation(() => { + reads += 1; + return reads === 1 ? wallStart : wallStart - 60_000 + reads; + }); + const kill = vi.spyOn(fake.terminal, "kill").mockImplementation(() => { + queueMicrotask(() => fake.deliverExit({ exitCode: 0, signal: 9 })); + }); + const fallback = setTimeout(() => controller.abort(), 1_500); + try { + const monotonicStarted = performance.now(); + const exit = observed( + await start(fake, { timeout_ms: 100 }, controller.signal), + ); + // The production clock rounds down to milliseconds, losing at most 1 ms. + expect(performance.now() - monotonicStarted).toBeGreaterThanOrEqual(99); + expect(exit.reason).toBe("timeout"); + expect(controller.signal.aborted).toBe(false); + expect(kill).toHaveBeenCalledOnce(); + } finally { + clearTimeout(fallback); + kill.mockRestore(); + wall.mockRestore(); + } +}); + it("records attempts when a self-exit notification arrives after escalation", async () => { const fake = fakeTerminal(); const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 }); diff --git a/src/process/capture/ProcessCaptureObservation.test.ts b/src/process/capture/ProcessCaptureObservation.test.ts index ace89ca938..de071301b0 100644 --- a/src/process/capture/ProcessCaptureObservation.test.ts +++ b/src/process/capture/ProcessCaptureObservation.test.ts @@ -1,6 +1,9 @@ import { expect, it } from "vitest"; -import { snapshotRoots } from "./FilesystemSnapshot.js"; +import { + ProcessCaptureSnapshotFailure, + snapshotRoots, +} from "./FilesystemSnapshot.js"; import { buildCaptureResult } from "./ProcessCaptureLifecycle.js"; import { isInitializedPtyRoot } from "./ProcessSampling.js"; import { readLinuxChildren } from "./LinuxProcessSampling.js"; @@ -271,17 +274,33 @@ it("admits PTY samples only after stable session and token setup", () => { it("cancels filesystem snapshots before traversing declared roots", async () => { const controller = new AbortController(); controller.abort(); - await expect( - snapshotRoots( - parseProcessScenario({ - ...base, - filesystem_observation_paths: ["/tmp"], - }), - controller.signal, - ), - ).rejects.toMatchObject({ + const failure: unknown = await snapshotRoots( + parseProcessScenario({ + ...base, + filesystem_observation_paths: ["/tmp"], + }), + controller.signal, + ).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(ProcessCaptureSnapshotFailure); + expect(failure).toMatchObject({ + _tag: "ProcessCaptureError", reason: "cancelled", - cause: controller.signal.reason, - snapshot: { files: [], truncated: true }, + userCategory: "cancelled", + cause: expect.objectContaining({ name: "AbortError" }), + snapshot: { + files: [], + truncated: true, + completeRoots: [], + coverage: { + hashed_bytes: 0, + enumeration_omissions: [], + hash_omissions: [], + }, + }, + cleanupLeases: [], + readFailures: [], }); + expect(failure instanceof Error ? failure.cause : undefined).toBe( + controller.signal.reason, + ); }); diff --git a/tests/acceptance/applications/javascriptCapturedGenericInvocation.test.ts b/tests/acceptance/applications/javascriptCapturedGenericInvocation.test.ts new file mode 100644 index 0000000000..d5dae895ae --- /dev/null +++ b/tests/acceptance/applications/javascriptCapturedGenericInvocation.test.ts @@ -0,0 +1,69 @@ +import { expect } from "vitest"; + +import { capturedGenericInvocations } from "../../fixtures/javascriptCapturedGenericInvocations.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + capturedGenericInvocations + .filter(({ chain }) => chain === "call.call" || chain === "apply.call") + .map((testCase) => ({ surface, ...testCase })), + ), +)( + "retains competing application owners for $name through $surface", + async (testCase, { cli }) => { + const root = await createTestTempDirectory("rea-captured-generic-"); + await writeFixtureFiles(root, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": "require('../shared/window.cjs');", + "a/index.html": "first application page", + "b/package.json": JSON.stringify({ main: "main.cjs" }), + "b/main.cjs": testCase.source, + "b/index.html": "second application page", + "shared/window.cjs": + "const {BrowserWindow}=require('electron');new BrowserWindow().loadFile('index.html');", + "shared/index.html": "source-directory decoy", + }); + const { graph, statistics } = await analyzeJavaScriptDirectory( + testCase.surface, + cli, + root, + ); + expect(statistics.parse_failures).toBe(0); + const renderer = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(renderer).toHaveLength(1); + expect(renderer[0]?.observations).toContainEqual( + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: testCase.host ? null : "a/index.html", + resolution_status: testCase.host ? "unavailable" : "resolved", + }), + }), + ); + const unknownLoads = graph.edges.filter( + ({ relation, properties }) => + relation === "imports" && properties.specifier === null, + ); + expect(unknownLoads.length > 0).toBe(testCase.host); + if (testCase.host) + expect(unknownLoads).toContainEqual( + expect.objectContaining({ + properties: expect.objectContaining({ + expression: testCase.expression, + }), + }), + ); + const mapped = graph.edges.filter( + ({ relation, source_node_id }) => + relation === "maps_to" && source_node_id === renderer[0]?.node_id, + ); + expect(mapped.map(({ properties }) => properties.resolved_path)).toEqual( + testCase.host ? [] : ["a/index.html"], + ); + }, +); diff --git a/tests/acceptance/applications/javascriptDependencyMainHostPaths.test.ts b/tests/acceptance/applications/javascriptDependencyMainHostPaths.test.ts new file mode 100644 index 0000000000..b2b29fa1ca --- /dev/null +++ b/tests/acceptance/applications/javascriptDependencyMainHostPaths.test.ts @@ -0,0 +1,68 @@ +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + [ + { main: "/shared/window.cjs", status: "unavailable" }, + { main: "../../shared/window.cjs", status: "resolved" }, + { main: "sub\\entry.cjs", status: "unavailable" }, + ].map((testCase) => ({ surface, ...testCase })), + ), +)( + "keeps dependency main $main in its lookup domain through $surface", + async ({ surface, main, status }, { cli }) => { + const root = await createTestTempDirectory("rea-dependency-host-main-"); + await writeFixtureFiles(root, { + "a/package.json": '{"main":"main.cjs"}', + "a/main.cjs": + 'throw new Error("Static fixture must never execute");require("fixture");', + "a/index.html": "first application page", + "b/package.json": '{"main":"main.cjs"}', + "b/main.cjs": + 'throw new Error("Static fixture must never execute");require("../shared/window.cjs");', + "b/index.html": "second application page", + "node_modules/fixture/package.json": JSON.stringify({ main }), + ...(main.includes("\\") + ? { + "node_modules/fixture/sub/entry.cjs": + 'throw new Error("Static fixture must never execute");require("../../../shared/window.cjs");', + } + : {}), + "shared/window.cjs": + 'throw new Error("Static fixture must never execute");const {BrowserWindow}=require("electron");new BrowserWindow().loadFile("index.html");', + "shared/index.html": "source-relative decoy", + "index.html": "artifact-root decoy", + }); + const { graph, statistics } = await analyzeJavaScriptDirectory( + surface, + cli, + root, + ); + expect(statistics.parse_failures).toBe(0); + const imports = graph.edges.filter( + ({ relation, properties }) => + relation === "imports" && properties.specifier === "fixture", + ); + expect(imports.length).toBeGreaterThan(0); + for (const edge of imports) + expect(edge.properties).toMatchObject({ + resolution_status: status, + resolved_path: status === "resolved" ? "shared/window.cjs" : null, + }); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + expect( + graph.edges.filter( + ({ relation, source_node_id }) => + relation === "maps_to" && source_node_id === roles[0]?.node_id, + ), + ).toEqual([]); + }, +); diff --git a/tests/acceptance/applications/javascriptEscapedArtifactImports.test.ts b/tests/acceptance/applications/javascriptEscapedArtifactImports.test.ts new file mode 100644 index 0000000000..94595dc593 --- /dev/null +++ b/tests/acceptance/applications/javascriptEscapedArtifactImports.test.ts @@ -0,0 +1,63 @@ +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + [ + { + request: "../../rea-artifact/shared/window.cjs", + status: "rejected", + }, + { request: "../shared/window.cjs", status: "resolved" }, + ].map((testCase) => ({ surface, ...testCase })), + ), +)( + "keeps ESM request $request within its real artifact boundary through $surface", + async ({ surface, request, status }, { cli }) => { + const root = await createTestTempDirectory("rea-esm-artifact-escape-"); + await writeFixtureFiles(root, { + "a/package.json": '{"main":"main.mjs"}', + "a/main.mjs": `throw new Error('Static fixture must never execute');import ${JSON.stringify(request)};`, + "a/index.html": "first application page", + "b/package.json": '{"main":"main.mjs"}', + "b/main.mjs": + 'throw new Error("Static fixture must never execute");import "../shared/window.cjs";', + "b/index.html": "second application page", + "shared/window.cjs": + 'throw new Error("Static fixture must never execute");const {BrowserWindow}=require("electron");new BrowserWindow().loadFile("index.html");', + "shared/index.html": "source-relative decoy", + "index.html": "artifact-root decoy", + }); + const { graph, statistics } = await analyzeJavaScriptDirectory( + surface, + cli, + root, + ); + expect(statistics.parse_failures).toBe(0); + const imports = graph.edges.filter( + ({ relation, properties }) => + relation === "imports" && properties.specifier === request, + ); + expect(imports.length).toBeGreaterThan(0); + for (const edge of imports) + expect(edge.properties).toMatchObject({ + resolution_status: status, + resolved_path: status === "resolved" ? "shared/window.cjs" : null, + }); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + const mapped = graph.edges.filter( + ({ relation, source_node_id }) => + relation === "maps_to" && source_node_id === roles[0]?.node_id, + ); + expect(mapped).toHaveLength(status === "rejected" ? 1 : 0); + if (status === "rejected") + expect(mapped[0]?.properties.resolved_path).toBe("b/index.html"); + }, +); diff --git a/tests/acceptance/applications/javascriptExportedAppPathChanges.test.ts b/tests/acceptance/applications/javascriptExportedAppPathChanges.test.ts new file mode 100644 index 0000000000..8ca0672b05 --- /dev/null +++ b/tests/acceptance/applications/javascriptExportedAppPathChanges.test.ts @@ -0,0 +1,81 @@ +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +const cases = [ + ...(["ESM", "CommonJS", "Object.assign"] as const).flatMap((format) => + [false, true].map((replaced) => ({ format, replaced })), + ), + { format: "local exports Object.assign", replaced: false }, + { format: "detached exports Object.assign", replaced: false }, +] as const; + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + cases.map((testCase) => ({ surface, ...testCase })), + ), +)( + "retains $format exported app setter uncertainty (replaced=$replaced) through $surface", + async ({ surface, format, replaced }, { cli }) => { + const root = await createTestTempDirectory("rea-exported-app-path-"); + const connected = + format !== "local exports Object.assign" && + format !== "detached exports Object.assign"; + const possible = connected && !replaced; + const publication = + format === "CommonJS" + ? "module.exports={change,app};" + : format === "local exports Object.assign" + ? "function publish(exports){Object.assign(exports,{change,app})}publish({});" + : `${format === "detached exports Object.assign" ? "module.exports={};" : ""}Object.assign(exports,{change,app});`; + const files: Record = + format !== "ESM" + ? { + "a/package.json": '{"main":"main.cjs"}', + "a/setter.cjs": `throw new Error('Static fixture must never execute');const {app}=require('electron');${replaced ? "app.setAppPath=()=>{};" : ""}const change=app.setAppPath;${publication}`, + "a/main.cjs": `throw new Error('Static fixture must never execute');const {change,app}=require('./setter.cjs');const path=require('node:path');${connected ? "" : "if(typeof change==='function')"}change.call(app,path.join(__dirname,'lib'));require('./win.cjs');`, + "a/win.cjs": `throw new Error('Static fixture must never execute');const {BrowserWindow}=require('electron');new BrowserWindow().loadFile('index.html');`, + } + : { + "a/package.json": '{"main":"main.mjs"}', + "a/main.mjs": `throw new Error('Static fixture must never execute');import {app,change} from '../shared/change.mjs';import {BrowserWindow} from 'electron';change.call(app,'./lib');new BrowserWindow().loadFile('index.html');`, + "shared/change.mjs": `throw new Error('Static fixture must never execute');import {app} from 'electron';${replaced ? "app.setAppPath=()=>{};" : ""}const change=app.setAppPath;export {app,change};`, + }; + await writeFixtureFiles(root, { + ...files, + "a/index.html": "application page", + "a/lib/index.html": "changed application page", + "shared/index.html": "source-relative decoy", + "index.html": "artifact-root decoy", + }); + const { graph, statistics } = await analyzeJavaScriptDirectory( + surface, + cli, + root, + ); + expect(statistics.parse_failures).toBe(0); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + expect(roles[0]?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolution_status: possible ? "unavailable" : "resolved", + }), + }), + ]), + ); + const mapped = graph.edges.filter( + ({ relation, source_node_id }) => + relation === "maps_to" && source_node_id === roles[0]?.node_id, + ); + expect(mapped).toHaveLength(possible ? 0 : 1); + if (!possible) + expect(mapped[0]?.properties.resolved_path).toBe("a/index.html"); + }, +); diff --git a/tests/acceptance/applications/javascriptFactoryCompletion.test.ts b/tests/acceptance/applications/javascriptFactoryCompletion.test.ts index 80b46ccc9f..3aa44d3d7f 100644 --- a/tests/acceptance/applications/javascriptFactoryCompletion.test.ts +++ b/tests/acceptance/applications/javascriptFactoryCompletion.test.ts @@ -378,7 +378,12 @@ cliTest( input_path: directory, format: "directory", }); - expect(response.isError, scenario.name).not.toBe(true); + expect( + response.isError, + response.isError + ? `${scenario.name}: ${JSON.stringify(response.content)}` + : scenario.name, + ).not.toBe(true); const evidence = contract.outputSchema.parse(response.structuredContent); verifyApplication( javascriptApplicationAnalysisResultSchema.parse( diff --git a/tests/acceptance/applications/javascriptIndirectLoaderOwners.test.ts b/tests/acceptance/applications/javascriptIndirectLoaderOwners.test.ts new file mode 100644 index 0000000000..7725003d7f --- /dev/null +++ b/tests/acceptance/applications/javascriptIndirectLoaderOwners.test.ts @@ -0,0 +1,280 @@ +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +const cases = [ + { + name: "typed member store retains returned module ownership", + mainFile: "main.ts", + load: 'const box={make:()=>({require(){}})};(box.make as any)=()=>module;box.make().require("../shared/window.cjs");', + possible: true, + }, + { + name: "typed local member store stays inert", + mainFile: "main.ts", + load: 'const box={make:()=>module};(box.make as any)=()=>({require(){}});box.make().require("../shared/window.cjs");', + possible: false, + }, + { + name: "assignment result retains module receiver", + load: 'let holder;(holder=module).require("../shared/window.cjs");', + possible: true, + }, + { + name: "assignment result retains local receiver", + load: 'let holder;(holder={require(){}}).require("../shared/window.cjs");', + possible: false, + }, + { + name: "captured wrapper require retains loader ownership", + load: 'const load=require;load("../shared/window.cjs");', + possible: true, + }, + { + name: "captured local callable stays inert", + load: 'const local=()=>{};const load=local;load("../shared/window.cjs");', + possible: false, + }, + { + name: "require.resolve does not execute its resolved module", + load: 'require.resolve("../shared/window.cjs");', + possible: false, + }, + { + name: "require.main is not an implicit module loader", + load: 'require.main("../shared/window.cjs");', + possible: false, + }, + { + name: "createRequire alias retains its source module base", + mainFile: "main.mjs", + load: 'import {createRequire} from "node:module";const load=createRequire(import.meta.url);load("../shared/window.cjs");', + possible: true, + }, + { + name: "conditional opaque factory retains competing loader ownership", + mainFile: "main.mjs", + load: 'import {createRequire as native} from "node:module";const make=flag?native:getFactory();const load=make(import.meta.url);load("./window.cjs");', + possible: true, + }, + { + name: "only local returned factory stays inert", + mainFile: "main.mjs", + load: 'const getFactory=()=>()=>()=>({});const make=getFactory();const load=make(import.meta.url);load("../shared/window.cjs");', + possible: false, + }, + { + name: "local createRequire name stays inert", + mainFile: "main.mjs", + load: 'const createRequire=()=>()=>{};const load=createRequire(import.meta.url);load("../shared/window.cjs");', + possible: false, + }, + { + name: "replaced createRequire factory stays inert", + mainFile: "main.mjs", + load: 'import * as mod from "node:module";mod.default.createRequire=()=>()=>({});const load=mod.default.createRequire(import.meta.url);load("../shared/window.cjs");', + possible: false, + }, + { + name: "CommonJS module has no synthetic default factory", + load: 'const load=require("node:module").default.createRequire(import.meta.url);load("../shared/window.cjs");', + possible: false, + }, + { + name: "conditional call replacement retains native loader alternative", + mainFile: "main.mjs", + load: 'import {createRequire as make} from "node:module";const load=make(import.meta.url);if(flag)load.call=()=>{};load.call(null,"../shared/window.cjs");', + possible: true, + }, + { + name: "conditional apply replacement retains native loader alternative", + mainFile: "main.mjs", + load: 'import {createRequire as make} from "node:module";const load=make(import.meta.url);if(flag)load.apply=()=>{};load.apply(null,["../shared/window.cjs"]);', + possible: true, + }, + { + name: "replaced created-loader call stays inert", + mainFile: "main.mjs", + load: 'import {createRequire as make} from "node:module";const load=make(import.meta.url);load.call=()=>{};load.call(null,"../shared/window.cjs");', + possible: false, + }, + { + name: "changed Function call retains loader argument uncertainty", + mainFile: "main.mjs", + load: 'import {createRequire as make} from "node:module";const load=make(import.meta.url);Function.prototype.call=()=>({});load.call(null,"./inert.cjs");', + possible: true, + }, + { + name: "changed Function apply retains loader argument uncertainty", + mainFile: "main.mjs", + load: 'import {createRequire as make} from "node:module";const load=make(import.meta.url);Function.prototype.apply=()=>({});load.apply(null,["./inert.cjs"]);', + possible: true, + }, + { + name: "custom array iterator preserves loader ownership", + load: 'Array.prototype[Symbol.iterator]=function*(){yield "../shared/window.cjs"};require(...["./inert.cjs"]);', + possible: true, + }, + { + name: "Reflect.apply retains module delegate ownership", + load: 'Reflect.apply(module.require,module,["../shared/window.cjs"]);', + possible: true, + }, + { + name: "local Reflect.apply stays inert", + load: 'const Reflect={apply(){}};Reflect.apply(module.require,module,["../shared/window.cjs"]);', + possible: false, + }, + { + name: "noncallable generic receiver excludes callable children", + load: 'const get=()=>({});const invoke=get.call;const holder={invoke,other:()=>module};holder.invoke(null).require("../shared/window.cjs");', + possible: false, + }, + { + name: "callable generic receiver retains its return", + load: 'const get=()=>({});const invoke=get.call;const holder=()=>module;holder.invoke=invoke;holder.other=()=>({});holder.invoke(null).require("../shared/window.cjs");', + possible: true, + }, + { + name: "destructured module alias retains own copied delegate", + load: 'let selected;({selected}={selected:module});selected.require=module.require;const copy={...module};const load=copy.require;load.call(module,"../shared/window.cjs");', + possible: true, + }, + { + name: "destructured local alias does not change module copies", + load: 'let selected;({selected}={selected:{require(){}}});selected.require=module.require;const copy={...module};const load=copy.require;load.call(module,"../shared/window.cjs");', + possible: false, + }, + { + name: "inherited delegate excluded from rest", + load: 'const {...h}=module;h.require.call(module,"../shared/window.cjs");', + possible: false, + }, + { + name: "own delegate retained in spread", + load: 'module.require=module.require;const h={...module};h.require.call(module,"../shared/window.cjs");', + possible: true, + }, + { + name: "nested module retained in spread", + load: 'const h={...{loader:module}};h.loader.require("../shared/window.cjs");', + possible: true, + }, + { + name: "require-named delegate", + load: 'const h={require:module.require};h.require.call(module,"../shared/window.cjs");', + possible: true, + }, + { + name: "require-named local", + load: 'const h={require(){}};h.require.call(module,"../shared/window.cjs");', + possible: false, + }, + { + name: "require-named applied delegate", + load: 'const h={require:module.require};h.require.apply(module,["../shared/window.cjs"]);', + possible: true, + }, + { + name: "require-named applied local", + load: 'const h={require(){}};h.require.apply(module,["../shared/window.cjs"]);', + possible: false, + }, + { + name: "module receiver", + load: '(0,module).require("../shared/window.cjs");', + possible: true, + }, + { + name: "main module receiver", + load: 'require.main.require("../shared/window.cjs");', + possible: true, + }, + { + name: "process main module receiver", + load: 'process.mainModule.require("../shared/window.cjs");', + possible: true, + }, + { + name: "local process main module receiver", + load: 'const process={mainModule:{require(){}}};process.mainModule.require("../shared/window.cjs");', + possible: false, + }, + { + name: "local main module receiver", + load: 'const require={main:{require(){}}};require.main.require("../shared/window.cjs");', + possible: false, + }, + { + name: "captured delegate", + load: 'const load=(0,module.require);load("../shared/window.cjs");', + possible: true, + }, + { + name: "local receiver", + load: '(module,{require(){}}).require("../shared/window.cjs");', + possible: false, + }, + { + name: "local final delegate", + load: 'const load=(module.require,()=>{});load("../shared/window.cjs");', + possible: false, + }, +] as const; + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + cases.map((test) => ({ ...test, surface })), + ), +)( + "preserves $name loader owner uncertainty through $surface", + async (testCase, { cli }) => { + const { surface, load, possible } = testCase; + const mainFile = "mainFile" in testCase ? testCase.mainFile : "main.cjs"; + const root = await createTestTempDirectory("rea-sequence-loader-"); + await writeFixtureFiles(root, { + "a/package.json": JSON.stringify({ main: mainFile }), + [`a/${mainFile}`]: `throw new Error('Static fixture must never execute');${load}`, + "a/inert.cjs": + "throw new Error('Static inert decoy must never execute');", + "a/window.cjs": + "throw new Error('Static factory-base decoy must never execute');", + "a/index.html": "possible page", + "b/package.json": '{"main":"main.cjs"}', + "b/main.cjs": `throw new Error('Static fixture must never execute');require('../shared/window.cjs');`, + "b/index.html": "proven page", + "shared/window.cjs": `throw new Error('Static fixture must never execute');const {BrowserWindow}=require('electron');new BrowserWindow().loadFile('index.html');`, + "shared/index.html": "source-relative decoy", + "index.html": "artifact-root decoy", + }); + const { graph, statistics } = await analyzeJavaScriptDirectory( + surface, + cli, + root, + ); + expect(statistics.parse_failures).toBe(0); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + expect(roles[0]?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolution_status: possible ? "unavailable" : "resolved", + }), + }), + ]), + ); + const mapped = graph.edges.filter( + ({ relation, source_node_id }) => + relation === "maps_to" && source_node_id === roles[0]?.node_id, + ); + expect(mapped).toHaveLength(possible ? 0 : 1); + if (!possible) + expect(mapped[0]?.properties.resolved_path).toBe("b/index.html"); + }, +); diff --git a/tests/acceptance/applications/javascriptLiteralPercentPaths.test.ts b/tests/acceptance/applications/javascriptLiteralPercentPaths.test.ts new file mode 100644 index 0000000000..31fb7ae6df --- /dev/null +++ b/tests/acceptance/applications/javascriptLiteralPercentPaths.test.ts @@ -0,0 +1,1403 @@ +import { expect } from "vitest"; +import { copyFile } from "node:fs/promises"; +import { join } from "node:path"; +import { createPackageWithOptions } from "@electron/asar"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { aliasSelectionBudgetSource } from "../../fixtures/javascriptAliasSelectionBudget.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +const fixtureImports = (loader: string) => { + switch (loader) { + case "pattern alias": + return { "#window/*": "./proxy/*.cjs" }; + case "recursive alias": + return { "#entry": "#window", "#window": "./proxy/window.cjs" }; + case "external-package alias": + return { "#entry": "fixture-window" }; + case "builtin alias": + return { "#entry": "fs" }; + case "electron alias": + return { "#entry": "electron" }; + case "electron main alias": + return { "#entry": "electron/main" }; + default: + return { "#entry": null }; + } +}; + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + [ + "direct", + "alias", + "call", + "apply", + "bind", + "reflect", + "captured reflect", + "local", + ].map((setter) => ({ + surface, + setter, + })), + ), +)( + "keeps loadFile roots unknown after an Electron application path setter through $surface ($setter)", + async ({ surface, setter }, { cli }) => { + const root = await createTestTempDirectory("rea-app-path-setter-"); + const change = + setter === "direct" + ? 'const { app } = require("electron"); app.setAppPath("/runtime/changed");' + : setter === "alias" + ? 'const { app: runtime } = require("electron/main"); const current = runtime; current["setAppPath"]("/runtime/changed");' + : setter === "call" + ? 'const { app } = require("electron"); app.setAppPath.call(app, "/runtime/changed");' + : setter === "apply" + ? 'const { app } = require("electron"); app.setAppPath.apply(app, ["/runtime/changed"]);' + : setter === "bind" + ? 'const { app } = require("electron"); const change = app.setAppPath.bind(app); change("/runtime/changed");' + : setter === "reflect" + ? 'const { app } = require("electron"); Reflect.apply(app.setAppPath, app, ["/runtime/changed"]);' + : setter === "captured reflect" + ? 'const { app } = require("electron"); const invoke = Reflect.apply; invoke(app.setAppPath, app, ["/runtime/changed"]);' + : 'const app = { setAppPath() {} }; app.setAppPath("/runtime/changed");'; + await writeFixtureFiles(root, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": `${change} require("./window.cjs");`, + "a/window.cjs": + 'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");', + "a/index.html": "package-root page", + "index.html": "artifact-root decoy", + }); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + const role = graph.nodes.find(({ kind }) => kind === "electron-renderer"); + const target = setter === "local" ? "a/index.html" : null; + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolution_status: target === null ? "unavailable" : "resolved", + resolved_path: target, + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ source_node_id, relation }) => + source_node_id === role?.node_id && relation === "maps_to", + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(target === null ? [] : [target]); + }, +); + +const unprovenLoaderSources: Readonly> = { + "custom iterator assignment module loader": + 'const values = { *[Symbol.iterator]() { yield module; } }; const holder = {}; [holder.loader] = values; holder.loader.require("../shared/window.cjs");', + "replaced iterator assignment module loader": + 'const values = [{}]; values[Symbol.iterator] = function* () { yield module; }; const holder = {}; [holder.loader] = values; holder.loader.require("../shared/window.cjs");', + "custom iterator rest declaration module loader": + 'const values = { *[Symbol.iterator]() { yield module; } }; const [...holder] = values; holder[0].require("../shared/window.cjs");', + "native array assignment local loader": + 'const holder = { loader: module }; [holder.loader] = [{ require() {} }]; holder.loader.require("../shared/window.cjs");', + "logical rewrite assignment module loader": + 'const holder = {}; holder.loader ??= module; holder.loader.require = replacement; require("../shared/window.cjs");', + "object rest declaration module loader": + 'const { ...holder } = { loader: module }; holder.loader.require("../shared/window.cjs");', + "array rest declaration module loader": + 'const [...holder] = [module]; holder[0].require("../shared/window.cjs");', + "object assignment module loader": + 'const holder = {}; ({ loader: holder.loader } = { loader: module }); holder.loader.require("../shared/window.cjs");', + "array assignment module loader": + 'const holder = {}; [holder.loader] = [module]; holder.loader.require("../shared/window.cjs");', + "nested assignment module loader": + 'const holder = {}; ({ inner: { loader: holder.loader } } = { inner: { loader: module } }); holder.loader.require("../shared/window.cjs");', + "default assignment module loader": + 'const holder = {}; ({ loader: holder.loader = module } = {}); holder.loader.require("../shared/window.cjs");', + "rest array assignment module loader": + 'const holder = {}; let ignored; [ignored, ...holder.loaders] = [{}, module]; holder.loaders[0].require("../shared/window.cjs");', + "delegate assignment module loader": + 'const holder = {}; ({ load: holder.load } = { load: module.require }); holder.load.call(module, "../shared/window.cjs");', + "rewrite assignment module loader": + 'const holder = {}; ({ loader: holder.loader } = { loader: module }); holder.loader.require = replacement; require("../shared/window.cjs");', + "exports assignment local loader": + 'const holder = {}; ({ exports: holder.loader } = module); holder.loader.require("../shared/window.cjs");', + "rest assignment local loader": + 'const holder = {}; ({ ...holder.loader } = module); holder.loader.require("../shared/window.cjs");', + "excluded assignment local loader": + 'const holder = {}; let ignored; ({ loader: ignored, ...holder.copy } = { loader: module }); holder.copy.loader.require("../shared/window.cjs");', + "inactive default assignment local loader": + 'const holder = {}; ({ loader: holder.loader = module } = { loader: { require() {} } }); holder.loader.require("../shared/window.cjs");', + "overwritten assignment local loader": + 'const holder = { loader: module }; ({ loader: holder.loader } = { loader: { require() {} } }); holder.loader.require("../shared/window.cjs");', + "bounded alias selection": aliasSelectionBudgetSource(), + "captured module delegate": + 'const load = module.require; load.call(module, "../shared/window.cjs");', + "destructured module delegate": + 'const { require: load } = module; load.apply(module, ["../shared/window.cjs"]);', + "selected local delegate slots": + 'const table = { x: {}, y: {}, z: module }; const target = flag ? table.x : table.y; target.require = replacement; require("../shared/window.cjs");', + "local assigned loader": + 'const holder = {}; holder.loader = { require() {} }; holder.loader.require("../shared/window.cjs");', + "overwritten local loader": + 'const holder = { loader: module }; holder.loader = { require() {} }; holder.loader.require("../shared/window.cjs");', + "normalized absolute ESM": 'import "/discard%2F/../shared/window.cjs";', + "normalized absolute dynamic import": + 'import("/discard%5C/../shared/window.cjs");', + "rewritten CommonJS delegate": + 'module.require = replacement; require("../shared/window.cjs");', + "nested rewritten CommonJS delegate": + 'const holder = { loader: module }; holder.loader.require = replacement; require("../shared/window.cjs");', + "assigned rewritten CommonJS delegate": + 'const holder = {}; holder.loader = module; holder.loader.require = replacement; require("../shared/window.cjs");', + "assigned module loader": + 'const holder = {}; holder.loader = module; holder.loader.require("../shared/window.cjs");', + "nested module loader": + 'const holder = { loader: module }; holder.loader.require("../shared/window.cjs");', + "destructured module loader": + 'const holder = { loader: module }; const { loader } = holder; loader.require("../shared/window.cjs");', + "array module loader": + 'const holder = [module]; holder[0].require("../shared/window.cjs");', + "bound module loader": + 'const load = module.require.bind(module); load("../shared/window.cjs");', + "bound module alias loader": + 'const current = module; const load = current.require.bind(current); load("../shared/window.cjs");', + "computed bound module loader": + 'const load = module["require"].bind(module); load("../shared/window.cjs");', + "fallback module alias": + 'const { loader = module } = options; loader.require("../shared/window.cjs");', + "inactive fallback": + 'const { loader = module } = { loader: {} }; loader.require("../shared/window.cjs");', + "absolute ESM": 'import "/shared/window.cjs";', + "invalid absolute ESM": 'import "/shared%2Fwindow.cjs";', + "invalid absolute dynamic import": 'import("/shared%5Cwindow.cjs");', + "absolute dynamic import": 'import("/shared/window.cjs");', + "absolute file URL": 'import "file:///shared/window.cjs";', + "dynamic file URL": 'import("file:///shared/window.cjs");', + "createRequire base": + 'import { createRequire } from "node:module"; const require = createRequire(new URL("../other/nested/main.cjs", import.meta.url)); require("../shared/window.cjs");', +}; + +const uncheckedModuleLoaders = new Set([ + "custom iterator assignment module loader", + "replaced iterator assignment module loader", + "custom iterator rest declaration module loader", + "logical rewrite assignment module loader", + "object rest declaration module loader", + "array rest declaration module loader", + "object assignment module loader", + "array assignment module loader", + "nested assignment module loader", + "default assignment module loader", + "rest array assignment module loader", + "delegate assignment module loader", + "rewrite assignment module loader", + "bounded alias selection", + "captured module delegate", + "destructured module delegate", + "rewritten CommonJS delegate", + "nested rewritten CommonJS delegate", + "assigned rewritten CommonJS delegate", + "assigned module loader", + "module.require", + "computed module.require", + "module alias", + "later reassigned module alias", + "conditional module alias", + "fallback module alias", + "bound module loader", + "bound module alias loader", + "computed bound module loader", + "nested module loader", + "destructured module loader", + "array module loader", +]); + +const fixtureMainPath = (loader: string): string => { + switch (loader) { + case "absolute ESM": + case "normalized absolute ESM": + case "normalized absolute dynamic import": + case "invalid absolute ESM": + case "invalid absolute dynamic import": + case "absolute dynamic import": + case "absolute file URL": + case "dynamic file URL": + case "createRequire base": + return "main.mjs"; + case "absolute main": + return "/opt/app/main.cjs"; + case "Windows relative main": + return "sub\\main.cjs"; + case "Windows drive-relative main": + return "C:main.cjs"; + case "percent main": + case "single percent main": + return "main%2Fentry.cjs"; + default: + return "main.cjs"; + } +}; + +const fixtureMainInventoryPath = (loader: string, main: string): string => { + if (loader === "absolute main") return "opt/app/main.cjs"; + if (loader === "Windows relative main") return "b/sub/main.cjs"; + if (loader === "Windows drive-relative main") return "b/main.cjs"; + return `b/${main}`; +}; + +const fixtureLoaderSource = (loader: string): string => + unprovenLoaderSources[loader] ?? + (loader === "resolve-only" + ? 'require.resolve("../shared/window.cjs");' + : loader === "missing wildcard alias" + ? 'require("#missing/*");' + : loader === "absolute require" + ? 'require("/shared/window.cjs");' + : loader === "projected module" + ? 'const { exports: api } = module; api.require = () => {}; api.require("../shared/window.cjs");' + : loader === "absolute main" + ? 'require("../../shared/window.cjs");' + : loader === "later reassigned module alias" + ? 'let current = module; current.require("../shared/window.cjs"); current = other;' + : loader === "earlier reassigned module alias" + ? 'let current = module; current = { require() {} }; current.require("../shared/window.cjs");' + : loader === "conditional module alias" + ? 'let current = module; if (flag) current = other; current.require("../shared/window.cjs");' + : loader === "module.require" + ? 'module.require("../shared/window.cjs");' + : loader === "computed module.require" + ? 'module["require"]("../shared/window.cjs");' + : loader === "module alias" + ? 'const current = module; current.require("../shared/window.cjs");' + : loader === "local module" + ? 'function local(module) { module.require("../shared/window.cjs"); }' + : loader === "pattern alias" + ? 'require("#window/window");' + : loader.endsWith("alias") + ? 'require("#entry");' + : 'require("../shared/window.cjs");'); + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + [ + "module.require", + "computed module.require", + "module alias", + "later reassigned module alias", + "earlier reassigned module alias", + "conditional module alias", + "absolute main", + "Windows relative main", + "Windows drive-relative main", + "percent main", + "pattern alias", + "recursive alias", + "external-package alias", + "builtin alias", + "electron alias", + "electron main alias", + "blocked alias", + "local module", + "projected module", + "resolve-only", + "missing wildcard alias", + "absolute require", + ...Object.keys(unprovenLoaderSources), + "single percent main", + ].map((loader) => ({ surface, loader })), + ), +)( + "preserves competing loader ownership for $loader through $surface", + async ({ surface, loader }, { cli }) => { + const root = await createTestTempDirectory("rea-loader-owner-"); + const single = loader === "single percent main"; + const inert = [ + "native array assignment local loader", + "exports assignment local loader", + "rest assignment local loader", + "excluded assignment local loader", + "inactive default assignment local loader", + "overwritten assignment local loader", + "local assigned loader", + "overwritten local loader", + "invalid absolute ESM", + "invalid absolute dynamic import", + "builtin alias", + "electron alias", + "electron main alias", + "inactive fallback", + "blocked alias", + "local module", + "projected module", + "resolve-only", + "missing wildcard alias", + "earlier reassigned module alias", + ].includes(loader); + const absolute = loader === "absolute main"; + const main = fixtureMainPath(loader); + const imports = fixtureImports(loader); + const load = fixtureLoaderSource(loader); + await writeFixtureFiles(root, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": + single || absolute + ? "module.exports = 1;" + : 'require("../shared/window.cjs");', + "b/package.json": JSON.stringify({ main, imports }), + [fixtureMainInventoryPath(loader, main)]: load, + "b/proxy/window.cjs": 'require("../../shared/window.cjs");', + "b/node_modules/fixture-window/package.json": JSON.stringify({ + main: "index.cjs", + }), + "b/node_modules/fixture-window/index.cjs": + 'require("../../../shared/window.cjs");', + "shared/window.cjs": + 'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");', + "a/index.html": "first application page", + "b/index.html": "second application page", + "shared/index.html": "source-relative decoy", + "index.html": "artifact-root decoy", + }); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + if (loader.startsWith("Windows ")) + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "loads", + properties: expect.objectContaining({ + declared_path: main, + resolution_status: "unavailable", + resolved_path: null, + }), + }), + ]), + ); + if ( + ["absolute require", "absolute ESM", "absolute dynamic import"].includes( + loader, + ) + ) + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + specifier: "/shared/window.cjs", + resolution_status: "unavailable", + resolved_path: null, + }), + }), + ]), + ); + if (loader === "createRequire base") + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + kind: "require", + specifier: null, + resolved_path: null, + }), + }), + ]), + ); + if (loader.endsWith("alias") && !inert && !loader.includes("module alias")) + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + specifier: + loader === "pattern alias" ? "#window/window" : "#entry", + resolution_status: "unavailable", + }), + }), + ]), + ); + if (uncheckedModuleLoaders.has(loader)) + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + kind: "require", + specifier: null, + resolved_path: null, + }), + }), + ]), + ); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + const role = roles[0]; + const target = single ? "b/index.html" : inert ? "a/index.html" : null; + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: target, + resolution_status: target === null ? "unavailable" : "resolved", + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ source_node_id, relation }) => + source_node_id === role?.node_id && relation === "maps_to", + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(target === null ? [] : [target]); + }, +); + +cliTest.for(["CLI", "stdio MCP"] as const)( + "propagates the application owner into an inventoried ASAR through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-asar-inward-"); + const source = join(root, "source"); + const application = join(root, "application"); + await writeFixtureFiles(source, { + "window.cjs": + 'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");', + "index.html": "archive-root decoy", + }); + await writeFixtureFiles(application, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": 'require("../resources/code.asar/window.cjs");', + "a/index.html": "proven application page", + }); + await createPackageWithOptions( + source, + join(application, "resources/code.asar"), + {}, + ); + const { graph } = await analyzeJavaScriptDirectory( + surface, + cli, + application, + ); + const role = graph.nodes.find(({ kind }) => kind === "electron-renderer"); + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolution_status: "resolved", + resolved_path: "a/index.html", + }), + }), + ]), + ); + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + specifier: "../resources/code.asar/window.cjs", + resolution_status: "resolved", + resolved_path: "resources/code.asar/window.cjs", + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ source_node_id, relation }) => + source_node_id === role?.node_id && relation === "maps_to", + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(["a/index.html"]); + }, +); + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + [false, true].map((dynamic) => ({ surface, dynamic })), + ), +)( + "retains possible owners for a module across a nested ASAR boundary through $surface (dynamic=$dynamic)", + async ({ surface, dynamic }, { cli }) => { + const root = await createTestTempDirectory("rea-asar-module-gap-"); + const source = join(root, "source"); + const application = join(root, "application"); + await writeFixtureFiles(source, { + "package.json": JSON.stringify({ main: "main.cjs" }), + "main.cjs": dynamic + ? "require(process.env.WINDOW_MODULE);" + : 'require("../window.cjs");', + }); + await writeFixtureFiles(application, { + "other/package.json": JSON.stringify({ main: "main.cjs" }), + "other/main.cjs": 'require("../resources/window.cjs");', + "resources/window.cjs": + 'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");', + "other/index.html": "false unique owner decoy", + }); + await createPackageWithOptions( + source, + join(application, "resources/app.asar"), + {}, + ); + const { graph } = await analyzeJavaScriptDirectory( + surface, + cli, + application, + ); + const role = graph.nodes.find(({ kind }) => kind === "electron-renderer"); + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolution_status: "unavailable", + resolved_path: null, + }), + }), + ]), + ); + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + specifier: dynamic ? null : "../window.cjs", + ...(dynamic ? {} : { resolution_status: "resolved" }), + resolved_path: dynamic ? null : "resources/window.cjs", + }), + }), + ]), + ); + expect( + graph.edges.filter( + ({ source_node_id, relation }) => + source_node_id === role?.node_id && relation === "maps_to", + ), + ).toEqual([]); + }, +); + +cliTest.for(["CLI", "stdio MCP"] as const)( + "resolves a proven application page across a nested ASAR boundary through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-asar-outer-page-"); + const source = join(root, "source"); + const application = join(root, "application"); + await writeFixtureFiles(source, { + "package.json": JSON.stringify({ main: "main.cjs" }), + "main.cjs": + 'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("../page.html");', + }); + await writeFixtureFiles(application, { + "resources/page.html": "owned outer page", + }); + await createPackageWithOptions( + source, + join(application, "resources/app.asar"), + {}, + ); + const { graph } = await analyzeJavaScriptDirectory( + surface, + cli, + application, + ); + const role = graph.nodes.find(({ kind }) => kind === "electron-renderer"); + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: "resources/page.html", + resolution_status: "resolved", + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ source_node_id, relation }) => + source_node_id === role?.node_id && relation === "maps_to", + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(["resources/page.html"]); + }, +); + +const cases = [ + ...[ + "C:/app/preload.cjs", + "C:\\app\\preload.cjs", + "\\\\server\\share\\preload.cjs", + ].map((path, index) => ({ + directory: `windows-absolute-${index}`, + expression: JSON.stringify(path), + target: null, + status: "unavailable", + })), + { + directory: "absolute-runtime", + expression: '"/outside%2Fname.cjs"', + target: null, + status: "unavailable", + }, + { + directory: "literal", + expression: 'path.join(__dirname, "preload%2F%5C%2E.cjs")', + target: "preload%2F%5C%2E.cjs", + status: "resolved", + }, + { + directory: "converted", + expression: + 'fileURLToPath(new URL("./preload%252Fname.cjs", import.meta.url))', + target: "preload%2Fname.cjs", + status: "resolved", + }, + { + directory: "escaped", + expression: 'path.join(__dirname, "../../outside%2Fname.cjs")', + target: null, + status: "rejected", + }, + { + directory: "ignored-prefix", + expression: + 'path.resolve(new URL("./", import.meta.url), path.dirname(fileURLToPath(import.meta.url)), "preload%2Fname.cjs")', + target: "preload%2Fname.cjs", + status: "resolved", + }, +] as const; + +const writeRuntimeFixture = async (root: string): Promise => { + const inert = 'throw new Error("fixture source must never execute");'; + const loadFileSource = `const { BrowserWindow } = require("electron"); + const win = new BrowserWindow(); + win.loadFile("dist%2Findex.html"); + win.loadFile("plain.html"); + win.loadFile("absent.html");`; + await writeFixtureFiles(root, { + "package.json": JSON.stringify({ private: true, workspaces: ["app"] }), + ...Object.fromEntries( + cases.map(({ directory, expression }) => [ + `${directory}/main.mjs`, + `import { BrowserWindow, utilityProcess } from "electron"; + import path from "node:path"; + import { fileURLToPath } from "node:url"; + new BrowserWindow({ webPreferences: { preload: ${expression} } }); + utilityProcess.fork(${expression});`, + ]), + ), + "literal/preload%2F%5C%2E.cjs": inert, + "converted/preload%2Fname.cjs": inert, + "converted/preload%252Fname.cjs": inert, + "converted/preload/name.cjs": inert, + "outside%2Fname.cjs": inert, + "ignored-prefix/preload%2Fname.cjs": inert, + "sub/main.cjs": loadFileSource, + "dist%2Findex.html": "application target", + "plain.html": "application target", + "sub/dist%2Findex.html": "nested decoy", + "sub/plain.html": "nested decoy", + "sub/absent.html": "nested-only decoy", + "app/package.json": JSON.stringify({ main: "sub/main.cjs" }), + "app/sub/package.json": JSON.stringify({ type: "commonjs" }), + "app/sub/main.cjs": loadFileSource, + "app/dist%2Findex.html": "wrapped application target", + "app/plain.html": "wrapped application target", + "app/sub/dist%2Findex.html": "wrapped source decoy", + "app/sub/plain.html": "wrapped source decoy", + "app/sub/absent.html": "wrapped missing-target decoy", + "external-app/package.json": JSON.stringify({ main: "../shared/main.cjs" }), + "shared/main.cjs": loadFileSource, + "external-app/dist%2Findex.html": "external-main application target", + "external-app/plain.html": "external-main application target", + "competition/package.json": JSON.stringify({ main: "main.cjs" }), + "competition/main.cjs": 'require("./bridge.cjs");', + "competition/bridge.cjs": 'require("./nested/main.cjs");', + "competition/nested/package.json": JSON.stringify({ main: "main.cjs" }), + "competition/nested/main.cjs": `${loadFileSource}\nrequire("../main.cjs");`, + "competition/dist%2Findex.html": "running application page", + "competition/nested/dist%2Findex.html": "competing application page", + "import-owner/package.json": JSON.stringify({ + main: "../imported/start.cjs", + }), + "imported/start.cjs": 'require("./child.cjs");', + "imported/child.cjs": loadFileSource, + "import-owner/dist%2Findex.html": "imported application target", + "import-owner/plain.html": "imported application target", + "default-app/package.json": JSON.stringify({ name: "default-entry-app" }), + "default-app/index.js": `${loadFileSource}\nrequire("./nested/main.cjs");`, + "default-app/nested/package.json": JSON.stringify({ main: "main.cjs" }), + "default-app/nested/main.cjs": loadFileSource, + "default-app/nested/dist%2Findex.html": "competing default-entry page", + "default-app/dist%2Findex.html": "default-entry application target", + "default-app/plain.html": "default-entry application target", + "ambiguous/package.json": JSON.stringify({ main: "sub/main.cjs" }), + "ambiguous/sub/package.json": JSON.stringify({ main: "main.cjs" }), + "ambiguous/third/package.json": JSON.stringify({ main: "../sub/main.cjs" }), + "ambiguous/sub/main.cjs": loadFileSource, + "ambiguous/dist%2Findex.html": "unproven application target", + "ambiguous/sub/dist%2Findex.html": "unproven nested application target", + }); + const archiveSource = await createTestTempDirectory("rea-loadfile-asar-"); + await writeFixtureFiles(archiveSource, { + "package.json": JSON.stringify({ main: "sub/main.cjs" }), + "sub/package.json": JSON.stringify({ type: "commonjs" }), + "sub/main.cjs": loadFileSource, + "dist%2Findex.html": "application target", + "plain.html": "application target", + "sub/dist%2Findex.html": "nested decoy", + "sub/plain.html": "nested decoy", + "sub/absent.html": "nested-only decoy", + "external-app/package.json": JSON.stringify({ main: "../shared/main.cjs" }), + "shared/main.cjs": loadFileSource, + "external-app/dist%2Findex.html": "external-main application target", + "external-app/plain.html": "external-main application target", + }); + await createPackageWithOptions(archiveSource, join(root, "a.asar"), {}); + // Equal container digests must not select the other archive occurrence. + await copyFile(join(root, "a.asar"), join(root, "b.asar")); +}; + +cliTest.for(["CLI", "stdio MCP"] as const)( + "does not invent a nested default-entry owner without index.js through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-empty-default-owner-"); + for (const [directory, metadata] of [ + ["empty", { main: "" }], + ["absent", {}], + ] as const) + await writeFixtureFiles(root, { + [`${directory}/package.json`]: JSON.stringify(metadata), + [`${directory}/helper.cjs`]: + 'const {BrowserWindow} = require("electron"); const w = new BrowserWindow(); w.loadFile("index.html");', + [`${directory}/index.html`]: "nested decoy without launch entry", + "index.html": "artifact-root page", + }); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(2); + for (const role of roles) { + expect(role.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolution_status: "resolved", + resolved_path: "index.html", + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ source_node_id, relation }) => + source_node_id === role.node_id && relation === "maps_to", + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(["index.html"]); + } + }, +); + +cliTest.for(["CLI", "stdio MCP"] as const)( + "retains non-string main evidence while following Node index entries through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-invalid-main-owner-"); + for (const [index, main] of [123, false, null, {}, []].entries()) { + await writeFixtureFiles(root, { + [`a${index}/package.json`]: JSON.stringify({ + name: `invalid-${index}`, + main, + }), + [`a${index}/index.js`]: `require("../shared/window${index}.cjs");`, + [`shared/window${index}.cjs`]: + 'const { BrowserWindow } = require("electron"); const win = new BrowserWindow(); win.loadFile("index.html");', + [`a${index}/index.html`]: "package index launch page", + "index.html": "artifact root decoy", + }); + } + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + const packages = graph.nodes.filter(({ kind }) => kind === "package"); + expect(packages).toHaveLength(5); + for (const node of packages) + expect(node.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + main: null, + main_status: "invalid", + parse_status: "included", + name: expect.stringMatching(/^invalid-\d$/u), + }), + }), + ]), + ); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(5); + for (const role of roles) { + const location = role.observations[0]?.evidence.location; + const source = + location?.available && location.value.kind === "source-range" + ? location.value.source + : null; + const match = /^shared\/window(\d)\.cjs$/u.exec(source ?? ""); + expect(match).not.toBeNull(); + const target = `a${match?.[1]}/index.html`; + expect(role.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: target, + resolution_status: "resolved", + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ source_node_id, relation }) => + source_node_id === role.node_id && relation === "maps_to", + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual([target]); + } + }, +); + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => [ + { surface, source: "app/sub/main.cjs", selectedOwner: false }, + { surface, source: "shared/main.cjs", selectedOwner: false }, + { surface, source: "shared/main.cjs", selectedOwner: true }, + ]), +)( + "keeps unselected package ownership unknown through $surface for $source (selected owner: $selectedOwner)", + async ({ surface, source, selectedOwner }, { cli }) => { + const root = await createTestTempDirectory("rea-selected-loadfile-"); + await writeRuntimeFixture(root); + if (selectedOwner) + await writeFixtureFiles(root, { + "other-owner/package.json": JSON.stringify({ + main: "../shared/main.cjs", + }), + "other-owner/dist%2Findex.html": "unselected competing page", + }); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root, { + sourcePaths: [ + "package.json", + ...(selectedOwner ? ["external-app/package.json"] : []), + source, + ], + }); + const roles = graph.nodes.filter( + ({ kind, observations }) => + kind === "electron-renderer" && + observations.some( + ({ properties }) => + properties.runtime_path_base === "application-root", + ), + ); + expect(roles).toHaveLength(3); + for (const role of roles) { + expect(role.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: null, + resolution_status: "unavailable", + }), + }), + ]), + ); + expect( + graph.edges.filter( + ({ source_node_id, relation }) => + source_node_id === role.node_id && relation === "maps_to", + ), + ).toEqual([]); + } + }, +); + +cliTest.for(["CLI", "stdio MCP"] as const)( + "keeps omitted launch owners unknown between identical ASAR occurrences through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-selected-asar-owner-"); + await writeRuntimeFixture(root); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root, { + sourcePaths: [ + "a.asar/package.json", + "a.asar/sub/package.json", + "a.asar/external-app/package.json", + "a.asar/sub/main.cjs", + "a.asar/shared/main.cjs", + ], + }); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(6); + for (const role of roles) { + expect(role.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + evidence: expect.objectContaining({ + location: expect.objectContaining({ + available: true, + value: expect.objectContaining({ + source: expect.stringMatching(/^a\.asar\//u), + }), + }), + }), + }), + ]), + ); + expect(role.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: null, + resolution_status: "unavailable", + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ source_node_id, relation }) => + source_node_id === role.node_id && relation === "maps_to", + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual([]); + } + }, +); + +cliTest.for(["CLI", "stdio MCP"] as const)( + "retains a selected entry owner and confines loadFile after applying its root through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-proven-partial-owner-"); + await writeFixtureFiles(root, { + "app/package.json": JSON.stringify({ main: "main.cjs" }), + "app/main.cjs": + 'const { BrowserWindow } = require("electron"); const win = new BrowserWindow(); win.loadFile("index.html"); win.loadFile("../shared/index.html"); win.loadFile("../../escape.html");', + "app/helper.cjs": 'module.exports = "unrelated omitted module";', + "index.html": "artifact-root decoy", + "app/index.html": "proven application page", + "shared/index.html": "valid sibling page", + "escape.html": "escape decoy", + }); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root, { + sourcePaths: ["app/package.json", "app/main.cjs"], + }); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(3); + for (const [declaration, target, status] of [ + ["index.html", "app/index.html", "resolved"], + ["../shared/index.html", "shared/index.html", "resolved"], + ["../../escape.html", null, "rejected"], + ] as const) { + const role = roles.find(({ observations }) => + observations.some( + ({ properties }) => properties.declared_path === declaration, + ), + ); + expect(role).toBeDefined(); + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: target, + resolution_status: status, + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ source_node_id, relation }) => + source_node_id === role?.node_id && relation === "maps_to", + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(target === null ? [] : [target]); + } + }, +); + +cliTest.for(["CLI", "stdio MCP"] as const)( + "keeps absolute loadFile host paths unknown through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-loadfile-absolute-"); + await writeFixtureFiles(root, { + "app/package.json": JSON.stringify({ main: "main.cjs" }), + "app/main.cjs": + 'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("/dist/index.html");', + "app/dist/index.html": "application-relative decoy", + "dist/index.html": "artifact-root decoy", + }); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + const role = roles[0]; + if (role === undefined) throw new Error("Missing renderer declaration"); + expect(role.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + declared_path: "/dist/index.html", + resolved_path: null, + resolution_status: "unavailable", + }), + }), + ]), + ); + expect( + graph.edges.filter( + ({ source_node_id, relation }) => + source_node_id === role.node_id && relation === "maps_to", + ), + ).toEqual([]); + }, +); + +cliTest.for(["CLI", "stdio MCP"] as const)( + "preserves literal-percent runtime paths through %s without executing source", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-runtime-percent-"); + await writeRuntimeFixture(root); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + for (const [sourcePath, targetRoot, declared] of ( + [ + ["sub/main.cjs", ""], + ["app/sub/main.cjs", "app/"], + ["shared/main.cjs", "external-app/"], + ["default-app/index.js", "default-app/"], + ["default-app/nested/main.cjs", null], + ["ambiguous/sub/main.cjs", null], + ["competition/nested/main.cjs", null], + ["imported/child.cjs", "import-owner/"], + ["a.asar/sub/main.cjs", "a.asar/"], + ["b.asar/sub/main.cjs", "b.asar/"], + ["a.asar/shared/main.cjs", "a.asar/external-app/"], + ["b.asar/shared/main.cjs", "b.asar/external-app/"], + ] as const + ).flatMap(([sourcePath, targetRoot]) => + ["dist%2Findex.html", "plain.html", "absent.html"].map( + (declared) => [sourcePath, targetRoot, declared] as const, + ), + )) { + const role = graph.nodes.find( + ({ kind, observations }) => + kind === "electron-renderer" && + observations.some( + ({ properties, evidence }) => + properties.declared_path === declared && + evidence.location.available && + evidence.location.value.kind === "source-range" && + evidence.location.value.source === sourcePath, + ), + ); + expect(role).toBeDefined(); + if (role === undefined) throw new Error(`Missing loadFile ${declared}`); + const expected = + declared === "absent.html" || targetRoot === null + ? null + : `${targetRoot}${declared}`; + expect(role.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + declared_path: declared, + runtime_path_base: "application-root", + resolved_path: expected, + resolution_status: + targetRoot === null + ? "unavailable" + : expected === null + ? "not-found" + : "resolved", + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ relation, source_node_id }) => + relation === "maps_to" && source_node_id === role.node_id, + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(expected === null ? [] : [expected]); + } + for (const { directory, target, status } of cases) { + for (const kind of ["electron-preload", "electron-utility"] as const) { + const nodes = graph.nodes.filter( + ({ kind: nodeKind, observations }) => + nodeKind === kind && + observations.some( + ({ evidence }) => + evidence.location.available && + evidence.location.value.kind === "source-range" && + evidence.location.value.source === `${directory}/main.mjs`, + ), + ); + expect(nodes).toHaveLength(1); + const node = nodes[0]; + if (node === undefined) throw new Error(`Missing ${directory} ${kind}`); + const resolutions = node.observations.filter( + ({ properties }) => "resolution_status" in properties, + ); + expect(resolutions).not.toHaveLength(0); + for (const { properties } of resolutions) { + expect(properties).toMatchObject({ + resolved_path: target === null ? null : `${directory}/${target}`, + resolution_status: status, + }); + } + const mappedPaths = graph.edges + .filter( + ({ relation, source_node_id }) => + relation === "maps_to" && source_node_id === node.node_id, + ) + .map(({ properties }) => properties.resolved_path); + expect([...new Set(mappedPaths)]).toEqual( + target === null ? [] : [`${directory}/${target}`], + ); + } + } + }, +); + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + ["unread", "included-inert", "included-competing"].map((owner) => ({ + surface, + owner, + })), + ), +)( + "retains unread competing launch owners across an ASAR boundary through $surface ($owner)", + async ({ surface, owner }, { cli }) => { + const root = await createTestTempDirectory("rea-unread-asar-owner-"); + const source = join(root, "source"); + const application = join(root, "application"); + await writeFixtureFiles(source, { + "window.cjs": + 'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");', + "index.html": "archive-root decoy", + }); + await writeFixtureFiles(application, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": 'require("../resources/shared.asar/window.cjs");', + "a/index.html": "first application page", + "b/package.json": JSON.stringify({ main: "main.cjs" }), + "b/main.cjs": + owner === "included-inert" + ? "module.exports = 1;" + : 'require("../resources/shared.asar/window.cjs");', + "b/index.html": "competing application page", + }); + await createPackageWithOptions( + source, + join(application, "resources/shared.asar"), + {}, + ); + const { graph } = await analyzeJavaScriptDirectory( + surface, + cli, + application, + { + sourcePaths: [ + "a/package.json", + "a/main.cjs", + "a/index.html", + "resources/shared.asar/window.cjs", + "resources/shared.asar/index.html", + ...(owner === "unread" + ? [] + : ["b/package.json", "b/main.cjs", "b/index.html"]), + ], + }, + ); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + const role = roles[0]; + const target = owner === "included-inert" ? "a/index.html" : null; + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: target, + resolution_status: target === null ? "unavailable" : "resolved", + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ relation, source_node_id }) => + relation === "maps_to" && source_node_id === role?.node_id, + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(target === null ? [] : [target]); + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + specifier: "../resources/shared.asar/window.cjs", + resolution_status: "resolved", + resolved_path: "resources/shared.asar/window.cjs", + }), + }), + ]), + ); + }, +); + +cliTest.for(["CLI", "stdio MCP"] as const)( + "keeps every application root unknown when three owners import one setter through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-shared-app-setter-"); + await writeFixtureFiles(root, { + "shared/change.cjs": + 'const { app } = require("electron"); app.setAppPath("/runtime/changed");', + ...Object.fromEntries( + ["a", "b", "c"].flatMap((owner) => [ + [`${owner}/package.json`, JSON.stringify({ main: "main.cjs" })], + [ + `${owner}/main.cjs`, + 'require("../shared/change.cjs"); require("./window.cjs");', + ], + [ + `${owner}/window.cjs`, + `// Application ${owner}\nconst { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");`, + ], + [`${owner}/index.html`, `application ${owner} page`], + ]), + ), + }); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(3); + for (const role of roles) { + expect(role.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolution_status: "unavailable", + resolved_path: null, + }), + }), + ]), + ); + expect( + graph.edges.filter( + ({ relation, source_node_id }) => + relation === "maps_to" && source_node_id === role.node_id, + ), + ).toEqual([]); + } + }, +); + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + ["relative", "bare"].map((request) => ({ surface, request })), + ), +)( + "retains package main ownership after an ASAR escape through $surface ($request)", + async ({ surface, request }, { cli }) => { + const root = await createTestTempDirectory("rea-asar-metadata-"); + const source = join(root, "source"); + const application = join(root, "application"); + const directory = + request === "bare" ? "resources/node_modules/fixture" : "resources/pkg"; + await writeFixtureFiles(source, { + "package.json": JSON.stringify({ main: "main.cjs" }), + "main.cjs": + request === "bare" ? 'require("fixture");' : 'require("../pkg");', + }); + await writeFixtureFiles(application, { + [`${directory}/package.json`]: JSON.stringify({ main: "actual.cjs" }), + [`${directory}/actual.cjs`]: + request === "bare" + ? 'require("../../window.cjs");' + : 'require("../window.cjs");', + [`${directory}/index.js`]: "module.exports = 1;", + "other/package.json": JSON.stringify({ main: "main.cjs" }), + "other/main.cjs": 'require("../resources/window.cjs");', + "resources/window.cjs": + 'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");', + "other/index.html": "false unique owner decoy", + }); + await createPackageWithOptions( + source, + join(application, "resources/app.asar"), + {}, + ); + const { graph } = await analyzeJavaScriptDirectory( + surface, + cli, + application, + ); + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + specifier: request === "bare" ? "fixture" : "../pkg", + resolution_status: "resolved", + resolved_path: `${directory}/actual.cjs`, + }), + }), + ]), + ); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + expect(roles[0]?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolution_status: "unavailable", + resolved_path: null, + }), + }), + ]), + ); + expect( + graph.edges.filter( + ({ relation, source_node_id }) => + relation === "maps_to" && source_node_id === roles[0]?.node_id, + ), + ).toEqual([]); + }, +); diff --git a/tests/acceptance/applications/javascriptLoaderReviewBoundaries.test.ts b/tests/acceptance/applications/javascriptLoaderReviewBoundaries.test.ts new file mode 100644 index 0000000000..f0cd68ee54 --- /dev/null +++ b/tests/acceptance/applications/javascriptLoaderReviewBoundaries.test.ts @@ -0,0 +1,594 @@ +import { createPackageWithOptions } from "@electron/asar"; +import { join } from "node:path"; +import { expect } from "vitest"; + +import type { JavaScriptApplicationGraph } from "../../../src/domain/javascript/javascriptApplicationGraph.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { + findSourceModule, + writeFixtureFiles, +} from "../../support/javascriptApplicationFixture.js"; + +const windowSource = + 'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");'; + +const expectRendererTarget = ( + graph: JavaScriptApplicationGraph, + target: string | null, +): void => { + const roles = graph.nodes.filter(({ kind }) => kind === "electron-renderer"); + expect(roles).toHaveLength(1); + const role = roles[0]; + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolution_status: target === null ? "unavailable" : "resolved", + resolved_path: target, + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ relation, source_node_id }) => + relation === "maps_to" && source_node_id === role?.node_id, + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(target === null ? [] : [target]); +}; + +const loaderCases = [ + { + name: "returned module", + source: 'const get=()=>module;get().require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "returned local", + source: + 'const get=()=>({require(){}});get().require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "returned module call wrapper", + source: + 'const get=()=>module;get.call(null).require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + requiredUnknownExpression: 'get.call(null).require("../shared/window.cjs")', + }, + { + name: "returned module apply wrapper", + source: + 'const get=()=>module;get.apply(null,[]).require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + requiredUnknownExpression: + 'get.apply(null,[]).require("../shared/window.cjs")', + }, + { + name: "local call wrapper override", + source: + 'const get=()=>module;get.call=()=>({require(){}});get.call(null).require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "local apply wrapper override", + source: + 'const get=()=>module;get.apply=()=>({require(){}});get.apply(null,[]).require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "lost call wrapper receiver", + source: + 'const get=()=>module;(0,get.call)(null).require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "lost apply wrapper receiver", + source: + 'const get=()=>module;(0,get.apply)(null,[]).require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "captured call wrapper host receiver", + source: + 'const get=()=>module;const invoke=get.call;invoke.call(get,null).require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + requiredUnknownExpression: + 'invoke.call(get,null).require("../shared/window.cjs")', + }, + { + name: "captured apply wrapper host receiver", + source: + 'const get=()=>module;const invoke=get.apply;invoke.call(get,null,[]).require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + requiredUnknownExpression: + 'invoke.call(get,null,[]).require("../shared/window.cjs")', + }, + { + name: "captured call wrapper local receiver", + source: + 'const get=()=>module;const local=()=>({require(){}});const invoke=get.call;invoke.call(local,null).require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "captured apply wrapper local receiver", + source: + 'const get=()=>module;const local=()=>({require(){}});const invoke=get.apply;invoke.call(local,null,[]).require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "inactive callable default", + source: + 'function host(){return module;}const {make=host}={make(){return {};}};make().require=()=>{};require("./local.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "active callable default", + source: + 'function host(){return module;}const {make=host}={};make().require=()=>{};require("./local.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "inactive array callable default", + source: + 'function host(){return module;}const [make=host]=[()=>({})];make().require=()=>{};require("./local.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "iterator callable default", + source: + 'function host(){return module;}const values=[()=>({})];values[Symbol.iterator]=function*(){yield undefined;};const [make=host]=values;make().require=()=>{};require("./local.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "generator local result", + source: + 'function* get(){yield module;}const api=get();api.require=()=>{};api.require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "async local result", + source: + 'async function get(){return module;}const api=get();api.require=()=>{};api.require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "awaited module result", + source: + '(async()=>{const get=async()=>module;(await get()).require("../shared/window.cjs");})();', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "awaited pending alias", + source: + '(async()=>{const get=async()=>module;const pending=get();const api=await pending;api.require("../shared/window.cjs");})();', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "awaited generator local", + source: + '(async()=>{function* get(){yield module;}const api=await get();api.require=()=>{};api.require("../shared/window.cjs");})();', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "awaited async local", + source: + '(async()=>{const get=async()=>({require(){}});const api=await get();api.require("../shared/window.cjs");})();', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "prototype extra host write", + source: + 'const [api]=[module];Array.prototype.extra=1;api.require=replacement;require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "prototype extra local write", + source: + 'const [api]=[{}];Array.prototype.extra=1;api.require=()=>{};api.require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "custom iterator host write", + source: + 'const values={*[Symbol.iterator](){yield module;}};const [api]=values;api.require=replacement;require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "constructor cycle iterator", + source: + 'Array.prototype.constructor.prototype[Symbol.iterator]=function*(){yield module;};const [api]=[];api.require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "inherited array hole", + source: + 'Array.prototype[0]=module;const [api]=[,];api.require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "Object prototype array hole", + source: + 'Object.prototype[0]=module;const [api]=[,];api.require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "Object prototype extra local", + source: + 'Object.prototype.extra=module;const [api]=[{require(){}}];api.require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "replaced host method", + source: + 'const box={make(){return {};}};box.make=()=>module;box.make().require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "replaced local method", + source: + 'const box={make(){return module;}};box.make=()=>({require(){}});box.make().require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "captured factory host", + source: + 'let make=()=>module;const box={make};make=()=>({});box.make().require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "captured factory local", + source: + 'let make=()=>({require(){}});const box={make};make=()=>module;box.make().require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "captured array factory", + source: + 'let make=()=>module;const values=[make];make=()=>({});values[0]().require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "captured stored factory", + source: + 'let make=()=>module;const box={};box.make=make;make=()=>({});box.make().require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "copied host factory", + source: + 'const old={make:()=>module};const box={...old};old.make=()=>({});box.make().require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "copied local factory", + source: + 'const old={make:()=>({require(){}})};const box={...old};old.make=()=>module;box.make().require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "wrapper before pattern replacement", + source: + 'var require;require("../shared/window.cjs");({require}={require:()=>({})});', + unknownOwner: true, + unknownLoad: false, + }, + { + name: "wrapper after pattern replacement", + source: + 'var require;({require}={require:()=>({})});require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + noProvenLoad: true, + }, + { + name: "shared deep child host write", + source: + 'const old={child:{inner:{make:()=>({})}}};const box={...old};old.child={inner:box.child.inner};old.child.inner.make=()=>module;box.child.inner.make().require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + requiredUnknownExpression: + 'box.child.inner.make().require("../shared/window.cjs")', + }, + { + name: "shared deep child local write", + source: + 'const old={child:{inner:{make:()=>module}}};const box={...old};old.child={inner:box.child.inner};old.child.inner.make=()=>({require(){}});box.child.inner.make().require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + absentUnknownExpression: + 'box.child.inner.make().require("../shared/window.cjs")', + }, + { + name: "fresh deep child local capture", + source: + 'const old={child:{inner:{make:()=>({require(){}})}}};const box={...old};old.child={inner:{make:()=>module}};old.child.inner.make=()=>module;box.child.inner.make().require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "fresh deep child host capture", + source: + 'const old={child:{inner:{make:()=>module}}};const box={...old};old.child={inner:{make:()=>({})}};old.child.inner.make=()=>({});box.child.inner.make().require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "twice copied host factory", + source: + 'const old={make:()=>module};const middle={...old};old.make=()=>({});const box={...middle};middle.make=()=>({});box.make().require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "twice copied local factory", + source: + 'const old={make:()=>({require(){}})};const middle={...old};old.make=()=>module;const box={...middle};middle.make=()=>module;box.make().require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "assigned rest captured module", + source: + 'const holder={};const old={loader:module};({...holder.box}=old);old.loader={};holder.box.loader.require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "assigned rest captured local", + source: + 'const holder={};const old={loader:{require(){}}};({...holder.box}=old);old.loader=module;holder.box.loader.require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "copied child later host factory", + source: + 'const old={child:{make:()=>({})}};const box={...old};old.child.make=()=>module;box.child.make().require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "copied child later local factory", + source: + 'const old={child:{make:()=>module}};const box={...old};old.child.make=()=>({require(){}});box.child.make().require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "captured child later module", + source: + 'const inner={loader:{}};const box={inner};inner.loader=module;box.inner.loader.require("../shared/window.cjs");', + unknownOwner: true, + unknownLoad: true, + }, + { + name: "captured child later local", + source: + 'const inner={loader:module};const box={inner};inner.loader={require(){}};box.inner.loader.require("../shared/window.cjs");', + unknownOwner: false, + unknownLoad: false, + }, + { + name: "Windows relative require", + source: `require(${JSON.stringify("..\\shared\\window.cjs")});`, + unknownOwner: true, + unknownLoad: false, + }, + { + name: "invalid builtin backslash", + source: `require(${JSON.stringify("node:\\fs")});`, + unknownOwner: false, + unknownLoad: false, + }, + { + name: "unrelated import pattern", + source: 'require("#foo");', + imports: { "#bar/*": "./proxy/*.cjs" }, + unknownOwner: false, + unknownLoad: false, + }, + { + name: "matching import pattern", + source: 'require("#foo/window");', + imports: { "#foo/*": "./proxy/*.cjs" }, + unknownOwner: true, + unknownLoad: false, + }, + { + name: "nonmatching import suffix", + source: 'require("#foo/window.js");', + imports: { "#foo/*.cjs": "./proxy/*.cjs" }, + unknownOwner: false, + unknownLoad: false, + }, +] as const; + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + loaderCases.map((testCase) => ({ surface, ...testCase })), + ), +)( + "preserves loader boundaries: $name through $surface", + async (testCase, { cli }) => { + const root = await createTestTempDirectory("rea-loader-review-"); + await writeFixtureFiles(root, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": 'require("../shared/window.cjs");', + "a/index.html": "first application page", + "b/package.json": JSON.stringify({ + main: "main.cjs", + ...("imports" in testCase ? { imports: testCase.imports } : {}), + }), + "b/main.cjs": testCase.source, + "b/local.cjs": "module.exports = {};", + "b/proxy/window.cjs": 'require("../../shared/window.cjs");', + "b/index.html": "second application page", + "shared/window.cjs": windowSource, + "shared/index.html": "source-directory decoy", + "index.html": "artifact-root decoy", + }); + const { graph } = await analyzeJavaScriptDirectory( + testCase.surface, + cli, + root, + ); + expectRendererTarget(graph, testCase.unknownOwner ? null : "a/index.html"); + const unknownLoads = graph.edges.filter( + ({ relation, properties }) => + relation === "imports" && properties.specifier === null, + ); + expect(unknownLoads.length > 0).toBe(testCase.unknownLoad); + if ("requiredUnknownExpression" in testCase) + expect(unknownLoads).toContainEqual( + expect.objectContaining({ + properties: expect.objectContaining({ + expression: testCase.requiredUnknownExpression, + }), + }), + ); + if ("absentUnknownExpression" in testCase) + expect( + unknownLoads.filter( + ({ properties }) => + properties.expression === testCase.absentUnknownExpression, + ), + ).toEqual([]); + if ("noProvenLoad" in testCase) { + const source = findSourceModule(graph, "b/main.cjs"); + expect(source).toBeDefined(); + expect( + graph.edges.filter( + ({ relation, source_node_id, properties }) => + relation === "imports" && + source_node_id === source?.node_id && + properties.specifier === "../shared/window.cjs", + ), + ).toEqual([]); + } + if (testCase.name === "Windows relative require") + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + specifier: "..\\shared\\window.cjs", + resolution_status: "unavailable", + resolved_path: null, + }), + }), + ]), + ); + }, +); + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + (["app.asar", "APP.ASAR", "app.AsAr"] as const).flatMap((archive) => + (["inward", "outward"] as const).map((direction) => ({ + surface, + archive, + direction, + })), + ), + ), +)( + "preserves $direction ASAR ownership for $archive through $surface", + async ({ surface, archive, direction }, { cli }) => { + const root = await createTestTempDirectory("rea-asar-case-"); + const source = join(root, "source"); + const application = join(root, "application"); + const inward = direction === "inward"; + await writeFixtureFiles( + source, + inward + ? { "window.cjs": windowSource, "index.html": "archive-root decoy" } + : { + "package.json": JSON.stringify({ main: "main.cjs" }), + "main.cjs": 'require("../window.cjs");', + "index.html": "archive application page", + }, + ); + await writeFixtureFiles( + application, + inward + ? { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": `require(${JSON.stringify(`../resources/${archive}/window.cjs`)});`, + "a/index.html": "outer application page", + } + : { "resources/window.cjs": windowSource }, + ); + await createPackageWithOptions( + source, + join(application, `resources/${archive}`), + {}, + ); + const { graph } = await analyzeJavaScriptDirectory( + surface, + cli, + application, + ); + expectRendererTarget( + graph, + inward ? "a/index.html" : `resources/${archive}/index.html`, + ); + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + specifier: inward + ? `../resources/${archive}/window.cjs` + : "../window.cjs", + resolution_status: "resolved", + resolved_path: inward + ? `resources/${archive}/window.cjs` + : "resources/window.cjs", + }), + }), + ]), + ); + }, +); diff --git a/tests/acceptance/applications/javascriptNonStringMainControls.test.ts b/tests/acceptance/applications/javascriptNonStringMainControls.test.ts new file mode 100644 index 0000000000..ca5aab58e0 --- /dev/null +++ b/tests/acceptance/applications/javascriptNonStringMainControls.test.ts @@ -0,0 +1,67 @@ +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + [false, true].map((invalidJson) => ({ surface, invalidJson })), + ), +)( + "separates an absent default main from malformed package JSON through $surface (invalidJson=$invalidJson)", + async ({ surface, invalidJson }, { cli }) => { + const root = await createTestTempDirectory("rea-nonstring-main-controls-"); + await writeFixtureFiles(root, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": [ + 'throw new Error("Static fixture must never execute");', + 'const {BrowserWindow}=require("electron");new BrowserWindow().loadFile("index.html");', + ].join("\n"), + "a/index.html": "known first application page", + "b/package.json": invalidJson ? "{" : JSON.stringify({ main: false }), + ...(invalidJson ? { "b/index.js": "module.exports={};" } : {}), + "index.html": "artifact-root decoy", + }); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + const packages = graph.nodes.filter(({ kind }) => kind === "package"); + expect(packages).toContainEqual( + expect.objectContaining({ + observations: expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + path: "b/package.json", + main_status: invalidJson ? "unavailable" : "invalid", + parse_status: invalidJson ? "invalid" : "included", + }), + }), + ]), + }), + ); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + const role = roles[0]; + const target = invalidJson ? null : "a/index.html"; + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolution_status: invalidJson ? "unavailable" : "resolved", + resolved_path: target, + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ source_node_id, relation }) => + source_node_id === role?.node_id && relation === "maps_to", + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(target === null ? [] : [target]); + }, +); diff --git a/tests/acceptance/applications/javascriptPackageExportPatternOwners.test.ts b/tests/acceptance/applications/javascriptPackageExportPatternOwners.test.ts new file mode 100644 index 0000000000..fe969f05f6 --- /dev/null +++ b/tests/acceptance/applications/javascriptPackageExportPatternOwners.test.ts @@ -0,0 +1,121 @@ +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { + analyzeJavaScriptDirectory, + runNodeJsonControl, +} from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +const cases = [ + { + name: "matching wildcard", + exports: { "./*": "./*.cjs" }, + status: "unavailable", + possibleOwner: true, + nodeAllowed: true, + }, + { + name: "unrelated wildcard", + exports: { "./other/*": "./*.cjs" }, + status: "external", + possibleOwner: false, + nodeAllowed: false, + }, + { + name: "exact key before wildcard", + exports: { "./*": "./*.cjs", "./window": "./window.cjs" }, + status: "resolved", + possibleOwner: true, + nodeAllowed: true, + }, + { + name: "exact null before wildcard", + exports: { "./*": "./*.cjs", "./window": null }, + status: "external", + possibleOwner: false, + nodeAllowed: false, + }, + { + name: "specific null before broader wildcard", + exports: { "./*": "./*.cjs", "./w*": null }, + status: "external", + possibleOwner: false, + nodeAllowed: false, + }, +] as const; +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + cases.map((test) => ({ ...test, surface })), + ), +)( + "retains $name dependency ownership through $surface", + async ({ surface, exports, status, possibleOwner, nodeAllowed }, { cli }) => { + const root = await createTestTempDirectory("rea-exports-pattern-owner-"); + await writeFixtureFiles(root, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": + 'throw new Error("Static fixture must never execute");require("fixture/window");', + "a/index.html": "first app decoy", + "b/package.json": JSON.stringify({ main: "main.cjs" }), + "b/main.cjs": + 'throw new Error("Static fixture must never execute");require("../node_modules/fixture/window.cjs");', + "b/index.html": "second application page", + "node_modules/fixture/package.json": JSON.stringify({ exports }), + "node_modules/fixture/window.cjs": + 'throw new Error("Static fixture must never execute");const {BrowserWindow}=require("electron");new BrowserWindow().loadFile("index.html");', + "index.html": "artifact root decoy", + "oracle.cjs": + 'const {createRequire}=require("node:module");const loader=createRequire(require("node:path").join(__dirname,"a/main.cjs"));try{console.log(JSON.stringify({path:loader.resolve("fixture/window").split(require("node:path").sep).join("/")}));}catch(error){console.log(JSON.stringify({code:error.code}));}', + }); + // This authored control asks the native resolver for a path; it never + // executes the Electron source whose ownership is under analysis. + const oracle = await runNodeJsonControl(root, "oracle.cjs"); + expect(oracle).toEqual( + nodeAllowed + ? { path: expect.stringContaining("/node_modules/fixture/window.cjs") } + : { code: "ERR_PACKAGE_PATH_NOT_EXPORTED" }, + ); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root, { + sourcePaths: [ + "a/package.json", + "a/main.cjs", + "b/package.json", + "b/main.cjs", + "node_modules/fixture/package.json", + "node_modules/fixture/window.cjs", + ], + }); + const refs = graph.edges.filter( + ({ relation, properties }) => + relation === "imports" && properties.specifier === "fixture/window", + ); + expect(refs.length).toBeGreaterThan(0); + for (const ref of refs) + expect(ref.properties.resolution_status).toBe(status); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + const target = possibleOwner ? null : "b/index.html"; + expect(roles[0]?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: target, + resolution_status: possibleOwner ? "unavailable" : "resolved", + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ relation, source_node_id }) => + relation === "maps_to" && source_node_id === roles[0]?.node_id, + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(target === null ? [] : [target]); + }, +); diff --git a/tests/acceptance/applications/javascriptPackageLaunchBoundaries.test.ts b/tests/acceptance/applications/javascriptPackageLaunchBoundaries.test.ts new file mode 100644 index 0000000000..c7a3c05892 --- /dev/null +++ b/tests/acceptance/applications/javascriptPackageLaunchBoundaries.test.ts @@ -0,0 +1,165 @@ +import { createPackageWithOptions } from "@electron/asar"; +import { join } from "node:path"; +import { expect } from "vitest"; + +import type { JavaScriptApplicationGraph } from "../../../src/domain/javascript/javascriptApplicationGraph.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +const windowSource = + 'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");'; + +const expectRendererOwner = ( + graph: JavaScriptApplicationGraph, + competing: boolean, +): void => { + const renderers = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(renderers).toHaveLength(1); + const role = renderers[0]; + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolution_status: competing ? "unavailable" : "resolved", + resolved_path: competing ? null : "a/index.html", + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ relation, source_node_id }) => + relation === "maps_to" && source_node_id === role?.node_id, + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(competing ? [] : ["a/index.html"]); +}; + +const launchCases = [ + { + name: "package index fallback", + main: "missing.cjs", + files: { "b/index.js": 'require("../shared/window.cjs");' }, + competing: true, + entry: "b/index.js", + }, + { + name: "absent package index", + main: "missing.cjs", + files: {}, + competing: false, + }, + { + name: "main before package index", + main: "main.cjs", + files: { + "b/main.cjs": "module.exports = {};", + "b/index.js": 'require("../shared/window.cjs");', + }, + competing: false, + entry: "b/main.cjs", + }, + { + name: "inert JSON index fallback", + main: "missing.cjs", + files: { "b/index.json": "{}" }, + competing: false, + }, +] as const; + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + launchCases.map((testCase) => ({ surface, ...testCase })), + ), +)( + "keeps launch ownership for $name through $surface", + async (testCase, { cli }) => { + const root = await createTestTempDirectory("rea-package-launch-"); + await writeFixtureFiles(root, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": 'require("../shared/window.cjs");', + "a/index.html": "first application page", + "b/package.json": JSON.stringify({ main: testCase.main }), + "b/index.html": "second application page", + "shared/window.cjs": windowSource, + "shared/index.html": "source-directory decoy", + ...testCase.files, + }); + const { graph } = await analyzeJavaScriptDirectory( + testCase.surface, + cli, + root, + ); + expectRendererOwner(graph, testCase.competing); + if ("entry" in testCase) + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "loads", + properties: expect.objectContaining({ + declared_path: testCase.main, + resolved_path: testCase.entry, + resolution_status: "resolved", + }), + }), + ]), + ); + }, +); + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + ["code.asar", "APP.ASAR", "app.AsAr"].map((archive) => ({ + surface, + archive, + })), + ), +)( + "keeps competing package launch owners across $archive through $surface", + async ({ surface, archive }, { cli }) => { + const root = await createTestTempDirectory("rea-package-archive-main-"); + const source = join(root, "source"); + const application = join(root, "application"); + const declaredMain = `../resources/${archive}/window.cjs`; + await writeFixtureFiles(source, { + "window.cjs": windowSource, + "index.html": "archive-root decoy", + }); + await writeFixtureFiles(application, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": `require(${JSON.stringify(declaredMain)});`, + "a/index.html": "first application page", + "b/package.json": JSON.stringify({ main: declaredMain }), + "b/index.js": "module.exports = {};", + "b/index.html": "second application page", + }); + await createPackageWithOptions( + source, + join(application, `resources/${archive}`), + {}, + ); + const { graph } = await analyzeJavaScriptDirectory( + surface, + cli, + application, + ); + expectRendererOwner(graph, true); + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "loads", + properties: expect.objectContaining({ + declared_path: declaredMain, + resolved_path: `resources/${archive}/window.cjs`, + resolution_status: "resolved", + }), + }), + ]), + ); + }, +); diff --git a/tests/acceptance/applications/javascriptPossibleSetterOwners.test.ts b/tests/acceptance/applications/javascriptPossibleSetterOwners.test.ts new file mode 100644 index 0000000000..7609414d26 --- /dev/null +++ b/tests/acceptance/applications/javascriptPossibleSetterOwners.test.ts @@ -0,0 +1,68 @@ +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + [false, true].map((unknownLoader) => ({ surface, unknownLoader })), + ), +)( + "retains possible application path setter owners through $surface (unknown loader: $unknownLoader)", + async ({ surface, unknownLoader }, { cli }) => { + const root = await createTestTempDirectory("rea-possible-setter-owner-"); + await writeFixtureFiles(root, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": 'require("../shared/change.cjs");', + "b/package.json": JSON.stringify({ main: "main.cjs" }), + "b/main.cjs": `${unknownLoader ? "require(process.env.LOADER);" : ""} require("./window.cjs");`, + "b/window.cjs": + 'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");', + "b/index.html": "second application page", + "shared/change.cjs": + 'const { app } = require("electron"); app.setAppPath("/runtime/changed");', + "shared/index.html": "setter-relative decoy", + "index.html": "artifact-root decoy", + }); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + const role = roles[0]; + const target = unknownLoader ? null : "b/index.html"; + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: target, + resolution_status: unknownLoader ? "unavailable" : "resolved", + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ relation, source_node_id }) => + relation === "maps_to" && source_node_id === role?.node_id, + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(target === null ? [] : [target]); + if (unknownLoader) + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + kind: "require", + specifier: null, + resolved_path: null, + }), + }), + ]), + ); + }, +); diff --git a/tests/acceptance/applications/javascriptPreloadResolveOwners.test.ts b/tests/acceptance/applications/javascriptPreloadResolveOwners.test.ts new file mode 100644 index 0000000000..17ea57552c --- /dev/null +++ b/tests/acceptance/applications/javascriptPreloadResolveOwners.test.ts @@ -0,0 +1,126 @@ +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + [ + { + name: "local helper", + declaration: "const helper={resolve(value){return value}};", + expression: 'helper.resolve(runtimePath,__dirname,"decoy.cjs")', + preload: null, + }, + { + name: "Node path resolver", + declaration: 'const path=require("node:path");', + expression: 'path.resolve(runtimePath,__dirname,"decoy.cjs")', + preload: "decoy.cjs", + }, + { + name: "untouched CommonJS directory parameter", + declaration: 'var __dirname;const path=require("node:path");', + expression: 'path.join(__dirname,"decoy.cjs")', + preload: "decoy.cjs", + }, + { + name: "local dirname helper", + declaration: + 'import path from "node:path";const helper={dirname(){return "/local"}};', + expression: + 'path.resolve(runtimePath,helper.dirname(import.meta.url),"decoy.cjs")', + preload: null, + mainFile: "main.mjs", + }, + { + name: "Node dirname and source URL", + declaration: + 'import path from "node:path";import {fileURLToPath} from "node:url";', + expression: + 'path.resolve(runtimePath,path.dirname(fileURLToPath(import.meta.url)),"decoy.cjs")', + preload: "decoy.cjs", + mainFile: "main.mjs", + }, + { + name: "separate default path write", + declaration: + 'import changed from "node:path";import path from "path";changed.resolve=first=>first;', + expression: 'path.resolve(runtimePath,__dirname,"decoy.cjs")', + preload: null, + mainFile: "main.mjs", + }, + { + name: "separate default URL write", + declaration: + 'import changed from "node:url";import url from "url";import path from "node:path";changed.fileURLToPath=()=>"/changed/main.mjs";', + expression: + 'path.resolve(runtimePath,path.dirname(url.fileURLToPath(import.meta.url)),"decoy.cjs")', + preload: null, + mainFile: "main.mjs", + }, + { + name: "shadowed directory", + declaration: 'const path=require("node:path");', + expression: 'path.resolve(runtimePath,__dirname,"decoy.cjs")', + preload: null, + scopeStart: "function build(__dirname){", + scopeEnd: "}build('local-directory');", + }, + ].map((testCase) => ({ + surface, + scopeStart: "", + scopeEnd: "", + mainFile: "main.cjs", + ...testCase, + })), + ), +)( + "requires the $name owner for preload resolve semantics through $surface", + async ( + { + surface, + declaration, + expression, + preload, + scopeStart, + scopeEnd, + mainFile, + }, + { cli }, + ) => { + const root = await createTestTempDirectory("rea-preload-resolve-owner-"); + await writeFixtureFiles(root, { + "package.json": JSON.stringify({ main: mainFile }), + [mainFile]: `throw new Error('Static fixture must never execute');${mainFile.endsWith(".mjs") ? "import {BrowserWindow} from 'electron';" : "const {BrowserWindow}=require('electron');"}${declaration}${scopeStart}new BrowserWindow({webPreferences:{preload:${expression}}});${scopeEnd}`, + "decoy.cjs": "throw new Error('Static preload must never execute');", + }); + const { graph, statistics } = await analyzeJavaScriptDirectory( + surface, + cli, + root, + ); + expect(statistics.parse_failures).toBe(0); + const windows = graph.nodes.filter(({ kind }) => kind === "browser-window"); + expect(windows).toHaveLength(1); + expect(windows[0]?.observations[0]?.properties.preload_path).toBe(preload); + const associations = graph.edges.filter( + ({ source_node_id, evidence }) => + source_node_id === windows[0]?.node_id && + evidence.extractor.operation === "associate-browser-window-preload", + ); + if (preload === null) expect(associations).toEqual([]); + else + expect(associations).toEqual([ + expect.objectContaining({ + relation: "loads", + properties: expect.objectContaining({ + resolved_path: preload, + resolution_status: "resolved", + }), + }), + ]); + }, +); diff --git a/tests/acceptance/applications/javascriptPublishedAppOwners.test.ts b/tests/acceptance/applications/javascriptPublishedAppOwners.test.ts new file mode 100644 index 0000000000..7bcabac108 --- /dev/null +++ b/tests/acceptance/applications/javascriptPublishedAppOwners.test.ts @@ -0,0 +1,99 @@ +import { expect } from "vitest"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { + analyzeJavaScriptDirectory, + runNodeJsonControl, +} from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +const cases = [ + { + name: "whole app ESM", + body: "export {app};", + esm: true, + changed: true, + oracle: "module.exports={app};", + }, + { name: "whole app CommonJS", body: "module.exports={app};", changed: true }, + { + name: "later local assign", + body: "Object.assign(exports,{change:app.setAppPath},{change(){}});", + changed: false, + }, + { + name: "later native assign", + body: "Object.assign(exports,{change(){}},{change:app.setAppPath});", + changed: true, + }, + { + name: "inherited setter", + body: "const source={__proto__:{change:app.setAppPath}};Object.assign(exports,source);", + changed: false, + }, + { + name: "own setter", + body: "const source={__proto__:{change(){}},change:app.setAppPath};Object.assign(exports,source);", + changed: true, + }, +] as const; +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + cases.map((test) => ({ ...test, surface })), + ), +)( + "retains only the actually published $name app capability through $surface", + async (test, { cli }) => { + const { surface, body, changed } = test; + const esm = "esm" in test && test.esm; + const root = await createTestTempDirectory("rea-published-app-owner-"); + const prefix = esm + ? 'import {app} from "electron";' + : 'const {app}=require("electron");'; + const helper = `shared/helper.${esm ? "mjs" : "cjs"}`; + const oracleBody = "oracle" in test ? test.oracle : body; + await writeFixtureFiles(root, { + "app/package.json": JSON.stringify({ + main: esm ? "main.mjs" : "main.cjs", + }), + [`app/main.${esm ? "mjs" : "cjs"}`]: esm + ? 'import * as api from "../shared/helper.mjs";import {BrowserWindow} from "electron";api.app.setAppPath("./other");new BrowserWindow().loadFile("index.html");' + : 'const api=require("../shared/helper.cjs");const {BrowserWindow}=require("electron");if(api.app)api.app.setAppPath("./other");else if(api.change)api.change("./other");new BrowserWindow().loadFile("index.html");', + [helper]: prefix + body, + "app/index.html": "package root decoy for native setters", + "shared/index.html": "helper-relative decoy", + "oracle.cjs": `const vm=require('node:vm');let changed=0;const app={setAppPath(){changed++}};const fixtureModule={exports:{}};vm.runInNewContext(${JSON.stringify(oracleBody)},{app,module:fixtureModule,exports:fixtureModule.exports});const value=fixtureModule.exports;if(value.app)value.app.setAppPath('./other');else if(value.change)value.change.call(app,'./other');console.log(JSON.stringify({changed:changed>0}));`, + }); + expect(await runNodeJsonControl(root, "oracle.cjs")).toEqual({ changed }); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root, { + sourcePaths: [ + "app/package.json", + `app/main.${esm ? "mjs" : "cjs"}`, + helper, + ], + }); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + const target = changed ? null : "app/index.html"; + expect(roles[0]?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: target, + resolution_status: changed ? "unavailable" : "resolved", + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ relation, source_node_id }) => + relation === "maps_to" && source_node_id === roles[0]?.node_id, + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(target === null ? [] : [target]); + }, +); diff --git a/tests/acceptance/applications/javascriptReturnedAppPathChanges.test.ts b/tests/acceptance/applications/javascriptReturnedAppPathChanges.test.ts new file mode 100644 index 0000000000..de571a4f65 --- /dev/null +++ b/tests/acceptance/applications/javascriptReturnedAppPathChanges.test.ts @@ -0,0 +1,124 @@ +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +const cases = [ + { + name: "returned app", + setter: "const getApp=()=>app;getApp().setAppPath('./lib');", + owners: 1, + changed: true, + }, + { + name: "awaited app", + setter: "const getApp=async()=>app;(await getApp()).setAppPath('./lib');", + owners: 1, + changed: true, + }, + { + name: "local app", + setter: "const getApp=()=>({setAppPath(){}});getApp().setAppPath('./lib');", + owners: 1, + changed: false, + }, + { + name: "replaced app setter", + setter: "app.setAppPath=()=>{};app.setAppPath('/changed');", + owners: 1, + changed: false, + }, + { + name: "conditional setter replacement", + setter: "if(flag)app.setAppPath=()=>{};app.setAppPath('/changed');", + owners: 1, + changed: true, + }, + { + name: "original setter captured before replacement", + setter: + "const change=app.setAppPath;app.setAppPath=()=>{};change('/changed');", + owners: 1, + changed: true, + }, + { + name: "local setter captured after replacement", + setter: + "app.setAppPath=()=>{};const change=app.setAppPath;change('/changed');", + owners: 1, + changed: false, + }, + { + name: "assigned app", + setter: "let runtime;(runtime=app).setAppPath('/changed');", + owners: 1, + changed: true, + }, + { + name: "assigned local app", + setter: "let runtime;(runtime={setAppPath(){}}).setAppPath('/changed');", + owners: 1, + changed: false, + }, + { + name: "shared returned app", + setter: "const getApp=()=>app;getApp().setAppPath('./lib');", + owners: 3, + changed: true, + }, +] as const; + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + cases.map((test) => ({ ...test, surface })), + ), +)( + "preserves $name path uncertainty for $owners application owners through $surface", + async ({ surface, setter, owners, changed }, { cli }) => { + const root = await createTestTempDirectory("rea-returned-app-path-"); + const files: Record = { + "shared/change.mjs": `throw new Error('Static fixture must never execute');import {app} from 'electron';${setter}`, + "shared/index.html": "setter-relative decoy", + "index.html": "artifact-root decoy", + }; + for (let index = 0; index < owners; index++) { + const directory = `app-${index}`; + files[`${directory}/package.json`] = JSON.stringify({ main: "main.mjs" }); + files[`${directory}/main.mjs`] = + `throw new Error('Static fixture must never execute');import '../shared/change.mjs';import {BrowserWindow} from 'electron';new BrowserWindow().loadFile('index.html');`; + files[`${directory}/index.html`] = `page-${index}`; + } + await writeFixtureFiles(root, files); + const { graph, statistics } = await analyzeJavaScriptDirectory( + surface, + cli, + root, + ); + expect(statistics.parse_failures).toBe(0); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(owners); + for (const role of roles) { + expect(role.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolution_status: changed ? "unavailable" : "resolved", + ...(changed ? { resolved_path: null } : {}), + }), + }), + ]), + ); + const mapped = graph.edges.filter( + ({ relation, source_node_id }) => + relation === "maps_to" && source_node_id === role.node_id, + ); + expect(mapped).toHaveLength(changed ? 0 : 1); + if (!changed) + expect(mapped[0]?.properties.resolved_path).toBe("app-0/index.html"); + } + }, +); diff --git a/tests/acceptance/applications/javascriptReturnedReferenceCycles.test.ts b/tests/acceptance/applications/javascriptReturnedReferenceCycles.test.ts index 981806e4dd..74b7a8a478 100644 --- a/tests/acceptance/applications/javascriptReturnedReferenceCycles.test.ts +++ b/tests/acceptance/applications/javascriptReturnedReferenceCycles.test.ts @@ -7,13 +7,14 @@ import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/j import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; +import { findSourceModule } from "../../support/javascriptApplicationFixture.js"; const verify = (value: unknown) => { const result = javascriptApplicationAnalysisResultSchema.parse( parseEvidence(value).normalized_result, ); expect(result.statistics).toMatchObject({ - parsed_javascript_files: 3, + parsed_javascript_files: 5, skipped_javascript_files: 0, parse_failures: 0, }); @@ -38,6 +39,28 @@ const verify = (value: unknown) => { interpretation: "candidate", }); expect(endpoints.some(({ value }) => value === "/intrinsic")).toBe(false); + expect(endpoints).toContainEqual({ + value: "/copied-factory", + interpretation: undefined, + }); + expect(endpoints).toContainEqual({ + value: "/copied-module", + interpretation: undefined, + }); + for (const [path, expression] of [ + ["copied-factory.cjs", 'box.make().require("./possible.cjs")'], + ["copied-module.cjs", 'box.loader.require("./possible.cjs")'], + ] as const) { + const source = findSourceModule(result.graph, path); + expect(source).toBeDefined(); + expect(result.graph.edges).toContainEqual( + expect.objectContaining({ + source_node_id: source?.node_id, + relation: "imports", + properties: expect.objectContaining({ specifier: null, expression }), + }), + ); + } expect(result.semantic_graph.unknowns).toContainEqual( expect.objectContaining({ reason: "dynamic-property", @@ -79,6 +102,20 @@ function walk(flag) { patch(cursor.target()); } new Map().get("/growing");`, + ); + await writeFile( + join(root, "copied-factory.cjs"), + `let box={make:()=>module}; +while(flag) box={...box}; +box.make().require("./possible.cjs"); +fetch("/copied-factory");`, + ); + await writeFile( + join(root, "copied-module.cjs"), + `let box={loader:module}; +while(flag) box={...box}; +box.loader.require("./possible.cjs"); +fetch("/copied-module");`, ); const response = await cli.run({ arguments: [ diff --git a/tests/acceptance/applications/javascriptUnreadAppPathChanges.test.ts b/tests/acceptance/applications/javascriptUnreadAppPathChanges.test.ts new file mode 100644 index 0000000000..95b915ba9d --- /dev/null +++ b/tests/acceptance/applications/javascriptUnreadAppPathChanges.test.ts @@ -0,0 +1,109 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +const cases = [ + "unselected", + "transitive-unselected", + "invalid-utf8", + "parse-failed", + "read-inert", + "unreferenced", + "json", +] as const; +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + cases.map((gap) => ({ surface, gap })), + ), +)( + "retains possible app path changes from a reachable $gap module through $surface", + async ({ surface, gap }, { cli }) => { + const root = await createTestTempDirectory("rea-unread-app-path-change-"); + const window = + 'const {BrowserWindow}=require("electron");new BrowserWindow().loadFile("index.html");'; + await writeFixtureFiles(root, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": `${gap === "unreferenced" ? "" : gap === "transitive-unselected" ? 'require("./middle.cjs");' : `require("./change.${gap === "json" ? "json" : "cjs"}");`}${window}`, + "a/change.cjs": + gap === "parse-failed" + ? "const = ;" + : gap === "read-inert" + ? "module.exports=42;" + : 'const {app}=require("electron");app.setAppPath("/changed");', + "a/change.json": '{"inert":true}', + "a/middle.cjs": 'require("./change.cjs");', + "a/index.html": "first application page", + "b/package.json": JSON.stringify({ main: "main.cjs" }), + "b/main.cjs": window, + "b/index.html": "unrelated second application page", + "index.html": "artifact root decoy", + }); + if (gap === "invalid-utf8") + await writeFile( + join(root, "a/change.cjs"), + Buffer.from([0xff, 0xfe, 0x80]), + ); + const sourcePaths = [ + "a/package.json", + "a/main.cjs", + "b/package.json", + "b/main.cjs", + ...(gap === "unselected" || + gap === "unreferenced" || + gap === "transitive-unselected" + ? [] + : [`a/change.${gap === "json" ? "json" : "cjs"}`]), + ...(gap === "transitive-unselected" ? ["a/middle.cjs"] : []), + ]; + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root, { + sourcePaths, + }); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(2); + for (const app of ["a", "b"]) { + const role = roles.find(({ observations }) => + observations.some( + ({ evidence }) => + evidence.location.available && + evidence.location.value.kind === "source-range" && + evidence.location.value.source === `${app}/main.cjs`, + ), + ); + // The unread source could itself import b/main, so the existing loader + // owner policy also retains competition for that module. In contrast an + // unreferenced omitted source must not invalidate either proven root. + const changed = [ + "unselected", + "transitive-unselected", + "invalid-utf8", + "parse-failed", + ].includes(gap); + const expected = changed ? null : `${app}/index.html`; + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: expected, + resolution_status: changed ? "unavailable" : "resolved", + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ source_node_id, relation }) => + source_node_id === role?.node_id && relation === "maps_to", + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(expected === null ? [] : [expected]); + } + }, +); diff --git a/tests/acceptance/applications/javascriptWindowsLoadFilePaths.test.ts b/tests/acceptance/applications/javascriptWindowsLoadFilePaths.test.ts new file mode 100644 index 0000000000..c000c602fa --- /dev/null +++ b/tests/acceptance/applications/javascriptWindowsLoadFilePaths.test.ts @@ -0,0 +1,68 @@ +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +const cases = [ + { declared: "sub\\index.html", status: "unavailable", path: null }, + { declared: "C:index.html", status: "unavailable", path: null }, + { + declared: "sub/index.html", + status: "resolved", + path: "app/sub/index.html", + }, + { declared: "sub\\index\0.html", status: "rejected", path: null }, +] as const; + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + cases.map((testCase) => ({ surface, ...testCase })), + ), +)( + "preserves loadFile host path semantics through $surface: $declared", + async ({ surface, declared, status, path }, { cli }) => { + const root = await createTestTempDirectory("rea-windows-loadfile-path-"); + await writeFixtureFiles(root, { + "app/package.json": JSON.stringify({ main: "main.cjs" }), + "app/main.cjs": [ + 'throw new Error("Static fixture must never execute");', + 'const {BrowserWindow}=require("electron");', + `new BrowserWindow().loadFile(${JSON.stringify(declared)});`, + ].join("\n"), + "app/sub/index.html": "application page", + "sub/index.html": "artifact-root decoy", + }); + const { graph, statistics } = await analyzeJavaScriptDirectory( + surface, + cli, + root, + ); + expect(statistics.parse_failures).toBe(0); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + const role = roles[0]; + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + declared_path: declared, + resolution_status: status, + resolved_path: path, + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ source_node_id, relation }) => + source_node_id === role?.node_id && relation === "maps_to", + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(path === null ? [] : [path]); + }, +); diff --git a/tests/acceptance/applications/javascriptWrapperElectronCaptures.test.ts b/tests/acceptance/applications/javascriptWrapperElectronCaptures.test.ts new file mode 100644 index 0000000000..fb85848f99 --- /dev/null +++ b/tests/acceptance/applications/javascriptWrapperElectronCaptures.test.ts @@ -0,0 +1,100 @@ +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { + findSourceModule, + writeFixtureFiles, +} from "../../support/javascriptApplicationFixture.js"; + +const writes = [ + { + name: "object pattern", + source: "({require}={require:()=>({BrowserWindow:LocalBrowserWindow})});", + }, + { + name: "array pattern", + source: "[require]=[()=>({BrowserWindow:LocalBrowserWindow})];", + }, +] as const; + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + writes.flatMap((write) => + ([false, true] as const).map((capturedBeforeWrite) => ({ + surface, + ...write, + capturedBeforeWrite, + })), + ), + ), +)( + "preserves Electron capture ownership through $surface after $name with capturedBeforeWrite=$capturedBeforeWrite", + async ({ surface, source, capturedBeforeWrite }, { cli }) => { + const root = await createTestTempDirectory("rea-wrapper-electron-capture-"); + const capture = 'const {BrowserWindow}=require("electron");'; + await writeFixtureFiles(root, { + "package.json": JSON.stringify({ main: "main.cjs" }), + "main.cjs": [ + 'throw new Error("Static fixture must never execute");', + "class LocalBrowserWindow{}", + "var require;", + ...(capturedBeforeWrite ? [capture, source] : [source, capture]), + 'new BrowserWindow({webPreferences:{preload:"./bridge.cjs"}});', + ].join("\n"), + "bridge.cjs": 'throw new Error("Preload fixture must never execute");', + }); + const result = await analyzeJavaScriptDirectory(surface, cli, root); + expect(result.statistics.parse_failures).toBe(0); + const expectedRoles = capturedBeforeWrite ? 1 : 0; + expect(result.summary).toMatchObject({ + browser_windows: expectedRoles, + preload_entrypoints: expectedRoles, + }); + const { graph } = result; + const windows = graph.nodes.filter(({ kind }) => kind === "browser-window"); + expect(windows).toHaveLength(expectedRoles); + // Generic `property:preload` discovery is separate from a proven + // BrowserWindow identity; only the latter may create this association. + expect( + graph.nodes.filter( + ({ kind, observations }) => + kind === "electron-preload" && + observations.some( + ({ properties }) => + properties.mechanism === "BrowserWindow:webPreferences.preload", + ), + ), + ).toHaveLength(expectedRoles); + const main = findSourceModule(graph, "main.cjs"); + expect(main).toBeDefined(); + const imports = graph.edges.filter( + ({ relation, source_node_id }) => + relation === "imports" && source_node_id === main?.node_id, + ); + expect( + imports.some(({ properties }) => properties.specifier === "electron"), + ).toBe(capturedBeforeWrite); + expect( + imports.some(({ properties }) => properties.specifier === null), + ).toBe(!capturedBeforeWrite); + const preloads = graph.edges.filter( + ({ evidence }) => + evidence.extractor.operation === "associate-browser-window-preload", + ); + if (capturedBeforeWrite) + expect(preloads).toMatchObject([ + { + relation: "loads", + source_node_id: windows[0]?.node_id, + properties: { + declared_path: "./bridge.cjs", + resolved_path: "bridge.cjs", + resolution_status: "resolved", + }, + }, + ]); + else expect(preloads).toEqual([]); + }, +); diff --git a/tests/acceptance/applications/packageSubpathsCli.test.ts b/tests/acceptance/applications/packageSubpathsCli.test.ts index 869a8deba7..1230ec44bf 100644 --- a/tests/acceptance/applications/packageSubpathsCli.test.ts +++ b/tests/acceptance/applications/packageSubpathsCli.test.ts @@ -1,7 +1,6 @@ import { createHash } from "node:crypto"; -import { mkdir, writeFile } from "node:fs/promises"; import { createRequire } from "node:module"; -import { dirname, join, relative } from "node:path"; +import { join, relative } from "node:path"; import { pathToFileURL } from "node:url"; import { expect } from "vitest"; @@ -11,6 +10,7 @@ import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/j import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/workspace/writeFixtureFiles.js"; const createPackageSubpathFixture = async () => { const root = await createTestTempDirectory("rea-package-subpaths-"); @@ -85,13 +85,7 @@ const createPackageSubpathFixture = async () => { "node_modules/scalar.js": "module.exports = 'scalar sibling';", "node_modules/scalar/package.json": JSON.stringify({ exports: false }), }; - await Promise.all( - Object.entries(files).map(async ([path, content]) => { - const absolute = join(root, path); - await mkdir(dirname(absolute), { recursive: true }); - await writeFile(absolute, content); - }), - ); + await writeFixtureFiles(root, files); return { root, source, selfSource, importSource }; }; @@ -189,8 +183,8 @@ cliTest.for(["CLI", "stdio MCP"] as const)( properties: expect.objectContaining({ module_link_kind: "import", specifier: "FILE:///src/uri-dep.cjs?token=%2F#entry", - resolved_path: "src/uri-dep.cjs", - resolution_status: "resolved", + resolved_path: null, + resolution_status: "unavailable", }), evidence: sourceEvidence("src/imports.mjs", importSource), }), diff --git a/tests/acceptance/applications/packageUrlUnicodeCli.test.ts b/tests/acceptance/applications/packageUrlUnicodeCli.test.ts index a50b4c837d..f7447feae4 100644 --- a/tests/acceptance/applications/packageUrlUnicodeCli.test.ts +++ b/tests/acceptance/applications/packageUrlUnicodeCli.test.ts @@ -32,10 +32,10 @@ const targets = { }, external: { value: "another-package", - imports: "external", + imports: "unavailable", exports: "rejected", }, - recursive: { value: "#another", imports: "external", exports: "rejected" }, + recursive: { value: "#another", imports: "unavailable", exports: "rejected" }, invalid: { value: "../feature.js", imports: "rejected", exports: "rejected" }, scalar: { value: false, imports: "rejected", exports: "rejected" }, numeric: { value: { 0: null }, imports: "rejected", exports: "rejected" }, @@ -47,7 +47,7 @@ const targets = { }, arrayExternal: { value: [false, "another-package"], - imports: "external", + imports: "unavailable", exports: "rejected", }, arrayRelative: { @@ -62,7 +62,7 @@ const targets = { }, escapeExternal: { value: [escapingTarget, "another-package"], - imports: "external", + imports: "unavailable", exports: "rejected", }, escapeNull: { @@ -82,7 +82,7 @@ const targets = { }, escapeNested: { value: [{ node: escapingTarget }, "another-package"], - imports: "external", + imports: "unavailable", exports: "rejected", }, escapeConfig: { @@ -92,12 +92,12 @@ const targets = { }, wtf8Reentry: { value: ["./%2\te%2\te/%ED%A0%80/feature.js", "another-package"], - imports: "external", + imports: "unavailable", exports: "rejected", }, opaqueReentry: { value: ["./%2\te%2\te/rea-op\taque-0/feature.js", "another-package"], - imports: "external", + imports: "unavailable", exports: "rejected", }, structuralReentry: { @@ -246,7 +246,7 @@ cliTest.for(["CLI", "stdio MCP"] as const)( }), evidence: expect.objectContaining({ limitations: - status === "unavailable" + status === "unavailable" && request.status !== "unavailable" ? expect.arrayContaining([ expect.stringContaining("unpaired UTF-16 surrogate"), ]) diff --git a/tests/boundary/cli/cliIntegerArguments.test.ts b/tests/boundary/cli/cliIntegerArguments.test.ts index 9f07e5777b..202d0898a9 100644 --- a/tests/boundary/cli/cliIntegerArguments.test.ts +++ b/tests/boundary/cli/cliIntegerArguments.test.ts @@ -1,6 +1,6 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect } from "vitest"; -import { serveCli } from "./cliServeHarness.js"; +import { cliTest, type CliResult } from "../../support/cli/cliFixture.js"; const SESSION_ID = "27d3e3f1-f1e5-49ae-91ec-95af1f343a5a"; const NON_INTEGER_TEXT = [ @@ -17,10 +17,7 @@ const NON_INTEGER_TEXT = [ ]; const NON_INTEGER_OPTION_TEXT = [...NON_INTEGER_TEXT, "-0"]; -const expectCliValidationError = ( - result: Awaited>, - field: string, -): void => { +const expectCliValidationError = (result: CliResult, field: string): void => { expect(result.exitCode).toBe(1); expect(JSON.parse(result.stdout)).toMatchObject({ code: "VALIDATION_ERROR", @@ -31,65 +28,70 @@ const expectCliValidationError = ( }; describe("CLI integer arguments", () => { - it.each(NON_INTEGER_TEXT)( + cliTest.for(NON_INTEGER_TEXT)( "rejects %j as a Cutter expected generation", - async (value) => { - const result = await serveCli([ - "cutter-command", - SESSION_ID, - value, - "pdi", - "--json", - ]); + async (value, { cli }) => { + const result = await cli.run({ + arguments: ["cutter-command", SESSION_ID, value, "pdi", "--json"], + }); expectCliValidationError(result, "expectedGeneration"); }, ); - it.each(NON_INTEGER_TEXT)( + cliTest.for(NON_INTEGER_TEXT)( "rejects %j as a web module script index", - async (value) => { - const result = await serveCli([ - "trace-web-module-imports", - "/nonexistent/manifest.json", - value, - "--json", - ]); + async (value, { cli }) => { + const result = await cli.run({ + arguments: [ + "trace-web-module-imports", + "/nonexistent/manifest.json", + value, + "--json", + ], + }); expectCliValidationError(result, "scriptIndex"); }, ); - it.each(NON_INTEGER_OPTION_TEXT)( + cliTest.for(NON_INTEGER_OPTION_TEXT)( "rejects %j in a repeated web capture record selector", - async (value) => { - const result = await serveCli([ - "inspect-web-network-capture", - "/nonexistent/capture.qlog", - "qlog", - "--record", - "0", - "--record", - value, - "--json", - ]); + async (value, { cli }) => { + const result = await cli.run({ + arguments: [ + "inspect-web-network-capture", + "/nonexistent/capture.qlog", + "qlog", + "--record", + "0", + "--record", + value, + "--json", + ], + }); expectCliValidationError(result, "record"); }, ); - it("keeps the one-based source line lower bound at the CLI boundary", async () => { - const result = await serveCli([ - "trace-web-source-location", - "/nonexistent/manifest.json", - "0", - "/nonexistent/app.js.map", - "https://example.test/app.js.map", - "0", - "0", - "--json", - ]); + cliTest( + "keeps the one-based source line lower bound at the CLI boundary", + async ({ cli }) => { + const result = await cli.run({ + arguments: [ + "trace-web-source-location", + "/nonexistent/manifest.json", + "0", + "/nonexistent/app.js.map", + "https://example.test/app.js.map", + "0", + "0", + "--json", + ], + }); - expectCliValidationError(result, "line"); - }); + expectCliValidationError(result, "line"); + }, + ); }); diff --git a/tests/boundary/cli/electronWrappedCallPaths.test.ts b/tests/boundary/cli/electronWrappedCallPaths.test.ts index 89b024be37..2a712e7874 100644 --- a/tests/boundary/cli/electronWrappedCallPaths.test.ts +++ b/tests/boundary/cli/electronWrappedCallPaths.test.ts @@ -281,7 +281,7 @@ cliTest( surface, cli, root, - connection?.call, + { mcpCall: connection?.call }, ); expect(application.input_path).toBe(root); expect(artifacts.has(application.root_artifact_sha256)).toBe(false); @@ -521,7 +521,7 @@ for (const surface of ["CLI", "stdio MCP"] as const) { surface, cli, root, - connection?.call, + { mcpCall: connection?.call }, ); expect(application.input_path).toBe(root); return application; diff --git a/tests/boundary/filesystem/artifactLiteralBackslash.test.ts b/tests/boundary/filesystem/artifactLiteralBackslash.test.ts index 02105ad604..b56bdb6327 100644 --- a/tests/boundary/filesystem/artifactLiteralBackslash.test.ts +++ b/tests/boundary/filesystem/artifactLiteralBackslash.test.ts @@ -1,7 +1,10 @@ import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; import { access, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { Readable } from "node:stream"; +import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; import { createPackage } from "@electron/asar"; import { expect, it } from "vitest"; @@ -15,6 +18,44 @@ import { import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { analyzeJavaScriptApplication } from "../../support/javascriptApplicationScope.js"; import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import type { JavaScriptApplicationGraph } from "../../../src/domain/javascript/javascriptApplicationGraph.js"; + +const loaderImports = (graph: JavaScriptApplicationGraph) => + graph.edges.filter( + ({ relation, properties }) => + relation === "imports" && + properties.specifier === "./b.js" && + (properties.kind === "static-import" || + properties.kind === "dynamic-import" || + properties.kind === "require"), + ); + +const forwardedExports = (graph: JavaScriptApplicationGraph) => + graph.edges.filter( + ({ relation, properties }) => + relation === "imports" && + properties.specifier === "./b.js" && + (properties.module_link_kind === "export" || + properties.module_link_kind === "commonjs-export"), + ); + +const writeLoaderExportFixture = async ( + source: string, + directory: string, + targetBody: string, +): Promise => { + await mkdir(join(source, directory), { recursive: true }); + for (const [name, body] of [ + [ + "a.mjs", + "import { marker } from './b.js'; export { marker }; import('./b.js');\n", + ], + ["a.cjs", "module.exports = require('./b.js');\n"], + ["a.ts", "import { marker } from './b.js'; export { marker };\n"], + ["b.js", targetBody], + ] as const) + await writeFile(join(source, directory, name), body); +}; const NAMES = [ "literal\\name.txt", @@ -167,24 +208,98 @@ it.skipIf(process.platform === "win32").each(["directory", "asar"] as const)( }, ); +it("rejects inherited encoded separators with the real Node ESM loader before file lookup", async () => { + const root = await createTestTempDirectory("rea-backslash-esm-url-"); + const source = pathToFileURL(join(root, "a.mjs")); + source.pathname = source.pathname.replace(/a\.mjs$/u, "literal%5Cdir/a.mjs"); + const target = new URL("./b.mjs", source); + expect(target.pathname).toContain("literal%5Cdir/b.mjs"); + // Use the actual Node loader, outside Vitest's module runner. + const observed = execFileSync( + process.execPath, + [ + "--input-type=module", + "--eval", + "try { await import(process.argv[1]); console.log(JSON.stringify({loaded:true})); } catch(error) { console.log(JSON.stringify({code:error.code})); }", + target.href, + ], + { encoding: "utf8", timeout: 5_000 }, + ); + expect(JSON.parse(observed) as unknown).toEqual({ + code: "ERR_INVALID_MODULE_SPECIFIER", + }); +}); + +it("keeps loader modes separate from forwarded export relationships", async () => { + const root = await createTestTempDirectory("rea-loader-export-contract-"); + const source = join(root, "source"); + await writeLoaderExportFixture( + source, + "ordinary", + "export const marker = 'target';\n", + ); + const result = await analyzeJavaScriptApplication({ + input_path: source, + format: "directory", + }); + if (!result.ok) throw result.error; + const { graph } = javascriptApplicationAnalysisResultSchema.parse( + result.value.normalized_result, + ); + const imports = loaderImports(graph); + expect(imports).toHaveLength(4); + expect( + imports.map(({ properties }) => [ + properties.kind, + properties.resolution_mode, + ]), + ).toEqual( + expect.arrayContaining([ + ["require", "require"], + ["dynamic-import", "import"], + ["static-import", "import"], + ["static-import", "compiler-dependent"], + ]), + ); + const exports = forwardedExports(graph); + expect(exports).toHaveLength(3); + expect( + exports.map(({ properties }) => [ + properties.module_link_kind, + properties.exported_name, + properties.resolution_mode, + ]), + ).toEqual( + expect.arrayContaining([ + ["export", "marker", undefined], + ["export", "marker", "compiler-dependent"], + ["commonjs-export", "default", undefined], + ]), + ); + for (const { properties } of [...imports, ...exports]) + expect(properties).toMatchObject({ + resolved_path: "ordinary/b.js", + resolution_status: "resolved", + }); +}); + it.skipIf(process.platform === "win32").each(["directory", "asar"] as const)( - "resolves ordinary relative imports inside a literal POSIX directory from %s", + "preserves loader-specific relative paths inside a literal POSIX directory from %s", async (format) => { const root = await createTestTempDirectory("rea-backslash-import-base-"); const source = join(root, "source"); const literal = "literal\\dir"; - await mkdir(join(source, literal), { recursive: true }); await mkdir(join(source, "literal", "dir"), { recursive: true }); const body = "export const marker = 'literal';\n"; - await writeFile( - join(source, literal, "a.js"), - "import { marker } from './b.js'; export { marker };\n", - ); - await writeFile(join(source, literal, "b.js"), body); + await writeLoaderExportFixture(source, literal, body); await writeFile( join(source, "literal", "dir", "b.js"), "export const marker = 'decoy';\n", ); + // Node's CommonJS resolver uses the literal filesystem base on POSIX. + expect( + createRequire(join(source, literal, "a.cjs")).resolve("./b.js"), + ).toBe(join(source, literal, "b.js")); const input = format === "directory" ? source : join(root, "input.asar"); if (format === "asar") await createPackage(source, input); const result = await analyzeJavaScriptApplication({ @@ -195,13 +310,60 @@ it.skipIf(process.platform === "win32").each(["directory", "asar"] as const)( const parsed = javascriptApplicationAnalysisResultSchema.parse( result.value.normalized_result, ); - const imports = parsed.graph.edges.filter( - ({ properties }) => properties.specifier === "./b.js", - ); - expect(imports.length).toBeGreaterThan(0); - expect(imports.map(({ properties }) => properties.resolved_path)).toEqual( - imports.map(() => `${literal}/b.js`), - ); + const imports = loaderImports(parsed.graph); + expect(imports).toHaveLength(4); + expect( + new Set(imports.map(({ properties }) => properties.resolution_mode)), + ).toEqual(new Set(["require", "import", "compiler-dependent"])); + for (const { properties } of imports) { + const mode = properties.resolution_mode; + expect(properties).toMatchObject({ + resolved_path: mode === "require" ? `${literal}/b.js` : null, + resolution_status: + mode === "require" + ? "resolved" + : mode === "import" + ? "rejected" + : "unavailable", + }); + if (mode === "compiler-dependent") + expect(properties.conditional_resolutions).toEqual([ + expect.objectContaining({ + mode: "import", + resolution_status: "rejected", + resolved_path: null, + }), + expect.objectContaining({ + mode: "require", + resolution_status: "resolved", + resolved_path: `${literal}/b.js`, + }), + ]); + } + const exports = forwardedExports(parsed.graph); + expect(exports).toHaveLength(3); + for (const exported of exports) { + const sourceNode = parsed.graph.nodes.find( + ({ node_id }) => node_id === exported.source_node_id, + ); + const path = sourceNode?.observations.find( + ({ properties }) => typeof properties.module_path === "string", + )?.properties.module_path; + expect([ + `${literal}/a.cjs`, + `${literal}/a.mjs`, + `${literal}/a.ts`, + ]).toContain(path); + expect(exported.properties).toMatchObject({ + resolved_path: path === `${literal}/a.cjs` ? `${literal}/b.js` : null, + resolution_status: + path === `${literal}/a.cjs` + ? "resolved" + : path === `${literal}/a.mjs` + ? "rejected" + : "unavailable", + }); + } expect(parsed.graph.nodes).toEqual( expect.arrayContaining([ expect.objectContaining({ diff --git a/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts index 8d355d8c1f..798cc3258c 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts @@ -50,6 +50,404 @@ import { } from "../../../../src/domain/javascript/javascriptSemanticGraph.js"; import { parseJavaScriptSemanticGraph } from "../../../../src/domain/javascript/javascriptSemanticGraphSerialization.js"; import { writeJavaScriptArtifactFixture } from "../../../fixtures/javascriptArtifactApplication.js"; +import { writeFixtureFiles } from "../../../support/javascriptApplicationFixture.js"; + +it("does not let a known-absent optional import erase another application's page", async () => { + const root = await createTestTempDirectory("rea-absent-optional-owner-"); + await writeFixtureFiles(root, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "b/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": + 'const {BrowserWindow} = require("electron"); const w = new BrowserWindow(); w.loadFile("index.html");', + "b/main.cjs": 'try { require("./optional.cjs"); } catch {}', + "a/index.html": "valid page", + "index.html": "root decoy", + }); + const result = await reconstructJavaScriptArtifact({ + input_path: root, + format: "directory", + }); + const role = result.graph.nodes.find( + ({ kind }) => kind === "electron-renderer", + ); + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: "a/index.html", + resolution_status: "resolved", + }), + }), + ]), + ); + expect( + result.graph.edges.filter( + ({ source_node_id, relation }) => + source_node_id === role?.node_id && relation === "maps_to", + ), + ).toHaveLength(1); +}); + +it.each([ + "unselected", + "unavailable", + "dynamic", + "unavailable-json", + "unavailable-package-json", + "unavailable-uppercase-json", + "unavailable-main", + "unavailable-default-main", + "unavailable-json-main", + "resolved-html", + "resolved-uppercase-json", + "resolved-json", + "resolved-package-json", + "resolved-html-main", + "unavailable-package-metadata", + "unavailable-directory-main", +] as const)( + "distinguishes hidden imports from inert data through %s", + async (omission) => { + const root = await createTestTempDirectory("rea-competing-gap-"); + const targetPaths: Partial> = { + "unavailable-default-main": "b/index.js", + "unavailable-json-main": "b/config.json", + "unavailable-main": "b/main.cjs", + "unavailable-json": "b/config.json", + "unavailable-package-json": "b/node_modules/pkg/package.json", + "unavailable-uppercase-json": "shared/bridge.JSON", + "resolved-html": "shared/bridge.html", + "resolved-uppercase-json": "shared/bridge.JSON", + "resolved-json": "b/config.json", + "resolved-package-json": "b/node_modules/pkg/package.json", + "resolved-html-main": "b/main.html", + "unavailable-package-metadata": "b/node_modules/pkg/package.json", + "unavailable-directory-main": "b/node_modules/pkg/package.json", + }; + const targetPath = targetPaths[omission] ?? "shared/bridge.cjs"; + const inertJson = [ + "unavailable-json", + "unavailable-package-json", + "unavailable-json-main", + "resolved-json", + "resolved-package-json", + ].includes(omission); + await writeFixtureFiles(root, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "b/package.json": JSON.stringify( + omission === "unavailable-default-main" + ? {} + : { + main: + omission === "unavailable-directory-main" + ? "node_modules/pkg" + : omission === "resolved-html-main" + ? "main.html" + : omission === "unavailable-json-main" + ? "config.json" + : "main.cjs", + }, + ), + "b/index.js": 'require("../shared/window.cjs");', + "b/main.html": 'require("../shared/window.cjs");', + "a/main.cjs": 'require("../shared/window.cjs");', + "b/main.cjs": + omission === "unavailable-package-metadata" + ? 'require("pkg");' + : omission === "unavailable-main" + ? 'require("../shared/window.cjs");' + : omission === "dynamic" + ? "require(process.env.CHILD);" + : `require(${JSON.stringify(targetPath.startsWith("b/") ? `./${targetPath.slice(2)}` : `../${targetPath}`)});`, + "b/config.json": "{}", + "b/node_modules/pkg/package.json": "{}", + "b/node_modules/pkg/index.js": 'require("../../../shared/window.cjs");', + "shared/bridge.JSON": 'require("./window.cjs");', + "shared/bridge.html": 'require("./window.cjs");', + "shared/bridge.cjs": 'require("./window.cjs");', + "shared/window.cjs": + 'const {BrowserWindow} = require("electron"); const w = new BrowserWindow(); w.loadFile("index.html");', + "a/index.html": "wrong unique owner", + "b/index.html": "competing owner", + "index.html": "root decoy", + }); + const snapshot = await scanCanonicalArtifactInventory(root, {}); + const reader = createJavaScriptArtifactReader(root, "directory"); + try { + const files = await readJavaScriptArtifactFiles( + reader, + snapshot, + undefined, + omission === "unselected" + ? [ + "a/package.json", + "b/package.json", + "a/main.cjs", + "b/main.cjs", + "shared/window.cjs", + ] + : undefined, + ); + const bridge = files.files.find(({ path }) => path === targetPath); + if (bridge === undefined) throw new Error("Missing bridge"); + const partial = !omission.startsWith("unavailable") + ? files + : { + ...files, + files: files.files.filter(({ path }) => path !== bridge.path), + unavailable_relevant_files: [ + ...files.unavailable_relevant_files, + { + path: bridge.path, + container_sha256: bridge.container_sha256, + sha256: null, + bytes: null, + inventory_artifact_id: bridge.inventory_artifact_id, + kind: bridge.kind, + hash_status: "unavailable" as const, + }, + ], + }; + const graph = buildJavaScriptArtifactGraph( + snapshot, + partial, + analyzeJavaScriptArtifactFiles(partial), + ); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + expect(roles[0]?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: inertJson ? "a/index.html" : null, + resolution_status: inertJson ? "resolved" : "unavailable", + }), + }), + ]), + ); + expect( + graph.edges.filter( + ({ source_node_id, relation }) => + source_node_id === roles[0]?.node_id && relation === "maps_to", + ), + ).toHaveLength(inertJson ? 1 : 0); + } finally { + await reader.close(); + } + }, +); + +it.each(["unselected", "unavailable"] as const)( + "keeps imported loadFile ownership unknown across an %s module", + async (omission) => { + const root = await createTestTempDirectory("rea-missing-module-owner-"); + await writeFixtureFiles(root, { + "app/package.json": JSON.stringify({ main: "../shared/main.cjs" }), + "shared/main.cjs": 'require("./window.cjs");', + "shared/window.cjs": + 'const { BrowserWindow } = require("electron"); const win = new BrowserWindow(); win.loadFile("index.html");', + "index.html": "artifact-root decoy", + "app/index.html": "unproven application page", + }); + const snapshot = await scanCanonicalArtifactInventory(root, {}); + const reader = createJavaScriptArtifactReader(root, "directory"); + try { + const files = await readJavaScriptArtifactFiles( + reader, + snapshot, + undefined, + omission === "unselected" + ? ["app/package.json", "shared/window.cjs"] + : undefined, + ); + const entry = files.files.find(({ path }) => path === "shared/main.cjs"); + if (entry === undefined) throw new Error("Missing inventoried entry"); + // The unavailable variant injects the reader boundary, not an actual I/O failure. + const partial = + omission === "unselected" + ? files + : { + ...files, + files: files.files.filter(({ path }) => path !== entry.path), + unavailable_relevant_files: [ + ...files.unavailable_relevant_files, + { + path: entry.path, + container_sha256: entry.container_sha256, + sha256: null, + bytes: null, + inventory_artifact_id: entry.inventory_artifact_id, + kind: entry.kind, + hash_status: "unavailable" as const, + }, + ], + }; + const graph = buildJavaScriptArtifactGraph( + snapshot, + partial, + analyzeJavaScriptArtifactFiles(partial), + ); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + const role = roles[0]; + if (role === undefined) throw new Error("Missing renderer"); + expect(role.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: null, + resolution_status: "unavailable", + }), + }), + ]), + ); + expect( + graph.edges.filter( + ({ source_node_id, relation }) => + source_node_id === role.node_id && relation === "maps_to", + ), + ).toEqual([]); + } finally { + await reader.close(); + } + }, +); + +it("keeps imported loadFile ownership unknown across an application projection omission", async () => { + const root = await createTestTempDirectory("rea-omitted-app-owner-"); + await writeFixtureFiles(root, { + "app/package.json": JSON.stringify({ main: "../shared/main.cjs" }), + "shared/main.cjs": 'require("./window.cjs");', + "shared/window.cjs": + 'const { BrowserWindow } = require("electron"); const win = new BrowserWindow(); win.loadFile("index.html");', + "index.html": "artifact-root decoy", + "app/index.html": "unproven application page", + }); + const snapshot = await scanCanonicalArtifactInventory(root, {}); + const reader = createJavaScriptArtifactReader(root, "directory"); + try { + const files = await readJavaScriptArtifactFiles(reader, snapshot); + const analysis = analyzeJavaScriptArtifactFiles(files); + const entry = analysis.files.find( + ({ file }) => file.path === "shared/main.cjs", + ); + expect( + entry?.javascript?.references.some( + ({ specifier }) => specifier === "./window.cjs", + ), + ).toBe(true); + // Exercise the production omission boundary without bypassing its memory guard. + const graph = buildJavaScriptArtifactGraph(snapshot, files, { + ...analysis, + files: analysis.files.map((file) => + file.file.path === "shared/main.cjs" + ? { + ...file, + application_projection_failure: { + reason: "fixture projection omitted", + error: null, + limits: null, + }, + } + : file, + ), + }); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + const role = roles[0]; + if (role === undefined) throw new Error("Missing renderer"); + expect(role.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: null, + resolution_status: "unavailable", + }), + }), + ]), + ); + expect( + graph.edges.filter( + ({ source_node_id, relation }) => + source_node_id === role.node_id && relation === "maps_to", + ), + ).toEqual([]); + } finally { + await reader.close(); + } +}); + +it("keeps a loadFile owner unknown when a sibling manifest is unavailable in the inventory projection", async () => { + const root = await createTestTempDirectory("rea-unavailable-app-owner-"); + await writeFixtureFiles(root, { + "app/package.json": JSON.stringify({ main: "../shared/main.cjs" }), + "shared/main.cjs": + 'const { BrowserWindow } = require("electron"); const win = new BrowserWindow(); win.loadFile("index.html");', + "index.html": "artifact-root decoy", + "app/index.html": "unproven app page", + }); + const snapshot = await scanCanonicalArtifactInventory(root, {}); + const reader = createJavaScriptArtifactReader(root, "directory"); + try { + const files = await readJavaScriptArtifactFiles(reader, snapshot); + const manifest = files.files.find( + ({ path }) => path === "app/package.json", + ); + if (manifest === undefined) throw new Error("Missing inventoried manifest"); + // Inject the reader boundary's unavailable projection, not a fabricated graph. + const partial = { + ...files, + files: files.files.filter(({ path }) => path !== manifest.path), + unavailable_relevant_files: [ + ...files.unavailable_relevant_files, + { + path: manifest.path, + container_sha256: manifest.container_sha256, + sha256: null, + bytes: null, + inventory_artifact_id: manifest.inventory_artifact_id, + kind: manifest.kind, + hash_status: "unavailable" as const, + }, + ], + }; + const graph = buildJavaScriptArtifactGraph( + snapshot, + partial, + analyzeJavaScriptArtifactFiles(partial), + ); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + const role = roles[0]; + if (role === undefined) throw new Error("Missing renderer"); + expect(role.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: null, + resolution_status: "unavailable", + }), + }), + ]), + ); + expect( + graph.edges.filter( + ({ source_node_id, relation }) => + source_node_id === role.node_id && relation === "maps_to", + ), + ).toEqual([]); + } finally { + await reader.close(); + } +}); it("keeps excluded non-JavaScript files out of both semantic construction paths", async () => { const root = await createTestTempDirectory("rea-js-semantic-applicability-"); diff --git a/tests/boundary/filesystem/javascript-applications/javascriptFileUrlPaths.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptFileUrlPaths.test.ts index 89c582c508..e2cf14aa05 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptFileUrlPaths.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptFileUrlPaths.test.ts @@ -8,10 +8,12 @@ import { describe, expect, it } from "vitest"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; import { javascriptApplicationAnalysisResultSchema } from "../../../../src/domain/javascript/javascriptApplicationAnalysis.js"; import { staticPath } from "../../../../src/domain/javascript/javascriptStaticAnalysisHelpers.js"; +import { analyzeJavaScriptStaticSource } from "../../../../src/domain/javascript/javascriptStaticAnalysis.js"; import { analyzeJavaScriptApplication } from "../../../support/javascriptApplicationScope.js"; const urlCases = [ ["./preload%20name.cjs", "preload name.cjs", "main.mjs"], + ["./preload%252Fname.cjs", "preload%2Fname.cjs", "main.mjs"], ["./pr%C3%A9load.cjs", "préload.cjs", "main.mjs"], ["./preload%3F%23.cjs", "preload?#.cjs", "main.mjs"], ["./preload.cjs?cache=1#v2", "preload.cjs", "main.mjs"], @@ -60,7 +62,7 @@ describe("JavaScript file URL paths", () => { expect(await readFile(nativePath, "utf8")).toContain("real preload"); await writeFile( sourceFile, - `import { fileURLToPath } from "node:url";\nnew BrowserWindow({ webPreferences: { preload: fileURLToPath(new URL(${JSON.stringify(reference)}, import.meta.url)) } });\n`, + `import { fileURLToPath } from "node:url";\nimport { BrowserWindow } from "electron";\nnew BrowserWindow({ webPreferences: { preload: fileURLToPath(new URL(${JSON.stringify(reference)}, import.meta.url)) } });\n`, ); const result = await analyzeJavaScriptApplication({ @@ -85,7 +87,7 @@ describe("JavaScript file URL paths", () => { }, ); - it("resolves a decoded literal file URL at the filesystem boundary", async () => { + it("keeps a decoded absolute file URL unmapped without host ownership", async () => { const root = await createTestTempDirectory("rea-literal-file-url-"); await mkdir(join(root, "app"), { recursive: true }); await writeFile( @@ -109,10 +111,16 @@ describe("JavaScript file URL paths", () => { expect(preload?.observations).not.toHaveLength(0); for (const observation of preload?.observations ?? []) expect(observation.properties).toMatchObject({ - resolved_path: "app/preload name.cjs", - resolution_status: "resolved", + resolved_path: null, + resolution_status: "unavailable", resolution_context: "filesystem-expression", }); + expect( + graph.edges.filter( + ({ source_node_id, relation }) => + source_node_id === preload?.node_id && relation === "maps_to", + ), + ).toEqual([]); }); it.each(unknownExpressions)("leaves unsupported %s unknown", (expression) => { @@ -122,9 +130,15 @@ describe("JavaScript file URL paths", () => { }); it("does not URL-decode literal filesystem punctuation", () => { - const expression = 'path.join(__dirname, "preload%20?#.cjs")'; - expect( - staticPath(parseExpression(expression, { sourceType: "module" })), - ).toBe("preload%20?#.cjs"); + const analysis = analyzeJavaScriptStaticSource(` + const path = require('node:path'); + const {BrowserWindow} = require('electron'); + new BrowserWindow({webPreferences:{preload:path.join(__dirname,"preload%20?#.cjs")}}); + `); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.electron.browser_windows).toHaveLength(1); + expect(analysis.electron.browser_windows[0]?.preload_path).toBe( + "preload%20?#.cjs", + ); }); }); diff --git a/tests/boundary/filesystem/javascript-applications/javascriptModuleRelationships.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptModuleRelationships.test.ts index 3e45278733..9a89e3d81b 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptModuleRelationships.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptModuleRelationships.test.ts @@ -38,6 +38,59 @@ const relationshipGraph = async (root: string) => { } }; +it.each(["browser", "module", "renderer", "main"] as const)( + "distinguishes %s package metadata from an imported module's application owner", + async (field) => { + const root = await createTestTempDirectory("rea-frontend-package-owner-"); + await writeFixtureFiles(root, { + "package.json": JSON.stringify({ main: "main.cjs" }), + "main.cjs": 'require("./lib/window.cjs");', + "lib/package.json": JSON.stringify({ [field]: "window.cjs" }), + "lib/window.cjs": + 'const {BrowserWindow} = require("electron"); const w = new BrowserWindow(); w.loadFile("index.html");', + "lib/orphan.cjs": + 'const {BrowserWindow} = require("electron"); const w = new BrowserWindow(); w.loadFile("orphan.html");', + "index.html": "root application page", + "lib/index.html": "frontend package decoy", + "orphan.html": "root application fallback", + "lib/orphan.html": "frontend package fallback decoy", + }); + const graph = await relationshipGraph(root); + const actualMain = field === "main"; + for (const declaredPath of ["index.html", "orphan.html"]) { + const renderer = graph.nodes.find( + ({ kind, observations }) => + kind === "electron-renderer" && + observations.some( + ({ properties }) => properties.declared_path === declaredPath, + ), + ); + expect(renderer?.observations[0]?.properties).toMatchObject({ + resolved_path: actualMain ? null : declaredPath, + resolution_status: actualMain ? "unavailable" : "resolved", + }); + expect( + graph.edges.filter( + ({ source_node_id, relation }) => + source_node_id === renderer?.node_id && relation === "maps_to", + ), + ).toHaveLength(actualMain ? 0 : 1); + } + if (!actualMain) + expect( + graph.nodes.some( + ({ kind, observations }) => + kind === "electron-renderer" && + observations.some( + ({ properties }) => + properties.mechanism === "package.json:renderer" && + properties.resolved_path === "lib/window.cjs", + ), + ), + ).toBe(true); + }, +); + it.each([ ["getter-expansion", getterExpansionSource()], [ diff --git a/tests/boundary/filesystem/javascript-applications/javascriptModuleSpecifierPaths.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptModuleSpecifierPaths.test.ts index 5e62327d0f..d8a86f0da6 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptModuleSpecifierPaths.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptModuleSpecifierPaths.test.ts @@ -398,15 +398,32 @@ describe("package exports URL path rejections", () => { ...crossContainer, container_sha256: "c".repeat(64), }); - expect( + const resolveTarget = () => resolveArtifactPathByContext({ declaredPath: "fixture", sourcePath: "app/main.js", context: "module-specifier", moduleKind, files, - }), - ).toMatchObject({ resolved_path: null, resolution_status: "not-found" }); + }); + expect(resolveTarget()).toMatchObject({ + resolved_path: null, + resolution_status: "unavailable", + candidate_path: target, + limitations: [ + expect.stringContaining("crosses an artifact container boundary"), + ], + }); + files.delete(target); + expect(resolveTarget()).toMatchObject({ + resolved_path: null, + resolution_status: "not-found", + }); + files.set(target, crossContainer); + expect(resolveTarget()).toMatchObject({ + resolved_path: target, + resolution_status: "resolved", + }); }, ); }); diff --git a/tests/boundary/filesystem/javascript-applications/javascriptNonStringPackageMain.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptNonStringPackageMain.test.ts new file mode 100644 index 0000000000..cbf0fb1f21 --- /dev/null +++ b/tests/boundary/filesystem/javascript-applications/javascriptNonStringPackageMain.test.ts @@ -0,0 +1,141 @@ +import { createRequire } from "node:module"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { analyzeJavaScriptArtifactFiles } from "../../../../src/application/javascript/JavaScriptArtifactAnalysis.js"; +import { buildJavaScriptArtifactGraph } from "../../../../src/application/javascript/JavaScriptArtifactGraphBuilder.js"; +import { resolveArtifactPathByContext } from "../../../../src/application/javascript/JavaScriptArtifactPathResolution.js"; +import { createJavaScriptArtifactReader } from "../../../../src/artifacts/javascript/JavaScriptArtifactReader.js"; +import { scanCanonicalArtifactInventory } from "../../../fixtures/artifactInventory.js"; +import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { readJavaScriptArtifactFiles } from "../../../support/javascriptApplicationScope.js"; +import { writeFixtureFiles } from "../../../support/javascriptApplicationFixture.js"; + +const windowSource = + 'const {BrowserWindow}=require("electron");new BrowserWindow().loadFile("index.html");'; + +it.each( + [null, false, 123, {}, []].flatMap((main) => + [false, true].map((index) => ({ main, index, request: "./b" })), + ), +)( + "follows Node index admission for main=$main with index=$index", + async ({ main, index, request }) => { + const root = await createTestTempDirectory("rea-nonstring-package-main-"); + await writeFixtureFiles(root, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": index ? "module.exports={};" : windowSource, + "a/index.html": "first application page", + "b/package.json": JSON.stringify({ main }), + ...(index ? { "b/index.js": windowSource } : {}), + "b/index.html": "second application page", + "index.html": "artifact-root decoy", + }); + const require = createRequire(join(root, "node-control.cjs")); + if (index) expect(require.resolve(request)).toBe(join(root, "b/index.js")); + else + expect(() => require.resolve(request)).toThrow( + expect.objectContaining({ code: "MODULE_NOT_FOUND" }), + ); + const snapshot = await scanCanonicalArtifactInventory(root, {}); + const reader = createJavaScriptArtifactReader(root, "directory"); + try { + const files = await readJavaScriptArtifactFiles(reader, snapshot); + const analysis = analyzeJavaScriptArtifactFiles(files); + expect( + analysis.packages.find(({ path }) => path === "b/package.json"), + ).toMatchObject({ main_status: "invalid", main_entrypoint: "" }); + const graph = buildJavaScriptArtifactGraph(snapshot, files, analysis); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + expect(roles[0]?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolution_status: "resolved", + resolved_path: index ? "b/index.html" : "a/index.html", + }), + }), + ]), + ); + } finally { + await reader.close(); + } + }, +); + +it.each( + ( + [ + ["{", false], + ["[]", false], + ["[{}]", false], + ["123", false], + ['"root"', false], + ["null", false], + ["false", false], + ["{}", true], + ['{"main":[]}', true], + ] as const + ).map(([text, valid]) => ({ text, valid, request: "./b" })), +)( + "distinguishes malformed package root $text from a valid index fallback", + async ({ text, valid, request }) => { + const root = await createTestTempDirectory("rea-invalid-package-main-"); + await writeFixtureFiles(root, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": windowSource, + "a/index.html": "first application page", + "b/package.json": text, + "b/index.js": "module.exports={};", + "index.html": "artifact-root decoy", + }); + const require = createRequire(join(root, "node-control.cjs")); + if (valid) expect(require.resolve(request)).toBe(join(root, "b/index.js")); + else + expect(() => require.resolve(request)).toThrow( + expect.objectContaining({ code: "ERR_INVALID_PACKAGE_CONFIG" }), + ); + const snapshot = await scanCanonicalArtifactInventory(root, {}); + const reader = createJavaScriptArtifactReader(root, "directory"); + try { + const files = await readJavaScriptArtifactFiles(reader, snapshot); + const analysis = analyzeJavaScriptArtifactFiles(files); + expect( + analysis.packages.find(({ path }) => path === "b/package.json"), + ).toMatchObject({ + status: valid ? "included" : "invalid", + main_entrypoint: valid ? "" : null, + }); + expect( + resolveArtifactPathByContext({ + sourcePath: "a/main.cjs", + declaredPath: "../b", + context: "module-specifier", + moduleKind: "require", + files: new Map(files.files.map((file) => [file.path, file])), + }), + ).toMatchObject({ + resolution_status: valid ? "resolved" : "unavailable", + resolved_path: valid ? "b/index.js" : null, + }); + const graph = buildJavaScriptArtifactGraph(snapshot, files, analysis); + const role = graph.nodes.find(({ kind }) => kind === "electron-renderer"); + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolution_status: valid ? "resolved" : "unavailable", + resolved_path: valid ? "a/index.html" : null, + }), + }), + ]), + ); + } finally { + await reader.close(); + } + }, +); diff --git a/tests/boundary/filesystem/javascript-applications/javascriptRuntimeReconciliation.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptRuntimeReconciliation.test.ts index 643b8dcdd4..d6e25854a2 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptRuntimeReconciliation.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptRuntimeReconciliation.test.ts @@ -6,6 +6,7 @@ import { expect, it } from "vitest"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; import { createElectronActiveEvidence } from "../../../../src/application/javascript/ElectronActiveEvidence.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../../src/domain/javascript/javascriptApplicationAnalysis.js"; import { reconcileJavaScriptRuntimeEvidence } from "../../../../src/application/javascript/JavaScriptRuntimeReconciliationService.js"; import { createEvidence } from "../../../../src/domain/evidence.js"; import { mappedRuntimePaths } from "../../../../src/domain/javascript/javascriptRuntimeStaticCandidates.js"; @@ -728,6 +729,80 @@ it.skipIf(process.platform === "win32")( }, ); +it.each([false, true])( + "merges equivalent owned page declarations when target presence is %s", + async (present) => { + const root = await createTestTempDirectory("rea-generated-page-runtime-"); + await mkdir(join(root, "app"), { recursive: true }); + await writeFile( + join(root, "app", "package.json"), + JSON.stringify({ main: "main.cjs" }), + ); + await writeFile( + join(root, "app", "main.cjs"), + 'const {BrowserWindow} = require("electron"); const w = new BrowserWindow(); w.loadFile("generated.html"); w.loadFile("./generated.html");', + ); + if (present) + await writeFile(join(root, "app", "generated.html"), "generated page"); + const analysis = await analyzeFixture(root); + const { graph } = javascriptApplicationAnalysisResultSchema.parse( + analysis.normalized_result, + ); + const roles = graph.nodes.filter( + ({ kind }) => kind === "electron-renderer", + ); + expect(roles).toHaveLength(1); + expect( + roles[0]?.observations + .map(({ properties }) => properties.declared_path) + .sort(), + ).toEqual(["./generated.html", "generated.html"]); + const reconcileTarget = (path: string) => + reconcileInput({ + static_layers: [{ analysis, role: "application" }], + runtime_observations: [rendererCapture(root, path)], + }).reconciliations.find(({ entity_kind }) => entity_kind === "target"); + expect(reconcileTarget(join(root, "app", "generated.html"))).toMatchObject({ + status: "matched", + candidate_static_count: 1, + }); + expect(reconcileTarget(join(root, "generated.html"))).toMatchObject({ + status: "unmatched", + }); + }, +); + +it("matches a resolved nested application path without an operator mapping", async () => { + const root = await createTestTempDirectory("rea-owned-loadfile-runtime-"); + await mkdir(join(root, "app", "dist"), { recursive: true }); + await mkdir(join(root, "dist"), { recursive: true }); + await writeFile( + join(root, "app", "package.json"), + JSON.stringify({ main: "main.cjs" }), + ); + await writeFile( + join(root, "app", "main.cjs"), + "const { BrowserWindow } = require('electron'); const win = new BrowserWindow(); win.loadFile('dist/index.html');", + ); + await writeFile( + join(root, "app", "dist", "index.html"), + "application target", + ); + await writeFile(join(root, "dist", "index.html"), "root decoy"); + const analysis = await analyzeFixture(root); + const reconcileTarget = (path: string) => + reconcileInput({ + static_layers: [{ analysis, role: "application" }], + runtime_observations: [rendererCapture(root, path)], + }).reconciliations.find(({ entity_kind }) => entity_kind === "target"); + expect( + reconcileTarget(join(root, "app", "dist", "index.html")), + ).toMatchObject({ status: "matched" }); + expect(reconcileTarget(join(root, "dist", "index.html"))).toMatchObject({ + status: "unmatched", + }); +}); + it("keeps a raw nested loadFile target application-relative when the producer also resolves a nested decoy", async () => { const root = await createTestTempDirectory("rea-raw-loadfile-ownership-"); await mkdir(join(root, "dist")); diff --git a/tests/boundary/filesystem/javascriptArtifactGraphObservationLookup.test.ts b/tests/boundary/filesystem/javascriptArtifactGraphObservationLookup.test.ts index afccab83de..4ec345dab7 100644 --- a/tests/boundary/filesystem/javascriptArtifactGraphObservationLookup.test.ts +++ b/tests/boundary/filesystem/javascriptArtifactGraphObservationLookup.test.ts @@ -62,6 +62,8 @@ it("uses the first structured observation when paths repeat", async () => { { ...packageObservation, status: "invalid", + main_status: "unavailable", + main_entrypoint: null, name: null, version: null, main: null, diff --git a/tests/boundary/javascript/loaderInvocationMethodAuthority.test.ts b/tests/boundary/javascript/loaderInvocationMethodAuthority.test.ts new file mode 100644 index 0000000000..216f57fbab --- /dev/null +++ b/tests/boundary/javascript/loaderInvocationMethodAuthority.test.ts @@ -0,0 +1,50 @@ +import { execFileSync } from "node:child_process"; +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "../../../src/domain/javascript/javascriptStaticAnalysis.js"; + +it.each( + (["call", "apply"] as const).flatMap((method) => + (["unmodified", "mutated", "shadowed"] as const).map((authority) => ({ + method, + authority, + })), + ), +)( + "preserves native $method authority when Function is $authority", + ({ method, authority }) => { + const setup = + "import {createRequire as make} from 'node:module';const load=make(import.meta.url);"; + const mutation = + authority === "unmodified" + ? "" + : `${authority === "shadowed" ? "const Function={prototype:{}};" : ""}Function.prototype.${method}=()=>({kind:'local'});`; + const invocation = `load.${method}(null,${method === "call" ? "'node:path'" : "['node:path']"})`; + const observed = execFileSync( + process.execPath, + [ + "--input-type=module", + "--eval", + `${setup}const original=globalThis.Function.prototype.${method};${mutation}let out;try{out=${invocation};}finally{globalThis.Function.prototype.${method}=original;}console.log(JSON.stringify({native:typeof out.join==='function',local:out.kind==='local'}));`, + ], + { encoding: "utf8", timeout: 5_000 }, + ); + // Native createRequire and Function.prototype belong to the same child + // realm; injecting a host function into a VM would test another prototype. + expect(JSON.parse(observed) as unknown).toEqual({ + native: authority !== "mutated", + local: authority === "mutated", + }); + const analysis = analyzeJavaScriptStaticSource( + `${setup}${mutation}${invocation};`, + ); + expect(analysis.parse_error_count).toBe(0); + expect( + analysis.references.filter(({ kind }) => kind === "require"), + ).toEqual([ + expect.objectContaining({ + specifier: authority === "mutated" ? null : "node:path", + }), + ]); + }, +); diff --git a/tests/boundary/javascript/mainModuleSource.test.ts b/tests/boundary/javascript/mainModuleSource.test.ts new file mode 100644 index 0000000000..2e1528ec1c --- /dev/null +++ b/tests/boundary/javascript/mainModuleSource.test.ts @@ -0,0 +1,50 @@ +import { execFileSync } from "node:child_process"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it.each([ + ["native main", "require.main.require('node:path');", 1], + ["captured main", "const main=require.main;main.require('node:path');", 1], + ["native process main", "process.mainModule.require('node:path');", 1], + [ + "captured process main", + "const main=process.mainModule;main.require('node:path');", + 1, + ], + [ + "captured process delegate", + "const main=process.mainModule;const load=main.require;load.call(main,'node:path');", + 1, + ], + [ + "shared deep child local write", + "const old={child:{inner:{make:()=>module}}};const box={...old};old.child={inner:box.child.inner};old.child.inner.make=()=>({require(){}});box.child.inner.make().require('node:path');", + 0, + ], + [ + "shared deep child host write", + "const old={child:{inner:{make:()=>({})}}};const box={...old};old.child={inner:box.child.inner};old.child.inner.make=()=>module;box.child.inner.make().require('node:path');", + 1, + ], +] as const)( + "observes real CommonJS Module loading for %s", + async (_name, source, loads) => { + const root = await createTestTempDirectory("rea-main-module-source-"); + const entry = join(root, "entry.cjs"); + await writeFile( + entry, + `if(process.mainModule!==module||require.main!==module)throw Error('Native main-module identity changed');const original=module.require;let loads=0;module.require=function(...args){loads++;return Reflect.apply(original,this,args)};const out=eval(${JSON.stringify(source)});process.stdout.write(JSON.stringify({loads,native:typeof out?.join==='function'}));`, + ); + const observed = execFileSync(process.execPath, [entry], { + encoding: "utf8", + timeout: 5_000, + }); + expect(JSON.parse(observed) as unknown).toEqual({ + loads, + native: loads === 1, + }); + }, +); diff --git a/tests/boundary/javascript/packageConditionalExports.test.ts b/tests/boundary/javascript/packageConditionalExports.test.ts index 64b762072b..e7512b847c 100644 --- a/tests/boundary/javascript/packageConditionalExports.test.ts +++ b/tests/boundary/javascript/packageConditionalExports.test.ts @@ -66,6 +66,7 @@ const compareWithNode = async ( readonly specifier?: string; readonly main?: string; readonly metadataText?: string; + readonly metadataAbsent?: boolean; readonly sourceWithinPackage?: string; readonly packageFiles?: Readonly>; } = {}, @@ -91,11 +92,11 @@ const compareWithNode = async ( exports: exportsMap, main: options.main, }); - await writeFile(join(packageDirectory, "package.json"), metadata); - const contents = new Map([ - [sourcePath, ""], - [`node_modules/${packageName}/package.json`, metadata], - ]); + const contents = new Map([[sourcePath, ""]]); + if (!options.metadataAbsent) { + await writeFile(join(packageDirectory, "package.json"), metadata); + contents.set(`node_modules/${packageName}/package.json`, metadata); + } if (options.siblingFile === true) { const filename = `${packageName}.js`; const source = `module.exports = ${JSON.stringify(filename)};`; @@ -320,13 +321,23 @@ describe("package exports before sibling files", () => { }, ); - it("does not let another container's exports override a local sibling file", async () => { - const { files } = await compareWithNode( - "./default.cjs", - "require", - TARGETS, - { siblingFile: true }, - ); + it("keeps an unmapped package container unknown despite a local sibling file", async () => { + const { + node, + node_error, + outcome: mapped, + files, + } = await compareWithNode("./default.cjs", "require", TARGETS, { + siblingFile: true, + }); + expect(node).toBe("default.cjs"); + expect(node_error).toBeNull(); + expect(mapped).toMatchObject({ + resolved_path: "node_modules/example/default.cjs", + resolution_status: "resolved", + }); + // The Node control has one runtime directory. A different inventoried + // container without its loader mapping cannot prove the metadata absent. const outcome = resolveArtifactPathByContext({ declaredPath: "example", sourcePath: "main.js", @@ -342,10 +353,37 @@ describe("package exports before sibling files", () => { ), }); expect(outcome).toMatchObject({ - resolved_path: "node_modules/example.js", - resolution_status: "resolved", + resolved_path: null, + resolution_status: "unavailable", + limitations: [ + expect.stringContaining( + "without an established runtime loader mapping", + ), + ], }); }); + + it.each([undefined, "main.js"])( + "uses the real sibling when metadata is absent with package-local source %s", + async (sourceWithinPackage) => { + const { node, node_error, outcome } = await compareWithNode( + "./default.cjs", + "require", + TARGETS, + { + siblingFile: true, + metadataAbsent: true, + ...(sourceWithinPackage === undefined ? {} : { sourceWithinPackage }), + }, + ); + expect(node).toBe("example.js"); + expect(node_error).toBeNull(); + expect(outcome).toMatchObject({ + resolved_path: "node_modules/example.js", + resolution_status: "resolved", + }); + }, + ); }); describe("scalar package exports metadata", () => { it.each([false, true, 0, 42])( @@ -564,13 +602,22 @@ describe("package self-reference scope boundaries", () => { }); }); - it("does not apply self exports from another container", async () => { - const { files } = await compareWithNode( - "./specific.cjs", - "require", - TARGETS, - { sourceWithinPackage: "main.js", siblingFile: true }, - ); + it("keeps unmapped self metadata unknown rather than selecting a sibling", async () => { + const { + node, + node_error, + outcome: mapped, + files, + } = await compareWithNode("./specific.cjs", "require", TARGETS, { + sourceWithinPackage: "main.js", + siblingFile: true, + }); + expect(node).toBe("specific.cjs"); + expect(node_error).toBeNull(); + expect(mapped).toMatchObject({ + resolved_path: "node_modules/example/specific.cjs", + resolution_status: "resolved", + }); const outcome = resolveArtifactPathByContext({ declaredPath: "example", sourcePath: "node_modules/example/main.js", @@ -586,8 +633,13 @@ describe("package self-reference scope boundaries", () => { ), }); expect(outcome).toMatchObject({ - resolved_path: "node_modules/example.js", - resolution_status: "resolved", + resolved_path: null, + resolution_status: "unavailable", + limitations: [ + expect.stringContaining( + "without an established runtime loader mapping", + ), + ], }); }); }); diff --git a/tests/boundary/javascript/packageImportAliases.test.ts b/tests/boundary/javascript/packageImportAliases.test.ts index 64da6c235b..3743c4f3ec 100644 --- a/tests/boundary/javascript/packageImportAliases.test.ts +++ b/tests/boundary/javascript/packageImportAliases.test.ts @@ -1,7 +1,6 @@ import { execFile } from "node:child_process"; -import { mkdir, writeFile } from "node:fs/promises"; import { createRequire } from "node:module"; -import { dirname, join, relative } from "node:path"; +import { join, relative } from "node:path"; import { promisify } from "node:util"; import { fileURLToPath } from "node:url"; @@ -14,6 +13,7 @@ import { } from "../../../src/application/javascript/JavaScriptArtifactPathResolution.js"; import type { JavaScriptArtifactFile } from "../../../src/domain/javascript/javascriptArtifactFiles.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { writeFixtureFiles } from "../../support/workspace/writeFixtureFiles.js"; const execute = promisify(execFile); const nodeOutcomeSchema = z.strictObject({ @@ -47,16 +47,8 @@ const writeTree = async ( root: string, files: Readonly>, ): Promise> => { - const contents = new Map(); - await Promise.all( - Object.entries(files).map(async ([path, value]) => { - const absolute = join(root, path); - await mkdir(dirname(absolute), { recursive: true }); - await writeFile(absolute, value); - contents.set(path, value); - }), - ); - return contents; + await writeFixtureFiles(root, files); + return new Map(Object.entries(files)); }; const nodeRequireTarget = async ( @@ -445,7 +437,7 @@ describe("package import target arrays compared with Node", () => { moduleKind, ); expect(result).toMatchObject({ - resolution_status: "external", + resolution_status: "unavailable", resolved_path: null, }); expect(result.limitations.join(" ")).toContain( @@ -458,7 +450,9 @@ describe("package import target arrays compared with Node", () => { describe("package import target URL normalization compared with Node", () => { it("normalizes targets before package-root validation and array selection", async () => { const root = await createTestTempDirectory("rea-package-target-url-"); - const directory = "app space#?%λ"; + // Windows forbids '?' in filenames; the suffix target below covers URL queries. + const directory = + process.platform === "win32" ? "app space#%λ" : "app space#?%λ"; const imports = { "#tab-escape": ["./.\t./outside.cjs", "./fallback.cjs"], "#newline-escape": ["./.%\n2e/outside.cjs", "./fallback.cjs"], @@ -710,9 +704,6 @@ describe("package import alias refusals compared with Node", () => { imports: { "#config": "./config.cjs", "#blocked": null, - "#": "./config.cjs", - "#/": "./config.cjs", - "#config/": "./config.cjs", }, }), "main.cjs": "module.exports = 1;", diff --git a/tests/boundary/javascript/packageMainHostPaths.test.ts b/tests/boundary/javascript/packageMainHostPaths.test.ts new file mode 100644 index 0000000000..5f5577e4a9 --- /dev/null +++ b/tests/boundary/javascript/packageMainHostPaths.test.ts @@ -0,0 +1,71 @@ +import { createRequire } from "node:module"; +import { join } from "node:path"; +import { expect, it } from "vitest"; + +import { resolveArtifactPathByContext } from "../../../src/application/javascript/JavaScriptArtifactPathResolution.js"; +import type { JavaScriptArtifactFile } from "../../../src/domain/javascript/javascriptArtifactFiles.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { writeFixtureFiles } from "../../support/workspace/writeFixtureFiles.js"; + +it.each(["absolute-host", "relative-package"] as const)( + "compares %s package main with the native Node resolver", + async (kind) => { + const root = await createTestTempDirectory("rea-package-main-host-"); + const artifact = join(root, "artifact"); + const host = join(root, "host/window.cjs"); + const main = kind === "absolute-host" ? host : "entry.cjs"; + const contents = { + "app/main.cjs": "throw new Error('Analyzed input must never execute');", + "app/node_modules/fixture/package.json": JSON.stringify({ main }), + "app/node_modules/fixture/entry.cjs": + "throw new Error('Package input must never execute');", + "app/node_modules/fixture/index.js": + "throw new Error('Index decoy must never execute');", + "shared/window.cjs": + "throw new Error('Artifact-root decoy must never execute');", + }; + await writeFixtureFiles(root, { + "host/window.cjs": "throw new Error('Host input must never execute');", + ...Object.fromEntries( + Object.entries(contents).map(([path, value]) => [ + `artifact/${path}`, + value, + ]), + ), + }); + const loader = createRequire(join(artifact, "app/main.cjs")); + expect(loader.resolve("fixture")).toBe( + kind === "absolute-host" + ? host + : join(artifact, "app/node_modules/fixture/entry.cjs"), + ); + const files = new Map( + Object.entries(contents).map(([path, value]) => [ + path, + { + path, + container_sha256: "a".repeat(64), + sha256: "b".repeat(64), + bytes: Buffer.byteLength(value), + inventory_artifact_id: `fixture-${path}`, + kind: path.endsWith("package.json") ? "package-json" : "javascript", + unpacked: false, + text: { included: true, value }, + }, + ]), + ); + expect( + resolveArtifactPathByContext({ + declaredPath: "fixture", + sourcePath: "app/main.cjs", + context: "module-specifier", + moduleKind: "require", + files, + }), + ).toMatchObject({ + resolution_status: kind === "absolute-host" ? "unavailable" : "resolved", + resolved_path: + kind === "absolute-host" ? null : "app/node_modules/fixture/entry.cjs", + }); + }, +); diff --git a/tests/boundary/process/processCaptureLifecycle.test.ts b/tests/boundary/process/processCaptureLifecycle.test.ts index 8775431b00..58e3c39ee8 100644 --- a/tests/boundary/process/processCaptureLifecycle.test.ts +++ b/tests/boundary/process/processCaptureLifecycle.test.ts @@ -11,7 +11,7 @@ import { import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; -import { expect, vi } from "vitest"; +import { expect, onTestFinished, vi } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { itWithCaptureCapability } from "./processCaptureCapability.js"; @@ -181,6 +181,16 @@ itWithCaptureCapability( const controller = new AbortController(); let fallback: ReturnType | undefined; let reachedRunning = false; + let fallbackFired = false; + const clearFallback = () => { + clearTimeout(fallback); + fallback = undefined; + }; + onTestFinished(() => { + controller.abort(); + clearFallback(); + now.mockRestore(); + }); try { const result = await captureProcessScenario( parseProcessScenario({ @@ -200,20 +210,30 @@ itWithCaptureCapability( if (update.phase === "running" && !reachedRunning) { reachedRunning = true; wallNow -= 60_000; - fallback = setTimeout(() => controller.abort(), 1_500); + // The deadline starts after native ownership preparation and + // executable inspection. Bound only the same running interval. + fallback = setTimeout(() => { + fallbackFired = true; + controller.abort(); + }, 1_500); } else if ( update.phase === "settling" || update.phase === "cleanup" ) - clearTimeout(fallback); + clearFallback(); }, }, ); expect(reachedRunning).toBe(true); + expect( + fallbackFired, + "the monotonic deadline precedes the caller safety abort", + ).toBe(false); const { capture } = captureObservations(result); expect(capture.exit.reason).toBe("timeout"); } finally { - clearTimeout(fallback); + controller.abort(); + clearFallback(); now.mockRestore(); } }, diff --git a/tests/fixtures/javascriptAliasSelectionBudget.ts b/tests/fixtures/javascriptAliasSelectionBudget.ts new file mode 100644 index 0000000000..16289b6875 --- /dev/null +++ b/tests/fixtures/javascriptAliasSelectionBudget.ts @@ -0,0 +1,10 @@ +/** Branching inert selectors exceed precise alias work without target execution. */ +export const aliasSelectionBudgetSource = (): string => { + const source = ["const a0 = {};"]; + for (let index = 1; index <= 30; index++) + source.push( + `const a${index} = flag ? a${index - 1}?.x : a${index - 1}?.y;`, + ); + source.push('a30?.require("./shared.cjs");'); + return source.join("\n"); +}; diff --git a/tests/fixtures/javascriptCapturedGenericInvocations.ts b/tests/fixtures/javascriptCapturedGenericInvocations.ts new file mode 100644 index 0000000000..45994226e8 --- /dev/null +++ b/tests/fixtures/javascriptCapturedGenericInvocations.ts @@ -0,0 +1,26 @@ +const callChains = [ + ["call.call", "other,null"], + ["apply.call", "other,null,[]"], + ["call.apply", "other,[null]"], + ["apply.apply", "other,[null,[]]"], + ["call.call.call", "other,null"], +] as const; + +/** Authored fixtures only; runtime execution is confined to their unit oracle. */ +export const capturedGenericInvocations = (["call", "apply"] as const).flatMap( + (captured) => + callChains.flatMap(([chain, arguments_]) => + [true, false].map((host) => { + const local = "({require(){return 'local';}})"; + const expression = `invoke.${chain}(${arguments_}).require('../shared/window.cjs')`; + return { + name: `captured ${captured} via ${chain} to ${host ? "host" : "local"}`, + captured, + chain, + host, + expression, + source: `const get=()=>${host ? local : "module"};const invoke=get.${captured};const other=()=>${host ? "module" : local};${expression};`, + }; + }), + ), +); diff --git a/tests/process-global/filesystemSnapshotFallbackCancellation.test.ts b/tests/process-global/filesystemSnapshotFallbackCancellation.test.ts index 8f13b7af2a..3370fa4d42 100644 --- a/tests/process-global/filesystemSnapshotFallbackCancellation.test.ts +++ b/tests/process-global/filesystemSnapshotFallbackCancellation.test.ts @@ -101,7 +101,12 @@ it.each(["hash", "snapshot"] as const)( }), controller.signal, ); - const failure = await result.catch((cause: unknown) => cause); + const failure: unknown = await result.catch((cause: unknown) => cause); + expect(failure).toMatchObject({ + _tag: "ProcessCaptureError", + reason: "cancelled", + userCategory: "cancelled", + }); if (operation === "hash") { expect(failure).toBeInstanceOf(ProcessCaptureReadFailure); const read = failure as ProcessCaptureReadFailure; diff --git a/tests/process-global/filesystemSnapshotRaces.test.ts b/tests/process-global/filesystemSnapshotRaces.test.ts index b684cf371d..bc216dd2e8 100644 --- a/tests/process-global/filesystemSnapshotRaces.test.ts +++ b/tests/process-global/filesystemSnapshotRaces.test.ts @@ -34,9 +34,7 @@ vi.mock("node:fs/promises", async (importOriginal) => { return original.readlink(...args); } if (path === race.target && race.mode === "readlink_eio") - throw Object.assign(new Error("injected unexpected readlink failure"), { - code: "EIO", - }); + throw race.reason; if (path === race.target && race.mode === "readlink_eacces") throw Object.assign(new Error("injected denied readlink"), { code: "EACCES", @@ -407,11 +405,17 @@ it.skipIf(process.platform === "win32")( }); race.mode = "readlink_eio"; race.target = link; + const reason = Object.assign( + new Error("injected unexpected readlink failure"), + { code: "EIO" }, + ); + race.reason = reason; const cause = await snapshotRoots(scenario).catch( (failure: unknown) => failure, ); expect(cause).toBeInstanceOf(ProcessCaptureSnapshotFailure); const failure = cause as ProcessCaptureSnapshotFailure; + expect(failure.cause).toBe(reason); expect(failure.cause).toMatchObject({ code: "EIO" }); expect(failure.userCategory).toBeUndefined(); expect(failure.snapshot).toMatchObject({ diff --git a/tests/process-global/javascriptRuntimeObservationCli.test.ts b/tests/process-global/javascriptRuntimeObservationCli.test.ts index 531d6e1c90..0490195e80 100644 --- a/tests/process-global/javascriptRuntimeObservationCli.test.ts +++ b/tests/process-global/javascriptRuntimeObservationCli.test.ts @@ -157,7 +157,7 @@ describe("JavaScript runtime observation CLI parity", () => { { exit: () => {}, stdout: (chunk) => (stdout += chunk) }, ); - expect(closeAttempts).toBe(1); + expect(closeAttempts, stdout).toBe(1); expect(JSON.parse(stdout)).toMatchObject({ code: "cleanup_incomplete", details: { diff --git a/tests/process-global/processSnapshotCleanupMcp.test.ts b/tests/process-global/processSnapshotCleanupMcp.test.ts index c277bdb381..7d5ef6ce8f 100644 --- a/tests/process-global/processSnapshotCleanupMcp.test.ts +++ b/tests/process-global/processSnapshotCleanupMcp.test.ts @@ -1,5 +1,4 @@ -import { mkdir, mkdtemp, open, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; +import { mkdir, open, rm, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; @@ -32,6 +31,7 @@ vi.mock("node:fs/promises", async (importOriginal) => { }; }); +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { createTestBinarySession } from "../fixtures/binarySession.js"; import { parseMcpToolError } from "../fixtures/mcpToolError.js"; import type { AnalysisClient } from "../../src/application/AnalysisProvider.js"; @@ -62,7 +62,7 @@ const connectSession = async () => { }; it("retains and retries the exact snapshot file handle through public MCP capture", async () => { - const root = await mkdtemp(join(tmpdir(), "rea-process-snapshot-cleanup-")); + const root = await createTestTempDirectory("rea-process-snapshot-cleanup-"); const observedRoot = join(root, "observed"); const filePath = join(observedRoot, "input.txt"); await mkdir(observedRoot); @@ -87,7 +87,7 @@ it("retains and retries the exact snapshot file handle through public MCP captur try { const failed = parseMcpToolError(await capture()); - expect(injected.opens).toBe(1); + expect(injected.opens, JSON.stringify(failed)).toBe(1); expect(injected.handle?.fd).toBeGreaterThanOrEqual(0); expect(failed.error).toMatchObject({ code: "cleanup_incomplete", @@ -120,7 +120,7 @@ it("retains and retries the exact snapshot file handle through public MCP captur expect(retryBlocked.error.details).toMatchObject({ resources: [injected.path], }); - expect(injected.opens).toBe(1); + expect(injected.opens, JSON.stringify(retryBlocked)).toBe(1); expect(injected.handle?.fd).toBeGreaterThanOrEqual(0); injected.failClose = false; diff --git a/tests/support/cli/javascriptApplication.ts b/tests/support/cli/javascriptApplication.ts index 0ec600e172..59bb317f2d 100644 --- a/tests/support/cli/javascriptApplication.ts +++ b/tests/support/cli/javascriptApplication.ts @@ -26,12 +26,26 @@ export const analyzeJavaScriptDirectory = async ( surface: "CLI" | "stdio MCP", cli: TestCli, root: string, - mcpCall?: Awaited>["call"], + { + sourcePaths = [], + mcpCall, + }: { + readonly sourcePaths?: readonly string[]; + readonly mcpCall?: + | Awaited>["call"] + | undefined; + } = {}, ) => { let document: unknown; if (surface === "CLI") { const response = await cli.run({ - arguments: ["analyze-javascript-application", root, "--format", "json"], + arguments: [ + "analyze-javascript-application", + root, + ...sourcePaths.flatMap((path) => ["--source-path", path]), + "--format", + "json", + ], environment: { REA_LOG_LEVEL: "silent" }, }); expect(response.exitCode, response.stderr).toBe(0); @@ -41,6 +55,7 @@ export const analyzeJavaScriptDirectory = async ( const response = await call("analyze_javascript_application", { input_path: root, format: "directory", + ...(sourcePaths.length === 0 ? {} : { source_paths: sourcePaths }), }); expect(response.isError, JSON.stringify(response)).not.toBe(true); document = response.structuredContent; diff --git a/tests/support/javascriptApplicationFixture.ts b/tests/support/javascriptApplicationFixture.ts index 45afb3a82f..b8f4704c08 100644 --- a/tests/support/javascriptApplicationFixture.ts +++ b/tests/support/javascriptApplicationFixture.ts @@ -1,6 +1,3 @@ -import { mkdir, writeFile } from "node:fs/promises"; -import { dirname, join } from "node:path"; - import { reconstructJavaScriptArtifact } from "./javascriptApplicationScope.js"; import type { ApplicationEdge, @@ -8,19 +5,7 @@ import type { } from "../../src/domain/javascript/javascriptApplicationGraphSchemas.js"; import type { JavaScriptApplicationGraph } from "../../src/domain/javascript/javascriptApplicationGraph.js"; -/** Write text fixture files, creating parent directories as needed. */ -export const writeFixtureFiles = async ( - root: string, - files: Readonly>, -): Promise => { - await Promise.all( - Object.entries(files).map(async ([relative, contents]) => { - const path = join(root, relative); - await mkdir(dirname(path), { recursive: true }); - await writeFile(path, contents); - }), - ); -}; +export { writeFixtureFiles } from "./workspace/writeFixtureFiles.js"; /** Reconstruct one JavaScript artifact fixture directory. */ export const reconstructJavaScriptFixture = ( diff --git a/tests/support/workspace/writeFixtureFiles.ts b/tests/support/workspace/writeFixtureFiles.ts new file mode 100644 index 0000000000..4d0c3715c8 --- /dev/null +++ b/tests/support/workspace/writeFixtureFiles.ts @@ -0,0 +1,20 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; + +/** Write text fixture files, creating parent directories as needed. */ +export const writeFixtureFiles = async ( + root: string, + files: Readonly>, +): Promise => { + const entries = Object.entries(files).map(([relative, contents]) => ({ + path: join(root, relative), + contents, + })); + // Concurrent recursive mkdir calls can race for shared ancestors on Windows. + for (const directory of new Set(entries.map(({ path }) => dirname(path)))) { + await mkdir(directory, { recursive: true }); + } + await Promise.all( + entries.map(({ path, contents }) => writeFile(path, contents)), + ); +}; From afde435c94e90b519b85aefeec4ec587e7a477ae Mon Sep 17 00:00:00 2001 From: ShuaixinHuang <1836656514@qq.com> Date: Tue, 20 Oct 2026 17:25:38 +0800 Subject: [PATCH 08/13] test(setup): use native Windows launcher in layered OpenCode fixture (#4947) --- .../setup/registrationRuntimeEnvironmentCli.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts b/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts index 9617e19b8b..7af5a60a7e 100644 --- a/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts +++ b/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts @@ -318,7 +318,13 @@ cliTest( mcp: { rea: clientRegistrationEntry( "opencode", - [resolve("scripts/rea.mjs"), "mcp"], + // Windows launches JavaScript through Node; otherwise setup repairs + // the command before this test can exercise layered environment data. + [ + ...(process.platform === "win32" ? [process.execPath] : []), + resolve("scripts/rea.mjs"), + "mcp", + ], env, ), }, From e84468c0bb0476412ac0c8161f7c2ecc9a6864ca Mon Sep 17 00:00:00 2001 From: ShuaixinHuang <1836656514@qq.com> Date: Tue, 20 Oct 2026 17:25:51 +0800 Subject: [PATCH 09/13] fix(setup): preserve Windows environment override identity (#4946) * fix(setup): preserve Windows environment override identity * fix(setup): preserve selected platform and YAML alias metadata * fix(setup): reconcile layered environment and projected YAML identities --- src/application/ClientConfigurationFiles.ts | 57 ++++- .../ClientRegistrationEnvironment.ts | 31 ++- src/application/ClientRegistrationStatus.ts | 9 +- src/application/DshCordisPatch.ts | 49 +++- src/application/DshCordisYaml.ts | 39 ++- src/application/SetupClientConfiguration.ts | 4 +- src/process/snapshotEnvironment.test.ts | 29 ++- src/process/snapshotEnvironment.ts | 41 ++- .../registrationRuntimeEnvironmentCli.test.ts | 64 +++++ .../clientConfigurationEnvironment.test.ts | 237 ++++++++++++++++++ ...entConfigurationLayeredEnvironment.test.ts | 46 ++++ 11 files changed, 562 insertions(+), 44 deletions(-) diff --git a/src/application/ClientConfigurationFiles.ts b/src/application/ClientConfigurationFiles.ts index 4781e1490d..35e02bcd9b 100644 --- a/src/application/ClientConfigurationFiles.ts +++ b/src/application/ClientConfigurationFiles.ts @@ -12,6 +12,8 @@ import type { SetupClient } from "./SupportedClients.js"; import { clientRegistrationEnvironment } from "./ClientRegistrationEnvironment.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import type { ConfigurationError } from "../domain/configurationErrors.js"; +import { mergeEnvironmentOverrides } from "../process/snapshotEnvironment.js"; +import { PRODUCT_IDENTITY } from "../identity.js"; /** A parsed layer retains its path so rereading the write target replaces only that layer. */ export interface ClientConfigurationSource extends ClientConfigurationDocument { @@ -88,6 +90,7 @@ export const readConfigurationFile = async ( export const readClientConfigurationFiles = async ( client: SetupClient, resources?: ArtifactResourceScope, + platform: NodeJS.Platform = process.platform, ): Promise< Result > => { @@ -104,6 +107,7 @@ export const readClientConfigurationFiles = async ( document.value, {}, "fragment", + platform, ); if (!environment.ok) return err(invalidEnvironment(path, environment.error)); @@ -111,7 +115,10 @@ export const readClientConfigurationFiles = async ( } } const environment = clientRegistrationEnvironment( - effectiveClientConfiguration(documents), + effectiveClientConfiguration(documents, undefined, platform), + {}, + "effective", + platform, ); if (!environment.ok) return err(invalidEnvironment(client.configPath, environment.error)); @@ -142,6 +149,7 @@ export const effectiveClientConfiguration = ( readonly client: Pick; readonly document: ClientConfigurationDocument | undefined; }, + platform: NodeJS.Platform = process.platform, ): ClientConfigurationDocument | undefined => { const documents = target === undefined @@ -164,11 +172,28 @@ export const effectiveClientConfiguration = ( ...Object.keys(document.legacyServers), ...Object.keys(document.servers), ]); - for (const name of names) - servers[name] = mergeConfigurationValue( - servers[name], - effectiveClientServer(document, name), - ); + for (const name of names) { + const previous = servers[name]; + const next = effectiveClientServer(document, name); + const merged = mergeConfigurationValue(previous, next); + const key = + document.dialect === "opencode" || document.dialect === "opencode_v2" + ? "environment" + : "env"; + if ( + name === PRODUCT_IDENTITY.mcpServerKey && + isObject(merged) && + isObject(merged[key]) + ) { + const before = + isObject(previous) && isObject(previous[key]) ? previous[key] : {}; + const after = isObject(next) && isObject(next[key]) ? next[key] : {}; + servers[name] = { + ...merged, + [key]: mergeEnvironmentOverrides(before, after, platform), + }; + } else servers[name] = merged; + } } return { ...last, servers, legacyServers: {} }; }; @@ -177,6 +202,7 @@ export const effectiveClientConfiguration = ( export const readEffectiveClientRegistrationConfiguration = async ( client: SetupClient, resources?: ArtifactResourceScope, + platform: NodeJS.Platform = process.platform, ): Promise< Result< { @@ -186,9 +212,17 @@ export const readEffectiveClientRegistrationConfiguration = async ( ClientConfigurationFileError > > => { - const nativeSources = await readClientConfigurationFiles(client, resources); + const nativeSources = await readClientConfigurationFiles( + client, + resources, + platform, + ); if (!nativeSources.ok) return nativeSources; - const nativeDocument = effectiveClientConfiguration(nativeSources.value); + const nativeDocument = effectiveClientConfiguration( + nativeSources.value, + undefined, + platform, + ); const fallbackPath = client.format === "zcode" ? client.zcodeFallbackPath : undefined; if ( @@ -207,9 +241,14 @@ export const readEffectiveClientRegistrationConfiguration = async ( const fallbackSources = await readClientConfigurationFiles( fallbackClient, resources, + platform, ); if (!fallbackSources.ok) return fallbackSources; - const fallbackDocument = effectiveClientConfiguration(fallbackSources.value); + const fallbackDocument = effectiveClientConfiguration( + fallbackSources.value, + undefined, + platform, + ); return ok( fallbackDocument === undefined ? { document: nativeDocument, configPath: client.configPath } diff --git a/src/application/ClientRegistrationEnvironment.ts b/src/application/ClientRegistrationEnvironment.ts index 7bbfc7aafb..979a867ff1 100644 --- a/src/application/ClientRegistrationEnvironment.ts +++ b/src/application/ClientRegistrationEnvironment.ts @@ -12,6 +12,10 @@ import { } from "./ClientConfigurationDocument.js"; import { PRODUCT_IDENTITY } from "../identity.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; +import { + mergeEnvironmentOverrides, + snapshotEnvironment, +} from "../process/snapshotEnvironment.js"; import { dshDocument } from "./DshCordisPatch.js"; import type { McpToolTimeoutInputs } from "../mcpStartupPolicy.js"; import type { AnalysisProviderSelector } from "../contracts/providerSelection.js"; @@ -31,13 +35,20 @@ export const clientRegistrationEnvironment = ( parsed: ClientConfigurationDocument | undefined, overrides: Readonly> = {}, validation: "fragment" | "effective" = "effective", + platform: NodeJS.Platform = process.platform, ): Result>, ConfigurationError> => { if (parsed?.dialect === "dsh") - return dshDocument(parsed.document).environment(overrides, validation); + return dshDocument(parsed.document).environment( + overrides, + validation, + platform, + ); const details = clientRegistrationEnvironmentDetails( parsed, overrides, validation, + undefined, + platform, ); return details.ok ? ok(details.value.values) : details; }; @@ -48,17 +59,20 @@ export const clientRegistrationEnvironmentDetails = ( overrides: Readonly> = {}, validation: "fragment" | "effective" = "effective", resolveProviderRequestBudget?: ProviderRequestBudgetResolver, + platform: NodeJS.Platform = process.platform, ): Result => { if (parsed?.dialect === "dsh") { const environment = dshDocument(parsed.document).environment( overrides, validation, + platform, ); if (!environment.ok) return environment; return validatedDetails( environment.value, validation, resolveProviderRequestBudget, + platform, ); } const key = @@ -88,11 +102,16 @@ export const clientRegistrationEnvironmentDetails = ( ); // Validate known settings with the runtime's parser, but retain the original // string overrides, including unknown client-specific keys and no defaults. - const environment = { ...existing.data, ...overrides }; + const environment = mergeEnvironmentOverrides( + existing.data, + overrides, + platform, + ); return validatedDetails( environment, validation, resolveProviderRequestBudget, + platform, ); }; @@ -100,16 +119,18 @@ const validatedDetails = ( environment: Readonly>, validation: "fragment" | "effective", resolveProviderRequestBudget?: ProviderRequestBudgetResolver, + platform: NodeJS.Platform = process.platform, ): Result => { + const runtimeEnvironment = snapshotEnvironment(environment, platform); const validated = validation === "fragment" - ? parseEnvironmentValues(environment) - : parseEnvironment(environment); + ? parseEnvironmentValues(runtimeEnvironment) + : parseEnvironment(runtimeEnvironment); if (!validated.ok) return validated; const providerRequestBudget = resolveProviderRequestBudget?.( validated.value.REA_ANALYSIS_PROVIDER, - environment, + runtimeEnvironment, ) ?? ok(undefined); if (!providerRequestBudget.ok) return providerRequestBudget; return ok({ diff --git a/src/application/ClientRegistrationStatus.ts b/src/application/ClientRegistrationStatus.ts index 68fc3c37a3..28643628cc 100644 --- a/src/application/ClientRegistrationStatus.ts +++ b/src/application/ClientRegistrationStatus.ts @@ -15,6 +15,7 @@ import { valid } from "semver"; import { RegularFileCleanupFailure } from "./RegularFileRead.js"; import { PRODUCT_IDENTITY } from "../identity.js"; +import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import { CURRENT_ENTRYPOINT_PATH } from "../process/CurrentEntrypointPath.js"; import { MCP_STARTUP_POLICY, @@ -185,6 +186,7 @@ export const readClientRegistrationInspections = async ( const configuration = await readEffectiveClientRegistrationConfiguration( client, options.resources, + options.platform, ); if (!configuration.ok) { if ( @@ -265,6 +267,7 @@ export const readClientRegistrationInspections = async ( {}, "effective", options.resolveProviderRequestBudget, + options.platform, ) : undefined; if ( @@ -297,8 +300,10 @@ export const readClientRegistrationInspections = async ( platform: options.platform ?? process.platform, kimiInputSchemaProfileSelected: registrationEnvironment?.ok === true && - registrationEnvironment.value.values.REA_MCP_INPUT_SCHEMA_PROFILE !== - undefined, + snapshotEnvironment( + registrationEnvironment.value.values, + options.platform, + ).REA_MCP_INPUT_SCHEMA_PROFILE !== undefined, }); record( configuredStatus( diff --git a/src/application/DshCordisPatch.ts b/src/application/DshCordisPatch.ts index a7d2458b87..630f7cce9f 100644 --- a/src/application/DshCordisPatch.ts +++ b/src/application/DshCordisPatch.ts @@ -7,6 +7,10 @@ import { ConfigurationError } from "../domain/configurationErrors.js"; import { err, ok, type Result } from "../domain/result.js"; import type { ClientConfigurationDocument } from "./ClientConfigurationDocument.js"; import { CordisExpression, DshCordisYaml } from "./DshCordisYaml.js"; +import { + mergeEnvironmentOverrides, + snapshotEnvironment, +} from "../process/snapshotEnvironment.js"; export const DSH_ENTRY_ID = "rea-mcp"; export const DSH_CLIENT_PACKAGE = "@deepseek-ai/dsh-mcp-client"; @@ -185,6 +189,7 @@ export class DshCordisDocument { environment( overrides: Readonly> = {}, validation: "fragment" | "effective" = "effective", + platform: NodeJS.Platform = process.platform, ): Result>, ConfigurationError> { const config = this.#field("config"); if (config instanceof CordisExpression) @@ -220,9 +225,11 @@ export class DshCordisDocument { ); const literals: Record = {}; const unresolvedSettings = new Set(); - for (const [key, value] of Object.entries(raw ?? {})) { + for (const [key, value] of Object.entries( + mergeEnvironmentOverrides(raw ?? {}, overrides, platform), + )) { if (value instanceof CordisExpression) { - if (overrides[key] === undefined) unresolvedSettings.add(key); + unresolvedSettings.add(key); continue; } if (typeof value !== "string") @@ -239,11 +246,20 @@ export class DshCordisDocument { ); literals[key] = value; } - const environment = { ...literals, ...overrides }; + const environment = literals; + const runtimeEnvironment = snapshotEnvironment(environment, platform); + const unresolved = new Set( + Object.keys( + snapshotEnvironment( + Object.fromEntries([...unresolvedSettings].map((key) => [key, ""])), + platform, + ), + ), + ); const validated = validation === "fragment" - ? parseEnvironmentValues(environment, unresolvedSettings) - : parseEnvironment(environment, unresolvedSettings); + ? parseEnvironmentValues(runtimeEnvironment, unresolved) + : parseEnvironment(runtimeEnvironment, unresolved); return validated.ok ? ok(environment) : validated; } @@ -255,10 +271,10 @@ export class DshCordisDocument { return err(this.#editabilityError); const environment = this.environment(overrides); if (!environment.ok) return environment; - const env = { - ...(isRecord(this.config?.env) ? this.config.env : {}), - ...overrides, - }; + const env = mergeEnvironmentOverrides( + isRecord(this.config?.env) ? this.config.env : {}, + overrides, + ); const desired = { ...this.config, ...dshClientRegistrationEntry(command) }; delete desired.env; if (Object.keys(env).length > 0) desired.env = env; @@ -373,6 +389,21 @@ export class DshCordisDocument { : this.#yaml.detach(env, invalidated); if (!isMap(nextEnv)) throw new ConfigurationError("Cordis environment is not a mapping"); + for (const pair of [...nextEnv.items]) { + const key = this.#yaml.key(pair.key); + if (Object.hasOwn(desiredEnv, key)) { + this.#yaml.renameKey(nextEnv, pair, key); + continue; + } + const alias = + process.platform === "win32" + ? Object.keys(desiredEnv).find( + (name) => name.toUpperCase() === key.toUpperCase(), + ) + : undefined; + if (alias === undefined) nextEnv.delete(pair.key); + else this.#yaml.renameKey(nextEnv, pair, alias); + } for (const [key, value] of Object.entries(desiredEnv)) { this.#yaml.setValue(nextEnv, key, value); } diff --git a/src/application/DshCordisYaml.ts b/src/application/DshCordisYaml.ts index ea84e5e65d..c7e88958f5 100644 --- a/src/application/DshCordisYaml.ts +++ b/src/application/DshCordisYaml.ts @@ -10,6 +10,7 @@ import { type Document, type Node, type YAMLMap, + type Pair, } from "yaml"; import { ConfigurationError } from "../domain/configurationErrors.js"; @@ -82,7 +83,7 @@ export class DshCordisYaml { else if (isMap(resolved)) { const entries: [string, unknown][] = []; for (const pair of resolved.items) { - entries.push([this.#key(pair.key), this.value(pair.value)]); + entries.push([this.key(pair.key), this.value(pair.value)]); } value = Object.fromEntries(entries); } @@ -91,7 +92,7 @@ export class DshCordisYaml { return value; } - #key(node: unknown): string { + key(node: unknown): string { const key = this.value(node); if (typeof key === "object" && key !== null) throw new ConfigurationError( @@ -101,13 +102,40 @@ export class DshCordisYaml { return key === null ? "" : String(key); } + /** A case-only rename retains source key/value comments on the chosen alias. */ + renameKey(map: YAMLMap, pair: Pair, key: string): void { + const existing = map.items.find( + (item) => item !== pair && this.key(item.key) === key, + ); + const replacement = existing ?? pair; + const carry = (source: unknown, target: unknown): void => { + if (!isNode(source) || !isNode(target) || source === target) return; + for (const name of ["comment", "commentBefore"] as const) { + const values = [target[name], source[name]].filter( + (value): value is string => value !== undefined, + ); + if (values.length > 0) target[name] = [...new Set(values)].join("\n"); + } + }; + if (existing !== undefined) { + carry(pair.key, existing.key); + carry(pair.value, existing.value); + map.items = map.items.filter((item) => item !== pair); + } else if (this.key(pair.key) !== key) { + const renamed = this.document.createNode(key); + carry(pair.key, renamed); + replacement.key = renamed; + } + } + field(map: YAMLMap | undefined, key: string): unknown { - return map?.items.findLast((pair) => this.#key(pair.key) === key)?.value; + return map?.items.findLast((pair) => this.key(pair.key) === key)?.value; } /** Preserve unchanged nodes and carry their comments when replacing values. */ setValue(map: YAMLMap, key: string, value: unknown): void { - const original = this.field(map, key); + const pair = map.items.findLast((item) => this.key(item.key) === key); + const original = pair?.value; if (isDeepStrictEqual(this.value(original), value)) return; const replacement = this.document.createNode(value); if (isNode(original) && isNode(replacement)) { @@ -116,7 +144,8 @@ export class DshCordisYaml { if (original.commentBefore !== undefined) replacement.commentBefore = original.commentBefore; } - map.set(key, replacement); + if (pair === undefined) map.set(key, replacement); + else pair.value = replacement; } /** Copy from the source before edits; materialized nodes own no anchors. */ diff --git a/src/application/SetupClientConfiguration.ts b/src/application/SetupClientConfiguration.ts index a2d7dfc769..e0a5a012d7 100644 --- a/src/application/SetupClientConfiguration.ts +++ b/src/application/SetupClientConfiguration.ts @@ -32,6 +32,7 @@ import { } from "./ClientRegistrationEnvironment.js"; import { PRODUCT_IDENTITY } from "../identity.js"; +import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import { npxRegistrationCommand } from "./ClientRegistrationIdentity.js"; import { MCP_STARTUP_POLICY, @@ -761,7 +762,8 @@ const clientConfigurationDesired = ( if (!environment.ok) throw environment.error; const registrationEnvironment = client.name === "kimi" && - environment.value.REA_MCP_INPUT_SCHEMA_PROFILE === undefined + snapshotEnvironment(environment.value).REA_MCP_INPUT_SCHEMA_PROFILE === + undefined ? { ...environment.value, REA_MCP_INPUT_SCHEMA_PROFILE: "compact", diff --git a/src/process/snapshotEnvironment.test.ts b/src/process/snapshotEnvironment.test.ts index 4012262993..d41f8fd8ab 100644 --- a/src/process/snapshotEnvironment.test.ts +++ b/src/process/snapshotEnvironment.test.ts @@ -1,6 +1,33 @@ import { expect, it } from "vitest"; -import { snapshotEnvironment } from "./snapshotEnvironment.js"; +import { + mergeEnvironmentOverrides, + snapshotEnvironment, +} from "./snapshotEnvironment.js"; + +it.each(["win32", "linux", "darwin"] as const)( + "merges explicit environment layers using %s key identity before sorting", + (platform) => { + const existing = { + rea_analysis_provider: "hopper", + CallerSetting: "retain", + }; + const overrides = { REA_ANALYSIS_PROVIDER: "ghidra", Path: "chosen" }; + const merged = mergeEnvironmentOverrides(existing, overrides, platform); + expect(merged).toEqual( + platform === "win32" + ? { + REA_ANALYSIS_PROVIDER: "ghidra", + CallerSetting: "retain", + Path: "chosen", + } + : { ...existing, ...overrides }, + ); + expect(Object.isFrozen(merged)).toBe(true); + expect(existing.rea_analysis_provider).toBe("hopper"); + expect(overrides.REA_ANALYSIS_PROVIDER).toBe("ghidra"); + }, +); it.each(["linux", "darwin"] as const)( "preserves case-sensitive %s variables in an immutable snapshot", diff --git a/src/process/snapshotEnvironment.ts b/src/process/snapshotEnvironment.ts index bfb57cd7d6..654328fff7 100644 --- a/src/process/snapshotEnvironment.ts +++ b/src/process/snapshotEnvironment.ts @@ -1,16 +1,33 @@ -/** Snapshot selected variables while preserving platform and caller precedence. */ -export const snapshotEnvironment = ( - environment: Readonly, +/** Merge explicit layers by platform identity while retaining the winning spelling/value. */ +export const mergeEnvironmentOverrides = ( + existing: Readonly>, + overrides: Readonly>, platform: NodeJS.Platform = process.platform, -): Readonly => { - if (platform !== "win32") return Object.freeze({ ...environment }); - const selected = new Map(); +): Readonly> => { + const selected = new Map(); + for (const layer of [existing, overrides]) + for (const [key, value] of Object.entries(layer)) + selected.set(platform === "win32" ? key.toUpperCase() : key, [ + key, + value, + ]); + return Object.freeze(Object.fromEntries(selected.values())); +}; + +/** Snapshot selected variables while preserving platform and caller precedence. */ +export const snapshotEnvironment = ( + environment: Readonly>, + platform: NodeJS.Platform = process.platform, +): Readonly> => { + const selected = mergeEnvironmentOverrides({}, environment, platform); + if (platform !== "win32") return selected; // Fold aliases in caller order so an explicit later override wins before // this canonical environment crosses a Windows process boundary. - for (const [key, value] of Object.entries(environment)) - selected.set(key.toUpperCase(), value); - const snapshot: NodeJS.ProcessEnv = {}; - for (const key of [...selected.keys()].sort()) - snapshot[key] = selected.get(key); - return Object.freeze(snapshot); + return Object.freeze( + Object.fromEntries( + Object.entries(selected) + .map(([key, value]) => [key.toUpperCase(), value] as const) + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)), + ), + ); }; diff --git a/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts b/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts index 7af5a60a7e..9971626460 100644 --- a/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts +++ b/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts @@ -19,6 +19,70 @@ const missingHopperAndGhidraEnvironment = (home: string) => ({ JAVA_HOME: join(home, "missing-java"), }); +cliTest.skipIf(process.platform !== "win32")( + "rejects lowercase Windows provider settings in doctor and setup without writing", + async ({ cli }) => { + const home = await createTestTempDirectory("rea-registration-case-"); + const config = join(home, ".cursor/mcp.json"); + await mkdir(dirname(config), { recursive: true }); + const original = JSON.stringify({ + mcpServers: { + rea: clientRegistrationEntry( + "json", + [process.execPath, resolve("scripts/rea.mjs"), "mcp"], + { rea_analysis_provider: "invalid provider" }, + ), + }, + }); + await writeFile(config, original); + const environment = { + HOME: home, + USERPROFILE: home, + XDG_CONFIG_HOME: join(home, ".config"), + ...missingHopperAndGhidraEnvironment(home), + }; + const doctor = await cli.run({ + arguments: ["doctor", "--client", "cursor", "--skill=false", "--json"], + cwd: home, + environment, + }); + expect(doctor.exitCode).toBe(1); + expect(doctor.json).toMatchObject({ + scope_checks: expect.arrayContaining([ + expect.objectContaining({ + name: "registration:cursor", + ok: false, + classification: "config_drift", + remediation: expect.stringContaining("REA_ANALYSIS_PROVIDER"), + }), + ]), + }); + const setup = await cli.run({ + arguments: [ + "setup", + "--client", + "cursor", + "--skill=false", + "--yes", + "--json", + ], + cwd: home, + environment, + }); + expect(setup.exitCode).toBe(1); + expect(setup.json).toMatchObject({ + status: "needs_human", + plannedActions: [], + appliedActions: [], + remediation: expect.stringContaining("REA_ANALYSIS_PROVIDER"), + }); + expect(await readFile(config, "utf8")).toBe(original); + await expect(access(`${config}.rea.backup`)).rejects.toMatchObject({ + code: "ENOENT", + }); + }, +); + const scenarios = [ { client: "cursor", diff --git a/tests/boundary/filesystem/clientConfigurationEnvironment.test.ts b/tests/boundary/filesystem/clientConfigurationEnvironment.test.ts index 10abf90325..bfcfe5c9d5 100644 --- a/tests/boundary/filesystem/clientConfigurationEnvironment.test.ts +++ b/tests/boundary/filesystem/clientConfigurationEnvironment.test.ts @@ -1,4 +1,5 @@ import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { spawnSync } from "node:child_process"; import { join } from "node:path"; import { expect, it } from "vitest"; @@ -15,6 +16,242 @@ import { } from "../../../src/application/SetupClientConfiguration.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; +import { clientRegistrationEnvironmentDetails } from "../../../src/application/ClientRegistrationEnvironment.js"; +import { snapshotEnvironment } from "../../../src/process/snapshotEnvironment.js"; +import { npxRegistrationCommand } from "../../../src/application/ClientRegistrationIdentity.js"; + +it.each(["win32", "linux"] as const)( + "uses the selected %s identity when inspecting lowercase registration settings", + async (platform) => { + const home = await createTestTempDirectory("rea-selected-platform-env-"); + const path = join(home, ".cursor/mcp.json"); + await mkdir(join(home, ".cursor")); + await writeFile( + path, + JSON.stringify({ + mcpServers: { + rea: clientRegistrationEntry( + "json", + npxRegistrationCommand(platform), + { rea_analysis_provider: "invalid provider" }, + ), + }, + }), + ); + const statuses = await readClientRegistrationStatuses(home, "rea", { + platform, + environment: {}, + }); + expect(statuses.find(({ client }) => client === "cursor")).toMatchObject({ + state: platform === "win32" ? "invalid" : "aligned", + }); + }, +); + +it.skipIf(process.platform !== "win32").each([ + { + key: "rea_analysis_provider", + override: "REA_ANALYSIS_PROVIDER", + value: "ghidra", + duplicate: false, + }, + { key: "true", override: "TRUE", value: "chosen", duplicate: false }, + { key: '"true"', override: "true", value: "chosen", duplicate: true }, + { + key: "rea_analysis_provider", + override: "REA_ANALYSIS_PROVIDER", + value: "ghidra", + duplicate: true, + }, +])( + "preserves DSH scalar-key identity and comments when replacing $key", + async ({ key, override, value, duplicate }) => { + const root = await createTestTempDirectory("rea-dsh-alias-comments-"); + const configPath = join(root, "cordis.patch.yml"); + const source = `- insert:\n - id: rea-mcp\n name: "@deepseek-ai/dsh-mcp-client"\n config:\n serverName: rea\n transport: stdio\n command: old-rea\n args: [mcp]\n env:\n # retain preceding rationale\n ${key}: "hopper" # retain inline rationale\n${duplicate ? ` ${override}: "ghidra" # retain winning rationale\n` : ""} CUSTOM: !!js caller.custom\n`; + await writeFile(configPath, source); + const client = { name: "dsh", format: "dsh", configPath } as const; + const result = await configureClientConfiguration( + client, + { [override]: value }, + ["rea", "mcp"], + ); + expect(result, JSON.stringify(result)).toMatchObject({ + status: "configured", + }); + const text = await readFile(configPath, "utf8"); + expect(text).toContain("# retain preceding rationale"); + expect(text).toContain("# retain inline rationale"); + if (duplicate) expect(text).toContain("# retain winning rationale"); + expect(text).toContain("!!js caller.custom"); + const env = clientRegistrationEnvironmentDetails( + parseClientConfiguration(text, "dsh"), + ); + expect(env.ok).toBe(true); + if (!env.ok) throw env.error; + expect(env.value.values).toHaveProperty(override, value); + expect( + Object.keys(env.value.values).filter( + (name) => name.toUpperCase() === override.toUpperCase(), + ), + ).toEqual([override]); + expect(await readFile(`${configPath}.rea.backup`, "utf8")).toBe(source); + expect( + await configureClientConfiguration(client, { [override]: value }, [ + "rea", + "mcp", + ]), + ).toEqual({ status: "unchanged" }); + }, +); + +it.skipIf(process.platform !== "win32")( + "preserves a lowercase explicit Kimi schema profile instead of inserting a conflicting default", + async () => { + const home = await createTestTempDirectory("rea-kimi-case-"); + await mkdir(join(home, ".kimi-code")); + const configPath = join(home, ".kimi-code/mcp.json"); + await writeFile( + configPath, + JSON.stringify({ + mcpServers: { + rea: { + command: "old-rea", + args: ["mcp"], + env: { rea_mcp_input_schema_profile: "full" }, + }, + }, + }), + ); + const client = { name: "kimi", format: "kimi", configPath } as const; + expect( + await configureClientConfiguration( + client, + {}, + npxRegistrationCommand(process.platform), + ), + ).toMatchObject({ status: "configured" }); + const parsed = parseClientConfiguration( + await readFile(configPath, "utf8"), + "kimi", + ); + expect(effectiveClientServer(parsed, "rea")).toHaveProperty("env", { + rea_mcp_input_schema_profile: "full", + }); + const statuses = await readClientRegistrationStatuses(home, "rea", { + environment: {}, + }); + expect(statuses.find(({ client }) => client === "kimi")).toMatchObject({ + state: "aligned", + }); + }, +); + +it + .skipIf(process.platform !== "win32") + .each(["hopper", "!!js caller.provider"])( + "replaces a Windows DSH environment alias without evaluating %s", + async (stored) => { + const root = await createTestTempDirectory("rea-dsh-case-"); + const configPath = join(root, "cordis.patch.yml"); + const source = `- insert:\n - id: rea-mcp\n name: "@deepseek-ai/dsh-mcp-client"\n config:\n serverName: rea\n transport: stdio\n command: old-rea\n args: [mcp]\n env:\n rea_analysis_provider: ${stored}\n CUSTOM: !!js caller.custom\n`; + await writeFile(configPath, source); + const client = { name: "dsh", format: "dsh", configPath } as const; + const overrides = { REA_ANALYSIS_PROVIDER: "ghidra" }; + const configured = await configureClientConfiguration(client, overrides, [ + "rea", + "mcp", + ]); + expect(configured, JSON.stringify(configured)).toMatchObject({ + status: "configured", + }); + const text = await readFile(configPath, "utf8"); + expect(text).not.toContain("rea_analysis_provider:"); + expect(text).toContain("REA_ANALYSIS_PROVIDER: ghidra"); + expect(text).toContain("!!js caller.custom"); + const details = clientRegistrationEnvironmentDetails( + parseClientConfiguration(text, "dsh"), + ); + expect(details.ok).toBe(true); + if (!details.ok) throw details.error; + expect(details.value.timeoutInputs.analysisProvider).toBe("ghidra"); + expect( + await configureClientConfiguration(client, overrides, ["rea", "mcp"]), + ).toEqual({ status: "unchanged" }); + }, +); + +it.skipIf(process.platform !== "win32").each([ + { + existing: { rea_analysis_provider: "hopper" }, + overrides: { REA_ANALYSIS_PROVIDER: "ghidra" }, + expected: "ghidra", + }, + { + existing: { REA_ANALYSIS_PROVIDER: "ghidra" }, + overrides: { rea_analysis_provider: "hopper" }, + expected: "hopper", + }, +])( + "keeps Windows override precedence consistent with the launched environment: $expected", + ({ existing, overrides, expected }) => { + const parsed = parseClientConfiguration( + JSON.stringify({ + mcpServers: { rea: { command: "rea", args: ["mcp"], env: existing } }, + }), + "json", + ); + const result = clientRegistrationEnvironmentDetails(parsed, overrides); + expect(result.ok).toBe(true); + if (!result.ok) throw result.error; + expect(result.value.timeoutInputs.analysisProvider).toBe(expected); + expect(snapshotEnvironment(result.value.values).REA_ANALYSIS_PROVIDER).toBe( + expected, + ); + expect(Object.keys(result.value.values)).toHaveLength(1); + const child = spawnSync( + process.execPath, + ["-e", "process.stdout.write(process.env.REA_ANALYSIS_PROVIDER ?? '')"], + { + env: result.value.values, + encoding: "utf8", + timeout: 5_000, + }, + ); + expect(child.status, child.stderr).toBe(0); + expect(child.stdout).toBe(expected); + }, +); + +it.skipIf(process.platform !== "win32")( + "rejects invalid lowercase known settings before writing a Windows registration", + async () => { + const root = await createTestTempDirectory( + "rea-invalid-lowercase-environment-", + ); + const configPath = join(root, "config.json"); + const original = JSON.stringify({ + mcpServers: { + rea: { + command: "rea", + args: ["mcp"], + env: { rea_analysis_provider: "invalid provider" }, + }, + }, + }); + await writeFile(configPath, original); + const result = await configureClientConfiguration( + { name: "cursor", format: "json", configPath }, + {}, + ["rea", "mcp"], + ); + expect(result).toMatchObject({ status: "failed" }); + expect(await readFile(configPath, "utf8")).toBe(original); + await expect(access(`${configPath}.rea.backup`)).rejects.toMatchObject({ + code: "ENOENT", + }); + }, +); it.each([ ["cursor", "json", "mcpServers", "env"], diff --git a/tests/boundary/filesystem/clientConfigurationLayeredEnvironment.test.ts b/tests/boundary/filesystem/clientConfigurationLayeredEnvironment.test.ts index 41f0b984c2..9d84f66679 100644 --- a/tests/boundary/filesystem/clientConfigurationLayeredEnvironment.test.ts +++ b/tests/boundary/filesystem/clientConfigurationLayeredEnvironment.test.ts @@ -14,6 +14,11 @@ import { import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; import { npxRegistrationCommand } from "../../../src/application/ClientRegistrationIdentity.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + effectiveClientConfiguration, + readClientConfigurationFiles, +} from "../../../src/application/ClientConfigurationFiles.js"; +import { clientRegistrationEnvironmentDetails } from "../../../src/application/ClientRegistrationEnvironment.js"; const command = npxRegistrationCommand(process.platform); const language = "x86:LE:32:default"; @@ -62,6 +67,47 @@ const layered = async (lower: unknown, upper: unknown) => { return { client, paths, sources, status, unchanged }; }; +it.skipIf(process.platform !== "win32").each([ + { REA_ANALYSIS_PROVIDER: "ghidra", rea_analysis_provider: "hopper" }, + { rea_analysis_provider: "hopper", REA_ANALYSIS_PROVIDER: "ghidra" }, +])( + "keeps explicit overrides above inherited Windows aliases: %j", + async (lower) => { + const fixture = await layered(lower, {}); + expect( + await configureClientConfiguration( + fixture.client, + { REA_ANALYSIS_PROVIDER: "ghidra" }, + command, + ), + ).toMatchObject({ status: "configured" }); + const sources = await readClientConfigurationFiles(fixture.client); + expect(sources.ok).toBe(true); + if (!sources.ok) throw new Error(sources.error.detail); + const environment = clientRegistrationEnvironmentDetails( + effectiveClientConfiguration(sources.value), + ); + expect(environment.ok).toBe(true); + if (!environment.ok) throw environment.error; + expect(environment.value.timeoutInputs.analysisProvider).toBe("ghidra"); + expect(environment.value.values).toHaveProperty( + "REA_ANALYSIS_PROVIDER", + "ghidra", + ); + expect(environment.value.values).not.toHaveProperty( + "rea_analysis_provider", + ); + expect(await readFile(fixture.paths[0]!, "utf8")).toBe(fixture.sources[0]); + expect( + await configureClientConfiguration( + fixture.client, + { REA_ANALYSIS_PROVIDER: "ghidra" }, + command, + ), + ).toEqual({ status: "unchanged" }); + }, +); + it("validates language/compiler settings in the effective OpenCode environment", async () => { const fixture = await layered( { REA_GHIDRA_LANGUAGE_ID: language }, From 1417bf943bf49e49b414e8a508bf28a31304be83 Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:48:52 +0800 Subject: [PATCH 10/13] feat(native): match retained function inventories across builds (#5390) Match retained native inventories through shared CLI/MCP workflows with conservative symbol/byte correspondence, unresolved structural candidates, complete Evidence provenance and bounded paged reads. Closes #5343. --- .../references/native-and-artifacts.md | 11 + docs/mcp-contracts.md | 4 + docs/native-function-matching.md | 120 +++ .../NativeFunctionMatchingService.test.ts | 707 ++++++++++++++++++ .../native/NativeFunctionMatchingService.ts | 250 +++++++ .../NativeFunctionMatchingViews.test.ts | 101 +++ .../native/NativeMatchingAdmission.ts | 60 ++ .../native/NativeMatchingBundleSources.ts | 165 ++++ src/cli.ts | 2 + src/cli/analysisEvidenceDelivery.ts | 11 +- src/cli/nativeFunctionMatchingCommands.ts | 103 +++ src/cliCommandNames.ts | 1 + src/contracts/evidenceInputContracts.ts | 23 +- .../native/functionMatchingContracts.ts | 30 + src/contracts/sessionToolContracts.ts | 8 +- src/contracts/toolContractExamples.ts | 4 + src/contracts/toolEffects.ts | 1 + src/contracts/toolOutputSchemaGroups.ts | 19 +- src/contracts/toolOutputSchemaPrimitives.ts | 17 +- src/domain/analysisView/analysisView.ts | 27 + .../nativeFunctionMatchingView.ts | 311 ++++++++ src/domain/evidenceReference.ts | 17 + src/domain/functionComparisonSchemas.ts | 6 +- src/domain/native/functionByteCoverage.ts | 193 +++++ src/domain/native/functionInventory.ts | 290 +++++++ src/domain/native/functionMatching.ts | 204 +++++ .../native/functionStructuralCandidate.ts | 54 ++ src/domain/native/matchFunctionInventories.ts | 197 +++++ src/domain/native/nativeByteRead.ts | 22 + src/domain/native/procedureInfo.ts | 18 + src/ghidra/GhidraFunctionValues.ts | 7 +- src/ghidra/GhidraInventoryValues.ts | 23 +- src/server/registerFunctionComparisonTool.ts | 12 +- .../registerNativeFunctionMatchingTool.ts | 65 ++ src/server/registerSessionTools.ts | 2 + .../tools/nativeFunctionMatching.test.ts | 276 +++++++ tests/fixtures/nativeFunctionMatching.c | 32 + tests/fixtures/nativeFunctionMatching.ts | 101 +++ 38 files changed, 3422 insertions(+), 72 deletions(-) create mode 100644 docs/native-function-matching.md create mode 100644 src/application/native/NativeFunctionMatchingService.test.ts create mode 100644 src/application/native/NativeFunctionMatchingService.ts create mode 100644 src/application/native/NativeFunctionMatchingViews.test.ts create mode 100644 src/application/native/NativeMatchingAdmission.ts create mode 100644 src/application/native/NativeMatchingBundleSources.ts create mode 100644 src/cli/nativeFunctionMatchingCommands.ts create mode 100644 src/contracts/native/functionMatchingContracts.ts create mode 100644 src/domain/analysisView/nativeFunctionMatchingView.ts create mode 100644 src/domain/evidenceReference.ts create mode 100644 src/domain/native/functionByteCoverage.ts create mode 100644 src/domain/native/functionInventory.ts create mode 100644 src/domain/native/functionMatching.ts create mode 100644 src/domain/native/functionStructuralCandidate.ts create mode 100644 src/domain/native/matchFunctionInventories.ts create mode 100644 src/domain/native/nativeByteRead.ts create mode 100644 src/domain/native/procedureInfo.ts create mode 100644 src/server/registerNativeFunctionMatchingTool.ts create mode 100644 tests/acceptance/tools/nativeFunctionMatching.test.ts create mode 100644 tests/fixtures/nativeFunctionMatching.c create mode 100644 tests/fixtures/nativeFunctionMatching.ts diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index 675de228cc..a9c49e748c 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -1,5 +1,16 @@ # Native, managed, and packaged artifacts +For release comparisons, export `list_procedures` Evidence from each target before +closing it. Close clears session records: import both bundles after collecting the +two targets, or supply portable inline Evidence. `match_native_functions` pairs unique exact names and, when supplied, +complete observed body bytes. Reuse optional same-context `list_names`, +`procedure_info`, `read_bytes` and existing `analyze_function` Evidence; avoid +decompiling every function merely to match it. Its summary retains a parent for +`inspect_analysis_view` `function-matches` collections `matches` and `functions`. +Select columns, batch IDs and page individual facts; aliases remain unresolved. +Unpaired observations do not prove additions/removals. Existing matched dossier +pairs include inputs for `compare_functions`; otherwise analyze just chosen pairs. + ## Desktop installation directories When an installation directory is supplied and its runtime is unknown, use diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 4f0dd8e6b5..c6e4d876d4 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -1,5 +1,9 @@ # MCP runtime contracts +For pairing native functions before comparing selected dossiers, see +[native function matching](native-function-matching.md). It operates on retained +inventories, preserves aliases/unknowns and exposes selected fact pages. + Use this reference to call REA tools and interpret their replies on one MCP connection. Start with [tool results](#tool-results), [retained Evidence](#retained-application-evidence-inputs), and [session discovery](#identity-and-discovery). See [transports](#transports) diff --git a/docs/native-function-matching.md b/docs/native-function-matching.md new file mode 100644 index 0000000000..c44f043e7d --- /dev/null +++ b/docs/native-function-matching.md @@ -0,0 +1,120 @@ +# Match native functions across builds + +`match_native_functions` pairs two existing `list_procedures` Evidence inventories. Select `list_procedures` `detail: "summary"` to retain +complete source rows without transferring the entire inventory. Its +`parent_evidence_id` is the reference matching consumes. +Targets may be opened sequentially; matching never opens a target, rereads its +file, executes it or starts a provider. Preserve each inventory before closing its +target, or import its Evidence bundle later. `close_binary` clears session +records. Export each bundle before closing; after collecting both sides and +closing the last target, import both bundles, then use their references. Keeping +only an Evidence ID across close is insufficient. Complete inline Evidence and +CLI `source_bundle` inputs are portable alternatives. Pass Evidence or exact +session references as `left` and `right`: + +```json +{ + "left": { "kind": "retained-evidence", "evidence_id": "ev_" }, + "right": { "kind": "retained-evidence", "evidence_id": "ev_" } +} +``` + +Unique exact reported names are paired first. Optional `list_names` Evidence +identifies dynamic/default names to exclude. If origin is unavailable, the +existing automatic-name exclusion rule applies and each function retains that +limitation. Exact names establish name correspondence, not unchanged code or +original-source identity. + +For unmatched candidates, optionally supply `left_observations` and +`right_observations`. `procedure_info` or existing `analyze_function` Evidence +gives exact inclusive body ranges; `read_bytes` supplies memory bytes. Observations +must belong to the side's exact target/provider/profile and document context. +Adjacent, consistent overlapping and partially returned reads compose by their +observed extent; conflicting intervals remain gaps. A byte match requires every +body byte, equal total size, equal ordered range lengths, a unique SHA-256 +fingerprint and the same known architecture. Extents are never guessed from the +next entry, and relocations are not normalized. A larger mapped read can serve +several observed bodies; only needed ranges need to be captured. + +This first slice reuses existing range observations. Stripped builds can require +many lightweight range queries when those facts are not retained yet. Matching +does not hide that acquisition cost or silently start those queries. Full +decompilation is unnecessary. + +Repeated candidates stay `unresolved` groups without a Cartesian pair expansion. +`only_left`/`only_right` mean no exact counterpart in the supplied observations; +renaming, inlining, stripping, changed bytes or incomplete discovery can explain +them. They do not prove source addition/removal. `source_coverage` explicitly keeps +inventory completeness unknown and separately counts known byte fingerprints and +resolved correspondence. Provider limitations remain accessible. + +Default `detail: "summary"` retains complete matching Evidence in MCP and returns +counts, both identities and coverage. Query its `parent_evidence_id`: + +```json +{ + "source": { + "kind": "retained-evidence", + "evidence_id": "ev_" + }, + "view": { + "kind": "function-matches", + "collection": "functions", + "states": ["matched"], + "sides": ["left"], + "fields": ["name", "body_sha256"], + "offset": 0, + "limit": 20 + } +} +``` + +`collection: "matches"` returns relation identities, states, basis and member +counts. Large member lists are omitted by default; select `left`/`right` columns +explicitly or page individual `functions`. Batch `match_ids`/`function_ids`, exact function `names`, state +and side filters intersect. Mandatory identities and provenance survive column +selection. Page coverage covers selected retained facts. Function identity binds +inventory Evidence and the owning inventory’s `source_index`, preserving same-address alternatives. +One validation, lazy member index and bounded recent filter are retained per +immutable parent; no live target state is cached. + +When both functions already have unambiguous `analyze_function` dossiers, a matched +relation includes `compare_functions_input` with directly usable retained +references. Otherwise analyze just the selected canonical addresses in their +respective targets, then compare those dossiers. Matching never claims behavioral +equivalence. Optional informative CFG/call/string fingerprints from supplied +dossiers connect unresolved candidate groups only. They require observed CFG, +callees and strings, complete observed call coverage, no unresolved calls, and +the same analysis-profile identity. Empty leaf-function buckets are excluded. +`structural_candidate_sha256` identifies that derived candidate signature; its +dossier link preserves the full supporting facts. Group membership does not +assert that every member pair is a candidate or equivalent. + +CLI uses the same service and JSON input: + +```bash +rea match-native-functions request.json \ + --source_bundle left-bundle.json --source_bundle right-bundle.json \ + --evidence_output matching-bundle.json --json +``` + +`source_bundle` resolves exact references from authenticated local bundles. +Default summary requires `evidence_output`, retaining full matching Evidence and +its selected source closure, including transitive Evidence links across supplied +bundles. Missing dependencies fail with an integrity error; unrelated records +are omitted. Publication never overwrites. Set `detail: "complete"` +in the JSON for all groups inline. Import the bundle into MCP for views, or +extract its matching Evidence record to a JSON file for the existing CLI +`inspect-analysis-view` `evidence-file` source. Input bundles are unchanged. + +Representation admission is bounded at 96 MiB before cloning selected observations +and expanding matching indexes and repeated per-function source links. Work yields to cancellation. No worker/provider +concurrency is introduced; this budget is not a bound on total process RSS. + +The self-owned source fixture is [nativeFunctionMatching.c](https://github.com/morluto/rea/blob/main/tests/fixtures/nativeFunctionMatching.c). +On Linux x86-64, compile with `cc -O2 -g0 -fno-pie -no-pie -DVERSION=1` and +`-DVERSION=2`; copy the second binary and run `strip --strip-all` on that copy. +It includes unchanged/changed code, a byte-identical rename, a removed/added +observation, an inlined function, identical-code aliases and a renamed changed +function with an observed call/string signature. Analyze without executing the +fixture. Full-provider verification still needs caller-supplied Ghidra and Java. diff --git a/src/application/native/NativeFunctionMatchingService.test.ts b/src/application/native/NativeFunctionMatchingService.test.ts new file mode 100644 index 0000000000..a8c3ef6987 --- /dev/null +++ b/src/application/native/NativeFunctionMatchingService.test.ts @@ -0,0 +1,707 @@ +import { expect, it } from "vitest"; +import { createHash } from "node:crypto"; +import { matchNativeFunctions } from "./NativeFunctionMatchingService.js"; +import { nativeFunctionMatchingSchema } from "../../domain/native/functionMatching.js"; +import { createEvidence } from "../../domain/evidence.js"; +import { createAnalysisProfile } from "../../domain/analysisProfile.js"; +import { functionDossierSchema } from "../../domain/hopperValues.js"; +import { + nativeMatchingInventory, + nativeMatchingDetails, + nativeMatchingTarget, + nativeMatchingProvider, +} from "../../../tests/fixtures/nativeFunctionMatching.js"; + +const compare = async (input: unknown) => { + const result = await matchNativeFunctions(input); + if (!result.ok) throw result.error; + return { + ...result.value, + report: nativeFunctionMatchingSchema.parse( + result.value.evidence.normalized_result, + ), + }; +}; + +it("retains transitive provenance without treating dependencies as native observations and fails closed for a missing ancestor", async () => { + const ancestor = createEvidence(undefined, nativeMatchingProvider, { + operation: "fixture_provenance", + parameters: {}, + result: { provenance: "Synthetic ancestor" }, + }); + const dependency = createEvidence(undefined, nativeMatchingProvider, { + operation: "fixture_dependency", + parameters: {}, + result: { provenance: "Synthetic dependency" }, + evidenceLinks: [ancestor.evidence_id], + }); + const left = nativeMatchingInventory( + "1", + [{ address: "0x10", value: "main" }], + [dependency.evidence_id], + ); + const right = nativeMatchingInventory( + "2", + [{ address: "0x20", value: "main" }], + [dependency.evidence_id], + ); + const dependencies = new Map( + [ancestor, dependency].map((record) => [record.evidence_id, record]), + ); + const result = await matchNativeFunctions({ left, right }, undefined, (id) => + dependencies.get(id), + ); + if (!result.ok) throw result.error; + expect( + new Set(result.value.sources.map((record) => record.evidence_id)), + ).toEqual( + new Set( + [left, right, dependency, ancestor].map((record) => record.evidence_id), + ), + ); + expect(new Set(result.value.evidence.evidence_links)).toEqual( + new Set([left.evidence_id, right.evidence_id]), + ); + expect( + nativeFunctionMatchingSchema.parse(result.value.evidence.normalized_result) + .summary.matched, + ).toBe(1); + const missing = await matchNativeFunctions( + { left, right }, + undefined, + (id) => (id === dependency.evidence_id ? dependency : undefined), + ); + expect(missing.ok).toBe(false); + if (missing.ok) throw new Error("Missing ancestor accepted"); + expect(missing.error).toMatchObject({ + evidenceId: ancestor.evidence_id, + reason: "missing", + }); +}); + +it("keeps symbol origin unknown when supplied metadata describes a different same-address alias", async () => { + const left = nativeMatchingInventory("1", [ + { address: "0x10", value: "reported_name" }, + ]); + const right = nativeMatchingInventory("2", [ + { address: "0x20", value: "reported_name" }, + ]); + const alias = createEvidence( + nativeMatchingTarget("1"), + nativeMatchingProvider, + { + operation: "list_names", + parameters: {}, + result: [ + { + address: "0x10", + value: "other_alias", + symbol: { + primary: false, + dynamic: false, + external: false, + type: "function", + source: "user_defined", + }, + }, + ], + }, + ); + const { report } = await compare({ left, right, left_observations: [alias] }); + expect(report.summary.matched).toBe(1); + expect(report.matches[0]?.left[0]?.limitations).toContain( + "Symbol origin unavailable; exact-name matching uses the reported name and the existing automatic-name exclusion rule.", + ); + expect(report.matches[0]?.left[0]?.evidence_links).toContain( + alias.evidence_id, + ); +}); + +it("preserves contradictory symbol origins from inventory/name facts or multiple name observations, independently of byte correspondence", async () => { + const observe = (operation: string, generated: boolean) => + createEvidence(nativeMatchingTarget("1"), nativeMatchingProvider, { + operation, + parameters: {}, + result: [ + { + address: "0x10", + value: "same_name", + symbol: { + primary: true, + dynamic: generated, + external: false, + type: "function", + source: generated ? "default" : "user_defined", + }, + }, + ], + }); + const right = nativeMatchingInventory("2", [ + { address: "0x20", value: "same_name" }, + ]); + for (const [left, names] of [ + [observe("list_procedures", false), [observe("list_names", true)]], + [observe("list_procedures", true), [observe("list_names", false)]], + [ + nativeMatchingInventory("1", [{ address: "0x10", value: "same_name" }]), + [observe("list_names", false), observe("list_names", true)], + ], + [ + nativeMatchingInventory("1", [{ address: "0x10", value: "same_name" }]), + [observe("list_names", true), observe("list_names", false)], + ], + ] as const) { + for (const withBytes of [false, true]) { + const { report } = await compare({ + left, + right, + left_observations: [ + ...names, + ...(withBytes ? nativeMatchingDetails("1", "0x10", "c3") : []), + ], + right_observations: withBytes + ? nativeMatchingDetails("2", "0x20", "c3") + : [], + }); + expect(report.summary.matched).toBe(withBytes ? 1 : 0); + expect(report.matches[0]!.basis).toBe( + withBytes ? "identical-bytes" : null, + ); + const member = report.matches[0]!.left[0]!; + expect(member.limitations).toContain( + "Conflicting generated and named symbol-origin observations; exact-name matching is unavailable. Independent complete body bytes remain eligible.", + ); + expect(member.limitations.join(" ")).not.toContain( + "Symbol origin unavailable", + ); + for (const source of names) + expect(member.evidence_links).toContain(source.evidence_id); + } + } +}); + +it("keeps informative CFG/call/string fingerprints as unresolved candidates and refuses incomplete structural observations", async () => { + const profile = createAnalysisProfile(nativeMatchingProvider, { + fixture: "structural-candidates-v1", + }); + const observe = ( + side: "1" | "2", + operation: string, + result: Parameters[2]["result"], + complete = true, + ) => + createEvidence(nativeMatchingTarget(side), nativeMatchingProvider, { + operation, + parameters: {}, + result, + analysisProfile: profile, + relationshipCoverage: complete + ? { status: "complete" } + : { status: "unknown", reason: "Missing direct call coverage" }, + }); + const left = observe("1", "list_procedures", [ + { address: "0x10", value: "old_behavior" }, + ]); + const right = observe("2", "list_procedures", [ + { address: "0x20", value: "new_behavior" }, + ]); + const dossier = (side: "1" | "2", address: string) => { + const value = functionDossierSchema.parse( + nativeMatchingDetails(side, address, "0102", true)[0]!.normalized_result, + ); + value.basic_blocks = [ + { + start: address, + end: `0x${(BigInt(address) + 2n).toString(16)}`, + successors: [], + }, + ]; + value.callees = [ + { + address: "0x500", + name: "puts", + classification: null, + body: { available: false, reason: "Synthetic external target" }, + }, + ]; + value.unresolved_calls = []; + value.referenced_strings = [ + { address: "0x900", value: "marker", source_address: address }, + ]; + return value; + }; + const l = dossier("1", "0x10"), + r = dossier("2", "0x20"); + const { report } = await compare({ + left, + right, + left_observations: [observe("1", "analyze_function", l)], + right_observations: [observe("2", "analyze_function", r)], + }); + expect(report.summary).toEqual({ + matched: 0, + only_left: 0, + only_right: 0, + unresolved: 1, + }); + expect(report.matches[0]?.left[0]?.structural_candidate_sha256).toMatch( + /^[a-f0-9]{64}$/u, + ); + expect(report.matches[0]?.left[0]?.structural_candidate_sha256).toBe( + report.matches[0]?.right[0]?.structural_candidate_sha256, + ); + expect(report.matches[0]?.compare_functions_input).toBeNull(); + const incomplete = await compare({ + left, + right, + left_observations: [observe("1", "analyze_function", l, false)], + right_observations: [observe("2", "analyze_function", r)], + }); + expect(incomplete.report.summary.unresolved).toBe(0); + expect(incomplete.report.source_coverage.left_structural_candidates).toBe(0); + const legacy = { ...l }; + delete legacy.facet_availability; + const missingFacets = await compare({ + left, + right, + left_observations: [observe("1", "analyze_function", legacy)], + right_observations: [observe("2", "analyze_function", r)], + }); + expect(missingFacets.report.source_coverage.left_structural_candidates).toBe( + 0, + ); + expect(missingFacets.report.summary.unresolved).toBe(0); + l.callees = []; + l.referenced_strings = []; + r.callees = []; + r.referenced_strings = []; + const uninformative = await compare({ + left, + right, + left_observations: [observe("1", "analyze_function", l)], + right_observations: [observe("2", "analyze_function", r)], + }); + expect(uninformative.report.source_coverage.left_structural_candidates).toBe( + 0, + ); + expect(uninformative.report.summary.unresolved).toBe(0); +}); + +it("matches symbols before bytes, retains exact renamed bytes, changed pairs, unpaired observations and duplicate candidates", async () => { + const left = nativeMatchingInventory("1", [ + { address: "0x1000", value: "stable" }, + { address: "0x1010", value: "changed" }, + { address: "0x1020", value: "old_name" }, + { address: "0x1030", value: "removed_observation" }, + { address: "0x1040", value: "duplicate" }, + { address: "0x1050", value: "duplicate" }, + ]); + const right = nativeMatchingInventory("2", [ + { address: "0x2000", value: "stable" }, + { address: "0x2010", value: "changed" }, + { address: "0x2020", value: "renamed" }, + { address: "0x2030", value: "added_observation" }, + { address: "0x2040", value: "duplicate" }, + ]); + const result = await compare({ + left, + right, + left_observations: [ + ...nativeMatchingDetails("1", "0x1010", "0102"), + ...nativeMatchingDetails("1", "0x1020", "030405", true), + ], + right_observations: [ + ...nativeMatchingDetails("2", "0x2010", "aabb"), + ...nativeMatchingDetails("2", "0x2020", "030405", true), + ], + }); + expect(result.report.summary).toEqual({ + matched: 3, + only_left: 1, + only_right: 1, + unresolved: 1, + }); + const renamed = result.report.matches.find( + (row) => row.basis === "identical-bytes", + )!; + expect(renamed.left[0]).toMatchObject({ + name: "old_name", + body_bytes: 3, + body_sha256: createHash("sha256") + .update(Buffer.from("030405", "hex")) + .digest("hex"), + }); + expect(renamed.right[0]?.name).toBe("renamed"); + expect(renamed.compare_functions_input?.left.evidence_id).toBe( + renamed.left[0]?.dossier_evidence_id, + ); + expect( + result.report.matches.find((row) => row.left[0]?.name === "changed"), + ).toMatchObject({ state: "matched", basis: "exact-symbol" }); + expect( + result.report.matches.find((row) => row.state === "unresolved"), + ).toMatchObject({ + left: [expect.anything(), expect.anything()], + right: [expect.anything()], + }); + expect(new Set(result.evidence.evidence_links)).toEqual( + new Set(result.sources.map((x) => x.evidence_id)), + ); + expect(result.report.limitations.join(" ")).toContain("never prove absence"); +}); + +it("does not force repeated identical-byte candidates after an exact name consumes an alias", async () => { + const left = nativeMatchingInventory("1", [ + { address: "0x10", value: "known" }, + { address: "0x20", value: "FUN_20" }, + ]); + const right = nativeMatchingInventory("2", [ + { address: "0x30", value: "known" }, + { address: "0x40", value: "FUN_40" }, + ]); + const { report } = await compare({ + left, + right, + left_observations: [ + ...nativeMatchingDetails("1", "0x10", "c3"), + ...nativeMatchingDetails("1", "0x20", "c3"), + ], + right_observations: [ + ...nativeMatchingDetails("2", "0x30", "c3"), + ...nativeMatchingDetails("2", "0x40", "c3"), + ], + }); + expect(report.summary).toEqual({ + matched: 1, + unresolved: 1, + only_left: 0, + only_right: 0, + }); + expect( + report.matches.find((x) => x.state === "unresolved")?.left[0]?.name, + ).toBe("FUN_20"); +}); + +it("reconstructs discontiguous bodies from adjacent byte reads without next-procedure guesses and keeps address-space/64-bit identity", async () => { + const address = "ram:0xfffffffffffffff0"; + const left = nativeMatchingInventory("1", [{ address, value: "FUN_f0" }]); + const right = nativeMatchingInventory("2", [ + { address: "0x20", value: "FUN_20" }, + ]); + const body = { + available: true, + provenance: "synthetic noncontiguous body", + ranges: [ + { start: address, end: "ram:0xfffffffffffffff1" }, + { start: "ram:0xfffffffffffffff8", end: "ram:0xfffffffffffffff8" }, + ], + total_bytes: 3, + span_bytes: 9, + non_contiguous: true, + contains_entry: true, + }; + const info = createEvidence( + nativeMatchingTarget("1"), + nativeMatchingProvider, + { + operation: "procedure_info", + parameters: {}, + result: { + name: "a", + entrypoint: address, + basicblock_count: 1, + length: 3, + signature: null, + locals: [], + classification: null, + body, + }, + }, + ); + const read = (address: string, hex: string) => + createEvidence(nativeMatchingTarget("1"), nativeMatchingProvider, { + operation: "read_bytes", + parameters: {}, + result: { + address, + requested_bytes: hex.length / 2, + returned_bytes: hex.length / 2, + bytes_hex: hex, + complete: true, + }, + }); + const rightDetails = nativeMatchingDetails("2", "0x20", "010203"); + const { report } = await compare({ + left, + right, + left_observations: [ + info, + read(address, "01"), + read("ram:0xfffffffffffffff1", "02"), + read("ram:0xfffffffffffffff8", "03"), + ], + right_observations: rightDetails, + }); + // Same concatenated bytes with a different body layout do not become a pair. + expect(report.summary.matched).toBe(0); + expect(report.matches[0]?.left[0]).toMatchObject({ + address, + body_bytes: 3, + body_sha256: createHash("sha256") + .update(Buffer.from("010203", "hex")) + .digest("hex"), + }); +}); + +it("preserves byte correspondence when consistent dossiers differ only in request identity, without choosing an arbitrary dossier", async () => { + const left = nativeMatchingInventory("1", [ + { address: "0x10", value: "old_name" }, + ]); + const right = nativeMatchingInventory("2", [ + { address: "0x20", value: "new_name" }, + ]); + const leftDetails = nativeMatchingDetails("1", "0x10", "0102", true); + const another = createEvidence( + nativeMatchingTarget("1"), + nativeMatchingProvider, + { + operation: "analyze_function", + parameters: { procedure: "old_name" }, + result: leftDetails[0]!.normalized_result, + relationshipCoverage: { status: "complete" }, + }, + ); + for (const observations of [ + [...leftDetails, another], + [another, ...leftDetails], + ]) { + const { report } = await compare({ + left, + right, + left_observations: observations, + right_observations: nativeMatchingDetails("2", "0x20", "0102", true), + }); + expect(report.summary.matched).toBe(1); + expect(report.matches[0]).toMatchObject({ + basis: "identical-bytes", + compare_functions_input: null, + left: [ + { + dossier_evidence_id: null, + body_bytes: 2, + body_sha256: createHash("sha256") + .update(Buffer.from("0102", "hex")) + .digest("hex"), + }, + ], + }); + expect(report.matches[0]!.left[0]!.evidence_links).toContain( + another.evidence_id, + ); + } + const conflicting = nativeMatchingDetails("1", "0x10", "010203", true)[0]!; + const { report } = await compare({ + left, + right, + left_observations: [...leftDetails, conflicting], + right_observations: nativeMatchingDetails("2", "0x20", "0102", true), + }); + expect(report.summary.matched).toBe(0); + expect(report.matches[0]!.left[0]!.body_sha256).toBeNull(); +}); + +it("rejects repeated per-occurrence source-link expansion within the shared representation budget", async () => { + const base = nativeMatchingDetails("1", "0x10", "c3")[0]!; + const observations = Array.from({ length: 250 }, (_, observation) => + createEvidence(nativeMatchingTarget("1"), nativeMatchingProvider, { + operation: "procedure_info", + parameters: { procedure: "0x10", observation }, + result: base.normalized_result, + }), + ); + const result = await matchNativeFunctions({ + left: nativeMatchingInventory( + "1", + Array.from({ length: 1000 }, (_, i) => ({ + address: "0x10", + value: `alias_${i}`, + })), + ), + right: nativeMatchingInventory("2", []), + left_observations: observations, + }); + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error._tag).toBe("AnalysisResourceConstraintError"); +}); + +it("retains missing bytes and conflicting overlapping observations as unknown instead of exact matches", async () => { + const left = nativeMatchingInventory("1", [ + { address: "0x10", value: "FUN_10" }, + ]); + const right = nativeMatchingInventory("2", [ + { address: "0x20", value: "FUN_20" }, + ]); + const observations = [ + ...nativeMatchingDetails("1", "0x10", "0102"), + nativeMatchingDetails("1", "0x10", "ffff")[1]!, + ]; + const { report } = await compare({ + left, + right, + left_observations: observations, + right_observations: nativeMatchingDetails("2", "0x20", "0102"), + }); + expect(report.summary.matched).toBe(0); + expect(report.matches[0]?.left[0]?.body_sha256).toBeNull(); +}); + +it("composes consistent overlapping and partial byte reads without losing complete body coverage", async () => { + const left = nativeMatchingInventory("1", [ + { address: "0x10", value: "FUN_10" }, + ]); + const right = nativeMatchingInventory("2", [ + { address: "0x20", value: "FUN_20" }, + ]); + const info = nativeMatchingDetails("1", "0x10", "01020304")[0]!; + const read = (address: string, hex: string, requested = hex.length / 2) => + createEvidence(nativeMatchingTarget("1"), nativeMatchingProvider, { + operation: "read_bytes", + parameters: { address, length: requested }, + result: { + address, + requested_bytes: requested, + returned_bytes: hex.length / 2, + bytes_hex: hex, + complete: requested === hex.length / 2, + }, + }); + const { report } = await compare({ + left, + right, + left_observations: [ + info, + read("0x10", "010203", 6), + read("0x11", "020304"), + ], + right_observations: nativeMatchingDetails("2", "0x20", "01020304"), + }); + expect(report.summary.matched).toBe(1); + expect(report.source_coverage).toMatchObject({ + status: "complete-within-inputs", + left_byte_fingerprints: 1, + right_byte_fingerprints: 1, + left_inventory_completeness: "unknown", + }); +}); + +it("does not use generated symbols and preserves literal named FUN_ symbols when producer metadata proves their origin", async () => { + const entries = [ + { address: "0x10", value: "FUN_10" }, + { address: "0x20", value: "apparently_named" }, + ]; + const names = (side: "1" | "2") => + createEvidence(nativeMatchingTarget(side), nativeMatchingProvider, { + operation: "list_names", + parameters: {}, + result: entries.map((entry, i) => ({ + ...entry, + symbol: { + primary: true, + dynamic: i === 1, + external: false, + type: "function", + source: i === 0 ? "user_defined" : "default", + }, + })), + }); + const { report } = await compare({ + left: nativeMatchingInventory("1", entries), + right: nativeMatchingInventory("2", entries), + left_observations: [names("1")], + right_observations: [names("2")], + }); + expect(report.summary.matched).toBe(1); + expect(report.matches[0]?.left[0]?.name).toBe("FUN_10"); + expect(report.matches[0]?.left[0]?.evidence_links).toContain( + names("1").evidence_id, + ); +}); + +it("rejects tampered, non-inventory and mixed-target observations with typed failures", async () => { + const left = nativeMatchingInventory("1", [ + { address: "0x10", value: "main" }, + ]); + for (const input of [ + { left: { ...left, normalized_result: [] }, right: left }, + { left: nativeMatchingDetails("1", "0x10", "c3")[0], right: left }, + { + left, + right: left, + left_observations: nativeMatchingDetails("2", "0x10", "c3"), + }, + { + left, + right: left, + left_observations: [ + { kind: "retained-evidence", evidence_id: `ev_${"a".repeat(64)}` }, + ], + }, + ]) { + const result = await matchNativeFunctions(input); + expect(result.ok).toBe(false); + if (!result.ok) + expect([ + "AnalysisInputError", + "EvidenceIntegrityError", + "EvidenceReferenceError", + ]).toContain(result.error._tag); + } +}); + +it("cooperatively preserves the exact cancellation reason and rejects expansion before cloning oversized input", async () => { + const controller = new AbortController(), + reason = new Error("stop matching"); + const left = nativeMatchingInventory( + "1", + Array.from({ length: 2000 }, (_, i) => ({ + address: `0x${i.toString(16)}`, + value: `item${i}`, + })), + ); + const pending = matchNativeFunctions( + { left, right: left }, + { signal: controller.signal }, + ); + setImmediate(() => controller.abort(reason)); + const cancelled = await pending; + expect(cancelled.ok).toBe(false); + if (!cancelled.ok) { + expect(cancelled.error._tag).toBe("AnalysisCancelledError"); + expect(cancelled.error.cause).toBe(reason); + } + const large = await matchNativeFunctions({ + left: { padding: "x".repeat(26 * 1024 * 1024) }, + right: left, + }); + expect(large.ok).toBe(false); + if (!large.ok) + expect(large.error._tag).toBe("AnalysisResourceConstraintError"); +}); + +it("keeps duplicate candidate storage linear and emits every input occurrence exactly once", async () => { + const entries = Array.from({ length: 1000 }, (_, i) => ({ + address: `0x${i.toString(16)}`, + value: "same", + })); + const { report } = await compare({ + left: nativeMatchingInventory("1", entries), + right: nativeMatchingInventory("2", entries), + }); + expect(report.matches).toHaveLength(1); + expect(report.matches[0]?.left).toHaveLength(1000); + expect(report.matches[0]?.right).toHaveLength(1000); + expect(new Set(report.matches[0]?.left.map((x) => x.function_id)).size).toBe( + 1000, + ); +}); diff --git a/src/application/native/NativeFunctionMatchingService.ts b/src/application/native/NativeFunctionMatchingService.ts new file mode 100644 index 0000000000..deff5d57d5 --- /dev/null +++ b/src/application/native/NativeFunctionMatchingService.ts @@ -0,0 +1,250 @@ +import { NativeMatchingAdmission } from "./NativeMatchingAdmission.js"; +import { loadNativeMatchingBundleSources } from "./NativeMatchingBundleSources.js"; +import { RegularFileCleanupFailure } from "../RegularFileRead.js"; +import { matchFunctionInventories } from "../../domain/native/matchFunctionInventories.js"; +import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; +import { setImmediate } from "node:timers/promises"; +import { matchNativeFunctionsInputSchema } from "../../contracts/native/functionMatchingContracts.js"; +import { retainedEvidenceReferenceSchema } from "../../domain/evidenceReference.js"; +import { + createEvidence, + parseEvidence, + type Evidence, +} from "../../domain/evidence.js"; +import { readFunctionInventory } from "../../domain/native/functionInventory.js"; +import { nativeFunctionMatchingSchema } from "../../domain/native/functionMatching.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisInputError, +} from "../../domain/analysisErrorCore.js"; +import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { + resolveEvidenceInput, + type EvidenceLookup, +} from "../EvidenceInputResolver.js"; +import type { ExecutionOptions } from "../AnalysisProvider.js"; +import { FUNCTION_COMPARISON_PROVIDER } from "../InvestigationProviders.js"; + +const OPERATION = "match_native_functions"; +export interface NativeFunctionMatchingResult { + readonly evidence: Evidence; + readonly sources: readonly Evidence[]; +} + +/** Pair existing observations through one source/analysis admission. */ +export const matchNativeFunctions = ( + rawInput: unknown, + options?: ExecutionOptions, + lookup?: EvidenceLookup, +): Promise> => + matchWithAdmission( + rawInput, + options, + lookup, + new NativeMatchingAdmission(OPERATION, options?.signal), + ); + +const matchWithAdmission = async ( + rawInput: unknown, + options: ExecutionOptions | undefined, + lookup: EvidenceLookup | undefined, + admission: NativeMatchingAdmission, +): Promise> => { + try { + const checkpoint = admission.checkpoint; + await setImmediate(); + await checkpoint(); + const parsed = matchNativeFunctionsInputSchema.safeParse(rawInput); + if (!parsed.success) + return err( + analysisInputErrorFromIssues(OPERATION, parsed.error.issues, rawInput), + ); + const sources = new Map(); + const resolve = async (input: unknown): Promise => { + const reference = retainedEvidenceReferenceSchema.safeParse(input); + let raw = input; + if (reference.success) { + const found = resolveEvidenceInput(reference.data, lookup); + if (!found.ok) throw found.error; + raw = found.value; + } + await admission.admit(raw); + const evidence = parseEvidence(raw); + const cached = sources.get(evidence.evidence_id); + if (cached !== undefined) return cached; + sources.set(evidence.evidence_id, evidence); + return evidence; + }; + const left = await resolve(parsed.data.left), + right = await resolve(parsed.data.right); + const observations = async (inputs: readonly unknown[]) => { + const result: Evidence[] = []; + const seen = new Set(); + for (const input of inputs) { + const record = await resolve(input); + if (!seen.has(record.evidence_id)) { + seen.add(record.evidence_id); + result.push(record); + } + } + return result; + }; + const leftObservations = await observations(parsed.data.left_observations); + const rightObservations = await observations( + parsed.data.right_observations, + ); + const directLinks = [...sources.keys()].sort(); + // Expand provenance separately from role observations: dependencies may be + // any Evidence operation and must not be interpreted as native body facts. + for (const source of sources.values()) + for (const id of source.evidence_links) + if (!sources.has(id)) + await resolve({ kind: "retained-evidence", evidence_id: id }); + const leftFunctions = await readFunctionInventory( + left, + leftObservations, + checkpoint, + ); + const rightFunctions = await readFunctionInventory( + right, + rightObservations, + checkpoint, + ); + const compatible = + left.subject?.architecture != null && + left.subject.architecture === right.subject?.architecture; + const { matches, summary } = await matchFunctionInventories( + leftFunctions, + rightFunctions, + { identicalBytesComparable: compatible, checkpoint }, + ); + const limitations = [ + ...new Set([ + ...left.limitations.map((x) => `Left: ${x}`), + ...right.limitations.map((x) => `Right: ${x}`), + "Coverage is restricted to the supplied provider-observed inventories; missing/truncated inventory facts never prove absence from either binary.", + "Only unique exact symbols and bytes are paired. Compatible observed CFG/call/string fingerprints propose unresolved candidates, never identity or equivalence. Missing structural coverage stays unknown.", + "Name or byte correspondence is not behavioral equivalence; use analyze_function for selected pairs, then compare_functions. Existing dossier pairs include directly usable retained-reference inputs.", + ...(!compatible + ? [ + "Identical-byte matching unavailable: input architectures are missing or differ.", + ] + : []), + ]), + ]; + const report = nativeFunctionMatchingSchema.parse({ + left: { + evidence_id: left.evidence_id, + sha256: left.subject!.digest.sha256, + functions: leftFunctions.length, + }, + right: { + evidence_id: right.evidence_id, + sha256: right.subject!.digest.sha256, + functions: rightFunctions.length, + }, + scope: "provider-observed-inputs", + source_coverage: { + status: + summary.only_left + summary.only_right + summary.unresolved > 0 + ? "partial" + : "complete-within-inputs", + left_inventory_completeness: "unknown", + right_inventory_completeness: "unknown", + left_byte_fingerprints: leftFunctions.filter( + (x) => x.fingerprint !== null, + ).length, + left_structural_candidates: leftFunctions.filter( + (x) => x.member.structural_candidate_sha256 !== null, + ).length, + right_structural_candidates: rightFunctions.filter( + (x) => x.member.structural_candidate_sha256 !== null, + ).length, + right_byte_fingerprints: rightFunctions.filter( + (x) => x.fingerprint !== null, + ).length, + }, + summary, + matches, + limitations, + }); + await checkpoint(); + const evidence = createEvidence(undefined, FUNCTION_COMPARISON_PROVIDER, { + operation: OPERATION, + predicateType: "rea.native-function-matching", + confidence: "derived", + authority: "analyst-inference", + parameters: { + left_evidence_id: left.evidence_id, + right_evidence_id: right.evidence_id, + left_observation_ids: leftObservations.map((x) => x.evidence_id), + right_observation_ids: rightObservations.map((x) => x.evidence_id), + algorithm: "exact-symbol-body-bytes-and-structural-candidates-v1", + }, + result: report, + limitations, + evidenceLinks: directLinks, + }); + await checkpoint(); + return ok({ evidence, sources: [...sources.values()] }); + } catch (cause: unknown) { + if (options?.signal?.aborted) + return err( + new AnalysisCancelledError(OPERATION, { cause: options.signal.reason }), + ); + if (cause instanceof AnalysisError) return err(cause); + return err( + new AnalysisInputError(OPERATION, { cause }, [ + { + path: [], + reason: "invalid_value", + message: + cause instanceof Error + ? cause.message + : "Invalid native matching observations", + }, + ]), + ); + } +}; + +/** Load selected portable records; do not retain unrelated records across bundles. */ +export const matchNativeFunctionsFromBundles = async ( + input: unknown, + paths: readonly string[], + resources: ArtifactResourceScope, + options?: ExecutionOptions, +): Promise> => { + try { + const parsed = matchNativeFunctionsInputSchema.safeParse(input); + if (!parsed.success) + return err( + analysisInputErrorFromIssues(OPERATION, parsed.error.issues, input), + ); + const admission = new NativeMatchingAdmission(OPERATION, options?.signal); + const loaded = await loadNativeMatchingBundleSources({ + input: parsed.data, + paths, + resources, + admission, + }); + return await matchWithAdmission( + loaded.input, + options, + loaded.lookup, + admission, + ); + } catch (cause: unknown) { + // This marker carries the exact descriptor owner; the caller must finish + // that owner before classifying cancellation or unexpected I/O failures. + if (cause instanceof RegularFileCleanupFailure) throw cause; + if (options?.signal?.aborted) + return err( + new AnalysisCancelledError(OPERATION, { cause: options.signal.reason }), + ); + if (cause instanceof AnalysisError) return err(cause); + return err(new AnalysisInputError(OPERATION, { cause })); + } +}; diff --git a/src/application/native/NativeFunctionMatchingViews.test.ts b/src/application/native/NativeFunctionMatchingViews.test.ts new file mode 100644 index 0000000000..72da044d7a --- /dev/null +++ b/src/application/native/NativeFunctionMatchingViews.test.ts @@ -0,0 +1,101 @@ +import { expect, it } from "vitest"; +import { matchNativeFunctions } from "./NativeFunctionMatchingService.js"; +import { inspectAnalysisView } from "../analysisView/AnalysisViewService.js"; +import { nativeMatchingInventory } from "../../../tests/fixtures/nativeFunctionMatching.js"; +import { analysisViewResultSchema } from "../../domain/analysisView/analysisView.js"; + +it("pages individual function facts without transferring an entire ambiguous alias group, preserving mandatory identity and source coverage", async () => { + const entries = Array.from({ length: 500 }, (_, i) => ({ + address: `0x${i.toString(16)}`, + value: "same", + })); + const matched = await matchNativeFunctions({ + left: nativeMatchingInventory("1", entries), + right: nativeMatchingInventory("2", entries), + }); + if (!matched.ok) throw matched.error; + const source = { kind: "inline", evidence: matched.value.evidence }; + const view = (query: unknown) => { + const result = inspectAnalysisView({ source, view: query }); + if (!result.ok) throw result.error; + return analysisViewResultSchema.parse(result.value.normalized_result); + }; + const groups = view({ + kind: "function-matches", + collection: "matches", + fields: ["basis"], + limit: 1, + }); + expect(groups).toMatchObject({ + kind: "function-matches", + coverage: { examined: 1, total: 1, exhausted: true }, + items: [ + { state: "unresolved", basis: null, left_count: 500, right_count: 500 }, + ], + }); + if (groups.kind !== "function-matches") throw new Error("Wrong view"); + expect(groups.items[0]).not.toHaveProperty("left"); + const member = `${matched.value.sources[0]!.evidence_id}/123`; + const functions = view({ + kind: "function-matches", + collection: "functions", + function_ids: [member], + fields: ["name"], + limit: 1, + }); + expect(functions).toMatchObject({ + kind: "function-matches", + coverage: { examined: 1, total: 1, exhausted: true }, + items: [ + { + function_id: member, + source_index: 123, + address: "0x7b", + name: "same", + side: "left", + state: "unresolved", + evidence_links: [matched.value.sources[0]!.evidence_id], + }, + ], + }); + expect(JSON.stringify(functions).length).toBeLessThan(6000); + expect( + view({ + kind: "function-matches", + collection: "functions", + names: [], + limit: 1, + }), + ).toMatchObject({ coverage: { total: 0 } }); + expect( + view({ + kind: "function-matches", + collection: "functions", + names: ["same"], + sides: ["left"], + limit: 1, + }), + ).toMatchObject({ coverage: { total: 500, next_offset: 1 } }); + expect( + view({ + kind: "function-matches", + collection: "functions", + sides: ["right"], + offset: 499, + limit: 2, + }), + ).toMatchObject({ + coverage: { examined: 1, total: 500, exhausted: true, next_offset: null }, + }); + expect( + view({ kind: "function-matches", collection: "functions", states: [] }), + ).toMatchObject({ coverage: { total: 0, exhausted: true } }); + expect(view({ kind: "summary" })).toMatchObject({ + summary: { + scope: "provider-observed-inputs", + summary: { unresolved: 1 }, + left: { functions: 500 }, + right: { functions: 500 }, + }, + }); +}); diff --git a/src/application/native/NativeMatchingAdmission.ts b/src/application/native/NativeMatchingAdmission.ts new file mode 100644 index 0000000000..1acb781bb1 --- /dev/null +++ b/src/application/native/NativeMatchingAdmission.ts @@ -0,0 +1,60 @@ +import { setImmediate } from "node:timers/promises"; +import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js"; + +const NATIVE_MATCHING_REPRESENTATION_BYTES = 96 * 1024 * 1024; + +/** Budget the actual selected observations, before cloning or index allocation. */ +export class NativeMatchingAdmission { + #remaining = NATIVE_MATCHING_REPRESENTATION_BYTES; + #work = 0; + readonly #seen = new WeakSet(); + constructor( + readonly operation: string, + readonly signal?: AbortSignal, + ) {} + readonly checkpoint = async (bytes = 0): Promise => { + this.#remaining -= bytes; + if (this.#remaining < 0) + throw new AnalysisResourceConstraintError( + this.operation, + "capacity", + "Native function matching exceeds representation capacity", + { maximum_bytes: NATIVE_MATCHING_REPRESENTATION_BYTES }, + ); + this.signal?.throwIfAborted(); + if (++this.#work % 256 === 0) + await setImmediate( + undefined, + this.signal === undefined ? {} : { signal: this.signal }, + ); + }; + async admit(root: unknown): Promise { + const pending: Iterator[] = [[root][Symbol.iterator]()]; + while (pending.length) { + const next = pending[pending.length - 1]!.next(); + if (next.done) { + pending.pop(); + continue; + } + const value = next.value; + if (typeof value === "object" && value !== null) { + if (this.#seen.has(value)) continue; + this.#seen.add(value); + await this.checkpoint(128); + // Iterator traversal avoids a second width-sized array of child values. + pending.push( + (function* () { + for (const key in value) + if (Object.hasOwn(value, key)) { + yield key; + yield (value as Record)[key]; + } + })(), + ); + } else + await this.checkpoint( + typeof value === "string" ? 64 + value.length * 4 : 128, + ); + } + } +} diff --git a/src/application/native/NativeMatchingBundleSources.ts b/src/application/native/NativeMatchingBundleSources.ts new file mode 100644 index 0000000000..378a823cff --- /dev/null +++ b/src/application/native/NativeMatchingBundleSources.ts @@ -0,0 +1,165 @@ +import { createHash } from "node:crypto"; +import type { z } from "zod"; +import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; +import type { EvidenceBundle } from "../../domain/evidenceBundle.js"; +import { immutableEvidence, type Evidence } from "../../domain/evidence.js"; +import { + EvidenceIntegrityError, + EvidenceReferenceError, +} from "../../domain/evidenceErrors.js"; +import { digestCanonicalValue } from "../../domain/canonicalDigest.js"; +import { retainedEvidenceReferenceSchema } from "../../domain/evidenceReference.js"; +import type { matchNativeFunctionsInputSchema } from "../../contracts/native/functionMatchingContracts.js"; +import { readImmutableEvidenceBundle } from "../EvidenceBundleFiles.js"; +import type { EvidenceLookup } from "../EvidenceInputResolver.js"; +import type { NativeMatchingAdmission } from "./NativeMatchingAdmission.js"; + +type Request = z.output; +interface SourceEntry { + readonly file: number; + readonly links: readonly string[]; +} +interface Options { + readonly input: Request; + readonly paths: readonly string[]; + readonly resources: ArtifactResourceScope; + readonly admission: NativeMatchingAdmission; +} +const reference = (evidence_id: string) => ({ + kind: "retained-evidence" as const, + evidence_id, +}); +// IDs and links have authenticated ASCII digest syntax. Own these short strings +// independently of the discarded bundle's potentially large JSON backing store. +const ownId = (value: string): string => + Buffer.from(value, "ascii").toString("ascii"); + +/** IDs bind semantic payloads; subjects add the display metadata IDs intentionally omit. */ +const recordSnapshot = async ( + bundle: EvidenceBundle, + admission: NativeMatchingAdmission, +): Promise => { + const hash = createHash("sha256"); + for (const record of bundle.records) { + await admission.checkpoint(); + hash + .update(record.evidence_id) + .update(digestCanonicalValue(record.subject)); + } + return hash.digest("hex"); +}; + +/** Index only graph metadata; keep at most the last whole bundle between passes. */ +export const loadNativeMatchingBundleSources = async ({ + input, + paths, + resources, + admission, +}: Options): Promise<{ input: Request; lookup: EvidenceLookup }> => { + const records = new Map(), + index = new Map(); + const rootIds: string[] = []; + for (const value of [ + input.left, + input.right, + ...input.left_observations, + ...input.right_observations, + ]) { + const parsed = retainedEvidenceReferenceSchema.safeParse(value); + if (parsed.success) rootIds.push(parsed.data.evidence_id); + else { + await admission.admit(value); + const record = immutableEvidence(value); + rootIds.push(record.evidence_id); + records.set(record.evidence_id, record); + index.set(record.evidence_id, { file: -1, links: record.evidence_links }); + } + } + const snapshots: string[] = []; + let last: { file: number; bundle: EvidenceBundle } | undefined; + for (let file = 0; file < paths.length; file++) { + last = undefined; + const loaded = await readImmutableEvidenceBundle( + paths[file]!, + admission.signal, + resources, + ); + if (!loaded.ok) throw loaded.error; + snapshots.push(await recordSnapshot(loaded.value, admission)); + for (const record of loaded.value.records) { + await admission.checkpoint( + 256 + + 4 * + (record.evidence_id.length + + record.evidence_links.reduce((sum, id) => sum + id.length, 0)), + ); + if (!index.has(record.evidence_id)) + index.set(ownId(record.evidence_id), { + file, + links: record.evidence_links.map(ownId), + }); + } + last = { file, bundle: loaded.value }; + } + const needed = new Set(rootIds), + files = new Set(); + // Set iteration includes newly discovered nodes. Shared ancestors and cycles + // are visited once, independent of source-file and record ordering. + for (const id of needed) { + await admission.checkpoint(128); + const entry = index.get(id); + if (entry === undefined) + throw new EvidenceReferenceError( + id, + "missing", + "Complete Evidence closure in the supplied bundles or inline inputs", + null, + ); + if (entry.file >= 0) files.add(entry.file); + for (const linked of entry.links) needed.add(linked); + } + const retain = async ( + bundle: EvidenceBundle, + file: number, + ): Promise => { + for (const record of bundle.records) + if ( + needed.has(record.evidence_id) && + index.get(record.evidence_id)?.file === file + ) { + await admission.admit(record); + records.set(record.evidence_id, record); + } + }; + if (last !== undefined && files.has(last.file)) { + await retain(last.bundle, last.file); + files.delete(last.file); + } + last = undefined; + for (const file of files) { + const loaded = await readImmutableEvidenceBundle( + paths[file]!, + admission.signal, + resources, + ); + if (!loaded.ok) throw loaded.error; + if ((await recordSnapshot(loaded.value, admission)) !== snapshots[file]) + throw new EvidenceIntegrityError( + `Evidence bundle records changed during closure lookup: ${paths[file]}`, + ); + await retain(loaded.value, file); + } + const left = rootIds[0]!, + right = rootIds[1]!; + const split = 2 + input.left_observations.length; + return { + input: { + ...input, + left: reference(left), + right: reference(right), + left_observations: rootIds.slice(2, split).map(reference), + right_observations: rootIds.slice(split).map(reference), + }, + lookup: (id) => records.get(id), + }; +}; diff --git a/src/cli.ts b/src/cli.ts index 8c31905328..d2a1a90d32 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,3 +1,4 @@ +import { registerNativeFunctionMatchingCommands } from "./cli/nativeFunctionMatchingCommands.js"; import { registerCaptureCommands } from "./cli/captureCommands.js"; import { registerSignalCommands } from "./cli/signalCommands.js"; import { registerWasmCommands } from "./cli/wasmCommands.js"; @@ -134,6 +135,7 @@ export const createCli = ( registerFirmwareCommands(cli, logger, environment); registerBinaryDiagnosticsCommands(cli, logger, environment); registerAnalysisViewCommands(cli, logger); + registerNativeFunctionMatchingCommands(cli, logger); registerFridaCommands(cli, logger, fridaInstrumentation); registerEvidenceCommands(cli, logger); registerEnvironmentCommands(cli, logger, environment); diff --git a/src/cli/analysisEvidenceDelivery.ts b/src/cli/analysisEvidenceDelivery.ts index 2c42544a4b..76f8b6be19 100644 --- a/src/cli/analysisEvidenceDelivery.ts +++ b/src/cli/analysisEvidenceDelivery.ts @@ -39,6 +39,8 @@ interface CliEvidenceDelivery { readonly signal: AbortSignal; readonly detail: "complete" | "summary"; readonly evidenceOutput: string | undefined; + /** Derived workflows publish their selected source closure with the result. */ + readonly sourceEvidence?: readonly Evidence[]; } export const cliEvidenceDeliveryInputError = ( @@ -115,7 +117,14 @@ export const publishCliAnalysisEvidence = async ( const delivered = await finishAnalysisCommand( async () => { const written = await writeEvidenceBundle( - createEvidenceBundle([complete]), + createEvidenceBundle([ + ...new Map( + [...(input.sourceEvidence ?? []), complete].map((record) => [ + record.evidence_id, + record, + ]), + ).values(), + ]), absoluteCallerPath(evidenceOutput), false, resources, diff --git a/src/cli/nativeFunctionMatchingCommands.ts b/src/cli/nativeFunctionMatchingCommands.ts new file mode 100644 index 0000000000..cc43069bd6 --- /dev/null +++ b/src/cli/nativeFunctionMatchingCommands.ts @@ -0,0 +1,103 @@ +import { z } from "incur"; +import type { Logger } from "pino"; +import { matchNativeFunctionsFromBundles } from "../application/native/NativeFunctionMatchingService.js"; +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { matchNativeFunctionsInputSchema } from "../contracts/native/functionMatchingContracts.js"; +import { CLI_COMMANDS } from "../cliCommandNames.js"; +import { parseCliJsonInput } from "../cliJsonInput.js"; +import { logCliCommand } from "../cliLogging.js"; +import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import type { Evidence } from "../domain/evidence.js"; +import { withCommandCancellation } from "./commandCancellation.js"; +import { + cliEvidenceDeliveryInputError, + publishCliAnalysisEvidence, +} from "./analysisEvidenceDelivery.js"; +import { finishAnalysisCommand } from "./finishAnalysisCommand.js"; +import type { CliInstance } from "./types.js"; + +/** Resolve portable input bundles, then use the same read-only matching service as MCP. */ +export const registerNativeFunctionMatchingCommands = ( + cli: CliInstance, + logger: Logger, +): void => { + cli.command(CLI_COMMANDS.matchNativeFunctions, { + description: + "Match two observed native function inventories without executing targets", + args: z.object({ + input: z + .string() + .describe( + "JSON or JSON file: left/right inventories, optional observations and detail", + ), + }), + options: z.object({ + source_bundle: z + .array(z.string()) + .default([]) + .describe( + "Canonical source Evidence bundle for retained references; repeat for each input", + ), + evidence_output: z + .string() + .min(1) + .optional() + .describe( + "Save full matching Evidence and selected sources; required for default summary; never overwrites", + ), + }), + run: ({ args, options }) => + withCommandCancellation((signal) => + logCliCommand(logger, CLI_COMMANDS.matchNativeFunctions, async () => { + const input = await parseCliJsonInput( + args.input, + "match_native_functions", + signal, + ); + if (!input.ok) return input.error; + const parsed = matchNativeFunctionsInputSchema.safeParse(input.value); + if (!parsed.success) + return projectAnalysisError( + analysisInputErrorFromIssues( + "match_native_functions", + parsed.error.issues, + input.value, + ), + ); + const delivery = { + providerId: "rea-function-comparison", + operation: "match_native_functions", + signal, + detail: parsed.data.detail, + evidenceOutput: options.evidence_output, + }; + const invalid = cliEvidenceDeliveryInputError(delivery); + if (invalid !== undefined) return projectAnalysisError(invalid); + const resources = new ArtifactResourceScope(); + let sources: readonly Evidence[] = []; + const complete = await finishAnalysisCommand( + async () => { + const result = await matchNativeFunctionsFromBundles( + parsed.data, + options.source_bundle, + resources, + { signal }, + ); + if (!result.ok) return result; + sources = result.value.sources; + return { ok: true as const, value: result.value.evidence }; + }, + resources, + delivery.providerId, + delivery.operation, + signal, + ); + return publishCliAnalysisEvidence(complete, { + ...delivery, + sourceEvidence: sources, + }); + }), + ), + }); +}; diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index 3df7981e3c..a400264d3d 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -4,6 +4,7 @@ export const CLI_COMMANDS = Object.freeze({ inspectPacketCapture: "inspect-packet-capture", inspectSignalDecoders: "inspect-signal-decoders", decodeSignalRecording: "decode-signal-recording", + matchNativeFunctions: "match-native-functions", compareWasmInterfaces: "compare-wasm-interfaces", inspectWasmArtifact: "inspect-wasm-artifact", inspectSqliteDatabase: "inspect-sqlite-database", diff --git a/src/contracts/evidenceInputContracts.ts b/src/contracts/evidenceInputContracts.ts index 02b2bd0e64..0419ba5d58 100644 --- a/src/contracts/evidenceInputContracts.ts +++ b/src/contracts/evidenceInputContracts.ts @@ -1,18 +1,5 @@ -import { z } from "zod"; -import { evidenceSchema } from "../domain/evidence.js"; -import { prefixedDigestSchema } from "../domain/digests.js"; - -/** Exact reference to Evidence already retained by the current session. */ -export const retainedEvidenceReferenceSchema = z.strictObject({ - kind: z.literal("retained-evidence"), - evidence_id: prefixedDigestSchema("ev"), -}); - -/** Portable inline Evidence or an exact, session-scoped reference. */ -export const evidenceInputSchema = z.union([ - evidenceSchema, - retainedEvidenceReferenceSchema, -]); - -/** Evidence input after parsing the advertised boundary contract. */ -export type EvidenceInput = z.output; +/** Canonical Evidence selection contracts are owned by the domain. */ +export { + evidenceInputSchema, + type EvidenceInput, +} from "../domain/evidenceReference.js"; diff --git a/src/contracts/native/functionMatchingContracts.ts b/src/contracts/native/functionMatchingContracts.ts new file mode 100644 index 0000000000..e8bc5c38c8 --- /dev/null +++ b/src/contracts/native/functionMatchingContracts.ts @@ -0,0 +1,30 @@ +import { z } from "zod"; +import { evidenceInputSchema } from "../evidenceInputContracts.js"; + +// Borrow inline payloads until application admission; advertise the exact public +// Evidence/reference union without a pre-admission recursive SDK clone. +const input = z.unknown().nonoptional(); +input._zod.processJSONSchema = evidenceInputSchema._zod.processJSONSchema; + +/** Inventories stay portable; optional observations avoid all-function decompilation. */ +export const matchNativeFunctionsInputSchema = z.strictObject({ + left: input.describe( + "Left list_procedures Evidence or exact retained reference", + ), + right: input.describe( + "Right list_procedures Evidence or exact retained reference", + ), + left_observations: z + .array(input) + .default([]) + .describe( + "Optional same-context list_names, procedure_info, read_bytes or analyze_function Evidence", + ), + right_observations: z + .array(input) + .default([]) + .describe( + "Optional same-context list_names, procedure_info, read_bytes or analyze_function Evidence", + ), + detail: z.enum(["summary", "complete"]).default("summary"), +}); diff --git a/src/contracts/sessionToolContracts.ts b/src/contracts/sessionToolContracts.ts index 215345c792..bbd12c5fe6 100644 --- a/src/contracts/sessionToolContracts.ts +++ b/src/contracts/sessionToolContracts.ts @@ -1,3 +1,4 @@ +import { matchNativeFunctionsInputSchema } from "./native/functionMatchingContracts.js"; import { z } from "zod"; import { isAbsolute } from "node:path"; @@ -107,9 +108,14 @@ export const SESSION_TOOL_CONTRACTS = [ "Compare complete artifact inventories by logical occurrence path, content identity, metadata, and graph relations. Pass the inventory Evidence nested in each inspect_artifact result as left and right, or pass its inspect_analysis_view summary while the complete parent Evidence remains retained in this session. Summary facts are not treated as a complete inventory; comparison resolves the exact retained parent. Every delta cites both complete inputs, and gaps yield truncated or unknown, never equivalence. Returns every change inline.", artifactComparisonInputSchema, ), + session( + "match_native_functions", + "Match two retained list_procedures inventories by unique exact reported symbols, then unique identical observed function bytes. Optional same-context list_names/procedure_info/read_bytes/analyze_function Evidence supplies symbol origin, exact bodies, bytes and dossier links as supplied static facts. Optional compatible CFG/call/string fingerprints from supplied dossiers propose unresolved groups only. Duplicate candidates remain unresolved groups; only_left/right mean unpaired within supplied observations, never proof of addition/removal. Default summary retains complete Evidence for filtered, selected-column, paged function-matches views. Complete detail returns every group. Existing dossier pairs provide direct retained-reference compare_functions inputs.", + matchNativeFunctionsInputSchema, + ), session( "compare_functions", - "Compare two explicit sets of analyze_function Evidence across identity, exact provider text, calls, references, strings, and address-normalized CFG topology. Missing or provider-incompatible facets remain truncated or unknown; every conclusion cites both Evidence sets.", + "Compare two explicit analyze_function Evidence records or exact retained session references across identity, exact provider text, calls, references, strings, and address-normalized CFG topology. Missing or provider-incompatible facets remain truncated or unknown; every conclusion cites both Evidence sets.", functionComparisonInputSchema, ), session( diff --git a/src/contracts/toolContractExamples.ts b/src/contracts/toolContractExamples.ts index dc6567e39f..e2f96d1583 100644 --- a/src/contracts/toolContractExamples.ts +++ b/src/contracts/toolContractExamples.ts @@ -74,6 +74,10 @@ export const TOOL_EXAMPLE_OVERRIDES: Readonly< left: ARTIFACT_COMPARISON_EXAMPLE.left, right: ARTIFACT_COMPARISON_EXAMPLE.right, }, + match_native_functions: { + left: { kind: "retained-evidence", evidence_id: `ev_${"1".repeat(64)}` }, + right: { kind: "retained-evidence", evidence_id: `ev_${"2".repeat(64)}` }, + }, compare_functions: { left: FUNCTION_COMPARISON_EXAMPLE.left, right: FUNCTION_COMPARISON_EXAMPLE.right, diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index d0e87c5c15..7bdb7fdedb 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -705,6 +705,7 @@ export const TOOL_EFFECTS: Readonly> = { compare_artifacts: sessionEvidence, compare_wasm_interfaces: sessionEvidence, compare_functions: sessionEvidence, + match_native_functions: sessionEvidence, compare_bundles: effects({ mutatesSession: true }), find_changed_behavior: sessionEvidence, build_call_path: sessionEvidence, diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index 0073793f45..c1c1ba5413 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -1,3 +1,6 @@ +import { nativeFunctionMatchingSchema } from "../domain/native/functionMatching.js"; +import { nativeFunctionMatchingSummaryViewResultSchema } from "../domain/analysisView/analysisView.js"; +import { nativeByteReadSchema } from "../domain/native/nativeByteRead.js"; import { analysisDetailOutputSchemaOf } from "./analysisDetail.js"; import { nativeInventorySummaryViewResultSchema } from "../domain/analysisView/analysisView.js"; import { nativeInventoryRowsSchemas } from "../domain/native/nativeInventory.js"; @@ -158,15 +161,7 @@ export const officialOutputSchemas: Readonly> = { procedure_callers: evidenceResultOf(addressList), procedure_info: procedureInfoOutput, read_function_instructions: evidenceResultOf(functionInstructionWindowSchema), - read_bytes: evidenceResultOf( - z.object({ - address: z.string(), - requested_bytes: z.number().int().min(1), - returned_bytes: z.number().int().min(0), - bytes_hex: z.string().regex(/^(?:[a-f0-9]{2})*$/u), - complete: z.boolean(), - }), - ), + read_bytes: evidenceResultOf(nativeByteReadSchema), address_to_file_offset: evidenceResultOf( z.object({ address: z.string(), @@ -595,6 +590,12 @@ export const sessionOutputSchemas = { compare_process_captures: evidenceResultOf(processCaptureComparisonSchema), compare_artifacts: evidenceResultOf(artifactComparisonResultSchema), compare_functions: evidenceResultOf(functionComparisonResultSchema), + match_native_functions: evidenceResultOf( + z.union([ + nativeFunctionMatchingSchema, + nativeFunctionMatchingSummaryViewResultSchema, + ]), + ), compare_bundles: evidenceResultOf(bundleComparisonResultSchema), find_changed_behavior: evidenceResultOf(changedBehaviorResultSchema), build_call_path: evidenceResultOf(callPathResultSchema), diff --git a/src/contracts/toolOutputSchemaPrimitives.ts b/src/contracts/toolOutputSchemaPrimitives.ts index 6221d57cb4..058388b201 100644 --- a/src/contracts/toolOutputSchemaPrimitives.ts +++ b/src/contracts/toolOutputSchemaPrimitives.ts @@ -1,11 +1,9 @@ +import { procedureInfoSchema } from "../domain/native/procedureInfo.js"; import { z } from "zod"; import { - procedureClassificationSchema, procedureIdentitySchema, - localVariableSchema, functionDossierSchema, - functionBodySchema, } from "../domain/hopperValues.js"; import { analysisProfileSchema } from "../domain/analysisProfile.js"; import { @@ -598,18 +596,7 @@ export const segmentOutput = evidenceResultOf( z.array(memoryRegionOutput.extend({ sections: z.array(memoryRegionOutput) })), ); -export const procedureInfoOutput = evidenceResultOf( - z.object({ - name: z.string(), - entrypoint: z.string(), - basicblock_count: z.number().int().min(0), - length: z.number().min(0), - signature: nullableText, - locals: z.array(localVariableSchema), - classification: procedureClassificationSchema.nullable(), - body: functionBodySchema, - }), -); +export const procedureInfoOutput = evidenceResultOf(procedureInfoSchema); export const symbolDiscoveryOutput = (property: "classes" | "protocols") => evidenceResultOf( diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 2ed46a7a28..ba9b70da48 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -1,3 +1,9 @@ +import { + projectNativeFunctionMatchingView, + nativeFunctionMatchingViewRequestSchema, + nativeFunctionMatchingViewItemsSchema, + nativeFunctionMatchingViewContextSchema, +} from "./nativeFunctionMatchingView.js"; import { protobufViewRequestSchema, protobufViewItemsSchema, @@ -243,6 +249,7 @@ const nativeViewSchema = z.strictObject({ /** Caller-selected projection of already completed analysis Evidence. */ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ + nativeFunctionMatchingViewRequestSchema, summaryViewSchema, desktopViewRequestSchema, facetViewSchema, @@ -377,6 +384,7 @@ const viewResultBase = { "inspect_dart_aot", "inspect_il2cpp_metadata", ...NATIVE_INVENTORY_OPERATIONS, + "match_native_functions", ]), parent_digest: digestSchema, view_digest: digestSchema, @@ -524,8 +532,25 @@ export const nativeInventorySummaryViewResultSchema = z.strictObject({ summary: nativeInventorySummarySchema, }); +/** Compact native matching discovery retains the complete matching report. */ +export const nativeFunctionMatchingSummaryViewResultSchema = z.strictObject({ + ...summaryViewResultBase, + artifact: z.null(), + parent_operation: z.literal("match_native_functions"), + summary: nativeFunctionMatchingViewContextSchema, +}); + /** Projected facts for one selected view of retained analysis Evidence. */ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + ...viewResultBase, + ...nativeFunctionMatchingViewContextSchema.shape, + kind: z.literal("function-matches"), + artifact: z.null(), + view: nativeFunctionMatchingViewRequestSchema, + items: nativeFunctionMatchingViewItemsSchema, + parent_operation: z.literal("match_native_functions"), + }), z.strictObject({ ...viewResultBase, ...apiSurfaceFields, @@ -574,6 +599,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ dartAotSummaryViewResultSchema, il2cppMetadataSummaryViewResultSchema, nativeInventorySummaryViewResultSchema, + nativeFunctionMatchingSummaryViewResultSchema, ]), z.strictObject({ ...viewResultBase, @@ -844,6 +870,7 @@ const retainedViewProjectors = new Map([ ["inspect_android_package", projectAndroidPackageView], ["inspect_dart_aot", projectDartAotView], ["inspect_il2cpp_metadata", projectIl2cppMetadataView], + ["match_native_functions", projectNativeFunctionMatchingView], ["inspect_artifact", projectArtifactInventoryView], ["inventory_artifact", projectArtifactInventoryView], ]); diff --git a/src/domain/analysisView/nativeFunctionMatchingView.ts b/src/domain/analysisView/nativeFunctionMatchingView.ts new file mode 100644 index 0000000000..fc0b11e03e --- /dev/null +++ b/src/domain/analysisView/nativeFunctionMatchingView.ts @@ -0,0 +1,311 @@ +import { z } from "zod"; +import { + nativeFunctionMatchingSchema, + nativeFunctionMatchingContextSchema, + nativeFunctionMatchSchema, + nativeFunctionMatchMemberSchema, + nativeMatchStateSchema, + type NativeFunctionMatching, +} from "../native/functionMatching.js"; +import { isImmutableJsonSnapshot } from "../immutableJson.js"; +import { parseAnalysisViewResult } from "./analysisViewParent.js"; +import { analysisViewInputError } from "./analysisViewErrors.js"; +import { + boundedViewMessages, + completeWithinViewCoverage, + pageViewCoverage, +} from "./analysisViewCoverage.js"; +import { err, ok, type Result } from "../result.js"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import type { + AnalysisViewParent, + AnalysisViewRequest, + UnsignedAnalysisView, +} from "./analysisView.js"; +import type { JsonValue } from "../jsonValue.js"; + +const matchRow = nativeFunctionMatchSchema.extend({ + left_count: z.number().int().nonnegative(), + right_count: z.number().int().nonnegative(), +}); +const functionRow = nativeFunctionMatchMemberSchema.extend({ + match_id: nativeFunctionMatchSchema.shape.match_id, + state: nativeMatchStateSchema, + side: z.enum(["left", "right"]), +}); +const page = { + kind: z.literal("function-matches"), + states: z.array(nativeMatchStateSchema).optional(), + match_ids: z.array(z.string()).optional(), + function_ids: z.array(z.string()).optional(), + offset: z.number().int().nonnegative().default(0), + limit: z.number().int().positive().default(64), +}; +export const nativeFunctionMatchingViewRequestSchema = z.discriminatedUnion( + "collection", + [ + z.strictObject({ + ...page, + collection: z.literal("matches"), + fields: z.array(matchRow.keyof()).optional(), + }), + z.strictObject({ + ...page, + collection: z.literal("functions"), + sides: z.array(z.enum(["left", "right"])).optional(), + names: z.array(z.string()).optional(), + fields: z.array(functionRow.keyof()).optional(), + }), + ], +); +export const nativeFunctionMatchingViewItemsSchema = z.union([ + z.array( + matchRow.partial().required({ + match_id: true, + state: true, + left_count: true, + right_count: true, + }), + ), + z.array( + functionRow.partial().required({ + function_id: true, + match_id: true, + state: true, + side: true, + inventory_evidence_id: true, + source_index: true, + address: true, + evidence_links: true, + }), + ), +]); +export const nativeFunctionMatchingViewContextSchema = + nativeFunctionMatchingContextSchema; +interface FunctionRow { + readonly member: NativeFunctionMatching["matches"][number]["left"][number]; + readonly match_id: string; + readonly state: NativeFunctionMatching["matches"][number]["state"]; + readonly side: "left" | "right"; +} +interface Checked { + readonly report: NativeFunctionMatching; + functions?: readonly FunctionRow[]; + recent?: { readonly key: string; readonly rows: readonly number[] }; +} +const checked = new WeakMap(); + +type Request = z.output; +const checkedReport = ( + parent: AnalysisViewParent, +): Result => { + const source = parent.normalizedResult; + const reusable = + typeof source === "object" && + source !== null && + parent.sealedResult === true && + isImmutableJsonSnapshot(source); + const cached = reusable ? checked.get(source) : undefined; + if (cached !== undefined) return ok(cached); + const parsed = parseAnalysisViewResult(nativeFunctionMatchingSchema, parent); + if (!parsed.ok) return parsed; + const entry: Checked = { + report: reusable ? (source as NativeFunctionMatching) : parsed.value, + }; + if (reusable) checked.set(source, entry); + return ok(entry); +}; +const functionRows = (entry: Checked): readonly FunctionRow[] => { + if (entry.functions !== undefined) return entry.functions; + const functions: FunctionRow[] = []; + for (const match of entry.report.matches) + for (const side of ["left", "right"] as const) + for (const member of match[side]) + functions.push({ + member, + match_id: match.match_id, + state: match.state, + side, + }); + entry.functions = functions; + return functions; +}; +const sharedContext = ( + parent: AnalysisViewParent, + report: NativeFunctionMatching, +) => ({ + parent_evidence_id: parent.evidenceId, + parent_operation: "match_native_functions" as const, + parent_digest: parent.evidenceId.slice(3), + artifact: null, + limitations: boundedViewMessages( + [...parent.limitations, ...report.limitations], + parent.evidenceId, + "limitations", + ), + unknowns: [ + "Unpaired observations and page exhaustion do not establish binary-wide function absence or behavioral equivalence.", + ], +}); +const selected = (values: readonly string[] | undefined) => + values === undefined ? undefined : new Set(values); +const filterRows = (entry: Checked, view: Request): readonly number[] => { + const key = JSON.stringify([ + view.collection, + view.states, + view.match_ids, + view.function_ids, + view.collection === "functions" ? view.sides : null, + view.collection === "functions" ? view.names : null, + ]); + if (entry.recent?.key === key) return entry.recent.rows; + const states = selected(view.states), + ids = selected(view.match_ids), + functions = selected(view.function_ids), + sides = selected(view.collection === "functions" ? view.sides : undefined), + names = selected(view.collection === "functions" ? view.names : undefined); + const matchesIdentity = (row: { state: string; match_id: string }): boolean => + (states === undefined || states.has(row.state)) && + (ids === undefined || ids.has(row.match_id)); + const rows: number[] = []; + if (view.collection === "matches") + entry.report.matches.forEach((row, index) => { + if ( + matchesIdentity(row) && + (functions === undefined || + [...row.left, ...row.right].some((member) => + functions.has(member.function_id), + )) + ) + rows.push(index); + }); + else + functionRows(entry).forEach((row, index) => { + if ( + matchesIdentity(row) && + (functions === undefined || functions.has(row.member.function_id)) && + (sides === undefined || sides.has(row.side)) && + (names === undefined || names.has(row.member.name)) + ) + rows.push(index); + }); + // One recent selector and at most one row index per source occurrence. + const count = entry.report.left.functions + entry.report.right.functions; + if (key.length <= Math.max(256, count * 8)) entry.recent = { key, rows }; + return rows; +}; +const selectedItems = ( + entry: Checked, + view: Request, + rows: readonly number[], +) => { + const indices = rows.slice(view.offset, view.offset + view.limit); + const defaults = + view.collection === "matches" + ? [ + "match_id", + "state", + "basis", + "reason", + "left_count", + "right_count", + "compare_functions_input", + ] + : [ + "function_id", + "match_id", + "state", + "side", + "inventory_evidence_id", + "source_index", + "address", + "name", + "body_sha256", + "structural_candidate_sha256", + "body_bytes", + "dossier_evidence_id", + "evidence_links", + ]; + const fields = new Set(view.fields ?? defaults); + const columns = (row: Readonly>) => + Object.fromEntries([...fields].map((field) => [field, row[field]!])); + if (view.collection === "matches") + return indices.map((index) => { + const row = entry.report.matches[index]!, + left_count = row.left.length, + right_count = row.right.length; + return { + ...columns({ ...row, left_count, right_count }), + match_id: row.match_id, + state: row.state, + left_count, + right_count, + }; + }); + const functions = functionRows(entry); + return indices.map((index) => { + const row = functions[index]!, + member = row.member; + return { + ...columns({ + ...member, + match_id: row.match_id, + state: row.state, + side: row.side, + }), + function_id: member.function_id, + match_id: row.match_id, + state: row.state, + side: row.side, + inventory_evidence_id: member.inventory_evidence_id, + source_index: member.source_index, + address: member.address, + evidence_links: member.evidence_links, + }; + }); +}; + +/** Query immutable matching facts without copying full alias groups into selected pages. */ +export const projectNativeFunctionMatchingView = ( + parent: AnalysisViewParent, + view: AnalysisViewRequest, +): Result => { + if (view.kind !== "summary" && view.kind !== "function-matches") + return err( + analysisViewInputError([ + { + path: ["view"], + reason: "invalid_value", + message: + "Native matching supports summary and function-matches pages", + }, + ]), + ); + const parsed = checkedReport(parent); + if (!parsed.ok) return parsed; + const entry = parsed.value; + const { + matches: _matches, + limitations: _limitations, + ...context + } = entry.report; + const shared = sharedContext(parent, entry.report); + if (view.kind === "summary") + return ok({ + kind: "summary", + view, + ...shared, + summary: context, + coverage: completeWithinViewCoverage(1, 1), + }); + const rows = filterRows(entry, view), + items = selectedItems(entry, view, rows); + return ok({ + kind: "function-matches", + view, + ...shared, + ...context, + items, + coverage: pageViewCoverage(view.offset, items.length, rows.length), + }); +}; diff --git a/src/domain/evidenceReference.ts b/src/domain/evidenceReference.ts new file mode 100644 index 0000000000..61c24c1fb4 --- /dev/null +++ b/src/domain/evidenceReference.ts @@ -0,0 +1,17 @@ +import { z } from "zod"; +import { prefixedDigestSchema } from "./digests.js"; + +/** Exact reference to Evidence already retained by the current session. */ +export const retainedEvidenceReferenceSchema = z.strictObject({ + kind: z.literal("retained-evidence"), + evidence_id: prefixedDigestSchema("ev"), +}); + +import { evidenceSchema } from "./evidence.js"; + +/** Portable observation or exact retained identity; semantic authentication follows admission. */ +export const evidenceInputSchema = z.union([ + evidenceSchema, + retainedEvidenceReferenceSchema, +]); +export type EvidenceInput = z.output; diff --git a/src/domain/functionComparisonSchemas.ts b/src/domain/functionComparisonSchemas.ts index 844bdc9b9d..9c340028ee 100644 --- a/src/domain/functionComparisonSchemas.ts +++ b/src/domain/functionComparisonSchemas.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import { overallStatus } from "./functionComparisonResults.js"; -import { evidenceSchema } from "./evidence.js"; +import { evidenceInputSchema } from "./evidenceReference.js"; import { digestSchema } from "./../domain/digests.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; @@ -19,8 +19,8 @@ const evidenceIdSchema = prefixedDigestSchema("ev"); /** Inputs for explicit function-to-function comparison. */ export const functionComparisonInputSchema = z.strictObject({ - left: evidenceSchema, - right: evidenceSchema, + left: evidenceInputSchema, + right: evidenceInputSchema, }); const textDeltaSchema = z.object({ diff --git a/src/domain/native/functionByteCoverage.ts b/src/domain/native/functionByteCoverage.ts new file mode 100644 index 0000000000..b3889fc636 --- /dev/null +++ b/src/domain/native/functionByteCoverage.ts @@ -0,0 +1,193 @@ +import { createHash } from "node:crypto"; +import type { z } from "zod"; +import type { functionBodySchema } from "../hopperValues.js"; +import type { nativeByteReadSchema } from "./nativeByteRead.js"; +import { addressCoordinate } from "./addressCoordinate.js"; +import { parseNativeAddress } from "./nativeAddress.js"; + +export type NativeMatchingCheckpoint = (bytes?: number) => Promise; +interface Interval { + start: bigint; + end: bigint; +} +interface Piece extends Interval { + readonly hex: string; + readonly links: readonly string[]; +} +interface Fingerprint { + readonly sha256: string; + readonly key: string; + readonly links: readonly string[]; +} +const slice = (piece: Piece, start: bigint, end: bigint): Piece => ({ + ...piece, + start, + end, + hex: + start === piece.start && end === piece.end + ? piece.hex + : piece.hex.slice( + Number(start - piece.start) * 2, + Number(end - piece.start) * 2, + ), +}); +const firstOverlapping = (pieces: readonly Piece[], start: bigint): number => { + let lower = 0, + upper = pieces.length; + while (lower < upper) { + const middle = (lower + upper) >>> 1; + if (pieces[middle]!.end <= start) lower = middle + 1; + else upper = middle; + } + return lower; +}; +const mergeIntervals = (intervals: Interval[]): Interval[] => { + intervals.sort((a, b) => + a.start < b.start ? -1 : a.start > b.start ? 1 : 0, + ); + const merged: Interval[] = []; + for (const interval of intervals) { + const previous = merged.at(-1); + if (previous !== undefined && interval.start <= previous.end) { + if (interval.end > previous.end) previous.end = interval.end; + } else merged.push({ ...interval }); + } + return merged; +}; +const excludeConflicts = ( + pieces: readonly Piece[], + conflicts: readonly Interval[], +): Piece[] => { + const safe: Piece[] = []; + let first = 0; + for (const piece of pieces) { + while (first < conflicts.length && conflicts[first]!.end <= piece.start) + first++; + let cursor = piece.start; + for ( + let index = first; + index < conflicts.length && conflicts[index]!.start < piece.end; + index++ + ) { + const conflict = conflicts[index]!; + if (conflict.start > cursor) + safe.push(slice(piece, cursor, conflict.start)); + if (conflict.end > cursor) cursor = conflict.end; + if (cursor >= piece.end) break; + } + if (cursor < piece.end) safe.push(slice(piece, cursor, piece.end)); + } + return safe; +}; + +/** Own exact observed byte intervals, their conflicts and whole-body hashing. */ +export class FunctionByteCoverage { + readonly #pieces = new Map(); + #prepared = false; + constructor(readonly checkpoint: NativeMatchingCheckpoint) {} + async add( + read: z.output, + evidenceId: string, + ): Promise { + if (this.#prepared) + throw new TypeError( + "Cannot append byte observations after fingerprinting", + ); + if (read.returned_bytes === 0) return; + const coordinate = addressCoordinate(parseNativeAddress(read.address))!; + let group = this.#pieces.get(coordinate.space); + if (group === undefined) this.#pieces.set(coordinate.space, (group = [])); + await this.checkpoint(192 + read.bytes_hex.length * 2); + group.push({ + start: coordinate.offset, + end: coordinate.offset + BigInt(read.returned_bytes), + hex: read.bytes_hex, + links: [evidenceId], + }); + } + async #conflict(left: Piece, right: Piece): Promise { + const start = left.start > right.start ? left.start : right.start; + const end = left.end < right.end ? left.end : right.end; + for (let cursor = start; cursor < end;) { + const next = cursor + 65536n < end ? cursor + 65536n : end; + await this.checkpoint(); + if (slice(left, cursor, next).hex !== slice(right, cursor, next).hex) + return { start, end }; + cursor = next; + } + return undefined; + } + async #prepareSpace(source: Piece[]): Promise { + source.sort((a, b) => + a.start < b.start + ? -1 + : a.start > b.start + ? 1 + : a.end > b.end + ? -1 + : a.end < b.end + ? 1 + : 0, + ); + const admitted: Piece[] = [], + conflicts: Interval[] = []; + for (const incoming of source) { + await this.checkpoint(); + for ( + let index = firstOverlapping(admitted, incoming.start); + index < admitted.length && admitted[index]!.start < incoming.end; + index++ + ) { + const conflict = await this.#conflict(admitted[index]!, incoming); + if (conflict !== undefined) conflicts.push(conflict); + } + const end = admitted.at(-1)?.end; + if (end === undefined || incoming.start >= end) admitted.push(incoming); + else if (incoming.end > end) + admitted.push(slice(incoming, end, incoming.end)); + } + return excludeConflicts(admitted, mergeIntervals(conflicts)); + } + async fingerprint( + body: z.output | undefined, + ): Promise { + if (!body?.available || !body.contains_entry || body.total_bytes === 0) + return null; + if (!this.#prepared) { + for (const [space, pieces] of this.#pieces) + this.#pieces.set(space, await this.#prepareSpace(pieces)); + this.#prepared = true; + } + const hash = createHash("sha256"), + lengths: string[] = [], + links = new Set(); + for (const range of body.ranges) { + const start = addressCoordinate(parseNativeAddress(range.start))!, + end = addressCoordinate(parseNativeAddress(range.end))!; + const pieces = this.#pieces.get(start.space) ?? []; + lengths.push(String(end.offset - start.offset + 1n)); + let index = firstOverlapping(pieces, start.offset), + cursor = start.offset; + while (cursor <= end.offset) { + const piece = pieces[index++]; + if (piece === undefined || piece.start > cursor || piece.end <= cursor) + return null; + const stop = piece.end < end.offset + 1n ? piece.end : end.offset + 1n; + for (let chunk = cursor; chunk < stop;) { + const next = chunk + 65536n < stop ? chunk + 65536n : stop; + await this.checkpoint(); + hash.update(Buffer.from(slice(piece, chunk, next).hex, "hex")); + chunk = next; + } + for (const link of piece.links) links.add(link); + cursor = stop; + } + } + const sha256 = hash.digest("hex"); + return { + sha256, + key: JSON.stringify([lengths, sha256]), + links: [...links], + }; + } +} diff --git a/src/domain/native/functionInventory.ts b/src/domain/native/functionInventory.ts new file mode 100644 index 0000000000..a89e09802b --- /dev/null +++ b/src/domain/native/functionInventory.ts @@ -0,0 +1,290 @@ +import { functionStructuralCandidate } from "./functionStructuralCandidate.js"; +import type { z } from "zod"; +import type { Evidence } from "../evidence.js"; +import { EvidenceIntegrityError } from "../evidenceErrors.js"; +import { digestCanonicalValue } from "../canonicalDigest.js"; +import { + functionBodySchema, + functionBodyEntryAgrees, + functionDossierSchema, +} from "../hopperValues.js"; +import { isAutoName } from "../functionComparisonNormalization.js"; +import { + addressedValueSchema, + nativeInventoryRowsSchemas, +} from "./nativeInventory.js"; +import { parseNativeAddress } from "./nativeAddress.js"; +import { nativeByteReadSchema } from "./nativeByteRead.js"; +import { procedureInfoSchema } from "./procedureInfo.js"; +import type { NativeFunctionMatchMember } from "./functionMatching.js"; +import { + FunctionByteCoverage, + type NativeMatchingCheckpoint, +} from "./functionByteCoverage.js"; + +type Body = z.infer; +type AddressedValue = z.infer; +export interface MatchingFunction { + readonly member: NativeFunctionMatchMember; + readonly symbol: string | null; + readonly fingerprint: string | null; +} +interface Detail { + body?: Body; + dossier?: string | null; + candidate?: string | null; + conflict: boolean; + readonly links: string[]; +} +interface Symbols { + readonly values: AddressedValue[]; + readonly links: Set; +} +interface Facts { + readonly details: Map; + readonly symbols: Map; + readonly bytes: FunctionByteCoverage; +} + +/** Supplementary observations must belong to this exact provider/target/profile/document. */ +const requireContext = (inventory: Evidence, observation: Evidence): void => { + if ( + observation.predicate_type !== "rea.analysis" || + digestCanonicalValue(observation.subject) !== + digestCanonicalValue(inventory.subject) || + digestCanonicalValue(observation.provider) !== + digestCanonicalValue(inventory.provider) || + digestCanonicalValue(observation.analysis_profile) !== + digestCanonicalValue(inventory.analysis_profile) || + (observation.parameters.document ?? null) !== + (inventory.parameters.document ?? null) + ) + throw new EvidenceIntegrityError( + "Function observation does not belong to its selected inventory context", + ); +}; +const addBody = (observation: Evidence, details: Facts["details"]): void => { + const dossier = + observation.operation === "analyze_function" + ? functionDossierSchema.parse(observation.normalized_result) + : undefined; + const procedure = + dossier?.procedure ?? + (() => { + const info = procedureInfoSchema.parse(observation.normalized_result); + return { address: info.entrypoint, body: info.body }; + })(); + const address = parseNativeAddress(procedure.address); + if (!functionBodyEntryAgrees(procedure.body, address)) + throw new EvidenceIntegrityError( + "Procedure body entry membership contradicts its address", + ); + let detail = details.get(address); + if (detail === undefined) + details.set(address, (detail = { conflict: false, links: [] })); + if ( + detail.body !== undefined && + digestCanonicalValue(detail.body) !== digestCanonicalValue(procedure.body) + ) + detail.conflict = true; + detail.body = procedure.body; + if (observation.operation === "analyze_function") { + const candidate = functionStructuralCandidate(observation, dossier!); + if (detail.dossier === undefined) { + detail.dossier = observation.evidence_id; + detail.candidate = candidate; + } else { + // Request identity and dossier selection are independent of body agreement. + if (detail.dossier !== observation.evidence_id) detail.dossier = null; + if (detail.candidate !== candidate) detail.candidate = null; + } + } + detail.links.push(observation.evidence_id); +}; +const addNames = async ( + observation: Evidence, + symbols: Facts["symbols"], + checkpoint: NativeMatchingCheckpoint, +): Promise => { + for (const symbol of nativeInventoryRowsSchemas.names.parse( + observation.normalized_result, + )) { + await checkpoint(128 + symbol.value.length * 2); + const address = parseNativeAddress(symbol.address); + let group = symbols.get(address); + if (group === undefined) + symbols.set(address, (group = { values: [], links: new Set() })); + group.values.push(symbol); + group.links.add(observation.evidence_id); + } +}; +const addBytes = async ( + observation: Evidence, + bytes: FunctionByteCoverage, +): Promise => { + const read = nativeByteReadSchema.parse(observation.normalized_result); + if ( + (typeof observation.parameters.address === "string" && + parseNativeAddress(observation.parameters.address) !== + parseNativeAddress(read.address)) || + (typeof observation.parameters.length === "number" && + observation.parameters.length !== read.requested_bytes) + ) + throw new EvidenceIntegrityError( + "Read-byte coordinates contradict the selected request", + ); + await bytes.add(read, observation.evidence_id); +}; +const reportedSymbol = ( + row: AddressedValue, + known: Symbols | undefined, +): { + readonly value: string | null; + readonly origin: "known" | "unknown" | "conflicting"; +} => { + const names = [ + ...(row.symbol === undefined ? [] : [row]), + ...(known?.values.filter((symbol) => symbol.value === row.value) ?? []), + ]; + const generated = names.some( + (symbol) => + symbol.symbol?.dynamic === true || symbol.symbol?.source === "default", + ); + const named = names.some( + (symbol) => + symbol.symbol?.dynamic === false && symbol.symbol.source !== "default", + ); + return { + value: + row.value.length === 0 || generated || (!named && isAutoName(row.value)) + ? null + : row.value, + origin: + generated && named + ? "conflicting" + : names.some((symbol) => symbol.symbol !== undefined) + ? "known" + : "unknown", + }; +}; +const functionFact = async ( + { + inventory, + row, + sourceIndex, + }: { inventory: Evidence; row: AddressedValue; sourceIndex: number }, + facts: Facts, + checkpoint: NativeMatchingCheckpoint, +): Promise => { + const address = parseNativeAddress(row.address), + detail = facts.details.get(address), + symbols = facts.symbols.get(address); + // Bound expansion before creating per-occurrence sets, arrays and subsequent + // schema/Evidence serialization copies. Same-address aliases repeat these links. + for (const link of detail?.links ?? []) + await checkpoint(256 + 8 * link.length); + for (const link of symbols?.links ?? []) + await checkpoint(256 + 8 * link.length); + await checkpoint(256 + 8 * inventory.evidence_id.length); + const links = new Set([inventory.evidence_id]); + for (const link of detail?.links ?? []) links.add(link); + for (const link of symbols?.links ?? []) links.add(link); + const body = detail?.conflict ? undefined : detail?.body; + const fingerprint = await facts.bytes.fingerprint(body); + for (const link of fingerprint?.links ?? []) { + await checkpoint(256 + 8 * link.length); + links.add(link); + } + const limitations: string[] = []; + const symbol = reportedSymbol(row, symbols); + if (fingerprint === null) + limitations.push( + detail?.conflict + ? "Conflicting procedure observations; body and dossier remain unknown." + : "No complete unambiguous bytes for the observed function body.", + ); + if (symbol.origin === "unknown") + limitations.push( + "Symbol origin unavailable; exact-name matching uses the reported name and the existing automatic-name exclusion rule.", + ); + if (symbol.origin === "conflicting") + limitations.push( + "Conflicting generated and named symbol-origin observations; exact-name matching is unavailable. Independent complete body bytes remain eligible.", + ); + if (detail?.dossier === null) + limitations.push( + "Multiple supplied dossiers describe this address; no dossier is selected automatically. Agreed body bytes and structural facts remain usable.", + ); + return { + member: { + function_id: `${inventory.evidence_id}/${sourceIndex}`, + inventory_evidence_id: inventory.evidence_id, + source_index: sourceIndex, + address, + name: row.value, + body_sha256: fingerprint?.sha256 ?? null, + structural_candidate_sha256: detail?.conflict + ? null + : (detail?.candidate ?? null), + body_bytes: body?.available ? body.total_bytes : null, + dossier_evidence_id: detail?.conflict ? null : (detail?.dossier ?? null), + evidence_links: [...links].sort(), + limitations, + }, + symbol: symbol.value, + fingerprint: fingerprint?.key ?? null, + }; +}; + +/** Consume normalized producer facts once; never infer body extents from neighbor entries. */ +export const readFunctionInventory = async ( + inventory: Evidence, + observations: readonly Evidence[], + checkpoint: NativeMatchingCheckpoint, +): Promise => { + if ( + inventory.operation !== "list_procedures" || + inventory.predicate_type !== "rea.analysis" || + inventory.subject === null + ) + throw new EvidenceIntegrityError( + "Expected artifact-bound list_procedures Evidence", + ); + const rows = nativeInventoryRowsSchemas.names.parse( + inventory.normalized_result, + ); + const facts: Facts = { + details: new Map(), + symbols: new Map(), + bytes: new FunctionByteCoverage(checkpoint), + }; + for (const observation of observations) { + await checkpoint(); + requireContext(inventory, observation); + switch (observation.operation) { + case "list_names": + await addNames(observation, facts.symbols, checkpoint); + break; + case "read_bytes": + await addBytes(observation, facts.bytes); + break; + case "procedure_info": + case "analyze_function": + addBody(observation, facts.details); + break; + default: + throw new EvidenceIntegrityError( + "Function matching observations must be list_names, procedure_info, read_bytes or analyze_function Evidence", + ); + } + } + const functions: MatchingFunction[] = []; + for (let sourceIndex = 0; sourceIndex < rows.length; sourceIndex++) { + const row = rows[sourceIndex]!; + await checkpoint(1024 + 4 * (row.address.length + row.value.length)); + functions.push( + await functionFact({ inventory, row, sourceIndex }, facts, checkpoint), + ); + } + return functions; +}; diff --git a/src/domain/native/functionMatching.ts b/src/domain/native/functionMatching.ts new file mode 100644 index 0000000000..e2b6039d31 --- /dev/null +++ b/src/domain/native/functionMatching.ts @@ -0,0 +1,204 @@ +import { digestCanonicalValue } from "../canonicalDigest.js"; +import { z } from "zod"; +import { digestSchema, prefixedDigestSchema } from "../digests.js"; +import { retainedEvidenceReferenceSchema } from "../evidenceReference.js"; + +const evidenceId = prefixedDigestSchema("ev"); +export const nativeMatchStateSchema = z.enum([ + "matched", + "only_left", + "only_right", + "unresolved", +]); +export const nativeFunctionMatchMemberSchema = z.strictObject({ + function_id: z.string().min(1), + inventory_evidence_id: evidenceId, + source_index: z.number().int().nonnegative(), + address: z.string(), + name: z.string(), + body_sha256: digestSchema.nullable(), + structural_candidate_sha256: digestSchema.nullable(), + body_bytes: z.number().int().nonnegative().nullable(), + dossier_evidence_id: evidenceId.nullable(), + evidence_links: z.array(evidenceId).min(1), + limitations: z.array(z.string()), +}); +export const nativeFunctionMatchSchema = z.strictObject({ + match_id: prefixedDigestSchema("match"), + state: nativeMatchStateSchema, + basis: z.enum(["exact-symbol", "identical-bytes"]).nullable(), + left: z.array(nativeFunctionMatchMemberSchema), + right: z.array(nativeFunctionMatchMemberSchema), + reason: z.string(), + compare_functions_input: z + .strictObject({ + left: retainedEvidenceReferenceSchema, + right: retainedEvidenceReferenceSchema, + }) + .nullable(), +}); +export const nativeFunctionMatchingSummarySchema = z.strictObject({ + matched: z.number().int().nonnegative(), + only_left: z.number().int().nonnegative(), + only_right: z.number().int().nonnegative(), + unresolved: z.number().int().nonnegative(), +}); +export const nativeFunctionMatchingContextSchema = z.strictObject({ + left: z.strictObject({ + evidence_id: evidenceId, + sha256: digestSchema, + functions: z.number().int().nonnegative(), + }), + right: z.strictObject({ + evidence_id: evidenceId, + sha256: digestSchema, + functions: z.number().int().nonnegative(), + }), + scope: z.literal("provider-observed-inputs"), + source_coverage: z.strictObject({ + status: z.enum(["complete-within-inputs", "partial"]), + left_inventory_completeness: z.literal("unknown"), + right_inventory_completeness: z.literal("unknown"), + left_byte_fingerprints: z.number().int().nonnegative(), + right_byte_fingerprints: z.number().int().nonnegative(), + left_structural_candidates: z.number().int().nonnegative(), + right_structural_candidates: z.number().int().nonnegative(), + }), + summary: nativeFunctionMatchingSummarySchema, +}); +export const nativeFunctionMatchingSchema = nativeFunctionMatchingContextSchema + .extend({ + matches: z.array(nativeFunctionMatchSchema), + limitations: z.array(z.string()), + }) + .superRefine((report, context) => { + const seen = new Set(); + const members = { left: new Set(), right: new Set() }; + const fingerprints = { left: 0, right: 0 }; + const candidates = { left: 0, right: 0 }; + const counts = { matched: 0, only_left: 0, only_right: 0, unresolved: 0 }; + let left = 0, + right = 0; + for (const match of report.matches) { + counts[match.state]++; + left += match.left.length; + right += match.right.length; + if (seen.has(match.match_id)) + context.addIssue({ + code: "custom", + message: "Repeated match identity", + }); + seen.add(match.match_id); + if ( + match.match_id !== + `match_${digestCanonicalValue({ state: match.state, basis: match.basis, left: match.left.map((x) => x.function_id), right: match.right.map((x) => x.function_id) })}` + ) + context.addIssue({ + code: "custom", + message: + "Match identity does not bind its selected occurrences and basis", + }); + for (const side of ["left", "right"] as const) + for (const member of match[side]) { + if ( + member.inventory_evidence_id !== report[side].evidence_id || + member.source_index >= report[side].functions || + member.function_id !== + `${report[side].evidence_id}/${member.source_index}` || + members[side].has(member.function_id) || + !member.evidence_links.includes(member.inventory_evidence_id) || + (member.dossier_evidence_id !== null && + !member.evidence_links.includes(member.dossier_evidence_id)) + ) + context.addIssue({ + code: "custom", + message: + "Function occurrence identity, provenance or uniqueness disagrees with its selected inventory", + }); + members[side].add(member.function_id); + if (member.body_sha256 !== null) fingerprints[side]++; + if (member.structural_candidate_sha256 !== null) candidates[side]++; + } + if ( + match.left.length + match.right.length === 0 || + (match.state !== "matched" && + (match.basis !== null || match.compare_functions_input !== null)) + ) + context.addIssue({ + code: "custom", + message: + "Unresolved/unpaired group cannot contain a definitive basis or comparison input", + }); + if ( + match.basis === "identical-bytes" && + (match.left[0]?.body_sha256 == null || + match.left[0]?.body_sha256 !== match.right[0]?.body_sha256 || + match.left[0]?.body_bytes !== match.right[0]?.body_bytes) + ) + context.addIssue({ + code: "custom", + message: + "Identical-byte match disagrees with its complete body digests and sizes", + }); + if ( + match.compare_functions_input !== null && + (match.compare_functions_input.left.evidence_id !== + match.left[0]?.dossier_evidence_id || + match.compare_functions_input.right.evidence_id !== + match.right[0]?.dossier_evidence_id) + ) + context.addIssue({ + code: "custom", + message: "Comparison input does not identify this pair's dossiers", + }); + + if ( + (match.state === "matched" && + (match.left.length !== 1 || + match.right.length !== 1 || + match.basis === null)) || + (match.state === "only_left" && + (match.left.length !== 1 || match.right.length !== 0)) || + (match.state === "only_right" && + (match.right.length !== 1 || match.left.length !== 0)) + ) + context.addIssue({ + code: "custom", + message: "Match state contradicts its occurrences", + }); + } + if ( + report.source_coverage.status !== + (counts.only_left + counts.only_right + counts.unresolved > 0 + ? "partial" + : "complete-within-inputs") || + report.source_coverage.left_byte_fingerprints !== fingerprints.left || + report.source_coverage.right_byte_fingerprints !== fingerprints.right || + report.source_coverage.left_structural_candidates !== candidates.left || + report.source_coverage.right_structural_candidates !== candidates.right + ) + context.addIssue({ + code: "custom", + message: "Source/matching coverage contradicts examined facts", + }); + if ( + left !== report.left.functions || + right !== report.right.functions || + Object.keys(counts).some( + (key) => + counts[key as keyof typeof counts] !== + report.summary[key as keyof typeof counts], + ) + ) + context.addIssue({ + code: "custom", + message: "Matching summary or examined inventory coverage disagrees", + }); + }); +export type NativeFunctionMatch = z.infer; +export type NativeFunctionMatchMember = z.infer< + typeof nativeFunctionMatchMemberSchema +>; +export type NativeFunctionMatching = z.infer< + typeof nativeFunctionMatchingSchema +>; diff --git a/src/domain/native/functionStructuralCandidate.ts b/src/domain/native/functionStructuralCandidate.ts new file mode 100644 index 0000000000..4897396e00 --- /dev/null +++ b/src/domain/native/functionStructuralCandidate.ts @@ -0,0 +1,54 @@ +import type { Evidence } from "../evidence.js"; +import type { FunctionDossier } from "../hopperValues.js"; +import { digestCanonicalValue } from "../canonicalDigest.js"; +import { + isAutoName, + normalizeCfg, +} from "../functionComparisonNormalization.js"; +import { assessFunctionCallCoverage } from "../functionCallCoverage.js"; +import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; + +/** An observed structural signature proposes candidates; it never proves identity. */ +export const functionStructuralCandidate = ( + evidence: Evidence, + dossier: FunctionDossier, +): string | null => { + if ( + evidence.subject?.architecture == null || + dossier.facet_availability?.cfg?.available !== true || + dossier.facet_availability?.callees?.available !== true || + dossier.facet_availability?.strings_names?.available !== true || + dossier.unresolved_calls === undefined || + assessFunctionCallCoverage(evidence.relationship_coverage, dossier).blockers + .length > 0 || + (dossier.swift_string_literal_omissions?.length ?? 0) > 0 || + evidence.analysis_profile === null + ) + return null; + const cfg = normalizeCfg(dossier.basic_blocks); + if ( + cfg === null || + cfg.length === 0 || + dossier.callees.some( + (callee) => isAutoName(callee.name) || callee.name.length === 0, + ) + ) + return null; + const calls = dossier.callees + .map((callee) => callee.name) + .sort(compareUnicodeCodePoints); + const strings = [ + ...dossier.referenced_strings.map((value) => value.value), + ...(dossier.swift_string_literals ?? []).map((value) => value.value), + ].sort(compareUnicodeCodePoints); + // Empty leaf functions otherwise produce huge uninformative CFG-only buckets. + if (calls.length === 0 && strings.length === 0) return null; + return digestCanonicalValue({ + algorithm: "observed-cfg-calls-strings-v1", + profile: evidence.analysis_profile.digest, + architecture: evidence.subject.architecture, + cfg, + calls, + strings, + }); +}; diff --git a/src/domain/native/matchFunctionInventories.ts b/src/domain/native/matchFunctionInventories.ts new file mode 100644 index 0000000000..8352150815 --- /dev/null +++ b/src/domain/native/matchFunctionInventories.ts @@ -0,0 +1,197 @@ +import { digestCanonicalValue } from "../canonicalDigest.js"; +import type { MatchingFunction } from "./functionInventory.js"; +import type { NativeMatchingCheckpoint } from "./functionByteCoverage.js"; +import type { + NativeFunctionMatch, + NativeFunctionMatching, +} from "./functionMatching.js"; + +type Groups = ReadonlyMap; +type PairingIndexes = readonly [Groups, Groups]; +interface MatchInput { + readonly state: NativeFunctionMatch["state"]; + readonly basis: NativeFunctionMatch["basis"]; + readonly left: MatchingFunction[]; + readonly right: MatchingFunction[]; + readonly reason: string; +} +const index = ( + items: readonly MatchingFunction[], + key: (item: MatchingFunction) => string | null, +): Groups => { + const groups = new Map(); + for (const item of items) { + const value = key(item); + if (value === null) continue; + let group = groups.get(value); + if (group === undefined) groups.set(value, (group = [])); + group.push(item); + } + return groups; +}; + +/** One request owns pairing state; ambiguous components never expand into pair products. */ +class InventoryMatcher { + readonly consumed = new Set(); + readonly matches: NativeFunctionMatch[] = []; + readonly #parent = new Map(); + readonly #ambiguous = new Set(); + constructor(readonly checkpoint: NativeMatchingCheckpoint) {} + add(input: MatchInput): void { + const left = input.left.map((x) => x.member), + right = input.right.map((x) => x.member); + const leftDossier = left[0]?.dossier_evidence_id, + rightDossier = right[0]?.dossier_evidence_id; + this.matches.push({ + match_id: `match_${digestCanonicalValue({ state: input.state, basis: input.basis, left: left.map((x) => x.function_id), right: right.map((x) => x.function_id) })}`, + state: input.state, + basis: input.basis, + left, + right, + reason: input.reason, + compare_functions_input: + input.state === "matched" && leftDossier != null && rightDossier != null + ? { + left: { kind: "retained-evidence", evidence_id: leftDossier }, + right: { kind: "retained-evidence", evidence_id: rightDossier }, + } + : null, + }); + for (const item of [...input.left, ...input.right]) this.consumed.add(item); + } + async pairUnique( + groups: PairingIndexes, + basis: NonNullable, + ): Promise { + for (const [key, left] of groups[0]) { + await this.checkpoint(); + const right = groups[1].get(key); + // Global multiplicity still matters after a different symbol consumes an alias. + if ( + left.length !== 1 || + right?.length !== 1 || + this.consumed.has(left[0]!) || + this.consumed.has(right[0]!) + ) + continue; + this.add({ + state: "matched", + basis, + left, + right, + reason: + basis === "exact-symbol" + ? "Unique exact reported symbol; this does not establish unchanged code or semantic equivalence." + : "Unique identical observed function bytes, total size and ordered range lengths; relocations are not normalized.", + }); + } + } + #find(item: MatchingFunction): MatchingFunction { + let root = item; + while (this.#parent.has(root) && this.#parent.get(root) !== root) + root = this.#parent.get(root)!; + let current = item; + while (this.#parent.has(current) && this.#parent.get(current) !== root) { + const next = this.#parent.get(current)!; + this.#parent.set(current, root); + current = next; + } + return root; + } + async connectCandidates(indexes: readonly PairingIndexes[]): Promise { + for (const groups of indexes) + for (const [key, left] of groups[0]) { + await this.checkpoint(); + const right = groups[1].get(key); + if (right === undefined) continue; + const pending = [...left, ...right].filter( + (item) => !this.consumed.has(item), + ), + first = pending[0]; + if (first === undefined) continue; + for (const item of pending) { + this.#parent.set(this.#find(item), this.#find(first)); + this.#ambiguous.add(item); + } + } + } + async finish( + left: readonly MatchingFunction[], + right: readonly MatchingFunction[], + ): Promise { + const groups = new Map< + MatchingFunction, + { left: MatchingFunction[]; right: MatchingFunction[] } + >(); + for (const [side, items] of [ + ["left", left], + ["right", right], + ] as const) + for (const item of items) { + await this.checkpoint(); + if (this.consumed.has(item)) continue; + if (!this.#ambiguous.has(item)) { + this.add({ + state: side === "left" ? "only_left" : "only_right", + basis: null, + left: side === "left" ? [item] : [], + right: side === "right" ? [item] : [], + reason: + "No exact correspondence in the supplied observed inventories. This is not proof of source addition/removal: renaming, inlining, stripping, byte changes or omitted functions can explain it.", + }); + continue; + } + const root = this.#find(item); + let group = groups.get(root); + if (group === undefined) + groups.set(root, (group = { left: [], right: [] })); + group[side].push(item); + } + for (const group of groups.values()) + this.add({ + state: "unresolved", + basis: null, + ...group, + reason: + "Repeated exact names/bytes, aliases or observed CFG/call/string fingerprints leave correspondence unresolved. Connected group membership does not imply pairwise equivalence. No candidate is forced into a pair.", + }); + } +} + +/** Match only exact unique facts, preserving every occurrence and all candidate ambiguity. */ +export const matchFunctionInventories = async ( + left: readonly MatchingFunction[], + right: readonly MatchingFunction[], + options: { + readonly identicalBytesComparable: boolean; + readonly checkpoint: NativeMatchingCheckpoint; + }, +): Promise<{ + matches: NativeFunctionMatch[]; + summary: NativeFunctionMatching["summary"]; +}> => { + const names: PairingIndexes = [ + index(left, (x) => x.symbol), + index(right, (x) => x.symbol), + ]; + const bytes: PairingIndexes = [ + index(left, (x) => + options.identicalBytesComparable ? x.fingerprint : null, + ), + index(right, (x) => + options.identicalBytesComparable ? x.fingerprint : null, + ), + ]; + const structural: PairingIndexes = [ + index(left, (x) => x.member.structural_candidate_sha256), + index(right, (x) => x.member.structural_candidate_sha256), + ]; + const matcher = new InventoryMatcher(options.checkpoint); + await matcher.pairUnique(names, "exact-symbol"); + await matcher.pairUnique(bytes, "identical-bytes"); + await matcher.connectCandidates([names, bytes, structural]); + await matcher.finish(left, right); + const summary = { matched: 0, only_left: 0, only_right: 0, unresolved: 0 }; + for (const match of matcher.matches) summary[match.state]++; + return { matches: matcher.matches, summary }; +}; diff --git a/src/domain/native/nativeByteRead.ts b/src/domain/native/nativeByteRead.ts new file mode 100644 index 0000000000..031cb21032 --- /dev/null +++ b/src/domain/native/nativeByteRead.ts @@ -0,0 +1,22 @@ +import { z } from "zod"; + +/** Provider-owned bytes and exact coverage of one requested memory interval. */ +export const nativeByteReadSchema = z + .object({ + address: z.string(), + requested_bytes: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), + returned_bytes: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + bytes_hex: z.string().regex(/^(?:[a-f0-9]{2})*$/u), + complete: z.boolean(), + }) + .superRefine((value, context) => { + if ( + value.returned_bytes * 2 !== value.bytes_hex.length || + value.returned_bytes > value.requested_bytes || + value.complete !== (value.returned_bytes === value.requested_bytes) + ) + context.addIssue({ + code: "custom", + message: "Read-byte length and completeness disagree", + }); + }); diff --git a/src/domain/native/procedureInfo.ts b/src/domain/native/procedureInfo.ts new file mode 100644 index 0000000000..f06ea4f09a --- /dev/null +++ b/src/domain/native/procedureInfo.ts @@ -0,0 +1,18 @@ +import { z } from "zod"; +import { + functionBodySchema, + localVariableSchema, + procedureClassificationSchema, +} from "../hopperValues.js"; + +/** Provider-neutral procedure metadata, including exact body ranges. */ +export const procedureInfoSchema = z.object({ + name: z.string(), + entrypoint: z.string(), + basicblock_count: z.number().int().min(0), + length: z.number().min(0), + signature: z.string().nullable(), + locals: z.array(localVariableSchema), + classification: procedureClassificationSchema.nullable(), + body: functionBodySchema, +}); diff --git a/src/ghidra/GhidraFunctionValues.ts b/src/ghidra/GhidraFunctionValues.ts index 8de099c083..2f2f7e8e7b 100644 --- a/src/ghidra/GhidraFunctionValues.ts +++ b/src/ghidra/GhidraFunctionValues.ts @@ -1,3 +1,4 @@ +import { procedureInfoSchema } from "../domain/native/procedureInfo.js"; import { z } from "zod"; import { nativeFunctionAnnotationsInputSchema, @@ -181,13 +182,11 @@ const procedureReferences = z ), }) .strict(); -const procedureInfo = z - .object({ +const procedureInfo = procedureInfoSchema + .safeExtend({ name: z.string().min(1), entrypoint: ghidraCanonicalAddressSchema, - basicblock_count: z.number().int().min(0), length: z.number().int().min(0), - signature: z.string().nullable(), locals: z.array(localVariable), classification, body: ghidraFunctionBody, diff --git a/src/ghidra/GhidraInventoryValues.ts b/src/ghidra/GhidraInventoryValues.ts index 36ddeb5159..938dea74bb 100644 --- a/src/ghidra/GhidraInventoryValues.ts +++ b/src/ghidra/GhidraInventoryValues.ts @@ -1,3 +1,4 @@ +import { nativeByteReadSchema } from "../domain/native/nativeByteRead.js"; import { z } from "zod"; import { bytePatternFields, @@ -320,25 +321,9 @@ const resultSchemas = { }); }), inspect_native_load_image: nativeLoadImageObservationSchema, - read_bytes: z - .strictObject({ - address: canonicalAddress, - requested_bytes: z.number().int().min(1), - returned_bytes: z.number().int().min(0), - bytes_hex: z.string().regex(/^(?:[a-f0-9]{2})*$/u), - complete: z.boolean(), - }) - .superRefine((value, context) => { - if ( - value.returned_bytes * 2 !== value.bytes_hex.length || - value.returned_bytes > value.requested_bytes || - value.complete !== (value.returned_bytes === value.requested_bytes) - ) - context.addIssue({ - code: "custom", - message: "Read-byte length and completeness disagree", - }); - }), + read_bytes: nativeByteReadSchema + .safeExtend({ address: canonicalAddress }) + .strict(), address_to_file_offset: z.strictObject({ address: canonicalAddress, file_offset: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), diff --git a/src/server/registerFunctionComparisonTool.ts b/src/server/registerFunctionComparisonTool.ts index 405efd713d..15e14d5cdd 100644 --- a/src/server/registerFunctionComparisonTool.ts +++ b/src/server/registerFunctionComparisonTool.ts @@ -1,3 +1,4 @@ +import { resolvePairedEvidenceRequest } from "../application/EvidenceInputResolver.js"; import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import type { BinarySessionPort } from "../application/binary/BinarySessionPort.js"; @@ -38,11 +39,18 @@ export const registerFunctionComparisonTool = ( contract.name, context.mcpReq.signal, async () => { + const resolved = resolvePairedEvidenceRequest( + input, + (id) => + session.evidenceForAnalysis?.(id) ?? session.evidenceById(id), + ); + if (!resolved.ok) + return server.delivery.toCallToolResult(resolved, contract); let leftEvidence: Evidence; let rightEvidence: Evidence; try { - leftEvidence = parseEvidence(input.left); - rightEvidence = parseEvidence(input.right); + leftEvidence = parseEvidence(resolved.value.left); + rightEvidence = parseEvidence(resolved.value.right); } catch (cause: unknown) { return server.delivery.toCallToolResult( err( diff --git a/src/server/registerNativeFunctionMatchingTool.ts b/src/server/registerNativeFunctionMatchingTool.ts new file mode 100644 index 0000000000..9d27ed723b --- /dev/null +++ b/src/server/registerNativeFunctionMatchingTool.ts @@ -0,0 +1,65 @@ +import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; +import type { BinarySessionPort } from "../application/binary/BinarySessionPort.js"; +import { toolContract } from "../contracts/toolContracts.js"; +import { matchNativeFunctions } from "../application/native/NativeFunctionMatchingService.js"; +import { summarizeRetainedAnalysis } from "../application/analysisView/AnalysisViewService.js"; +import { withAdmittedAnalysis } from "./analysisAdmission.js"; +import { createEvidenceToolHandler } from "./contractToolHandler.js"; +import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; +import { recordSessionEvidenceSources } from "./sessionEvidence.js"; +import { matchNativeFunctionsInputSchema } from "../contracts/native/functionMatchingContracts.js"; +import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; +import { err, ok } from "../domain/result.js"; +import type { Logger } from "pino"; + +/** Bind one named contract to the same retained-Evidence workflow used by CLI. */ +export const registerNativeFunctionMatchingTool = ( + server: EvidenceMcpServer, + session: BinarySessionPort, + logger: Logger, +): void => { + const contract = toolContract("match_native_functions"); + const evidenceById = (id: string) => + session.evidenceForAnalysis?.(id) ?? session.evidenceById(id); + const recordEvidence: BinarySessionPort["recordEvidence"] = (evidence) => + session.recordEvidence(evidence); + const handler = createEvidenceToolHandler( + server, + async (input, options) => { + const parsed = matchNativeFunctionsInputSchema.safeParse(input); + if (!parsed.success) + return err( + analysisInputErrorFromIssues( + contract.name, + parsed.error.issues, + input, + ), + ); + const result = await matchNativeFunctions( + parsed.data, + options, + evidenceById, + ); + if (!result.ok) return result; + const recorded = recordSessionEvidenceSources( + recordEvidence, + result.value.sources, + ); + if (!recorded.ok) return recorded; + return parsed.data.detail === "summary" + ? summarizeRetainedAnalysis(result.value.evidence, { + evidenceById, + recordEvidence, + }) + : ok(result.value.evidence); + }, + logger, + recordEvidence, + withAdmittedAnalysis({ kind: "session", session }), + ); + server.registerTool( + contract.name, + toolRegistrationOptions(contract), + handler(contract), + ); +}; diff --git a/src/server/registerSessionTools.ts b/src/server/registerSessionTools.ts index dedc3ff590..68c6ddbaa6 100644 --- a/src/server/registerSessionTools.ts +++ b/src/server/registerSessionTools.ts @@ -1,3 +1,4 @@ +import { registerNativeFunctionMatchingTool } from "./registerNativeFunctionMatchingTool.js"; import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { workflowUnknownInput } from "../application/investigation/WorkflowUnknown.js"; @@ -375,6 +376,7 @@ export const registerSessionTools = ( registerProcessComparisonTool(server, session, compareContract); registerArtifactComparisonTool(server, session, compareArtifactsContract); registerFunctionComparisonTool(server, session, compareFunctionsContract); + registerNativeFunctionMatchingTool(server, session, logger); registerBundleComparisonTool( server, session, diff --git a/tests/acceptance/tools/nativeFunctionMatching.test.ts b/tests/acceptance/tools/nativeFunctionMatching.test.ts new file mode 100644 index 0000000000..dbab41cbd8 --- /dev/null +++ b/tests/acceptance/tools/nativeFunctionMatching.test.ts @@ -0,0 +1,276 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; +import { cliTest, type TestCli } from "../../support/cli/cliFixture.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { + nativeMatchingInventory, + nativeMatchingDetails, +} from "../../fixtures/nativeFunctionMatching.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { nativeMatchingProvider } from "../../fixtures/nativeFunctionMatching.js"; +import { + createEvidenceBundle, + parseEvidenceBundle, +} from "../../../src/domain/evidenceBundle.js"; +import { nativeFunctionMatchingSchema } from "../../../src/domain/native/functionMatching.js"; + +const createMatchingFixture = async () => { + const root = await createTestTempDirectory("rea-native-matching-"); + const ancestor = createEvidence(undefined, nativeMatchingProvider, { + operation: "fixture_provenance", + parameters: {}, + result: { source: "Synthetic ancestor" }, + }); + const unrelated = createEvidence(undefined, nativeMatchingProvider, { + operation: "fixture_unrelated", + parameters: {}, + result: { source: "Unselected synthetic record" }, + }); + const dependency = createEvidence(undefined, nativeMatchingProvider, { + operation: "fixture_dependency", + parameters: {}, + result: { source: "Synthetic dependency" }, + evidenceLinks: [ancestor.evidence_id], + }); + const left = nativeMatchingInventory( + "1", + [ + { address: "0x10", value: "main" }, + { address: "0x20", value: "old_name" }, + ], + [dependency.evidence_id], + ); + const right = nativeMatchingInventory("2", [ + { address: "0x30", value: "main" }, + { address: "0x40", value: "new_name" }, + ]); + const leftDetails = [ + ...nativeMatchingDetails("1", "0x10", "9000", true), + ...nativeMatchingDetails("1", "0x20", "c3", true), + ]; + const rightDetails = [ + ...nativeMatchingDetails("2", "0x30", "9001", true), + ...nativeMatchingDetails("2", "0x40", "c3", true), + ]; + const ancestorsPath = join(root, "ancestors.json"); + await writeFile( + ancestorsPath, + JSON.stringify(createEvidenceBundle([ancestor, unrelated])), + ); + const importedPath = join(root, "dependencies.json"); + await writeFile( + importedPath, + JSON.stringify(createEvidenceBundle([ancestor, dependency])), + ); + return { + root, + ancestor, + dependency, + unrelated, + left, + right, + leftDetails, + rightDetails, + ancestorsPath, + importedPath, + }; +}; +const verifyPortableCli = async ( + cli: TestCli, + fixture: Awaited>, + input: Record, + evidenceId: string, +) => { + const { + root, + ancestor, + dependency, + unrelated, + left, + right, + leftDetails, + rightDetails, + ancestorsPath, + } = fixture; + const bundlePath = join(root, "sources.json"), + inputPath = join(root, "request.json"), + outputPath = join(root, "result.json"); + await writeFile( + bundlePath, + JSON.stringify( + createEvidenceBundle([ + left, + right, + ...leftDetails, + ...rightDetails, + dependency, + ]), + ), + ); + const original = await readFile(bundlePath); + await writeFile(inputPath, JSON.stringify({ ...input, detail: "complete" })); + const cliFull = await cli.run({ + arguments: [ + "match-native-functions", + inputPath, + "--source_bundle", + ancestorsPath, + "--source_bundle", + bundlePath, + "--json", + ], + }); + expect(cliFull.exitCode, cliFull.stderr).toBe(0); + expect(parseEvidence(cliFull.json).evidence_id).toBe(evidenceId); + await writeFile(inputPath, JSON.stringify(input)); + const cliSummary = await cli.run({ + arguments: [ + "match-native-functions", + inputPath, + "--source_bundle", + ancestorsPath, + "--source_bundle", + bundlePath, + "--evidence_output", + outputPath, + "--json", + ], + }); + expect(cliSummary.exitCode, cliSummary.stderr).toBe(0); + expect(cliSummary.json).toMatchObject({ + summary: { + normalized_result: { parent_evidence_id: evidenceId }, + }, + evidence_bundle: { path: outputPath }, + }); + const bundle = parseEvidenceBundle( + JSON.parse(await readFile(outputPath, "utf8")), + ); + const ids = new Set(bundle.records.map((x) => x.evidence_id)); + for (const record of bundle.records) + for (const id of record.evidence_links) + expect( + ids.has(id), + `Missing dependency ${id} of ${record.evidence_id}`, + ).toBe(true); + expect(ids.has(ancestor.evidence_id)).toBe(true); + expect(ids.has(dependency.evidence_id)).toBe(true); + expect(ids.has(unrelated.evidence_id)).toBe(false); + expect(ids.has(evidenceId)).toBe(true); + expect(await readFile(bundlePath)).toEqual(original); + const incomplete = await cli.run({ + arguments: [ + "match-native-functions", + inputPath, + "--source_bundle", + bundlePath, + "--evidence_output", + join(root, "incomplete.json"), + "--json", + ], + }); + expect(incomplete.json).toMatchObject({ + code: "evidence_integrity_mismatch", + }); +}; + +cliTest( + "shared public CLI/MCP matching retains sources, pages exact facts, and feeds selected dossier pairs directly to compare_functions", + async ({ cli }) => { + const fixture = await createMatchingFixture(); + const { left, right, leftDetails, rightDetails, importedPath } = fixture; + const { call } = await connectLocalToolsMcp(); + const imported = await call("import_evidence_bundle", { + path: importedPath, + }); + expect(imported.isError, JSON.stringify(imported)).not.toBe(true); + const full = await call("match_native_functions", { + left, + right, + left_observations: leftDetails, + right_observations: rightDetails, + detail: "complete", + }); + expect(full.isError, JSON.stringify(full)).not.toBe(true); + const evidence = parseEvidence(full.structuredContent), + report = nativeFunctionMatchingSchema.parse(evidence.normalized_result); + expect(report.summary).toEqual({ + matched: 2, + only_left: 0, + only_right: 0, + unresolved: 0, + }); + const reference = (evidence_id: string) => ({ + kind: "retained-evidence", + evidence_id, + }); + const input = { + left: reference(left.evidence_id), + right: reference(right.evidence_id), + left_observations: leftDetails.map((x) => reference(x.evidence_id)), + right_observations: rightDetails.map((x) => reference(x.evidence_id)), + }; + const summary = await call("match_native_functions", input); + expect(summary.isError, JSON.stringify(summary)).not.toBe(true); + expect( + parseEvidence(summary.structuredContent).normalized_result, + ).toMatchObject({ + kind: "summary", + artifact: null, + parent_evidence_id: evidence.evidence_id, + summary: { summary: report.summary }, + }); + expect(parseEvidence(summary.structuredContent).locations).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: "artifact-path" }), + ]), + ); + const page = await call("inspect_analysis_view", { + source: reference(evidence.evidence_id), + view: { + kind: "function-matches", + collection: "functions", + sides: ["left"], + fields: ["name", "body_sha256"], + offset: 1, + limit: 1, + }, + }); + expect(page.isError, JSON.stringify(page)).not.toBe(true); + expect( + parseEvidence(page.structuredContent).normalized_result, + ).toMatchObject({ + artifact: null, + coverage: { total: 2, next_offset: null }, + items: [ + { + name: "old_name", + function_id: `${left.evidence_id}/1`, + address: "0x20", + }, + ], + }); + expect(parseEvidence(page.structuredContent).locations).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: "artifact-path" }), + ]), + ); + const compared = await call( + "compare_functions", + report.matches[0]!.compare_functions_input!, + ); + expect(compared.isError, JSON.stringify(compared)).not.toBe(true); + expect(parseEvidence(compared.structuredContent).operation).toBe( + "compare_functions", + ); + await verifyPortableCli(cli, fixture, input, evidence.evidence_id); + const missing = await call("match_native_functions", { + left: reference(`ev_${"a".repeat(64)}`), + right: reference(right.evidence_id), + }); + expect(missing.isError).toBe(true); + }, + 60000, +); diff --git a/tests/fixtures/nativeFunctionMatching.c b/tests/fixtures/nativeFunctionMatching.c new file mode 100644 index 0000000000..d7a062a958 --- /dev/null +++ b/tests/fixtures/nativeFunctionMatching.c @@ -0,0 +1,32 @@ +#include +#include +#ifndef VERSION +#define VERSION 1 +#endif +#define FN __attribute__((noinline, used)) +FN int stable(int x) { return x * 3 + 7; } +FN int changed(int x) { return x + VERSION * 5; } +#if VERSION == 1 +FN int old_behavior(int x) { puts("behavior marker"); return x + 7; } +FN int old_name(int x) { return x ^ 0x13579; } +FN int removed_observation(int x) { return x * 13 + 99; } +FN int formerly_separate(int x) { return x + 12345; } +#else +FN int new_behavior(int x) { puts("behavior marker"); return x + 8; } +FN int renamed(int x) { return x ^ 0x13579; } +FN int added_observation(int x) { return x * 17 + 77; } +static inline int formerly_separate(int x) { return x + 12345; } +#endif +FN int alias_one(int x) { return x + 42; } +FN int alias_two(int x) { return x + 42; } +int main(int argc, char **argv) { + int x = argc > 1 ? atoi(argv[1]) : 2; + int result = stable(x) + changed(x) + formerly_separate(x) + alias_one(x) + alias_two(x); +#if VERSION == 1 + result += old_name(x) + removed_observation(x) + old_behavior(x); +#else + result += renamed(x) + added_observation(x) + new_behavior(x); +#endif + printf("%d\n", result); + return 0; +} diff --git a/tests/fixtures/nativeFunctionMatching.ts b/tests/fixtures/nativeFunctionMatching.ts new file mode 100644 index 0000000000..f20dd77c00 --- /dev/null +++ b/tests/fixtures/nativeFunctionMatching.ts @@ -0,0 +1,101 @@ +import { createEvidence, type Evidence } from "../../src/domain/evidence.js"; +import { jsonValueSchema } from "../../src/domain/jsonValue.js"; +import { FUNCTION_COMPARISON_EXAMPLE } from "../../src/contracts/functionComparisonExample.js"; +import { functionDossierSchema } from "../../src/domain/hopperValues.js"; + +/** Explicit synthetic producer facts; they do not qualify any real provider. */ +export const nativeMatchingProvider = { + id: "matching-fixture", + name: "Synthetic native matching facts", + version: "1", +}; +export const nativeMatchingTarget = (side: "1" | "2") => ({ + path: `/tmp/matching-${side}.elf`, + sha256: side.repeat(64), + format: "elf" as const, + architecture: "x86_64" as const, +}); +export const nativeMatchingInventory = ( + side: "1" | "2", + entries: readonly { address: string; value: string }[], + evidenceLinks: readonly string[] = [], +) => + createEvidence(nativeMatchingTarget(side), nativeMatchingProvider, { + operation: "list_procedures", + parameters: {}, + result: jsonValueSchema.parse(entries), + evidenceLinks: [...evidenceLinks], + }); +export const nativeMatchingBody = (address: string, bytes: number) => ({ + available: true as const, + provenance: "explicit synthetic body", + ranges: [ + { + start: address, + end: `0x${(BigInt(address) + BigInt(bytes) - 1n).toString(16)}`, + }, + ], + total_bytes: bytes, + span_bytes: bytes, + non_contiguous: false, + contains_entry: true, +}); +export const nativeMatchingDetails = ( + side: "1" | "2", + address: string, + hex: string, + dossier = false, +): Evidence[] => { + const target = nativeMatchingTarget(side), + body = nativeMatchingBody(address, hex.length / 2); + const procedure = { + name: "fixture", + entrypoint: address, + basicblock_count: 1, + length: hex.length / 2, + signature: null, + locals: [], + classification: null, + body, + }; + let observation: Evidence; + if (dossier) { + const value = functionDossierSchema.parse( + FUNCTION_COMPARISON_EXAMPLE.left.normalized_result, + ); + value.procedure = { + address, + name: "fixture", + classification: null, + body, + signature: null, + locals: [], + }; + value.basic_blocks = []; + observation = createEvidence(target, nativeMatchingProvider, { + operation: "analyze_function", + parameters: { procedure: address }, + result: jsonValueSchema.parse(value), + relationshipCoverage: { status: "complete" }, + }); + } else + observation = createEvidence(target, nativeMatchingProvider, { + operation: "procedure_info", + parameters: { procedure: address }, + result: procedure, + }); + return [ + observation, + createEvidence(target, nativeMatchingProvider, { + operation: "read_bytes", + parameters: { address, length: hex.length / 2 }, + result: { + address, + requested_bytes: hex.length / 2, + returned_bytes: hex.length / 2, + bytes_hex: hex, + complete: true, + }, + }), + ]; +}; From c23e16162914e6bffc9549959c3faef0a66f6027 Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:09:28 +0800 Subject: [PATCH 11/13] perf(mcp): record catalog identities for all schema profiles (#5415) Share compiled schema conversion while recording all four built-in catalog profiles. Preserve canonical hashes, output validation and custom-budget fallback. Refs #723. Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- .../analysis-efficiency/mcp-profile-costs.py | 74 ++ .../mcp-profile-digest.mjs | 25 + .../mcp-profile-identity.json | 808 ++++++++++++++++++ .../mcp-profile-identity.md | 131 +++ .../mcp-profile-identity.py | 199 +++++ scripts/record-catalog-digests.mjs | 19 +- src/catalogIdentity.ts | 108 ++- src/recordedCatalogDigests.ts | 8 +- tests/boundary/mcp/mcpDoctor.test.ts | 36 + tests/boundary/mcp/serverIdentity.test.ts | 42 +- 10 files changed, 1395 insertions(+), 55 deletions(-) create mode 100644 experiments/analysis-efficiency/mcp-profile-costs.py create mode 100644 experiments/analysis-efficiency/mcp-profile-digest.mjs create mode 100644 experiments/analysis-efficiency/mcp-profile-identity.json create mode 100644 experiments/analysis-efficiency/mcp-profile-identity.md create mode 100644 experiments/analysis-efficiency/mcp-profile-identity.py diff --git a/experiments/analysis-efficiency/mcp-profile-costs.py b/experiments/analysis-efficiency/mcp-profile-costs.py new file mode 100644 index 0000000000..64cb1292d0 --- /dev/null +++ b/experiments/analysis-efficiency/mcp-profile-costs.py @@ -0,0 +1,74 @@ +"""Summarize recorded Codex discovery tasks; tiktoken==0.12.0 is a proxy only.""" +import argparse +import hashlib +import json +from pathlib import Path + +import tiktoken + + +def compact(value): + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + + +def inspect(path, encoding): + report = json.loads((path / "report.json").read_text()) + catalog = json.loads((path / "catalog.json").read_text()) + transcript = json.loads((path / "transcript.json").read_text()) + log = path.with_suffix(".log") + scope = next(json.loads(line) for line in reversed(log.read_text().splitlines()) + if line.startswith('{"resource_scope":')) + observed = scope.get("payload_resource_observation") or {} + completed = (report["status"] == "passed" and not report["cleanup_failures"] + and report["appserver_exit_code"] == 0 and scope["abort"] is None + and scope.get("payload_resource_observation_verified") is True + and observed.get("payload_cleanup_verified") is True + and observed.get("payload_exit_code") == 0) + tools = list(catalog["data"][0]["tools"].values()) + messages = [event["message"] for event in transcript] + ids = {m["id"] for m in messages if m.get("method") == "mcpServer/tool/call"} + replies = [m["result"] for m in messages if m.get("id") in ids and "result" in m] + arguments = [m["params"] for m in messages if m.get("method") == "mcpServer/tool/call"] + count = lambda text: len(encoding.encode(text, disallowed_special=())) + input_catalog = count(compact([{k: t[k] for k in ["name", "description", "inputSchema"]} for t in tools])) + argument_tokens = sum(count(compact(a)) for a in arguments) + text_tokens = sum(count(c["text"]) for r in replies for c in r["content"] if c["type"] == "text") + return { + "recording": path.name, "completed_with_verified_cleanup": completed, + "input_profile": report["input_profile"], "output_profile": report["output_profile"], + "catalog": report["catalog"], "calls": report["calls"], + # Codex returns tools as a map; JSON member order varies by process. + "catalog_semantic_sha256": hashlib.sha256(json.dumps( + catalog, ensure_ascii=False, sort_keys=True, separators=(",", ":"), + allow_nan=False).encode()).hexdigest(), + "digests": [r["structuredContent"]["result"]["server_identity"]["catalog"]["digests"] for r in replies], + "token_proxy": {"input_facing_catalog": input_catalog, "arguments": argument_tokens, + "text_replies": text_tokens, "combined": input_catalog + argument_tokens + text_tokens, + "complete_discovered_catalog": count(compact(catalog))}, + "wall_seconds": scope["elapsed_seconds"], + "cpu_seconds": observed["cpu_stat"]["usage_usec"] / 1e6, + "kernel_family_peak_bytes_including_file_cache": observed.get("memory_peak_bytes"), + "observed_resource_events": observed.get("resource_events"), + "final_whole_scope_events": scope.get("resource_events_final"), + "sha256": {p.name: hashlib.sha256(p.read_bytes()).hexdigest() + for p in [path / "report.json", path / "catalog.json", path / "transcript.json", log]}, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("cases", type=Path, nargs="+") + args = parser.parse_args() + encoding = tiktoken.get_encoding("cl100k_base") + records = [inspect(path, encoding) for path in args.cases] + result = {"tiktoken": tiktoken.__version__, "encoding": encoding.name, + "actual_model_usage": None, + "unmeasured": ["actual host model schema injection and rendering", "model input/output/cached tokens", "model reasoning and application investigation"], + "recordings": records} + args.output.write_text(json.dumps(result, indent=2) + "\n") + assert all(r["completed_with_verified_cleanup"] for r in records) + + +if __name__ == "__main__": + main() diff --git a/experiments/analysis-efficiency/mcp-profile-digest.mjs b/experiments/analysis-efficiency/mcp-profile-digest.mjs new file mode 100644 index 0000000000..ed780af903 --- /dev/null +++ b/experiments/analysis-efficiency/mcp-profile-digest.mjs @@ -0,0 +1,25 @@ +import { pathToFileURL } from "node:url"; +import { resolve } from "node:path"; +const [root, input, output] = process.argv.slice(2); +const start = performance.now(); +const { createCatalogIdentity } = await import( + pathToFileURL(resolve(root, "dist/catalogIdentity.js")) +); +const imported = performance.now(); +const cpu = process.cpuUsage(); +const identity = createCatalogIdentity( + input === "compact" ? { budgetBytes: 13 * 1024 } : undefined, + output, +); +const digests = identity.digests; +const end = performance.now(); +console.log( + JSON.stringify({ + input, + output, + import_ms: imported - start, + digest_ms: end - imported, + digest_cpu_us: process.cpuUsage(cpu), + digests, + }), +); diff --git a/experiments/analysis-efficiency/mcp-profile-identity.json b/experiments/analysis-efficiency/mcp-profile-identity.json new file mode 100644 index 0000000000..742d3b74d3 --- /dev/null +++ b/experiments/analysis-efficiency/mcp-profile-identity.json @@ -0,0 +1,808 @@ +{ + "issue": 723, + "source_base": "d532b18123916a9e88f98c7880ba4f984e23d715", + "runtimes": { + "baseline-runtime-d532b1812": { + "source_revision": "d532b18123916a9e88f98c7880ba4f984e23d715", + "archive_sha256": "79832e2571112400892ec1aae429b030de38c8c672a8acfa6680a2568362fed3", + "manifest_sha256": "89cbc1688e4702a47b9a5cc66ac38d18de46b74eb550e44227f0ec2119c5e7af" + }, + "candidate-runtime-1": { + "source_base": "d532b18123916a9e88f98c7880ba4f984e23d715", + "patch_sha256": "cfb0eee1ee7f3ceecff7cb77ac63e569dfcfc020e4252cb11869644628f4f743", + "archive_sha256": "7e4275b1db784545335a4eab373a3db11b1eb5cc6bac7611521a698b0cffce56", + "manifest_sha256": "356b3baa153650b330cea6bc7e28163565f3201b8d80aa2d6bb69f66faa8705a" + } + }, + "node": "24.18.0", + "npm": "11.16.0", + "mcp_sdk": "2.3.1", + "client": "Codex app-server 0.163.0", + "negotiated_protocol": "2025-06-18", + "model_turns_started": 0, + "actual_model_usage": null, + "task": "Complete 209-tool discovery, then verify no target open, no cleanup pending, no bound provider, package version and tool count twice.", + "run_order": [ + "baseline-d532-1", + "candidate-1-1 (reversed profiles)", + "baseline-d532-2 (reversed profiles)", + "candidate-1-2" + ], + "resource_bounds": { + "cpu_affinity": [15], + "cpu_quota_percent": 100, + "hard_memory_gib": 4, + "early_stop_gib": 3, + "node_heap_gib": 2, + "swap": 0, + "task_ceiling": 192, + "accounting": "Kernel cgroup process-family peak including file cache, Codex, controller and server; CPU counters at payload exit with only observer alive. Final whole-scope counters unavailable.", + "oom_events": 0, + "verified_payload_cleanup_recordings": 16 + }, + "tiktoken": "0.12.0", + "encoding": "cl100k_base", + "token_proxy_limit": "Input-facing catalog + RPC params + text result serialization; no model turn/usage, schema injection, prompts or reasoning measured. Identity paths/timestamps cause small reply token differences.", + "scripts_sha256": { + "codex-host-probe.py": "d0da9b39e0343b5e84f05328f56df3e82b712fd3fb32ee375cf3735a448fb80e", + "digest-probe.mjs": "e2f8950014bb40806aef147196ca5d432c6400941d3901e1c0a0e78ce5266bcf", + "build-digest-probe.mjs": "46060b75c014ca39f9efe317a0445a0220ea7166a08dbdacf04706670e958e6d", + "run-cohort.py": "1cc5db864b4fe1b2d657ddfe72d468d9e72bddf5118b0bf10c9fce344ec45e9c" + }, + "profile_digest_oracle": { + "full": { + "full": { + "tools_sha256": "c3e7950408c9005ae3ae0633966609bf1c0f72a36392bc0e0c0fdfe9ea584ae6", + "prompts_sha256": "7152916ab48bd2a9677a5ad9cdbde0a51dc53c113934952ad096e93f5f18aa7e", + "combined_sha256": "e0b2715af8312fb565f14d75296adc947b94252e53643f08d831ca8fc8aee941" + }, + "off": { + "tools_sha256": "5ad649af53472a1d789b26997cb26a4e8d5d1f2a97c587ad1a6b16030ef5d764", + "prompts_sha256": "7152916ab48bd2a9677a5ad9cdbde0a51dc53c113934952ad096e93f5f18aa7e", + "combined_sha256": "cd84cff526daa360e30e299002e7d14427fde9c2b00f59f622fccf89ff2535c7" + } + }, + "compact": { + "full": { + "tools_sha256": "774c1b7c2c2f57beea9bed62171f046da0f17f2b3bbd7900a90855ecdd26ba79", + "prompts_sha256": "7152916ab48bd2a9677a5ad9cdbde0a51dc53c113934952ad096e93f5f18aa7e", + "combined_sha256": "2cbc9ae351fe4f1204a4c2b5e2e92fff8a2ecd87bc14c7ef068d6691c17dff15" + }, + "off": { + "tools_sha256": "efe7a17bad5617171162414df88529ff54807b870b5d510e2838ce18e2ba8bd8", + "prompts_sha256": "7152916ab48bd2a9677a5ad9cdbde0a51dc53c113934952ad096e93f5f18aa7e", + "combined_sha256": "07d379697201b70d45579d0811e137a0df088be88498fc9fae6806adba26c976" + } + } + }, + "recordings": [ + { + "recording": "baseline-d532-1-full-full", + "verified_complete": true, + "profiles": ["full", "full"], + "catalog_tools": 209, + "catalog_bytes": 3322571, + "catalog_semantic_sha256": "799cc997a9573b5c3dfce3ac22e1fb5eb1637e81b489392f37cc81767b334359", + "discovery_ms": 5074.2612921167165, + "first_status_ms": 1699.8079400509596, + "warm_status_ms": 981.8687399383634, + "wall_seconds": 8.919, + "cpu_seconds": 8.480822, + "kernel_family_peak_bytes": 802246656, + "token_proxy": { + "input_facing_catalog": 187005, + "arguments": 86, + "text_replies": 22702, + "combined": 209793, + "complete_discovered_catalog": 845872 + }, + "raw_sha256": { + "report.json": "f36f889ab241f704977241443ad55176de6607805f6c82013a287e968dbb428d", + "catalog.json": "ad5dcfb828d4524f97b494884deed42389ab3e29e2f7839a12b5e95bd1c50ff1", + "transcript.json": "7d019f379100f45ff1c3f55e66d79e233f7b736886be1823da11e4b843d5fdc1", + "baseline-d532-1-full-full.log": "3709d55fe5412f49ee880bfee4be818711c977b026ad3133a3f8abb91111f9a7" + } + }, + { + "recording": "baseline-d532-2-full-full", + "verified_complete": true, + "profiles": ["full", "full"], + "catalog_tools": 209, + "catalog_bytes": 3322571, + "catalog_semantic_sha256": "799cc997a9573b5c3dfce3ac22e1fb5eb1637e81b489392f37cc81767b334359", + "discovery_ms": 5847.158181015402, + "first_status_ms": 1123.013640055433, + "warm_status_ms": 911.4040599670261, + "wall_seconds": 8.905, + "cpu_seconds": 8.452489, + "kernel_family_peak_bytes": 803254272, + "token_proxy": { + "input_facing_catalog": 187004, + "arguments": 86, + "text_replies": 22702, + "combined": 209792, + "complete_discovered_catalog": 845872 + }, + "raw_sha256": { + "report.json": "b85792959c6838010ebd622d1cf213d2eb1cd9562da14fc49a2555bda03a0eee", + "catalog.json": "f588658a13bc1b559008508954f2ce7f34815b326686b4c167c0c27b521cad1b", + "transcript.json": "5326f37ffdeae98ad4679cf53f462a41fddd71eb9891040753e716489dcc1b68", + "baseline-d532-2-full-full.log": "147000d226f6cfb3450af51ade419b20c6b0b7d7409c22e876c98830762b39eb" + } + }, + { + "recording": "baseline-d532-1-compact-full", + "verified_complete": true, + "profiles": ["compact", "full"], + "catalog_tools": 209, + "catalog_bytes": 2723927, + "catalog_semantic_sha256": "d18f02920b84d39b90dc488db89a8f403f6ec35139a39a3329375a18078af1ea", + "discovery_ms": 11450.75011299923, + "first_status_ms": 4571.937991073355, + "warm_status_ms": 1836.9912200141698, + "wall_seconds": 18.99, + "cpu_seconds": 16.522676, + "kernel_family_peak_bytes": 808927232, + "token_proxy": { + "input_facing_catalog": 43507, + "arguments": 82, + "text_replies": 22714, + "combined": 66303, + "complete_discovered_catalog": 702182 + }, + "raw_sha256": { + "report.json": "b27141e7af05b3dd43930c0df30f4872709d8db72cc953478150084e99e032f4", + "catalog.json": "8ff7b639d3409b52761d483283c674e3670e42169ddfda0c1e2a87a207c4dcbd", + "transcript.json": "1f91daa9703495ca6519459124e9564f0c96f5542b61563bf10029a1f6337926", + "baseline-d532-1-compact-full.log": "fb187f2b9255e4fc1ecd2c15f9df3c7f1efc76aaba528589f5eed81e7e80e835" + } + }, + { + "recording": "baseline-d532-2-compact-full", + "verified_complete": true, + "profiles": ["compact", "full"], + "catalog_tools": 209, + "catalog_bytes": 2723927, + "catalog_semantic_sha256": "d18f02920b84d39b90dc488db89a8f403f6ec35139a39a3329375a18078af1ea", + "discovery_ms": 5926.411310909316, + "first_status_ms": 2899.308810941875, + "warm_status_ms": 1365.2960499748588, + "wall_seconds": 11.148, + "cpu_seconds": 10.613801, + "kernel_family_peak_bytes": 816377856, + "token_proxy": { + "input_facing_catalog": 43507, + "arguments": 82, + "text_replies": 22714, + "combined": 66303, + "complete_discovered_catalog": 702182 + }, + "raw_sha256": { + "report.json": "df856c2a486c513923b3c67db30bc60ca28ba3b1d0e3418d28d9d6429eca9366", + "catalog.json": "407ff33fd8e57f2842b76ce190c2089930add6c9a4f618cc90320045f6a67ed2", + "transcript.json": "d11dfb4c28c5355e733943e3504d3b4e11f056f0f7625e835e5cdc916a451716", + "baseline-d532-2-compact-full.log": "0485a806e815ecb91862d37cb9200b7cb0877a3d8c69882e8299100d13769a57" + } + }, + { + "recording": "baseline-d532-1-full-off", + "verified_complete": true, + "profiles": ["full", "off"], + "catalog_tools": 209, + "catalog_bytes": 832191, + "catalog_semantic_sha256": "e537c521b5dd0ac369dee706815579d9d7803497a8fc7f7d683fec267042b81c", + "discovery_ms": 11546.154452953488, + "first_status_ms": 4629.906311165541, + "warm_status_ms": 2057.3992100544274, + "wall_seconds": 19.588, + "cpu_seconds": 16.963204, + "kernel_family_peak_bytes": 709857280, + "token_proxy": { + "input_facing_catalog": 187005, + "arguments": 86, + "text_replies": 22710, + "combined": 209801, + "complete_discovered_catalog": 195042 + }, + "raw_sha256": { + "report.json": "4963559eafd97d6aa72e2265651d9f6d3d0cd3a04965677d6fdedfe0378e7c11", + "catalog.json": "aef1822a088446790e14373a6b9b1039cda640598d19620fdd776f6d7c49d890", + "transcript.json": "203f7f00eca7617cdaa8c7fd45fdde3d4502cd29cf36ee0c724181ad3178410b", + "baseline-d532-1-full-off.log": "84e5a67f6f1e97fb3c429d96d7b3de33e08eddc56b21b453fa2b3abf19dbd15b" + } + }, + { + "recording": "baseline-d532-2-full-off", + "verified_complete": true, + "profiles": ["full", "off"], + "catalog_tools": 209, + "catalog_bytes": 832191, + "catalog_semantic_sha256": "e537c521b5dd0ac369dee706815579d9d7803497a8fc7f7d683fec267042b81c", + "discovery_ms": 5198.2202511280775, + "first_status_ms": 2726.173529867083, + "warm_status_ms": 1434.7798710223287, + "wall_seconds": 10.325, + "cpu_seconds": 9.75455, + "kernel_family_peak_bytes": 703328256, + "token_proxy": { + "input_facing_catalog": 187005, + "arguments": 86, + "text_replies": 22710, + "combined": 209801, + "complete_discovered_catalog": 195042 + }, + "raw_sha256": { + "report.json": "e603556898f606d6804c73a227ef6d809f09f516316255b316e5afb473170ff9", + "catalog.json": "8200cccc7a0d7bdcf288cab0225269acadbd532aed69da7d7e8a3b4cf34b9c16", + "transcript.json": "4d2a862a24282cac2581a36ca46fd99474835cba247203556c04015ea02096fe", + "baseline-d532-2-full-off.log": "90b720806f3f5756408501a1f9d5dd724108647706e38aca10cbc0a35e56a37a" + } + }, + { + "recording": "baseline-d532-1-compact-off", + "verified_complete": true, + "profiles": ["compact", "off"], + "catalog_tools": 209, + "catalog_bytes": 233547, + "catalog_semantic_sha256": "0b81bdd81251ddd527a9c9ecbb5e6048e9c8a3be1f6195465a803d206f03fe73", + "discovery_ms": 12976.338863139972, + "first_status_ms": 6152.441991027445, + "warm_status_ms": 3121.8184099998325, + "wall_seconds": 23.47, + "cpu_seconds": 19.234341, + "kernel_family_peak_bytes": 665423872, + "token_proxy": { + "input_facing_catalog": 43507, + "arguments": 82, + "text_replies": 22690, + "combined": 66279, + "complete_discovered_catalog": 51352 + }, + "raw_sha256": { + "report.json": "c3c21635cf212eba3902ad64f956244ddfe122425fa3b39ec280b5296ebe23c6", + "catalog.json": "e5a4444ea91913e68705a6bd6afee2b8ea0a43b2f91f8628ada597e57afab5db", + "transcript.json": "58df156c85a8b1ec325945db08e4e84e8174dfd739f48e4d83b2a3537f7a52b9", + "baseline-d532-1-compact-off.log": "8d5b2ddcf5b31ad4635269837e217dd2242a91d82ea961d01e2ab3a2076b9e35" + } + }, + { + "recording": "baseline-d532-2-compact-off", + "verified_complete": true, + "profiles": ["compact", "off"], + "catalog_tools": 209, + "catalog_bytes": 233547, + "catalog_semantic_sha256": "0b81bdd81251ddd527a9c9ecbb5e6048e9c8a3be1f6195465a803d206f03fe73", + "discovery_ms": 5907.094460912049, + "first_status_ms": 2939.780990127474, + "warm_status_ms": 1040.7937010750175, + "wall_seconds": 10.742, + "cpu_seconds": 10.176561, + "kernel_family_peak_bytes": 667668480, + "token_proxy": { + "input_facing_catalog": 43507, + "arguments": 82, + "text_replies": 22690, + "combined": 66279, + "complete_discovered_catalog": 51352 + }, + "raw_sha256": { + "report.json": "14aed217fcad50915b211e4ee9894ebf9cb4abd8d4c4a4e007cc6fe2b397d44a", + "catalog.json": "e55a5b2cba59b09ac4f63831c2af6682d162bf5b05259b3c0f9db8eb2bc67e44", + "transcript.json": "e9a9ac8b657da35501e577bf1d1e18c89923c5bcdbb1307c8254d512bbdc7b53", + "baseline-d532-2-compact-off.log": "e7c0c68bd1088f11107a6713df1efb74f01ff3479d1913977fe22935814c912c" + } + }, + { + "recording": "candidate-1-1-full-full", + "verified_complete": true, + "profiles": ["full", "full"], + "catalog_tools": 209, + "catalog_bytes": 3322571, + "catalog_semantic_sha256": "799cc997a9573b5c3dfce3ac22e1fb5eb1637e81b489392f37cc81767b334359", + "discovery_ms": 7035.046560922638, + "first_status_ms": 1280.1523089874536, + "warm_status_ms": 962.2103709261864, + "wall_seconds": 10.309, + "cpu_seconds": 9.82015, + "kernel_family_peak_bytes": 806117376, + "token_proxy": { + "input_facing_catalog": 187004, + "arguments": 84, + "text_replies": 22690, + "combined": 209778, + "complete_discovered_catalog": 845872 + }, + "raw_sha256": { + "report.json": "4e6d4cb20d4b435b47164535930c60902eed44c4efec0665d9eeb2bd0635fd39", + "catalog.json": "f0700961a2a999671107dc5e5274098a5d97047e834654c7c3c7ee0c420a91b5", + "transcript.json": "4d0b64606c913325ce83bd797b2286d0e9758f09c526dacfce422756cf5212d8", + "candidate-1-1-full-full.log": "9d37d44776cfd9f41811e0a608c79ed63257383218e01f602317c2dc5a9de10f" + } + }, + { + "recording": "candidate-1-2-full-full", + "verified_complete": true, + "profiles": ["full", "full"], + "catalog_tools": 209, + "catalog_bytes": 3322571, + "catalog_semantic_sha256": "799cc997a9573b5c3dfce3ac22e1fb5eb1637e81b489392f37cc81767b334359", + "discovery_ms": 5431.887832004577, + "first_status_ms": 1290.1408409234136, + "warm_status_ms": 1058.6841800250113, + "wall_seconds": 8.819, + "cpu_seconds": 8.06667, + "kernel_family_peak_bytes": 801071104, + "token_proxy": { + "input_facing_catalog": 187004, + "arguments": 78, + "text_replies": 22690, + "combined": 209772, + "complete_discovered_catalog": 845872 + }, + "raw_sha256": { + "report.json": "e2853484e579376a8edb168f84ad2add2ffdd30ce78e123b2dfcdd17d524344e", + "catalog.json": "5c918daeb3f8b10625927a87f3f95367a2a2823cc7c098aa8ad3d8d5b47aa6ad", + "transcript.json": "1e22387d227134741eac842b8000c5bb73a42e2368c3c53e21bc1d45c17ed819", + "candidate-1-2-full-full.log": "877a3bdfe67875bb5ab1981850da2fe519428708184eef55a28c5fa18bb727ea" + } + }, + { + "recording": "candidate-1-1-compact-full", + "verified_complete": true, + "profiles": ["compact", "full"], + "catalog_tools": 209, + "catalog_bytes": 2723927, + "catalog_semantic_sha256": "d18f02920b84d39b90dc488db89a8f403f6ec35139a39a3329375a18078af1ea", + "discovery_ms": 5128.74938105233, + "first_status_ms": 1351.6553710214794, + "warm_status_ms": 1321.080510970205, + "wall_seconds": 8.815, + "cpu_seconds": 8.478922, + "kernel_family_peak_bytes": 781271040, + "token_proxy": { + "input_facing_catalog": 43507, + "arguments": 80, + "text_replies": 22702, + "combined": 66289, + "complete_discovered_catalog": 702182 + }, + "raw_sha256": { + "report.json": "ee38fcca794a959271af78a6c65377f411fdbe3b7b97068835b36586ec804ddf", + "catalog.json": "db8f1cbaa97c09672537d87ac48457bdb599ae439bca2d8a9216a032edbbdd88", + "transcript.json": "22d7a8b00909ad82e40ca7beb21d793bff61dacd88ba0d345ef82eeacf177516", + "candidate-1-1-compact-full.log": "ae17a479c9f094e522e15b58617302bf098464a4f55cea21cc913d18ba02afd2" + } + }, + { + "recording": "candidate-1-2-compact-full", + "verified_complete": true, + "profiles": ["compact", "full"], + "catalog_tools": 209, + "catalog_bytes": 2723927, + "catalog_semantic_sha256": "d18f02920b84d39b90dc488db89a8f403f6ec35139a39a3329375a18078af1ea", + "discovery_ms": 5686.0627620480955, + "first_status_ms": 1266.2633110303432, + "warm_status_ms": 1025.3590901847929, + "wall_seconds": 8.893, + "cpu_seconds": 8.191988, + "kernel_family_peak_bytes": 780988416, + "token_proxy": { + "input_facing_catalog": 43507, + "arguments": 82, + "text_replies": 22702, + "combined": 66291, + "complete_discovered_catalog": 702182 + }, + "raw_sha256": { + "report.json": "1ae8930195ef5d5b6d5118ffaad3d0d2bedaaf114b3887a28357504be023821f", + "catalog.json": "7e65fcac2fe7257a2f1c058f2b9ad1200772072fd7cabd4ea315bf4c48307303", + "transcript.json": "5a0220b5fe36ae31af91f9eb7f8006290fcf790c774096b838f7287c472e94c1", + "candidate-1-2-compact-full.log": "2e4baa5d7e627be27f87739d03f2a4e762ae1675a63e14cc7b5adf883c24192c" + } + }, + { + "recording": "candidate-1-1-full-off", + "verified_complete": true, + "profiles": ["full", "off"], + "catalog_tools": 209, + "catalog_bytes": 832191, + "catalog_semantic_sha256": "e537c521b5dd0ac369dee706815579d9d7803497a8fc7f7d683fec267042b81c", + "discovery_ms": 5637.611771002412, + "first_status_ms": 1178.2668600790203, + "warm_status_ms": 1036.6536700166762, + "wall_seconds": 8.791, + "cpu_seconds": 8.201682, + "kernel_family_peak_bytes": 656801792, + "token_proxy": { + "input_facing_catalog": 187005, + "arguments": 82, + "text_replies": 22698, + "combined": 209785, + "complete_discovered_catalog": 195042 + }, + "raw_sha256": { + "report.json": "cd706d0ab9a3dd9662ec73ecc45deefdbd40a4a42cd9ee2735be0f8798cc1933", + "catalog.json": "9d1fd380d490edf4b6403e3daed015b72bc654b96dfb06dbe7d776e6a6146d76", + "transcript.json": "0e3f0c66a2b638729d341af8d8a379576d267b53c1dcea5563e1afa4f9245cc8", + "candidate-1-1-full-off.log": "1baf3976c762923680cd17cbef36778b3982e90005eb34bb8ec3ce8030ebd3fb" + } + }, + { + "recording": "candidate-1-2-full-off", + "verified_complete": true, + "profiles": ["full", "off"], + "catalog_tools": 209, + "catalog_bytes": 832191, + "catalog_semantic_sha256": "e537c521b5dd0ac369dee706815579d9d7803497a8fc7f7d683fec267042b81c", + "discovery_ms": 5316.555531928316, + "first_status_ms": 1383.681601146236, + "warm_status_ms": 1108.902801061049, + "wall_seconds": 8.698, + "cpu_seconds": 8.19166, + "kernel_family_peak_bytes": 662822912, + "token_proxy": { + "input_facing_catalog": 187004, + "arguments": 82, + "text_replies": 22698, + "combined": 209784, + "complete_discovered_catalog": 195042 + }, + "raw_sha256": { + "report.json": "89c5f579cb8f8a1860b3f605042ffc51db4aaae69f7a22213256f93e42b91672", + "catalog.json": "0301c40f667a17a9238ad8d9abdbb8e8c7b07a7d0dcc6ae062f5885e0b19dfc7", + "transcript.json": "7e884d99d3335707f9108a63528fd2830b1c21e7888cfc320852d75924d833cb", + "candidate-1-2-full-off.log": "ba34788aa92aa60e79e832659c48f304396d1df41f493a509ff474a152fa7c6c" + } + }, + { + "recording": "candidate-1-1-compact-off", + "verified_complete": true, + "profiles": ["compact", "off"], + "catalog_tools": 209, + "catalog_bytes": 233547, + "catalog_semantic_sha256": "0b81bdd81251ddd527a9c9ecbb5e6048e9c8a3be1f6195465a803d206f03fe73", + "discovery_ms": 4922.329220920801, + "first_status_ms": 813.2044598460197, + "warm_status_ms": 1415.0590698700398, + "wall_seconds": 7.982, + "cpu_seconds": 7.401054, + "kernel_family_peak_bytes": 629260288, + "token_proxy": { + "input_facing_catalog": 43507, + "arguments": 80, + "text_replies": 22678, + "combined": 66265, + "complete_discovered_catalog": 51352 + }, + "raw_sha256": { + "report.json": "89ae06a9c0e8f4c9b85aa711fc6f71204798c990e582f5702713f78612c4fce2", + "catalog.json": "d6a231b2babd7885d58f9712881f96fa9dc2b763cc7efd802bc5676dd71fe28a", + "transcript.json": "65fbdb48a9bc2aa65a815ea405d1210e298f908ddbab5e60f815c4e01a0f0e98", + "candidate-1-1-compact-off.log": "76501971a3698b13435b96647b59ee092689397012d453e62c821e4ecf40e988" + } + }, + { + "recording": "candidate-1-2-compact-off", + "verified_complete": true, + "profiles": ["compact", "off"], + "catalog_tools": 209, + "catalog_bytes": 233547, + "catalog_semantic_sha256": "0b81bdd81251ddd527a9c9ecbb5e6048e9c8a3be1f6195465a803d206f03fe73", + "discovery_ms": 4839.713891968131, + "first_status_ms": 1174.165149917826, + "warm_status_ms": 1409.5584598835558, + "wall_seconds": 8.289, + "cpu_seconds": 7.764389, + "kernel_family_peak_bytes": 627363840, + "token_proxy": { + "input_facing_catalog": 43507, + "arguments": 80, + "text_replies": 22678, + "combined": 66265, + "complete_discovered_catalog": 51352 + }, + "raw_sha256": { + "report.json": "a4ef1220b73c39421ccfc22c2102c613fd0a33900cbc8683dad1dc3bc034cf5a", + "catalog.json": "39ae1d7385fe8684f037e354ec2ab1126243fd31667be1b1cc559a9d1e40149c", + "transcript.json": "d60eae5fdbfb4095f040d54103a6e9b1d2ff8b8f64126976d14b0a3e024969f7", + "candidate-1-2-compact-off.log": "0e195ca040565e29e1f6d25b087a706d0d500ff1752a24bff54ad30704bd6d6b" + } + } + ], + "digest_phases": [ + { + "recording": "baseline-d532-1-compact-full-digest", + "input": "compact", + "output": "full", + "import_ms": 3827.9035209999997, + "digest_ms": 2916.01332, + "digest_cpu_us": { + "user": 2587474, + "system": 151462 + }, + "raw_sha256": "28a8e0b5c847978be9d71e4d5ed6953758e266e697b820f45f1d2141861c95e3" + }, + { + "recording": "baseline-d532-1-compact-off-digest", + "input": "compact", + "output": "off", + "import_ms": 3873.5359620000004, + "digest_ms": 3790.342601, + "digest_cpu_us": { + "user": 3121749, + "system": 212457 + }, + "raw_sha256": "b88dd91f159a1763d547f1e26637afaf3a03b28faad3295c2deff8c90e7d7707" + }, + { + "recording": "baseline-d532-1-full-full-digest", + "input": "full", + "output": "full", + "import_ms": 1766.4657209999998, + "digest_ms": 0.06419000000005326, + "digest_cpu_us": { + "user": 65, + "system": 12 + }, + "raw_sha256": "0f18c0d63def7df73b6183240c49011aca9cd85899a221fdb01908da1d79120f" + }, + { + "recording": "baseline-d532-1-full-off-digest", + "input": "full", + "output": "off", + "import_ms": 5377.792571, + "digest_ms": 3209.784441, + "digest_cpu_us": { + "user": 2585090, + "system": 252879 + }, + "raw_sha256": "2de37e6aac00f37003f619c4b4a5a747637e0b188e9892d5a37a091ae568400a" + }, + { + "recording": "baseline-d532-2-compact-full-digest", + "input": "compact", + "output": "full", + "import_ms": 1807.87952, + "digest_ms": 1512.0799399999999, + "digest_cpu_us": { + "user": 1421207, + "system": 88507 + }, + "raw_sha256": "a330124a6cec474e5337aa5a40c33d31fa0bdce304554c1c68f79954bf5cdd33" + }, + { + "recording": "baseline-d532-2-compact-off-digest", + "input": "compact", + "output": "off", + "import_ms": 1854.8867300000002, + "digest_ms": 1402.711411, + "digest_cpu_us": { + "user": 1284446, + "system": 116933 + }, + "raw_sha256": "02f19af3b9836796d85f81effd5e86bab0aa4978eeae0c59b8c35e57d03dd17b" + }, + { + "recording": "baseline-d532-2-full-full-digest", + "input": "full", + "output": "full", + "import_ms": 1572.84796, + "digest_ms": 0.0849799999998595, + "digest_cpu_us": { + "user": 85, + "system": 15 + }, + "raw_sha256": "a9fca1656b8763f5ef305548eceb7bbd27cdeb426c6d6934a891c82823a54f31" + }, + { + "recording": "baseline-d532-2-full-off-digest", + "input": "full", + "output": "off", + "import_ms": 1814.550491, + "digest_ms": 1280.248369, + "digest_cpu_us": { + "user": 1166910, + "system": 108321 + }, + "raw_sha256": "350a276f6f408de517db2b19af7cd7490c243b71338854d4acb54418f1a00b64" + }, + { + "recording": "candidate-1-1-compact-full-digest", + "input": "compact", + "output": "full", + "import_ms": 1760.5044, + "digest_ms": 0.08051999999997861, + "digest_cpu_us": { + "user": 76, + "system": 14 + }, + "raw_sha256": "3ebc2dac6cff195b54918e5afaedc0b73fe9f1a63ef12ebb253d9a1aca683c8c" + }, + { + "recording": "candidate-1-1-compact-off-digest", + "input": "compact", + "output": "off", + "import_ms": 1922.20413, + "digest_ms": 0.07567999999992026, + "digest_cpu_us": { + "user": 75, + "system": 11 + }, + "raw_sha256": "d5916e1e5760f9c7ce5a42ee8b7635477464a5a7cb7d379309666357b6fbe35f" + }, + { + "recording": "candidate-1-1-full-full-digest", + "input": "full", + "output": "full", + "import_ms": 1777.54332, + "digest_ms": 0.08724099999994905, + "digest_cpu_us": { + "user": 93, + "system": 13 + }, + "raw_sha256": "ed68a8c592c9556b7a279a90b7324d5549f9dc197fe593d27c55c5de24827481" + }, + { + "recording": "candidate-1-1-full-off-digest", + "input": "full", + "output": "off", + "import_ms": 2062.896171, + "digest_ms": 0.06988900000033027, + "digest_cpu_us": { + "user": 69, + "system": 12 + }, + "raw_sha256": "166dcdaa064c334dcc882429c2434c0353b0f6d2ccdea986138a2149ccccf10a" + }, + { + "recording": "candidate-1-2-compact-full-digest", + "input": "compact", + "output": "full", + "import_ms": 1813.35421, + "digest_ms": 0.06941099999994549, + "digest_cpu_us": { + "user": 73, + "system": 10 + }, + "raw_sha256": "86497ee4962fd03fcad4f2bcc31e95f5f57d9c4270cc0578d4b43a076ec01b53" + }, + { + "recording": "candidate-1-2-compact-off-digest", + "input": "compact", + "output": "off", + "import_ms": 2024.0317209999998, + "digest_ms": 0.08607000000029075, + "digest_cpu_us": { + "user": 89, + "system": 13 + }, + "raw_sha256": "82b47ed9bc203f1417f5f76de808d493c171f95598934fc9ebbd6f63be43b938" + }, + { + "recording": "candidate-1-2-full-full-digest", + "input": "full", + "output": "full", + "import_ms": 1917.6121309999999, + "digest_ms": 0.07732000000009975, + "digest_cpu_us": { + "user": 77, + "system": 11 + }, + "raw_sha256": "410395499363c55e53fe4d8488b5e83c54170786fe1814389fd3bf4ec317c6ac" + }, + { + "recording": "candidate-1-2-full-off-digest", + "input": "full", + "output": "off", + "import_ms": 1715.96008, + "digest_ms": 0.08907999999996719, + "digest_cpu_us": { + "user": 104, + "system": 15 + }, + "raw_sha256": "f6ea0fffc2974e02bdab1348736533c8dbd677c2311e339a945d760bd9337f9a" + } + ], + "build_digest_phase": [ + { + "phase": "before", + "operation": "createCatalogDigests", + "import_ms": 2103.978981, + "compute_ms": 1640.0212299999998, + "compute_cpu_us": { + "user": 1487510, + "system": 148234 + }, + "raw_sha256": "3a4bfe5e4297e95cf0ab3b832bb34c81edeca02e7f884db7dd5a6944c2ee1674" + }, + { + "phase": "after", + "operation": "createCatalogDigestProfiles", + "import_ms": 1752.475121, + "compute_ms": 2379.7143199999996, + "compute_cpu_us": { + "user": 2232836, + "system": 140061 + }, + "raw_sha256": "b25120373c7433457e30ff00925ca2771c0d61ba96828fe89e63be651b4f6e4a" + } + ], + "failed_attempts": [ + { + "recording": "codex-host-full-1", + "outcome": "Harness expected counts.tools rather than canonical counts.mcp_tools; corrected oracle, not a completed task.", + "raw_sha256": { + "report.json": "ac3d531a9dad31e31cb9a5957ed1f5e6415534fda1e52c64bd93a07a36ef7d0f", + "codex-host-full-1.log": "7c5f13e2976efef59306b9a2fabd2e2031277d3c5aed2b74f4ff2ad80c25a61c" + } + }, + { + "recording": "codex-host-matrix-compact-off-1", + "outcome": "Functional checks passed but MCP child remained at payload exit. Excluded from qualified cohort. Controller now waits for allocated scope to drain without targeting a PID.", + "raw_sha256": { + "report.json": "1cbc9df5aff972538629ed695271f0c3a66fcfe546208b392d09b3b7cf72f97c", + "codex-host-matrix-compact-off-1.log": "997e13a9ea7b3c5247d5037279fbef1d0dfd238eed0781748cc2ddf3afc706e7" + } + } + ], + "unverified": [ + "Model selection/injection and actual model tokens", + "Provider analysis, target execution and end-to-end reverse-engineering application quality", + "Source-mode speedup: absent build records still use dynamic fallback", + "Other MCP clients/OS compatibility not established by this Codex experiment" + ], + "validation": [ + { + "check": "check-profiles", + "payload_exit_code": 1, + "owned_cleanup_verified": true, + "raw_sha256": "04dd1ad8142e61e9c2960ec55a3a7f0421f4c22f5cb5b9702761a0d7f29d3cc4", + "source_head": "pre-rebase candidate", + "outcome": "Unused build-only export identified; existing output-policy coverage extended to the profile builder." + }, + { + "check": "build-latest", + "payload_exit_code": 0, + "owned_cleanup_verified": true, + "raw_sha256": "f430e38ecb205aa7bde17b48ff7ad97805b18511dbbfee8bdbab605f4f93b86e", + "source_head": "7ad06628df042a7c8034155a98fc51c35fa0704c", + "outcome": "passed" + }, + { + "check": "tests-latest", + "payload_exit_code": 0, + "owned_cleanup_verified": true, + "raw_sha256": "58b06f5be6ae5c0cb871cba6788b3f84581289a47ffe26aaad8897b606c94e2d", + "source_head": "7ad06628df042a7c8034155a98fc51c35fa0704c", + "outcome": "passed" + }, + { + "check": "check-latest", + "payload_exit_code": 0, + "owned_cleanup_verified": true, + "raw_sha256": "2373fec55a74515f6e3036d46becd4ed76149d9a70731a64c5dddacaeeb920cc", + "source_head": "7ad06628df042a7c8034155a98fc51c35fa0704c", + "outcome": "passed" + }, + { + "check": "docs-latest", + "payload_exit_code": 0, + "owned_cleanup_verified": true, + "raw_sha256": "23ef004935364b7a1ad2634610b67c1d739bde4f4c455552a473a05b346920ee", + "source_head": "7ad06628df042a7c8034155a98fc51c35fa0704c", + "outcome": "passed" + } + ], + "post_rebase_qualification": { + "head": "7ad06628df042a7c8034155a98fc51c35fa0704c", + "base": "e84468c0b", + "client": "Codex app-server 0.163.0", + "profiles": ["compact", "off"], + "outcome": "Complete discovery and first/warm status facts pass; allocated scope drains; no model turns. Qualification only; not relabeled as the pinned before/after measurements.", + "raw_sha256": { + "final-latest-compact-off/report.json": "ba6f549fe087f55dc48011daf92bd18f4bfc2a32e6185081ab2b7436bf39b1b8", + "final-latest-compact-off/transcript.json": "461a25723c2a4bb4c18675ffa613e98a8592a4e933054fa8643f1da73fc4686f", + "final-latest-compact-off.log": "4efc56e69b8f0d1111345bce10d0d0792b510e18901f52d10e538374257e1d61" + } + } +} diff --git a/experiments/analysis-efficiency/mcp-profile-identity.md b/experiments/analysis-efficiency/mcp-profile-identity.md new file mode 100644 index 0000000000..812473e677 --- /dev/null +++ b/experiments/analysis-efficiency/mcp-profile-identity.md @@ -0,0 +1,131 @@ +# MCP schema profile identity startup + +Continues [#723](https://github.com/morluto/rea/issues/723). The default catalog +identity was already recorded at build time (#4041). Compact input and omitted +output advertisement profiles still converted and hashed the complete catalog +on demand. This change records the four built-in combinations at the same +owning build step. Canonical tool facts and output-schema conversion are shared +while building those records. Only immutable hash triples reach the runtime: +the generated module grows from 411 to 1,320 bytes. + +Exact full/compact and full/off selections choose their matching record. Source +execution and custom compact budgets retain lazy dynamic calculation, including +snapshotting the caller's budget. The build consumes the compiled contracts and +pinned converter before bundling; rebuilding invalidates the records. No live +provider state is cached. Output-off identities still cover canonical output +contracts, as before. All four digest triples match the pre-change runtime. + +## Completed task and oracle + +A fresh **actual Codex app-server 0.163.0** discovers every tool, then reports +whether a target is open, cleanup is pending, a provider is bound, and whether +the package version and catalog count match, twice in the same session. +The oracle is the selected package metadata, the pinned 209-tool contract, +no-target state, text/structured-result agreement and original pre-change +identities. Canonicalized actual-client catalogs agree before/after within +each profile; JSON object order in Codex's tool map varies by process. + +The [driver](mcp-profile-identity.py) uses the installed client's version-specific +`initialize`, `thread/start`, `mcpServerStatus/list` and `mcpServer/tool/call` +protocol ([official app-server documentation](https://learn.chatgpt.com/docs/app-server)). +It uses a fresh temporary account/configuration and starts **no model turns**. +This establishes actual-client schema discovery and direct tool-call +compatibility, not model analysis quality or actual model token savings. +Existing compact/off choices remain explicit; neither becomes a new default. + +Baseline: main `d532b18123916a9e88f98c7880ba4f984e23d715`. Both packaged runtimes +are frozen before measurement; their source patch, package/manifests and raw +transcripts/resource observations are retained outside Git, pinned by the +[receipt](mcp-profile-identity.json). Later validation does not relabel timings. +Node 24.18.0/npm 11.16.0, MCP SDK 2.3.1, negotiated protocol `2025-06-18`, Linux +x86-64. Two repetitions alternate baseline/candidate cohorts and reverse profile +order. Every recording starts a fresh server and includes discovery, first and +warm status calls, verification and cleanup. OS caches are unflushed. + +The existing guard admits one CPU, a 4 GiB hard/3 GiB early memory boundary, +2 GiB Node heaps, no swap and 192 tasks. Kernel family peaks include file cache, +Codex, REA and controller; they are not provider heap measurements. All 16 +qualified recordings exit successfully with only the observer remaining, no +resource stop and zero observed OOM events. Final whole-scope counters are +unavailable. The host's CPU is shared with unrelated work; ranges retain the +observed variation. + +## Measurements + +Ranges show both repetitions. Times include the whole task and shutdown. + +| Input / output profile | Wall seconds before → after | CPU seconds before → after | Family peak MiB before → after | +| ---------------------- | --------------------------- | -------------------------- | ------------------------------ | +| full / full | 8.91–8.92 → 8.82–10.31 | 8.45–8.48 → 8.07–9.82 | 765–766 → 764–769 | +| compact / full | 11.15–18.99 → 8.82–8.89 | 10.61–16.52 → 8.19–8.48 | 772–779 → 745–745 | +| full / off | 10.33–19.59 → 8.70–8.79 | 9.76–16.96 → 8.19–8.20 | 671–677 → 626–632 | +| compact / off | 10.74–23.47 → 7.98–8.29 | 10.18–19.23 → 7.40–7.76 | 635–637 → 598–600 | + +First/warm call times remain separately recorded in the receipt. Across the +three non-default profiles the first call ranges from 2.73–6.15 s before to +0.81–1.38 s after; warm calls range from 1.04–3.12 s before to 1.03–1.42 s after. +The default profile has no established gain; one candidate repetition is slower. + +Fresh-process [identity probes](mcp-profile-digest.mjs) isolate the lazy read +from module loading. The non-default digest computation takes 1.28–3.79 s +before and under 0.2 ms after. Building all four identities shares schema +conversion: isolated compute CPU increases from 1.636 s (one identity) to +2.373 s (four). Full uncached builds were 42.59 → 81.09 s wall, 41.59 → 68.27 s +CPU, 3318.5 → 3216.4 MiB peak. These single builds include TypeScript, bundling +and concurrent host contention, so their difference is not attributed solely +to digest generation. The isolated phase is one observation per version. + +There is **no token reduction from this change**. Each profile's complete +catalog size and serialization-token estimate are unchanged. Full/full remains +3,322,571 bytes as exposed by this client; compact/off remains 233,547 bytes +through existing configuration. The input-facing catalog plus both argument +and text replies is about 209.8k proxy tokens with full input and 66.3k with +compact input, before and after. Small differences come from fresh timestamps +and runtime paths. The [cost script](mcp-profile-costs.py) uses +`tiktoken==0.12.0`/`cl100k_base`; these estimates are not actual model usage or a +claim about what the host injects. Output schemas, hidden prompts and model +reasoning must not be inferred from that projection. + +## Validation and remaining scope + +Executed: four files / 46 tests pass, including real production stdio doctor +for all four profiles, source catalog/schema consistency, output-off identity +coverage, malformed replies, startup cancellation/cleanup, generated immutable +record reuse and caller-selected custom budgets. The pre-change runtime is +also an independent oracle for all four complete hash triples. + +After rebasing onto main `e84468c0b`, the four files / 46 tests pass again, +as do `npm run check` (types, lint, format, knip, metadata and architecture) +and `npm run docs:check` (including five README consistency tests). Actual +Codex compact/off discovery and both status calls pass again with verified +cleanup. This final qualification does not replace the pinned timing cohort. +An initial static check identified the build-only export as untracked; the +existing output-policy regression now directly exercises its four profiles. + +Earlier exploratory attempts remain excluded: one controller expected the +wrong count key; one functional compact/off run left a child alive at payload +exit. The corrected controller waits for its allocated scope to drain without +killing individual PIDs; all later qualified runs verify that drain. Neither +failed attempt establishes a production regression. + +SDK 2.3.1 already supports a named `toolDefinition`, used by REA's doctor to +avoid preparing every output validator for its one probe. Generic external SDK +callers still incur their own catalog preparation. Input presentation and +output refs already have owning reuse; these were not duplicate fixes. +Target/provider investigations, other clients/platforms and actual model +analysis cost remain unverified by this no-target startup task. + +To reproduce with the pinned client and a built runtime, run under the existing +resource guard: + +```sh +python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \ + python3 experiments/analysis-efficiency/mcp-profile-identity.py \ + --output /tmp/rea-profile-case --runtime /absolute/built/rea \ + --node /absolute/node24/bin/node --codex /absolute/codex-0.163.0 \ + --input-profile compact --output-profile off +``` + +Retain the resource output as `/tmp/rea-profile-case.log` for the cost script. +The driver is pinned to Codex 0.163.0's experimental protocol and the 209-tool +catalog in this receipt; update its oracle deliberately for another revision. diff --git a/experiments/analysis-efficiency/mcp-profile-identity.py b/experiments/analysis-efficiency/mcp-profile-identity.py new file mode 100644 index 0000000000..78c9dcc862 --- /dev/null +++ b/experiments/analysis-efficiency/mcp-profile-identity.py @@ -0,0 +1,199 @@ +"""Exercise real Codex MCP discovery/calls without starting a model turn.""" +import argparse +import asyncio +import hashlib +import json +import os +from pathlib import Path +import re +import time +import traceback + + +async def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--runtime", type=Path, required=True) + parser.add_argument("--codex", type=Path, required=True) + parser.add_argument("--node", type=Path, required=True) + parser.add_argument("--input-profile", choices=["full", "compact"], default="full") + parser.add_argument("--output-profile", choices=["full", "off"], default="full") + args = parser.parse_args() + args.output.mkdir(mode=0o700) + audit_home = args.output / "account" + codex_state = audit_home / ".codex" + mcp_home = audit_home / "mcp" + workspace = audit_home / "workspace" + for directory in [codex_state, mcp_home, workspace]: + directory.mkdir(parents=True) + task_path = f"{args.node.parent}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + mcp_env = {"HOME": str(mcp_home), "PATH": task_path, + "NODE_OPTIONS": "--max-old-space-size=2048", "UV_THREADPOOL_SIZE": "1", + "REA_LOG_LEVEL": "silent", "REA_MCP_INPUT_SCHEMA_PROFILE": args.input_profile, + "REA_MCP_OUTPUT_SCHEMA_PROFILE": args.output_profile} + config = '[analytics]\nenabled = false\n[mcp_servers.rea_audit]\n' + config += f'command = {json.dumps(str(args.node))}\n' + config += f'args = {json.dumps([str(args.runtime / "scripts/rea.mjs"), "mcp"])}\n' + config += 'startup_timeout_sec = 60\ntool_timeout_sec = 60\n' + config += '[mcp_servers.rea_audit.env]\n' + config += ''.join(f'{key} = {json.dumps(value)}\n' for key, value in mcp_env.items()) + (codex_state / "config.toml").write_text(config) + (args.output / "declared-config.toml").write_text(config) + environment = {"HOME": str(audit_home), "CODEX_HOME": str(codex_state), "PATH": task_path, + "NODE_OPTIONS": "--max-old-space-size=2048", "UV_THREADPOOL_SIZE": "1"} + report = {"question": "Discover the complete canonical tool catalog and report no-target session/cleanup state, selected provider, package version and catalog tool count twice.", + "client": "Codex app-server 0.163.0", "model_turns_started": 0, + "actual_model_usage": None, "input_profile": args.input_profile, + "output_profile": args.output_profile, "calls": [], "observations": [], + "unexpected_server_requests": [], "cleanup_failures": []} + transcript = [] + pending = {} + next_id = 0 + stderr = (args.output / "server.log").open("wb") + process = await asyncio.create_subprocess_exec(str(args.codex), "app-server", "--stdio", + stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=stderr, + cwd=workspace, env=environment, limit=16 * 1024 * 1024) + + async def send(message): + encoded = (json.dumps(message, separators=(",", ":")) + "\n").encode() + transcript.append({"direction": "client", "bytes": len(encoded), "message": message}) + process.stdin.write(encoded) + await process.stdin.drain() + + async def read_messages(): + try: + while line := await process.stdout.readline(): + value = json.loads(line) + transcript.append({"direction": "server", "bytes": len(line), "message": value}) + if "id" in value and "method" not in value: + future = pending.pop(value["id"], None) + if future and not future.done(): + future.set_result(value) + elif "id" in value: + report["unexpected_server_requests"].append(value["method"]) + await send({"id": value["id"], "error": {"code": -32601, "message": "Unexpected request in read-only discovery probe"}}) + raise RuntimeError("app-server stdout closed") + except Exception as error: + for future in pending.values(): + if not future.done(): + future.set_exception(error) + + reader = asyncio.create_task(read_messages()) + + async def request(method, params): + nonlocal next_id + assert method in ["initialize", "thread/start", "mcpServerStatus/list", "mcpServer/tool/call"] + next_id += 1 + message = {"id": next_id, "method": method, "params": params} + future = asyncio.get_running_loop().create_future() + pending[next_id] = future + start = time.monotonic() + await send(message) + reply = await asyncio.wait_for(future, 90) + report["calls"].append({"method": method, "elapsed_ms": (time.monotonic() - start) * 1000, + "error": reply.get("error")}) + if "error" in reply: + raise RuntimeError(json.dumps(reply["error"])[:4000]) + return reply["result"] + + try: + await request("initialize", {"clientInfo": {"name": "rea_host_probe", "title": "REA host compatibility probe", "version": "1"}, + "capabilities": {"experimentalApi": True, "explicitGatewayOauth": True}}) + await send({"method": "initialized"}) + created = await request("thread/start", {"cwd": str(workspace), "ephemeral": True, + "approvalPolicy": "never", "sandbox": "read-only"}) + thread_id = created["thread"]["id"] + assert not created["thread"].get("turns", []) + inventory = await request("mcpServerStatus/list", {"threadId": thread_id, + "serverName": "rea_audit", "detail": "toolsAndAuthOnly", "limit": 10}) + assert inventory.get("nextCursor") is None + assert len(inventory["data"]) == 1 + server = inventory["data"][0] + assert server["name"] == "rea_audit" and server.get("toolsError") is None + tools = list(server["tools"].values()) + names = [tool["name"] for tool in tools] + assert len(names) == len(set(names)) == 209 + assert "binary_session" in names + output_schemas = sum(tool.get("outputSchema") is not None for tool in tools) + assert output_schemas == (209 if args.output_profile == "full" else 0) + raw = json.dumps(inventory, separators=(",", ":"), ensure_ascii=False).encode() + (args.output / "catalog.json").write_bytes(raw) + report["catalog"] = {"tools": len(tools), "output_schemas": output_schemas, + "bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest(), + "server_info": server.get("serverInfo"), "runtime_status": server.get("runtimeStatus")} + version = json.loads((args.runtime / "package.json").read_text())["version"] + for phase in ["first", "warm"]: + reply = await request("mcpServer/tool/call", {"threadId": thread_id, + "server": "rea_audit", "tool": "binary_session", "arguments": {}}) + assert reply.get("isError") is not True + structured = reply["structuredContent"] + texts = [c["text"] for c in reply["content"] if c.get("type") == "text"] + assert len(texts) == 1 and json.loads(texts[0]) == structured + facts = structured["result"] + assert facts["open"] is False and facts["cleanup_pending"] is False + assert facts["analysis_provider_binding"] is None + assert facts["server_identity"]["package"]["version"] == version + assert facts["server_identity"]["catalog"]["counts"]["mcp_tools"] == 209 + report["observations"].append({"phase": phase, "open": facts["open"], + "cleanup_pending": facts["cleanup_pending"], "selected_provider": facts["analysis_provider_binding"], + "package_version": version, "catalog_tools": 209, + "negotiated_protocol": facts["server_identity"]["negotiated_protocol_version"], + "structured_bytes": len(json.dumps(structured, separators=(",", ":"), ensure_ascii=False).encode())}) + selected_facts = lambda observation: {key: value for key, value in observation.items() + if key not in ["phase", "structured_bytes"]} + assert selected_facts(report["observations"][0]) == selected_facts(report["observations"][1]) + assert not report["unexpected_server_requests"] + report["status"] = "passed" + except Exception: + report["status"] = "failed" + report["failure"] = traceback.format_exc()[-6000:] + finally: + process.stdin.close() + try: + await asyncio.wait_for(process.wait(), 15) + except asyncio.TimeoutError: + report["cleanup_failures"].append("app-server did not exit after stdin close; terminated owned process") + process.terminate() + try: + await asyncio.wait_for(process.wait(), 5) + except asyncio.TimeoutError: + process.kill() + await process.wait() + await reader + # app-server can acknowledge exit before its MCP child finishes EOF + # cleanup. Wait on the allocated resource scope; never target a PID. + try: + groups = [line[3:] for line in Path("/proc/self/cgroup").read_text().splitlines() + if line.startswith("0::")] + assert len(groups) == 1 + group = Path("/sys/fs/cgroup") / groups[0].lstrip("/") + assert re.fullmatch(r"rea-efficiency-[0-9a-f]{32}\.scope", group.name) + observer = os.getppid() + observer_args = Path(f"/proc/{observer}/cmdline").read_bytes().split(b"\0") + assert any(Path(os.fsdecode(value)).name == "resource_payload.py" for value in observer_args if value) + expected = {os.getpid(), observer} + until = time.monotonic() + 10 + while True: + actors = {int(value) for value in (group / "cgroup.procs").read_text().splitlines()} + assert expected <= actors + remaining = sorted(actors - expected) + if not remaining or time.monotonic() >= until: + break + await asyncio.sleep(0.05) + report["owned_scope_drain"] = {"remaining_processes": remaining} + if remaining: + report["cleanup_failures"].append("owned MCP child did not settle after app-server exit") + except (OSError, ValueError, AssertionError) as error: + report["owned_scope_drain"] = {"status": "unavailable", "reason": type(error).__name__} + report["cleanup_failures"].append("allocated resource-scope cleanup could not be verified") + stderr.close() + report["appserver_exit_code"] = process.returncode + if process.returncode != 0: + report["cleanup_failures"].append(f"app-server exit code: {process.returncode}") + (args.output / "transcript.json").write_text(json.dumps(transcript) + "\n") + (args.output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps({key: report.get(key) for key in ["status", "failure", "catalog", "observations", "cleanup_failures", "appserver_exit_code"]})) + return 0 if report["status"] == "passed" and not report["cleanup_failures"] else 1 + + +raise SystemExit(asyncio.run(main())) diff --git a/scripts/record-catalog-digests.mjs b/scripts/record-catalog-digests.mjs index 82807c121f..6e3525b9b3 100644 --- a/scripts/record-catalog-digests.mjs +++ b/scripts/record-catalog-digests.mjs @@ -2,18 +2,31 @@ import { writeFile } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -// Record the default catalog digests into the compiled runtime. Run after +// Record built-in catalog profile digests into the compiled runtime. Run after // `tsc` and before bundling, so the digests come from the same compiled // contracts and pinned schema converter that the runtime serves. const compiledRoot = resolve( dirname(fileURLToPath(import.meta.url)), "../dist", ); -const { createCatalogDigests } = await import( +const { createCatalogDigestProfiles } = await import( pathToFileURL(join(compiledRoot, "catalogIdentity.js")).href ); +// Retain only hashes, with every shared record frozen. Schemas remain owned by +// the compiled contracts and are converted once for all four identities. +const profiles = Object.entries(createCatalogDigestProfiles()) + .map( + ([input, outputs]) => + `${JSON.stringify(input)}: Object.freeze({${Object.entries(outputs) + .map( + ([output, digests]) => + `${JSON.stringify(output)}: Object.freeze(${JSON.stringify(digests)})`, + ) + .join(",")}})`, + ) + .join(","); await writeFile( join(compiledRoot, "recordedCatalogDigests.js"), - `/** Recorded from the compiled default catalog by scripts/record-catalog-digests.mjs; do not edit. */\nexport const RECORDED_CATALOG_DIGESTS = Object.freeze(${JSON.stringify(createCatalogDigests())});\n`, + `/** Recorded from compiled catalog profiles by scripts/record-catalog-digests.mjs; do not edit. */\nexport const RECORDED_CATALOG_DIGEST_PROFILES = Object.freeze({${profiles}});\n`, "utf8", ); diff --git a/src/catalogIdentity.ts b/src/catalogIdentity.ts index 80425d26ee..8429783a90 100644 --- a/src/catalogIdentity.ts +++ b/src/catalogIdentity.ts @@ -17,7 +17,11 @@ import { type CompactInputSchemaPresentation, } from "./contracts/compactInputSchema.js"; import type { McpOutputSchemaProfile } from "./config/mcpOutputSchemaProfile.js"; -import { RECORDED_CATALOG_DIGESTS } from "./recordedCatalogDigests.js"; +import { + COMPACT_INPUT_SCHEMA_BUDGET_BYTES, + type McpInputSchemaProfile, +} from "./config/mcpInputSchemaProfile.js"; +import { RECORDED_CATALOG_DIGEST_PROFILES } from "./recordedCatalogDigests.js"; const sortByName = ( contracts: readonly ToolContract[], @@ -34,10 +38,7 @@ const sortedToolContracts = sortByName(TOOL_CONTRACTS); * that predates the untrusted-content framing must not digest equal to one * that emits `_meta.provenance`, or a stale process would report aligned. */ -const createToolCatalog = ( - contracts: readonly ToolContract[], - compactPresentation: CompactInputSchemaPresentation | undefined, -) => +const createToolCatalog = (contracts: readonly ToolContract[]) => contracts.map((contract) => ({ name: contract.name, title: contract.title, @@ -46,15 +47,26 @@ const createToolCatalog = ( effects: { ...contract.effects }, returns_target_content: contract.returnsTargetContent, annotations: contract.annotations, - input_schema: presentAdvertisedInputSchema( - advertisedInputJsonSchema(contract), - compactPresentation, - ), + input_schema: advertisedInputJsonSchema(contract), output_schema: toolOutputSchemaWithMetadata(contract)[ "~standard" ].jsonSchema.output({ target: "draft-2020-12" }), })); +const presentToolCatalog = ( + catalog: ReturnType, + presentation: CompactInputSchemaPresentation | undefined, +) => + presentation === undefined + ? catalog + : catalog.map((tool) => ({ + ...tool, + input_schema: presentAdvertisedInputSchema( + tool.input_schema, + presentation, + ), + })); + const createPromptCatalog = (contracts: readonly PromptContract[]) => contracts .map((contract) => ({ @@ -76,22 +88,15 @@ const digest = (value: unknown): string => { return createHash("sha256").update(encoded).digest("hex"); }; -/** - * Catalog digests over the given tool contracts. Exported so tests can prove - * which contract fields the identity covers; production uses - * {@link CATALOG_IDENTITY}. - */ -export const createCatalogDigests = ( - contracts: readonly ToolContract[] = TOOL_CONTRACTS, - promptContracts: readonly PromptContract[] = PROMPT_CONTRACTS, - compactPresentation: CompactInputSchemaPresentation | undefined = undefined, - outputSchemaProfile: McpOutputSchemaProfile = "full", +const digestCatalogs = ( + catalog: ReturnType, + prompts: ReturnType, + outputSchemaProfile: McpOutputSchemaProfile, ) => { const tools = { output_schema_profile: outputSchemaProfile, - catalog: createToolCatalog(sortByName(contracts), compactPresentation), + catalog, }; - const prompts = createPromptCatalog(promptContracts); return { tools_sha256: digest(tools), prompts_sha256: digest(prompts), @@ -103,7 +108,50 @@ export const createCatalogDigests = ( } as const; }; -export type CatalogDigests = ReturnType; +/** + * Catalog digests over the given tool contracts. Exported so tests can prove + * which contract fields the identity covers; production uses + * {@link CATALOG_IDENTITY}. + */ +export const createCatalogDigests = ( + contracts: readonly ToolContract[] = TOOL_CONTRACTS, + promptContracts: readonly PromptContract[] = PROMPT_CONTRACTS, + compactPresentation: CompactInputSchemaPresentation | undefined = undefined, + outputSchemaProfile: McpOutputSchemaProfile = "full", +) => + digestCatalogs( + presentToolCatalog( + createToolCatalog(sortByName(contracts)), + compactPresentation, + ), + createPromptCatalog(promptContracts), + outputSchemaProfile, + ); + +export type CatalogDigests = ReturnType; +export type CatalogDigestProfiles = Readonly< + Record< + McpInputSchemaProfile, + Readonly> + > +>; + +/** Build-time identities share schema conversion across the four supported profiles. */ +export const createCatalogDigestProfiles = ( + contracts: readonly ToolContract[] = TOOL_CONTRACTS, + promptContracts: readonly PromptContract[] = PROMPT_CONTRACTS, +): CatalogDigestProfiles => { + const full = createToolCatalog(sortByName(contracts)); + const compact = presentToolCatalog(full, { + budgetBytes: COMPACT_INPUT_SCHEMA_BUDGET_BYTES, + }); + const prompts = createPromptCatalog(promptContracts); + const outputProfiles = (catalog: typeof full) => ({ + full: digestCatalogs(catalog, prompts, "full"), + off: digestCatalogs(catalog, prompts, "off"), + }); + return { full: outputProfiles(full), compact: outputProfiles(compact) }; +}; export const createCatalogIdentity = ( compactPresentation: CompactInputSchemaPresentation | undefined = undefined, @@ -113,12 +161,18 @@ export const createCatalogIdentity = ( compactPresentation === undefined ? undefined : { budgetBytes: compactPresentation.budgetBytes }; - // Building and canonicalizing every advertised schema dominates doctor and - // server startup, so the compiled runtime records the default digests once. + // Only the exact built-in presentations have build-time identities. Source + // runs and custom budgets still compute from their selected schemas lazily. + const inputProfile = + selectedPresentation === undefined + ? "full" + : selectedPresentation.budgetBytes === COMPACT_INPUT_SCHEMA_BUDGET_BYTES + ? "compact" + : undefined; let catalogDigests: CatalogDigests | undefined = - selectedPresentation === undefined && outputSchemaProfile === "full" - ? RECORDED_CATALOG_DIGESTS - : undefined; + inputProfile === undefined + ? undefined + : RECORDED_CATALOG_DIGEST_PROFILES?.[inputProfile][outputSchemaProfile]; return { counts: { cli_commands: CLI_COMMAND_NAMES.length, diff --git a/src/recordedCatalogDigests.ts b/src/recordedCatalogDigests.ts index fc37c24d50..fc8ded8edb 100644 --- a/src/recordedCatalogDigests.ts +++ b/src/recordedCatalogDigests.ts @@ -1,8 +1,10 @@ -import type { CatalogDigests } from "./catalogIdentity.js"; +import type { CatalogDigestProfiles } from "./catalogIdentity.js"; /** - * Digests of the default catalog (full input and output schemas), recorded + * Digests of the four built-in input/output schema profiles, recorded * from the compiled contracts by `scripts/record-catalog-digests.mjs` after * compilation. Source runs leave this undefined and compute them on first use. */ -export const RECORDED_CATALOG_DIGESTS: CatalogDigests | undefined = undefined; +export const RECORDED_CATALOG_DIGEST_PROFILES: + | CatalogDigestProfiles + | undefined = undefined; diff --git a/tests/boundary/mcp/mcpDoctor.test.ts b/tests/boundary/mcp/mcpDoctor.test.ts index 12919a7fb2..f7cecc5bec 100644 --- a/tests/boundary/mcp/mcpDoctor.test.ts +++ b/tests/boundary/mcp/mcpDoctor.test.ts @@ -1,7 +1,9 @@ import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { execFile } from "node:child_process"; import { readFile, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; +import { promisify } from "node:util"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -22,6 +24,40 @@ import { MCP_STARTUP_POLICY } from "../../../src/mcpStartupPolicy.js"; afterEach(() => vi.useRealTimers()); describe("production MCP doctor", () => { + it("reuses immutable compiled profile digests and preserves custom-budget identity", async () => { + // Exercise the generated module with Node, outside Vitest's source loader. + await promisify(execFile)( + process.execPath, + [ + "--input-type=module", + "--eval", + ` + import assert from "node:assert/strict"; + import { createCatalogIdentity, createCatalogDigests } from "./dist/catalogIdentity.js"; + import { RECORDED_CATALOG_DIGEST_PROFILES as profiles } from "./dist/recordedCatalogDigests.js"; + import { COMPACT_INPUT_SCHEMA_BUDGET_BYTES as budgetBytes } from "./dist/config/mcpInputSchemaProfile.js"; + assert.ok(Object.isFrozen(profiles)); + for (const input of ["full", "compact"]) { + assert.ok(Object.isFrozen(profiles[input])); + for (const output of ["full", "off"]) { + const identity = createCatalogIdentity(input === "compact" ? { budgetBytes } : undefined, output); + assert.equal(identity.digests, profiles[input][output]); + assert.ok(Object.isFrozen(identity.digests)); + } + } + const selection = { budgetBytes: 1 }; + const custom = createCatalogIdentity(selection, "off"); + selection.budgetBytes = budgetBytes; + const expected = createCatalogDigests(undefined, undefined, { budgetBytes: 1 }, "off"); + assert.deepEqual(custom.digests, expected); + assert.notEqual(custom.digests.tools_sha256, profiles.compact.off.tools_sha256); + assert.equal(custom.digests, custom.digests); + `, + ], + { timeout: 30_000 }, + ); + }, 35_000); + it("preserves the full-output envelope for startup failures", async () => { const result = await runProductionMcpDoctorCli( ["--json", "--full-output"], diff --git a/tests/boundary/mcp/serverIdentity.test.ts b/tests/boundary/mcp/serverIdentity.test.ts index 2deef7abe5..68a726c29c 100644 --- a/tests/boundary/mcp/serverIdentity.test.ts +++ b/tests/boundary/mcp/serverIdentity.test.ts @@ -12,6 +12,7 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { machoImage } from "../../../src/artifacts/apple/MachoImage.fixture.js"; import { CATALOG_IDENTITY, + createCatalogDigestProfiles, createCatalogDigests, } from "../../../src/catalogIdentity.js"; import { CLI_COMMAND_NAMES } from "../../../src/cliCommandNames.js"; @@ -141,17 +142,7 @@ describe("server and catalog identity", () => { }); it("binds output schema profile and canonical output contracts into identity", () => { - const full = createCatalogDigests(); - const off = createCatalogDigests( - TOOL_CONTRACTS, - PROMPT_CONTRACTS, - undefined, - "off", - ); - expect(off.tools_sha256).not.toBe(full.tools_sha256); - expect(off.combined_sha256).not.toBe(full.combined_sha256); - expect(off.prompts_sha256).toBe(full.prompts_sha256); - + const profiles = createCatalogDigestProfiles(); const changedOutput = TOOL_CONTRACTS.map((contract, index) => index === 0 ? { @@ -160,17 +151,24 @@ describe("server and catalog identity", () => { } : contract, ); - const changedFull = createCatalogDigests(changedOutput); - const changedOff = createCatalogDigests( - changedOutput, - PROMPT_CONTRACTS, - undefined, - "off", - ); - expect(changedFull.tools_sha256).not.toBe(full.tools_sha256); - expect(changedOff.tools_sha256).not.toBe(off.tools_sha256); - expect(changedFull.combined_sha256).not.toBe(full.combined_sha256); - expect(changedOff.combined_sha256).not.toBe(off.combined_sha256); + const changed = createCatalogDigestProfiles(changedOutput); + for (const input of ["full", "compact"] as const) { + const { full, off } = profiles[input]; + expect(off.tools_sha256).not.toBe(full.tools_sha256); + expect(off.combined_sha256).not.toBe(full.combined_sha256); + expect(off.prompts_sha256).toBe(full.prompts_sha256); + for (const output of ["full", "off"] as const) { + expect(changed[input][output].tools_sha256).not.toBe( + profiles[input][output].tools_sha256, + ); + expect(changed[input][output].combined_sha256).not.toBe( + profiles[input][output].combined_sha256, + ); + expect(changed[input][output].prompts_sha256).toBe( + profiles[input][output].prompts_sha256, + ); + } + } }); it("changes catalog identity when advertised input examples change", () => { From aa314ab682e4bcd1e050a1e073bc90930660807f Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:18:17 +0800 Subject: [PATCH 12/13] ci(windows): run environment registration regressions (#5409) Retain 13 Windows-only environment/setup regressions in the curated native lane. Select Windows qualification for both their three test files and their client configuration, registration and DSH production owner families. Guard membership and actual Git-diff classification; qualify all 60 target cases on hosted Windows. --- .github/workflows/ci.yml | 3 ++ scripts/ci/scopes.mjs | 6 +++ .../boundary/filesystem/ciChangeScope.test.ts | 48 +++++++++++++++++++ .../process-global/curatedWindowsLane.test.ts | 10 ++++ 4 files changed, 67 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fffe2de4e9..7baf1f64e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -750,6 +750,9 @@ jobs: tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts tests/boundary/filesystem/setupUnavailableProvider.test.ts tests/boundary/filesystem/windowsNpxRegistration.test.ts + tests/boundary/filesystem/clientConfigurationEnvironment.test.ts + tests/boundary/filesystem/clientConfigurationLayeredEnvironment.test.ts + tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts tests/boundary/android/workspaceGuardianRetry.test.ts tests/acceptance/setup/homeDirectoryFallback.test.ts tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index fcb4a0dab0..32a15faaa3 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -130,6 +130,9 @@ export const SCOPES = { "src/config/**", "src/application/Setup*.ts", "src/application/Uninstall*.ts", + "src/application/ClientConfiguration*.ts", + "src/application/ClientRegistration*.ts", + "src/application/DshCordis*.ts", "src/cli/interactiveSetup.ts", "src/cli/setupCommands.ts", "tests/boundary/filesystem/binaryTarget.test.ts", @@ -138,6 +141,9 @@ export const SCOPES = { "tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts", "tests/boundary/filesystem/setupUnavailableProvider.test.ts", "tests/boundary/filesystem/windowsNpxRegistration.test.ts", + "tests/boundary/filesystem/clientConfigurationEnvironment.test.ts", + "tests/boundary/filesystem/clientConfigurationLayeredEnvironment.test.ts", + "tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts", "tests/boundary/android/workspaceGuardianRetry.test.ts", "src/android/JadxWorkspaceRoot.ts", "scripts/rea-android-workspace-guardian.mjs", diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 874b4ade29..395a597c7d 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -724,6 +724,54 @@ it.each([ "tests/boundary/filesystem/windowsNpxRegistration.test.ts", { code: true, package: true, runtime: true, apple: false }, ], + [ + "tests/boundary/filesystem/clientConfigurationEnvironment.test.ts", + { code: true, package: true, runtime: true, apple: false }, + ], + [ + "tests/boundary/filesystem/clientConfigurationLayeredEnvironment.test.ts", + { code: true, package: true, runtime: true, apple: false }, + ], + [ + "tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts", + { code: true, package: true, runtime: true, apple: false }, + ], + [ + "src/application/ClientRegistrationEnvironment.ts", + { code: true, package: true, runtime: true, apple: false }, + ], + [ + "src/application/ClientRegistrationStatus.ts", + { code: true, package: true, runtime: true, apple: false }, + ], + [ + "src/application/ClientRegistrationIdentity.ts", + { code: true, package: true, runtime: true, apple: false }, + ], + [ + "src/application/ClientConfigurationDocument.ts", + { code: true, package: true, runtime: true, apple: false }, + ], + [ + "src/application/ClientConfigurationFiles.ts", + { code: true, package: true, runtime: true, apple: false }, + ], + [ + "src/application/ClientConfigurationPublication.ts", + { code: true, package: true, runtime: true, apple: false }, + ], + [ + "src/application/ClientConfigurationCleanup.ts", + { code: true, package: true, runtime: true, apple: false }, + ], + [ + "src/application/DshCordisPatch.ts", + { code: true, package: true, runtime: true, apple: false }, + ], + [ + "src/application/DshCordisYaml.ts", + { code: true, package: true, runtime: true, apple: false }, + ], [ "scripts/verify-packed-bridge.mjs", { diff --git a/tests/process-global/curatedWindowsLane.test.ts b/tests/process-global/curatedWindowsLane.test.ts index c486b7a248..637ca8d23a 100644 --- a/tests/process-global/curatedWindowsLane.test.ts +++ b/tests/process-global/curatedWindowsLane.test.ts @@ -83,6 +83,16 @@ describe("curated Windows test lane", () => { ); }); + it("keeps Windows environment identity and layered setup regressions in the lane", async () => { + expect(await explicitPaths("windows")).toEqual( + expect.arrayContaining([ + "tests/boundary/filesystem/clientConfigurationEnvironment.test.ts", + "tests/boundary/filesystem/clientConfigurationLayeredEnvironment.test.ts", + "tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts", + ]), + ); + }); + it("keeps packed bridge verification under the retained native runtime lease in the lane", async () => { expect(await explicitPaths("windows")).toContain( "tests/boundary/process/packedBridgeVerification.test.ts", From 052956af3416fa01d93837e54fdec3ad2739f6d1 Mon Sep 17 00:00:00 2001 From: xjh <87483213+xiajiahang@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:32:12 +0800 Subject: [PATCH 13/13] fix(doctor): validate Hopper startup timeout in readiness checks (#3755) Fixes #3306. Reuse the runtime timeout parser in doctor, preserve scope semantics and separate setting validation from provider configuration. Co-authored-by: xiajiahang Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> --- src/application/Doctor.test.ts | 97 ++++++++++++++++++- src/application/Doctor.ts | 7 ++ src/application/DoctorDiagnostics.ts | 26 +++++ src/setupEnvironment.test.ts | 2 + .../cli/doctorHopperDiscovery.test.ts | 54 +++++++++++ 5 files changed, 185 insertions(+), 1 deletion(-) diff --git a/src/application/Doctor.test.ts b/src/application/Doctor.test.ts index 2a2013c914..b8aad49859 100644 --- a/src/application/Doctor.test.ts +++ b/src/application/Doctor.test.ts @@ -3,10 +3,105 @@ import { describe, expect, it } from "vitest"; import { SUPPORTED_NODE_VERSION_PROSE } from "../domain/runtimeVersion.js"; import type { ClientRegistrationStatus } from "./ClientRegistrationStatus.js"; import { createDoctorHostFixture as host } from "./Doctor.fixture.js"; -import { runDoctor } from "./Doctor.js"; +import { runDoctor, systemDoctorHost, type DoctorHost } from "./Doctor.js"; import { err } from "../domain/result.js"; import { HopperReleaseError } from "./HopperReleaseFeed.js"; +describe("doctor Hopper startup timeout", () => { + const timeoutHost = ( + value: string | undefined, + overrides: Partial = {}, + ): DoctorHost => ({ + ...systemDoctorHost({ + environment: + value === undefined ? {} : { REA_HOPPER_STARTUP_TIMEOUT_MS: value }, + }), + ...host({ + installationPaths: () => Promise.resolve([]), + clientRegistrations: () => Promise.resolve([]), + ...overrides, + }), + }); + + it.each([ + ["0", true, false], + ["120000", true, true], + ["0", false, false], + ["120000", false, true], + [undefined, false, true], + ] as const)( + "checks timeout %s with Hopper present=%s and an available peer", + async (value, hopperPresent, healthy) => { + const selectedHost = timeoutHost(value, { + executable: () => Promise.resolve(hopperPresent), + providerInspections: () => + Promise.resolve([ + { + id: "ghidra", + configured: true, + available: true, + providerVersion: null, + registrationEnvironment: {}, + checks: [], + }, + ]), + }); + const result = await runDoctor(undefined, selectedHost); + expect(result.healthy).toBe(healthy); + expect(result.environment_healthy).toBe(healthy); + const timeoutCheck = result.checks.find( + ({ name }) => name === "hopper-startup-timeout", + ); + if (value === undefined) expect(timeoutCheck).toBeUndefined(); + else expect(timeoutCheck).toMatchObject({ ok: value !== "0" }); + }, + ); + + it.each(["", "0", "-1", "1.5", "1e3", "invalid", "2147483648"])( + "rejects invalid Hopper startup timeout %j in the selected provider scope", + async (value) => { + const selectedHost = timeoutHost(value); + const result = await runDoctor(undefined, selectedHost, { + providers: ["hopper"], + }); + + expect(result.healthy).toBe(false); + expect(result.scope_checks).toContainEqual( + expect.objectContaining({ + name: "hopper-startup-timeout", + ok: false, + classification: "config_drift", + detail: expect.stringContaining("REA_HOPPER_STARTUP_TIMEOUT_MS"), + remediation: expect.stringContaining("REA_HOPPER_STARTUP_TIMEOUT_MS"), + }), + ); + }, + ); + + it.each([undefined, "1", "120000", "2147483647"])( + "keeps Hopper ready with the valid startup timeout %j", + async (value) => { + const selectedHost = timeoutHost(value); + expect( + (await runDoctor(undefined, selectedHost, { providers: ["hopper"] })) + .healthy, + ).toBe(true); + }, + ); + + it("keeps an invalid Hopper timeout informational outside the Hopper scope", async () => { + const selectedHost = timeoutHost("0"); + const result = await runDoctor(undefined, selectedHost, { providers: [] }); + expect(result.healthy).toBe(true); + expect(result.informational_checks).toContainEqual( + expect.objectContaining({ + name: "hopper-startup-timeout", + ok: false, + }), + ); + }); +}); + describe("doctor runtime check", () => { it("reports Node 25 as unsupported", async () => { const result = await runDoctor( diff --git a/src/application/Doctor.ts b/src/application/Doctor.ts index 1204391e17..face3e03d8 100644 --- a/src/application/Doctor.ts +++ b/src/application/Doctor.ts @@ -108,6 +108,7 @@ export interface DoctorHost { readonly runtime: JavaScriptRuntime; readonly homeDirectory: string; readonly configuredHopperPath?: string; + readonly configuredHopperStartupTimeout?: string; readonly configuredIlspyCmdPath?: string; macosVersion(): Promise; linuxDistribution(): Promise; @@ -523,6 +524,12 @@ export const systemDoctorHost = ( ...(environment.HOPPER_LAUNCHER_PATH === undefined ? {} : { configuredHopperPath: environment.HOPPER_LAUNCHER_PATH }), + ...(environment.REA_HOPPER_STARTUP_TIMEOUT_MS === undefined + ? {} + : { + configuredHopperStartupTimeout: + environment.REA_HOPPER_STARTUP_TIMEOUT_MS, + }), ...(environment.REA_ILSPY_CMD_PATH === undefined ? {} : { configuredIlspyCmdPath: environment.REA_ILSPY_CMD_PATH }), diff --git a/src/application/DoctorDiagnostics.ts b/src/application/DoctorDiagnostics.ts index a2e34d9757..9f9526ad8f 100644 --- a/src/application/DoctorDiagnostics.ts +++ b/src/application/DoctorDiagnostics.ts @@ -2,6 +2,7 @@ import { SUPPORTED_NODE_VERSION_PROSE, supportsNodeVersion, } from "../domain/runtimeVersion.js"; +import { providerStartupTimeoutSchema } from "../config/providerStartupTimeout.js"; import { linuxHopperLauncherCandidates, systemLinuxHopperLauncherCandidates, @@ -34,6 +35,8 @@ export const collectDoctorDiagnostics = async ( const hopperPath = await findHopper(host); checks.push(hopperCheck(host, hopperPath)); + const hopperTimeoutCheck = optionalHopperStartupTimeoutCheck(host); + if (hopperTimeoutCheck !== undefined) checks.push(hopperTimeoutCheck); const hopperVersionCheck = await optionalHopperVersionCheck(host, hopperPath); if (hopperVersionCheck !== undefined) checks.push(hopperVersionCheck); @@ -80,6 +83,8 @@ export const doctorHealthy = ( .every(({ ok }) => ok); const configuredProvidersHealthy = providerChecks .filter(({ name }) => { + // Validate the supplied setting independently of provider discovery. + if (name === "hopper-startup-timeout") return true; if (name.startsWith("hopper")) return providers.hopperConfigured; return providers.providerInspections?.some( (inspection) => @@ -204,6 +209,27 @@ const hopperCheck = ( }, ); +const optionalHopperStartupTimeoutCheck = ( + host: DoctorHost, +): DoctorCheck | undefined => { + if (host.configuredHopperStartupTimeout === undefined) return undefined; + const parsed = providerStartupTimeoutSchema.safeParse( + host.configuredHopperStartupTimeout, + ); + return check( + "hopper-startup-timeout", + parsed.success, + parsed.success + ? `REA_HOPPER_STARTUP_TIMEOUT_MS=${String(parsed.data)}` + : `REA_HOPPER_STARTUP_TIMEOUT_MS: ${parsed.error.issues.map(({ message }) => message).join("; ")}`, + { + classification: "config_drift", + remediation: + "Unset REA_HOPPER_STARTUP_TIMEOUT_MS to use the default, or set it to a positive decimal integer from 1 to 2147483647 milliseconds.", + }, + ); +}; + const optionalHopperVersionCheck = async ( host: DoctorHost, hopperPath: string | undefined, diff --git a/src/setupEnvironment.test.ts b/src/setupEnvironment.test.ts index 15d57a1319..278a03f044 100644 --- a/src/setupEnvironment.test.ts +++ b/src/setupEnvironment.test.ts @@ -183,6 +183,7 @@ it.each([ environment: { REA_ANALYSIS_PROVIDER: "auto", REA_GHIDRA_STARTUP_TIMEOUT_MS: "2700000", + REA_HOPPER_STARTUP_TIMEOUT_MS: "120000", REA_MCP_INPUT_SCHEMA_PROFILE: "full", }, providerConfigured: false, @@ -214,6 +215,7 @@ it.each([ expect(initial.providerEnvironment).toMatchObject({ REA_ANALYSIS_PROVIDER: "auto", REA_GHIDRA_STARTUP_TIMEOUT_MS: "2700000", + REA_HOPPER_STARTUP_TIMEOUT_MS: "120000", REA_MCP_INPUT_SCHEMA_PROFILE: "full", }); } diff --git a/tests/boundary/cli/doctorHopperDiscovery.test.ts b/tests/boundary/cli/doctorHopperDiscovery.test.ts index db4e442600..1a33db49cd 100644 --- a/tests/boundary/cli/doctorHopperDiscovery.test.ts +++ b/tests/boundary/cli/doctorHopperDiscovery.test.ts @@ -6,6 +6,60 @@ import { expect } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; +cliTest.skipIf(!["darwin", "linux"].includes(process.platform))( + "reports invalid Hopper startup timeouts without launching the discovered engine", + async ({ cli }) => { + const home = await createTestTempDirectory("rea-doctor-hopper-timeout-"); + const launcher = join(home, "hopper"); + const marker = join(home, "launched"); + await writeFile(launcher, `#!/bin/sh\nprintf launched > '${marker}'\n`); + await chmod(launcher, 0o700); + + for (const [timeout, expectedExit] of [ + ["0", 1], + ["120000", 0], + ] as const) { + const environment = { + ...(process.platform === "linux" + ? { + // Keep the foreign launcher unsupported without reaching the + // release network. This qualifies timeout diagnostics only. + NODE_OPTIONS: `--import=${new URL("../../fixtures/hopperRejectionFeed.mjs", import.meta.url).href}`, + } + : {}), + HOME: home, + HOPPER_LAUNCHER_PATH: launcher, + REA_HOPPER_STARTUP_TIMEOUT_MS: timeout, + GHIDRA_INSTALL_DIR: join(home, "missing-ghidra"), + }; + const providers = await cli.run({ + arguments: ["providers", "--json"], + environment, + }); + expect(providers.exitCode).toBe(expectedExit); + const doctor = await cli.run({ + arguments: ["doctor", "--provider", "hopper", "--json"], + environment, + }); + // Linux independently rejects this shell fixture as a Hopper build. + const healthy = process.platform === "darwin" && expectedExit === 0; + expect(doctor.exitCode).toBe(healthy ? 0 : 1); + expect(doctor.json).toMatchObject({ + healthy, + scope_checks: expect.arrayContaining([ + expect.objectContaining({ + name: "hopper-startup-timeout", + ok: expectedExit === 0, + classification: expectedExit === 0 ? "healthy" : "config_drift", + detail: expect.stringContaining("REA_HOPPER_STARTUP_TIMEOUT_MS"), + }), + ]), + }); + } + await expect(access(marker)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); + cliTest.skipIf(!["darwin", "linux"].includes(process.platform))( "retains a configured Hopper path in host diagnostics without launching it", async ({ cli }) => {