From 5f702dd6ccd9e8e505e224854b91678439c3b41f Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:01:16 +0800 Subject: [PATCH] fix(parsers): preserve format boundaries and query occurrence identity Normalize parsed facts at their owning boundaries across PE exports and certificates, ELF section names, GPU bundles, IL2CPP generic types, dyld slice selection, source maps, environment replay, and JavaScript mapped arguments. Preserve partial facts and explicit unsupported cases; clarify the corresponding documentation. Validation: 831 affected tests passed with one existing skip across 23 files; build, typecheck/lint, documentation, formatting, and 2133-file test discovery passed. Format controls are synthetic; real Unity, GPU, and ARM-device execution is unverified. Fixes #6709 Fixes #6688 Fixes #6719 Fixes #6718 Fixes #6704 Fixes #6703 Fixes #6692 Fixes #6723 Fixes #6687 Fixes #6726 Fixes #6696 Fixes #6691 Fixes #6727 --- docs/agent-environments.md | 28 ++- docs/apple-application-analysis.md | 13 +- docs/binary-diagnostics.md | 3 + docs/elf-notes.md | 13 +- docs/gpu-code-objects.md | 8 + docs/il2cpp-metadata.md | 6 +- docs/pe-certificates.md | 7 +- docs/web-source-location.md | 9 +- .../environment/environmentCompare.test.ts | 36 ++++ .../environment/environmentCompare.ts | 2 +- .../environment/environmentCompareMatch.ts | 20 +- .../environment/environmentReplay.test.ts | 83 ++++++++ .../environment/environmentReplay.ts | 177 ++++++++++++++-- .../environmentStateAnswer.test.ts | 26 +++ .../environment/environmentStateAnswer.ts | 63 ++++-- .../environment/environmentStateCallLines.ts | 15 +- .../environmentStateWriteAnswer.ts | 22 +- .../environment/environmentToolAnswer.ts | 18 +- .../environment/environmentTools.test.ts | 11 +- .../environment/environmentTools.ts | 26 ++- .../apple/MachoLoadCommandReader.test.ts | 101 +++++++++ src/browser/WebSourceMapFetcher.ts | 7 +- .../analysisView/environmentStateRules.ts | 10 +- .../analysisView/environmentStateView.test.ts | 12 ++ src/domain/apple/dyldPaths.ts | 40 +++- src/domain/apple/dylibResolution.test.ts | 28 +++ src/domain/apple/dylibResolution.ts | 16 +- ...avascriptSemanticAnalysis.dataflow.test.ts | 69 +++++- .../javascript/javascriptSemanticAnalysis.ts | 148 +++++++++++++ .../javascript/javascriptSourceMapPaths.ts | 31 ++- src/gpu/AmdGpuTarget.ts | 28 +++ src/gpu/ClangOffloadBundle.ts | 37 +++- src/gpu/CudaFatbinary.ts | 2 +- src/gpu/GpuElfKernels.ts | 9 +- src/gpu/GpuInventory.ts | 8 +- src/gpu/GpuPayloadCodec.ts | 10 +- src/il2cpp/Il2cppBinaryTypes.ts | 75 ++++++- src/il2cpp/Il2cppMetadataFile.ts | 36 ++++ src/il2cpp/Il2cppMetadataProvider.ts | 2 +- .../sourceMaps/TraceSourceMap.test.ts | 17 ++ src/javascript/sourceMaps/TraceSourceMap.ts | 13 +- src/native/elf/ElfNoteParser.test.ts | 46 ++++ src/native/elf/ElfNoteParser.ts | 43 +++- src/native/elf/ElfNotes.fixture.ts | 80 +++++++ src/native/elf/ElfNotesProvider.ts | 2 +- src/native/exports/NativeExportParser.test.ts | 69 ++++++ src/native/exports/NativeExportParser.ts | 73 ++++++- src/native/pe/PeCertificateAsn1.ts | 14 +- src/native/pe/PeCertificateParser.test.ts | 27 ++- src/native/pe/PeCertificateParser.ts | 10 +- tests/acceptance/elfNotesCli.test.ts | 27 ++- tests/acceptance/il2cppMetadataCli.test.ts | 99 ++++++++- .../filesystem/sourceToBundlePaths.test.ts | 38 +++- tests/boundary/gpu/codeObjects.test.ts | 96 +++++++++ .../boundary/il2cpp/il2cppBinaryFile.test.ts | 58 +++++- .../il2cpp/il2cppMetadataProvider.test.ts | 21 +- .../boundary/mcp/environmentServeMcp.test.ts | 60 +++++- tests/fixtures/gpu/codeObjects.ts | 19 +- tests/fixtures/il2cpp/binary.ts | 16 +- tests/fixtures/il2cpp/metadata.ts | 197 +++++++++++++++--- 60 files changed, 2069 insertions(+), 211 deletions(-) create mode 100644 src/gpu/AmdGpuTarget.ts diff --git a/docs/agent-environments.md b/docs/agent-environments.md index c260334040..e1d0689554 100644 --- a/docs/agent-environments.md +++ b/docs/agent-environments.md @@ -165,7 +165,10 @@ Each such MCP tool call selects one of the records its manifest tool cites and returns that record's recorded response. Selection compares the supplied path parameters against the record's path segments at the parameter indexes, then the supplied query values against the record's query string, and keeps the -record with the most matches; ties keep manifest order. The result reports +record with the most matches; ties keep manifest order. A query argument accepts +one string or an array of strings for repeated values; the array is matched +against every occurrence in captured order. The raw recorded query remains the +replay evidence. The result reports where the answer came from: ```json @@ -187,9 +190,14 @@ where the answer came from: } ``` -`exact_match` is true when every supplied path parameter and query value -equals the recorded one; it says nothing about parameters the caller omitted. -Recorded query parsing runs only when supplied query arguments affect selection. +`exact_match` is true when every supplied path parameter and query occurrence +matches the recorded one; it says nothing about parameters the caller omitted. +Comparison probes retain the raw query spelling and its decoded parameter pairs +in request order, including the interleaving of repeated values across names. +Standalone tool calls express repeated values as per-name arrays; when matching +records differ only in cross-name order, `exact_match` is false because those arguments cannot +identify that order. Recorded query parsing runs only when supplied query +arguments affect selection. A path value matches a recorded segment as written or after percent-decoding; when both kinds of match are available, the record whose segment equals the value as written is chosen. `recorded_query` keeps the `?` of an explicit empty @@ -365,7 +373,8 @@ shape and carries no annotations. Path parameters are required strings whose description names the segment index, the templating rule and the observed values (sampled within the list budget above; the manifest keeps them all); -query parameter names are optional strings; write methods accept an +query parameter names accept an optional string or array of strings (for +repeated values in captured order); write methods accept an optional `body` object described by the recorded top-level key types. Each input keeps its own argument: a query name equal to a path parameter's name is advertised as `query_`, a query name longer than 64 characters as @@ -429,6 +438,10 @@ Each read tool answers according to its view in the state: | `document` | When the arguments name the document's checked line exactly, the state's value for it; a tool error when the state holds none. | | `recorded` | The recording, as replay answers it. | +A read with repeated query values uses replay, because the state query model +does not infer repeated-parameter semantics. A repeated-query write is refused +before it can change state for the same reason. + An item or list view that always spells a clock field otherwise than the entity's stored value names that spelling in `clock_formats`, and a value it never recorded, such as a created entity's clock, is rendered in it: a list that @@ -573,7 +586,10 @@ id are unique; `view_fields` is optional and names item or list views of its type. Views render their values over the entity's fields, so a view's values must not set the type's identity field to a value whose text differs from the entity's `id`. Spell a collection as requests spell it, as `data/get` reports it: list -calls find entities under the checked line's recorded path. Each document +calls find entities under the checked line's recorded path. State collection +identity preserves repeated separators inside captured paths: `/api/books` +and `/api//books` are distinct. A single trailing slash continues to name the +same collection. Each document names a `document` view once, at its initial document's `path`. The `version`, and the `state_evidence_id` and `types` that `data/get` reports, must match the served state, so a `data/get` result loads back unchanged. A number without diff --git a/docs/apple-application-analysis.md b/docs/apple-application-analysis.md index 3f718d4cc7..73ce07e2ce 100644 --- a/docs/apple-application-analysis.md +++ b/docs/apple-application-analysis.md @@ -220,9 +220,16 @@ satisfy a dylib dependency. `platform-mismatch` means a matching-architecture dylib was found but its observed platform is incompatible with the process. These outcomes reject that candidate; dyld search may continue and resolve a later candidate. Candidate outcomes describe individual search paths, while -`resolution.status` describes the final edge. Slices follow dyld's graded -architectures, so an `x86_64h` process also loads `x86_64`. The edge's -`resolution` is one of: +`resolution.status` describes the final edge. For each selected process-root +slice, candidate slices are matched by CPU type and dyld's subtype preference +order, independently of FAT table order. An ARMv7s root slice prefers ARMv7s, +then ARMv7; other known ARM32 subtypes use their compatibility rows from +[Apple's dyld source](https://github.com/apple-oss-distributions/dyld/blob/dyld-852.2/src/dyld2.cpp#L2811-L3105). +An `x86_64h` process also loads generic `x86_64`. Unknown ARM32 subtype policy +remains undetermined when ARM32 candidate slices exist. This is a static model +keyed to the selected root slice; it does not observe the target device's host +CPU or prove which slice the device will execute. The edge's `resolution` is +one of: | Status | Meaning | | -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/docs/binary-diagnostics.md b/docs/binary-diagnostics.md index c4fddfd6d0..6b84224255 100644 --- a/docs/binary-diagnostics.md +++ b/docs/binary-diagnostics.md @@ -348,6 +348,9 @@ count and SHA-256. Unreadable names, out-of-range name ordinals and unreadable forwarders are listed in `issues`, their entries are not reported, and coverage is `partial`. +PE export names are bounded null-terminated byte strings and may continue +across adjacent file-backed section mappings when the virtual bytes are +unambiguous. Gaps, zero-fill and conflicting mappings remain unreadable names. Unresolved symbol versions keep their rows: the version name is null when no definition or requirement names the index, and `version` is `"unknown"` when the version table cannot be read; coverage is likewise diff --git a/docs/elf-notes.md b/docs/elf-notes.md index 2627e4149a..9d3ab30f18 100644 --- a/docs/elf-notes.md +++ b/docs/elf-notes.md @@ -24,8 +24,11 @@ requires an absolute filesystem path and preserves that exact selection: The inline Evidence result contains the artifact path, SHA-256, and byte length. `tables` lists each declared note program header or section, including the -section name when a string table is present. Each `entries[]` item reports owner, -type, descriptor SHA-256, and a hex preview for descriptors of 64 bytes or less. +section name when a string table is present. Zlib-compressed section-name +tables are decompressed within the selected file-size budget. An unsupported +compression type leaves section names unknown and adds a limitation; note bytes +remain independently examined. Each `entries[]` item reports owner, type, +descriptor SHA-256, and a hex preview for descriptors of 64 bytes or less. For GNU notes, `gnu.build_id_hex`, `gnu.abi_tag`, or `gnu.gold_version` are filled when the descriptor matches that type. Other owners keep `gnu` null. @@ -40,8 +43,10 @@ the image never carried notes in another container or overlay. The selected file is limited to 64 MiB by default (512 MiB maximum). At most 32 note entries are examined by default across every table. Retained note -names, section names, and GNU descriptor text share that file's byte budget, -and repeated table ranges count again. Each descriptor range is hashed once, +names, section names, and GNU descriptor text share a scan budget based on +the source bytes plus any decoded section-name table, capped at +`max_file_bytes`. The raw source and decoded table must also fit together +within that ceiling; repeated table ranges count again. Each descriptor range is hashed once, and each GNU version's trailing NUL run is scanned once; neither walk may exceed that byte ceiling. Malformed ELF headers, truncated header tables, declared note tables that extend past the file, and truncated diff --git a/docs/gpu-code-objects.md b/docs/gpu-code-objects.md index 512b4a264a..080a71af71 100644 --- a/docs/gpu-code-objects.md +++ b/docs/gpu-code-objects.md @@ -32,6 +32,14 @@ unexamined payloads remain `unexamined`. An exact AMDGPU target is normalized at its note owner for comparison with a Clang bundle declaration; raw values remain intact. Unresolved feature comparisons stay `unknown`. +HIP bundle recognition accepts both the legacy `amdgcn` triple and LLVM's +`amdgpu` subarchitecture triples, including declared generic processors such +as `gfx11-generic` and `gfx9-4-generic`. CUDA's legacy fatbinary compression +flag is retained as an unsupported codec with its encoded and decoded extents; +compressed bytes are not treated as PTX text. CCOB v1 has no stored total size, +so a following legacy bundle is located by the next `CCOB` signature and +zero-byte alignment padding remains part of the first container extent. + `cuda_target_header` retains the OS ABI, normalized raw `e_flags` and exact header location. CUDA OS ABI `0x33` and the observed `0x41` flags layout use different architecture fields; they are decoded at one ABI-aware owner. diff --git a/docs/il2cpp-metadata.md b/docs/il2cpp-metadata.md index 091120374f..ca59c01090 100644 --- a/docs/il2cpp-metadata.md +++ b/docs/il2cpp-metadata.md @@ -260,8 +260,10 @@ The result adds: definitions, `T[]` for vectors, `T[*]` for rank-one multidimensional arrays, `T[,]` for rank-two arrays and `T*`, generic instances as `System.Collections.Generic.List` and generic parameters by their - metadata name (`TKey`). A trailing `&` marks a by-reference type. `name` is - `null` only for type index `-1`. + metadata name (`TKey`). Nested generic arguments stay with the component + that declares them, for example `Game.Outer+Inner` or + `Game.Outer+Leaf` when the nested type adds no parameters. A trailing + `&` marks a by-reference type. `name` is `null` only for type index `-1`. `binary` also records `types_field` and `types_count`: the address of `Il2CppMetadataRegistration.typesCount` and its value. That record is found as diff --git a/docs/pe-certificates.md b/docs/pe-certificates.md index 4c9acf0c5f..9875e19f91 100644 --- a/docs/pe-certificates.md +++ b/docs/pe-certificates.md @@ -49,8 +49,11 @@ each `SignerInfo` must carry a non-empty signature value. Each embedded X.509 certificate must include `subjectPublicKeyInfo` and `signatureValue`. Decoded certificate name text, serials, validity spellings, and subject key identifiers share a 64 KiB budget per PKCS#7 payload. Unsupported certificate name -encodings, including TeletexString (T.61) and GeneralString, fail explicitly -rather than returning names decoded with an assumed character set. +encodings, including GeneralString, fail explicitly. TeletexString (T.61) +values are rendered with the ISO-8859-1 compatibility mapping used by OpenSSL; +this is not a full T.61 decoder, so the result includes a limitation whenever +that mapping is used. RFC 5280 retains TeletexString for backward compatibility +and describes its repertoire as ASCII-like Latin with additional characters. The structural CMS reader accepts definite lengths and BER indefinite lengths for constructed values, including the outer `ContentInfo` sequence. Primitive diff --git a/docs/web-source-location.md b/docs/web-source-location.md index 901662b431..bb6a87b6dc 100644 --- a/docs/web-source-location.md +++ b/docs/web-source-location.md @@ -29,9 +29,12 @@ zero-based UTF-16 column. The MCP arguments are: } ``` -The URL supplies the context for resolving `sources` and `sourceRoot`; it is not -an instruction to fetch. Absolute virtual or file contexts can be selected as -well. The map/script association is caller-selected: a declaration or local map +The URL supplies the context for resolving `sources` and `sourceRoot` using the +ECMA-426 URL parsing rule; it is not an instruction to fetch. Repeated path +separators remain part of the resolved URL identity. A non-null `sourceRoot` +prefixes each source, adding `/` only when the root does not already end with +one. An empty root therefore prefixes `/`; a null or omitted root adds no +prefix. Absolute virtual or file contexts can be selected as well. The map/script association is caller-selected: a declaration or local map does not prove deployment authenticity or execution. Use [import tracing](website-module-trace.md) to inspect outgoing native module edges. diff --git a/src/application/environment/environmentCompare.test.ts b/src/application/environment/environmentCompare.test.ts index ff898055f9..90a33e17df 100644 --- a/src/application/environment/environmentCompare.test.ts +++ b/src/application/environment/environmentCompare.test.ts @@ -65,6 +65,32 @@ const compare = (...probes: (readonly TimedCaptureRow[])[]) => { }; describe("matchProbeRequest", () => { + it("compares a repeated-query probe against the matching recorded response", () => { + const recorded = timedCaptureEvidence([ + ["GET", "/api/items?tag=a&tag=b", { selected: "b" }], + ["GET", "/api/items?tag=a&tag=c", { selected: "c" }], + ]); + const served = environmentStateServeFixture(recorded); + const report = compareEnvironment({ + source: environmentServeSource(served), + captureValue: recorded, + probes: [ + { + label: "repeated-query", + value: timedCaptureEvidence([ + ["GET", "/api/items?tag=a&tag=c", { selected: "c" }], + ]), + }, + ], + operation: "environment-compare", + }); + if (!report.ok) throw report.error; + expect(report.value.probes[0]?.results[0]).toMatchObject({ + equal: true, + ignored_query: [], + }); + }); + it("prefers literal segments, requires format-rule parameters to fit, and maps query and body arguments", () => { const item = matchProbeRequest(source.manifest, { method: "GET", @@ -103,6 +129,16 @@ describe("matchProbeRequest", () => { // The environment decodes this once, as replay decodes a recorded segment once. expect(item?.args).toEqual({ id: "%31" }); }); + + it("keeps every occurrence of a repeated query value", () => { + const item = matchProbeRequest(environmentServeSource().manifest, { + method: "GET", + path: "/api/search", + query: "?query=one&query=two&type=dash-db", + body: undefined, + }); + expect(item?.args).toEqual({ query: ["one", "two"], type: "dash-db" }); + }); }); const nestedUsers = { diff --git a/src/application/environment/environmentCompare.ts b/src/application/environment/environmentCompare.ts index b503606231..2c0e470599 100644 --- a/src/application/environment/environmentCompare.ts +++ b/src/application/environment/environmentCompare.ts @@ -343,7 +343,7 @@ const compareStep = ( const problem = check(args); if (problem !== undefined) return refusedWith(`Invalid arguments: ${problem}`); - const answered = run.answer(call.tool, args); + const answered = run.answer(call.tool, args, call.observedQuery); if (!answered.ok) return refusedWith(answered.error.message); if (live === undefined) return undefined; const env: Answer = { diff --git a/src/application/environment/environmentCompareMatch.ts b/src/application/environment/environmentCompareMatch.ts index f43855a38b..9eaae56547 100644 --- a/src/application/environment/environmentCompareMatch.ts +++ b/src/application/environment/environmentCompareMatch.ts @@ -34,6 +34,8 @@ export interface ProbeToolCall { readonly pagingArguments: readonly string[]; /** Query names the tool does not take; the call drops them. */ readonly ignoredQuery: readonly string[]; + /** Raw observed query, preserving delimiter, escaping, and pair order. */ + readonly observedQuery: string | null; } /** @@ -163,16 +165,28 @@ export const matchProbeRequest = ( } const ignoredQuery: string[] = []; const pagingArguments: string[] = []; + const queryValues = new Map(); if (request.query !== null) for (const [key, value] of new URLSearchParams(request.query.slice(1))) { const name = names.query.get(key); if (name === undefined) ignoredQuery.push(key); - else if (!Object.hasOwn(args, name)) { - args[name] = value; + else { + const values = queryValues.get(name) ?? []; + values.push(value); + queryValues.set(name, values); if (isPagingParameter(key)) pagingArguments.push(name); } } + for (const [name, values] of queryValues) + args[name] = values.length === 1 ? values[0]! : values; if (names.body !== undefined && request.body !== undefined) args[names.body] = request.body; - return { tool, args, pathArguments, pagingArguments, ignoredQuery }; + return { + tool, + args, + pathArguments, + pagingArguments, + ignoredQuery, + observedQuery: request.query, + }; }; diff --git a/src/application/environment/environmentReplay.test.ts b/src/application/environment/environmentReplay.test.ts index fcd1dd5f72..3b6cf5bbc6 100644 --- a/src/application/environment/environmentReplay.test.ts +++ b/src/application/environment/environmentReplay.test.ts @@ -126,6 +126,87 @@ describe("pathValueMatches", () => { }); describe("replayEnvironmentTool selection", () => { + it("uses observed cross-name query order and marks ambiguous key-wise replay inexact", () => { + const search = tool("get_search"); + const [first, second] = search.record_ordinals; + if (first === undefined || second === undefined) + throw new Error("Fixture search tool must cite two records"); + const exchanges = new Map([ + [ + first, + { + ordinal: first, + path: "/api/search", + query: "?query=a&type=b&query=c", + status: 200, + contentType: null, + bodyText: "first", + }, + ], + [ + second, + { + ordinal: second, + path: "/api/search", + query: "?query=a&query=c&type=b", + status: 200, + contentType: null, + bodyText: "second", + }, + ], + ]); + const args = { query: ["a", "c"], type: "b" }; + expect( + selectReplayExchange(search, exchanges, args, "?query=a&type=b&query=c"), + ).toMatchObject({ exchange: { ordinal: first }, exact: true }); + expect( + selectReplayExchange(search, exchanges, args, "?query=a&query=c&type=b"), + ).toMatchObject({ exchange: { ordinal: second }, exact: true }); + expect(selectReplayExchange(search, exchanges, args)).toMatchObject({ + exchange: { ordinal: first }, + exact: false, + }); + const translated = new Map([ + [ + first, + { + ...exchanges.get(first)!, + query: "?query=seed&type=b&query=c", + }, + ], + ]); + expect( + selectReplayExchange( + search, + translated, + { query: ["seed", "c"], type: "b" }, + "?query=live&type=b&query=c", + ), + ).toMatchObject({ exchange: { ordinal: first }, exact: true }); + }); + it("selects the record only when every repeated query occurrence matches", () => { + const base = source.exchanges.get(tool("get_search").record_ordinals[0]!); + if (base === undefined) throw new Error("Missing search fixture exchange"); + const search = { + ...tool("get_search"), + query_parameter_names: ["tag"], + record_ordinals: [901, 902], + }; + const exchanges = new Map([ + [901, { ...base, ordinal: 901, query: "?tag=a&tag=b" }], + [902, { ...base, ordinal: 902, query: "?tag=a&tag=c" }], + ]); + expect( + selectReplayExchange(search, exchanges, { tag: ["a", "c"] }), + ).toMatchObject({ + exchange: { ordinal: 902, query: "?tag=a&tag=c" }, + exact: true, + }); + expect( + selectReplayExchange(search, exchanges, { tag: ["a", "d"] }), + ).toMatchObject({ exchange: { ordinal: 901 }, exact: false }); + }); + it("replays the record whose path segments equal the supplied parameters as an exact match", () => { const result = replay("get_users_by_id", { id: "2" }); expect(result).toMatchObject({ @@ -184,7 +265,9 @@ describe("replayEnvironmentTool selection", () => { expect(unmatched.replay.exact_match).toBe(false); expect(unmatched.replay.recorded_query).toBe("?query=probe&type=dash-db"); }); +}); +describe("replay selection precedence", () => { it("prefers a recorded segment equal to the argument over one that only matches after decoding", () => { const users = { ...tool("get_users_by_id"), record_ordinals: [901, 902] }; const exchange = (ordinal: number, path: string): RecordedExchange => ({ diff --git a/src/application/environment/environmentReplay.ts b/src/application/environment/environmentReplay.ts index 342bc8a719..f11bca2e48 100644 --- a/src/application/environment/environmentReplay.ts +++ b/src/application/environment/environmentReplay.ts @@ -19,6 +19,8 @@ import { /** Caller arguments as the MCP tool receives them. */ export type ReplayArguments = Readonly>; +/** Raw query spelling from an observed request; null means no `?` delimiter. */ +export type ObservedQuery = string | null; /** What one tool call returns: the recorded response and where it came from. */ export interface ReplayResult { @@ -45,6 +47,7 @@ interface MatchScore { /** Path matches whose recorded segment equals the value as given, not only after decoding. */ readonly verbatimMatches: number; readonly queryMatches: number; + readonly observedOrderMatches: number; readonly exact: boolean; } @@ -59,6 +62,7 @@ const scoreExchange = ( queryArguments: ReadonlyMap, exchange: RecordedExchange, args: ReplayArguments, + observedQuery?: ObservedQuery, ): MatchScore => { const providedPaths: { index: number; value: string }[] = []; for (const param of tool.params) { @@ -82,26 +86,82 @@ const scoreExchange = ( if (segment !== undefined && pathValueMatches(segment, value)) pathMatches += 1; } - const supplied = new Map(); + const supplied = new Map(); for (const [name, argument] of queryArguments) { const value = args[argument]; - if (typeof value === "string") supplied.set(name, value); + if (typeof value === "string") supplied.set(name, [value]); + else if ( + Array.isArray(value) && + value.every((item) => typeof item === "string") + ) + supplied.set(name, value); + } + const recorded = new Map(); + const inspectQuery = supplied.size > 0 || observedQuery !== undefined; + const recordedPairs = inspectQuery + ? [...iterateUrlQueryParameters(exchange.query ?? "")].filter(([name]) => + queryArguments.has(name), + ) + : []; + for (const [name, value] of recordedPairs) { + const values = recorded.get(name) ?? []; + values.push(value); + recorded.set(name, values); } const matched = new Set(); - if (supplied.size > 0) - for (const [name, value] of iterateUrlQueryParameters( - exchange.query ?? "", - )) { - if (supplied.get(name) === value) matched.add(name); - if (matched.size === supplied.size) break; - } + for (const [name, values] of supplied) { + const actual = recorded.get(name); + if ( + actual !== undefined && + actual.length === values.length && + actual.every((value, index) => value === values[index]) + ) + matched.add(name); + } const queryProvided = supplied.size; const queryMatches = matched.size; + const observedOccurrences = new Map(); + const observedPairs = + observedQuery === undefined + ? undefined + : [...iterateUrlQueryParameters(observedQuery ?? "")] + .filter(([name]) => queryArguments.has(name)) + .map(([name, rawValue]) => { + const occurrence = observedOccurrences.get(name) ?? 0; + observedOccurrences.set(name, occurrence + 1); + const argument = queryArguments.get(name); + const value = argument === undefined ? undefined : args[argument]; + const normalized = + typeof value === "string" + ? occurrence === 0 + ? value + : rawValue + : Array.isArray(value) && typeof value[occurrence] === "string" + ? value[occurrence] + : rawValue; + return [name, normalized] as const; + }); + const observedOrderMatches = + observedPairs === undefined + ? 0 + : Number( + (observedQuery === null) === (exchange.query === null) && + observedPairs.length === recordedPairs.length && + observedPairs.every( + ([name, value], index) => + recordedPairs[index]?.[0] === name && + recordedPairs[index]?.[1] === value, + ), + ); return { pathMatches, verbatimMatches, queryMatches, - exact: pathMatches === pathProvided && queryMatches === queryProvided, + observedOrderMatches, + exact: + pathMatches === pathProvided && + queryMatches === queryProvided && + (observedPairs === undefined || observedOrderMatches === 1), }; }; @@ -110,13 +170,16 @@ const outranks = (score: MatchScore, best: MatchScore): boolean => ? score.pathMatches > best.pathMatches : score.verbatimMatches !== best.verbatimMatches ? score.verbatimMatches > best.verbatimMatches - : score.queryMatches > best.queryMatches; + : score.queryMatches !== best.queryMatches + ? score.queryMatches > best.queryMatches + : score.observedOrderMatches > best.observedOrderMatches; /** Choose the record whose path segments, then verbatim spellings, then query values best match the call; ties keep manifest order. */ export const selectReplayExchange = ( tool: EnvironmentManifestTool, exchanges: ReadonlyMap, args: ReplayArguments, + observedQuery?: ObservedQuery, ): | { readonly exchange: RecordedExchange; readonly exact: boolean } | undefined => { @@ -131,13 +194,63 @@ export const selectReplayExchange = ( argumentsMap.query, exchange, args, + observedQuery, ); if (best === undefined || outranks(score, best.score)) best = { exchange, score }; } - return best === undefined - ? undefined - : { exchange: best.exchange, exact: best.score.exact }; + if (best === undefined) return undefined; + const suppliedQueryNames = new Set( + [...argumentsMap.query].flatMap(([name, argument]) => + args[argument] === undefined ? [] : [name], + ), + ); + const queryWasSupplied = suppliedQueryNames.size > 0; + const canInspectQuery = observedQuery !== undefined || queryWasSupplied; + const bestPairs = canInspectQuery + ? [...iterateUrlQueryParameters(best.exchange.query ?? "")].filter( + ([name]) => suppliedQueryNames.has(name), + ) + : []; + const orderAmbiguous = + observedQuery === undefined && + queryWasSupplied && + best.score.exact && + tool.record_ordinals.some((ordinal) => { + const candidate = exchanges.get(ordinal); + if ( + candidate === undefined || + candidate.ordinal === best.exchange.ordinal + ) + return false; + const score = scoreExchange( + tool, + argumentsMap.path, + argumentsMap.query, + candidate, + args, + ); + if ( + score.pathMatches !== best.score.pathMatches || + score.verbatimMatches !== best.score.verbatimMatches || + !score.exact + ) + return false; + const pairs = [ + ...iterateUrlQueryParameters(candidate.query ?? ""), + ].filter(([name]) => suppliedQueryNames.has(name)); + return ( + pairs.length !== bestPairs.length || + pairs.some( + ([name, value], index) => + bestPairs[index]?.[0] !== name || bestPairs[index]?.[1] !== value, + ) + ); + }); + return { + exchange: best.exchange, + exact: best.score.exact && !orderAmbiguous, + }; }; /** Whether the arguments name one recorded request line exactly: every path value supplied and matching, every supplied query value matching, and every recorded query parameter supplied with its recorded value. */ @@ -145,6 +258,7 @@ export const selectsRecordedLine = ( tool: EnvironmentManifestTool, exchange: RecordedExchange, args: ReplayArguments, + observedQuery?: ObservedQuery, ): boolean => { const argumentsMap = environmentToolArguments(tool); if ( @@ -153,13 +267,34 @@ export const selectsRecordedLine = ( typeof args[argumentsMap.path.get(param.name) ?? param.name] === "string", ) || - !scoreExchange(tool, argumentsMap.path, argumentsMap.query, exchange, args) - .exact + !scoreExchange( + tool, + argumentsMap.path, + argumentsMap.query, + exchange, + args, + observedQuery, + ).exact ) return false; + const recorded = new Map(); for (const [name, value] of iterateUrlQueryParameters(exchange.query ?? "")) { + const values = recorded.get(name) ?? []; + values.push(value); + recorded.set(name, values); + } + for (const [name, values] of recorded) { const argument = argumentsMap.query.get(name); - if (argument === undefined || args[argument] !== value) return false; + const supplied = args[argument ?? ""]; + if ( + argument === undefined || + (typeof supplied === "string" + ? values.length !== 1 || values[0] !== supplied + : !Array.isArray(supplied) || + supplied.length !== values.length || + !supplied.every((value, index) => value === values[index])) + ) + return false; } return true; }; @@ -209,8 +344,14 @@ export const replayEnvironmentTool = ( source: EnvironmentSource, tool: EnvironmentManifestTool, args: ReplayArguments, + observedQuery?: ObservedQuery, ): Result => { - const selected = selectReplayExchange(tool, source.exchanges, args); + const selected = selectReplayExchange( + tool, + source.exchanges, + args, + observedQuery, + ); if (selected === undefined) return err( new AnalysisInputError( diff --git a/src/application/environment/environmentStateAnswer.test.ts b/src/application/environment/environmentStateAnswer.test.ts index 528a3d1225..53e1dff00c 100644 --- a/src/application/environment/environmentStateAnswer.test.ts +++ b/src/application/environment/environmentStateAnswer.test.ts @@ -243,6 +243,30 @@ describe("loaded later pages", () => { }); describe("list answers", () => { + it("replays repeated query values instead of applying scalar state-query semantics", () => { + const serving = servingFrom([ + ["GET", "/api/items?tag=a&tag=b", { selected: "b" }], + ["GET", "/api/items?tag=a&tag=c", { selected: "c" }], + ]); + const tool = serving.source.manifest.tools.find( + (candidate) => candidate.path_template === "/api/items", + ); + if (tool === undefined) throw new Error("Missing items query tool"); + const argument = tool.query_parameter_names.find((name) => name === "tag"); + if (argument === undefined) throw new Error("Missing tag query argument"); + const result = answerFromState(serving, tool, { [argument]: ["a", "c"] }); + if (!result.ok) throw result.error; + expect(result.value.body).toEqual({ selected: "c" }); + expect(result.value.replay).toMatchObject({ + exact_match: true, + recorded_query: "?tag=a&tag=c", + }); + expect(result.value.state).toEqual({ + source: "recording", + reason: "query-multiplicity", + }); + }); + it("lists every entity of the collection in state order when membership is all", () => { const serving = servingOf(); const args = { uid: "acb4v6" }; @@ -326,7 +350,9 @@ describe("list answers", () => { state: { source: "state", listed: 1 }, }); }); +}); +describe("list answer coverage and bounds", () => { it("answers an unfiltered call from a paging-only line of the parent its path arguments name", () => { const books = servingFrom([ ["GET", "/api/orgs/1/books?limit=50", [{ id: 1 }, { id: 2 }]], diff --git a/src/application/environment/environmentStateAnswer.ts b/src/application/environment/environmentStateAnswer.ts index d369c75e61..daf2e1f9a9 100644 --- a/src/application/environment/environmentStateAnswer.ts +++ b/src/application/environment/environmentStateAnswer.ts @@ -1,4 +1,7 @@ -import { environmentToolArguments } from "./environmentTools.js"; +import { + environmentToolArguments, + hasRepeatedQueryArgument, +} from "./environmentTools.js"; import { stateBodyJson } from "../../domain/analysisView/EnvironmentStateAccumulator.js"; import { collectionMatch, @@ -39,6 +42,7 @@ import { replayAnswer, replayEnvironmentTool, selectReplayExchange, + type ObservedQuery, type ReplayArguments, type ReplayResult, } from "./environmentReplay.js"; @@ -65,6 +69,7 @@ export type StateProvenance = readonly reason: | RecordedViewReason | "no-view" + | "query-multiplicity" | "unrecorded-request" | "page-exceeded" | "projection"; @@ -141,8 +146,9 @@ const fromRecording = ( tool: EnvironmentManifestTool, args: ReplayArguments, reason: Extract["reason"], + observedQuery?: ObservedQuery, ): Result => { - const replayed = replayEnvironmentTool(source, tool, args); + const replayed = replayEnvironmentTool(source, tool, args, observedQuery); return replayed.ok ? ok({ ...replayed.value, state: { source: "recording", reason } }) : replayed; @@ -295,6 +301,7 @@ const answerItem = ( tool: EnvironmentManifestTool, view: StateItemView, args: ReplayArguments, + observedQuery?: ObservedQuery, ): Result => { const { source, store } = serving; const call = itemCall(tool, args); @@ -309,7 +316,7 @@ const answerItem = ( }); // A projection of an entity the state holds replays as the capture's projected reads did. if (asksProjection(view, tool, args)) - return fromRecording(source, tool, args, "projection"); + return fromRecording(source, tool, args, "projection", observedQuery); const shown = shownEntity( tool, view, @@ -324,8 +331,12 @@ const answerItem = ( refused(tool, `Tool ${tool.name} has no usable recorded envelope.`), ); const exact = - selectReplayExchange(tool, new Map([[base.ordinal, base]]), args)?.exact === - true; + selectReplayExchange( + tool, + new Map([[base.ordinal, base]]), + args, + observedQuery, + )?.exact === true; return stateAnswer( serving, { exchange: base, exact }, @@ -476,21 +487,34 @@ const answerCheckedLine = ( tool: EnvironmentManifestTool, view: StateListView | StateDocumentView, args: ReplayArguments, + observedQuery?: ObservedQuery, ): Result => { const named = checkedLine( serving.source, tool, view.checked_record_ordinals, args, + observedQuery, ); const base = named ?? (view.shape === "list" - ? (unfilteredLine(serving.source, tool, allLines(view), args) ?? - queryLine(serving.source, tool, view, args)) + ? (unfilteredLine( + serving.source, + tool, + allLines(view), + args, + observedQuery, + ) ?? queryLine(serving.source, tool, view, args, observedQuery)) : undefined); if (base === undefined) - return fromRecording(serving.source, tool, args, "unrecorded-request"); + return fromRecording( + serving.source, + tool, + args, + "unrecorded-request", + observedQuery, + ); // An unfiltered call answers on a line it did not name, so its replay match is not exact. return view.shape === "list" ? answerList( @@ -507,17 +531,32 @@ export const answerFromState = ( serving: StateServing, tool: EnvironmentManifestTool, args: ReplayArguments, + observedQuery?: ObservedQuery, ): Result => { + if (hasRepeatedQueryArgument(tool, args)) + return fromRecording( + serving.source, + tool, + args, + "query-multiplicity", + observedQuery, + ); const view = serving.store.state.views.get(tool.name); if (view === undefined) - return fromRecording(serving.source, tool, args, "no-view"); + return fromRecording(serving.source, tool, args, "no-view", observedQuery); switch (view.shape) { case "recorded": - return fromRecording(serving.source, tool, args, view.reason); + return fromRecording( + serving.source, + tool, + args, + view.reason, + observedQuery, + ); case "item": - return answerItem(serving, tool, view, args); + return answerItem(serving, tool, view, args, observedQuery); case "list": case "document": - return answerCheckedLine(serving, tool, view, args); + return answerCheckedLine(serving, tool, view, args, observedQuery); } }; diff --git a/src/application/environment/environmentStateCallLines.ts b/src/application/environment/environmentStateCallLines.ts index a79737146f..16709cd655 100644 --- a/src/application/environment/environmentStateCallLines.ts +++ b/src/application/environment/environmentStateCallLines.ts @@ -9,6 +9,7 @@ import { pagesOnly } from "../../domain/analysisView/environmentStatePaging.js"; import { selectReplayExchange, selectsRecordedLine, + type ObservedQuery, type ReplayArguments, } from "./environmentReplay.js"; import type { @@ -48,14 +49,18 @@ export const checkedLine = ( tool: EnvironmentManifestTool, checked: readonly number[], args: ReplayArguments, + observedQuery?: ObservedQuery, ): RecordedExchange | undefined => { const lines = new Map(); for (const ordinal of checked) { const exchange = source.exchanges.get(ordinal); - if (exchange !== undefined && selectsRecordedLine(tool, exchange, args)) + if ( + exchange !== undefined && + selectsRecordedLine(tool, exchange, args, observedQuery) + ) lines.set(ordinal, exchange); } - return selectReplayExchange(tool, lines, args)?.exchange; + return selectReplayExchange(tool, lines, args, observedQuery)?.exchange; }; /** For a list of a whole collection, a call that supplies no query argument answers from a checked line of the collection its path arguments name that only pages it, chosen as replay chooses; a requested page answers from its recording. */ @@ -64,6 +69,7 @@ export const unfilteredLine = ( tool: EnvironmentManifestTool, checked: readonly number[], args: ReplayArguments, + observedQuery?: ObservedQuery, ): RecordedExchange | undefined => { const query = environmentToolArguments(tool).query; const segments = callSegments(tool, args); @@ -82,7 +88,7 @@ export const unfilteredLine = ( ) lines.set(ordinal, exchange); } - return selectReplayExchange(tool, lines, args)?.exchange; + return selectReplayExchange(tool, lines, args, observedQuery)?.exchange; }; /** A call's query arguments by parameter name. */ @@ -105,6 +111,7 @@ export const queryLine = ( tool: EnvironmentManifestTool, view: StateListView, args: ReplayArguments, + observedQuery?: ObservedQuery, ): RecordedExchange | undefined => { const query = view.query; const segments = callSegments(tool, args); @@ -125,5 +132,5 @@ export const queryLine = ( ) lines.set(ordinal, exchange); } - return selectReplayExchange(tool, lines, args)?.exchange; + return selectReplayExchange(tool, lines, args, observedQuery)?.exchange; }; diff --git a/src/application/environment/environmentStateWriteAnswer.ts b/src/application/environment/environmentStateWriteAnswer.ts index ccfdf104a9..114d9e8903 100644 --- a/src/application/environment/environmentStateWriteAnswer.ts +++ b/src/application/environment/environmentStateWriteAnswer.ts @@ -40,6 +40,7 @@ import { err, ok, type Result } from "../../domain/result.js"; import { replayAnswer, selectReplayExchange, + type ObservedQuery, type ReplayArguments, } from "./environmentReplay.js"; import type { EnvironmentManifestTool } from "./environmentSource.js"; @@ -61,7 +62,10 @@ import { unloadableMember, type HeldDocument, } from "./environmentStateDocument.js"; -import { environmentToolArguments } from "./environmentTools.js"; +import { + environmentToolArguments, + hasRepeatedQueryArgument, +} from "./environmentTools.js"; /** Attempts at a generated UUID or token before a collision with held values is refused. */ const GENERATION_ATTEMPTS = 64; @@ -71,6 +75,7 @@ interface WriteRequest { readonly tool: EnvironmentManifestTool; readonly plan: StateWritePlan; readonly args: ReplayArguments; + readonly observedQuery?: ObservedQuery; } /** The path values a call supplies, keyed by template segment as write plans name them. */ @@ -266,8 +271,12 @@ const writeAnswer = ( if (plan.response === null || exchange === undefined) return err(refused(tool, `Tool ${tool.name} has no recorded response.`)); const exact = - selectReplayExchange(tool, new Map([[exchange.ordinal, exchange]]), args) - ?.exact === true; + selectReplayExchange( + tool, + new Map([[exchange.ordinal, exchange]]), + args, + request.observedQuery, + )?.exact === true; if (plan.response.shape === "recorded") return ok({ ...replayAnswer(serving.source, exchange, exact), state }); const body = writtenResponse(plan.response, recordedJson(exchange), rendered); @@ -534,6 +543,13 @@ export const answerWrite = ( serving: StateServing, request: WriteRequest, ): Result => { + if (hasRepeatedQueryArgument(request.tool, request.args)) + return err( + refused( + request.tool, + "The state write model cannot apply repeated query parameters; the state is unchanged.", + ), + ); const body = callBody(request); if (!body.ok) return body; const call: WriteCall = { diff --git a/src/application/environment/environmentToolAnswer.ts b/src/application/environment/environmentToolAnswer.ts index 5b2e4bf010..16b4330f30 100644 --- a/src/application/environment/environmentToolAnswer.ts +++ b/src/application/environment/environmentToolAnswer.ts @@ -17,6 +17,7 @@ import { answerWrite } from "./environmentStateWriteAnswer.js"; export type EnvironmentToolAnswer = ( tool: EnvironmentManifestTool, args: ReplayArguments, + observedQuery?: string | null, ) => Result; /** The answer a served environment gives: replay without a state store; with one, a served write applies to the live state and a read answers from it. */ @@ -25,10 +26,19 @@ export const environmentToolAnswer = ( store: EnvironmentStateStore | undefined, ): EnvironmentToolAnswer => store === undefined - ? (tool, args) => replayEnvironmentTool(source, tool, args) - : (tool, args) => { + ? (tool, args, observedQuery) => + replayEnvironmentTool(source, tool, args, observedQuery) + : (tool, args, observedQuery) => { const plan = store.state.writes.get(tool.name); return plan?.served === true - ? answerWrite({ source, store }, { tool, plan, args }) - : answerFromState({ source, store }, tool, args); + ? answerWrite( + { source, store }, + { + tool, + plan, + args, + ...(observedQuery === undefined ? {} : { observedQuery }), + }, + ) + : answerFromState({ source, store }, tool, args, observedQuery); }; diff --git a/src/application/environment/environmentTools.test.ts b/src/application/environment/environmentTools.test.ts index f71e91a9ae..85ae00b887 100644 --- a/src/application/environment/environmentTools.test.ts +++ b/src/application/environment/environmentTools.test.ts @@ -92,9 +92,9 @@ describe("environmentToolDefinitions", () => { expect(search.additionalProperties).toBe(false); expect(Object.keys(search.properties).sort()).toEqual(["query", "type"]); expect(search.properties.query).toEqual({ - type: "string", + oneOf: [{ type: "string" }, { type: "array", items: { type: "string" } }], description: - "Query parameter query; optional, matched against the recorded query on replay.", + "Query parameter query; optional. Supply one string for one occurrence or an array to preserve repeated values in captured order; repeated values replay their recorded request.", }); expect("body" in search.properties).toBe(false); }); @@ -522,9 +522,12 @@ describe("environmentToolInputSchema argument names", () => { ]); expect(Object.hasOwn(schema.properties, "constructor")).toBe(true); expect(schema.properties.query_id?.description).toBe( - "Query parameter id; optional, matched against the recorded query on replay.", + "Query parameter id; optional. Supply one string for one occurrence or an array to preserve repeated values in captured order; repeated values replay their recorded request.", ); - expect(schema.properties.body?.type).toBe("string"); + expect(schema.properties.body?.oneOf).toEqual([ + { type: "string" }, + { type: "array", items: { type: "string" } }, + ]); expect(schema.properties.request_body?.type).toBe("object"); expect(schema.required).toEqual(["id"]); }); diff --git a/src/application/environment/environmentTools.ts b/src/application/environment/environmentTools.ts index 6165f9eb91..0ad2a42618 100644 --- a/src/application/environment/environmentTools.ts +++ b/src/application/environment/environmentTools.ts @@ -26,7 +26,12 @@ export const REPLAY_CATALOG_INSTRUCTIONS = /** JSON Schema advertised for one tool argument. */ export interface EnvironmentToolProperty { - readonly type: "string" | "object"; + readonly type?: "string" | "object" | "array"; + readonly items?: { readonly type: "string" }; + readonly oneOf?: readonly ( + | { readonly type: "string" } + | { readonly type: "array"; readonly items: { readonly type: "string" } } + )[]; readonly description: string; } @@ -306,6 +311,18 @@ export const environmentToolArguments = ( }; }; +/** Whether the caller supplied multiple occurrences for a captured query name. */ +export const hasRepeatedQueryArgument = ( + tool: EnvironmentManifestTool, + args: Readonly>, +): boolean => { + for (const argument of environmentToolArguments(tool).query.values()) { + const value = args[argument]; + if (Array.isArray(value) && value.length > 1) return true; + } + return false; +}; + /** Lazily render properties so admission precedes retaining a whole schema. */ function* environmentToolProperties( tool: EnvironmentManifestTool, @@ -324,8 +341,11 @@ function* environmentToolProperties( yield [ argument, { - type: "string", - description: `Query parameter ${describeText(name, "query name")}; optional, matched against the recorded query on replay.`, + oneOf: [ + { type: "string" }, + { type: "array", items: { type: "string" } }, + ], + description: `Query parameter ${describeText(name, "query name")}; optional. Supply one string for one occurrence or an array to preserve repeated values in captured order; repeated values replay their recorded request.`, }, false, ]; diff --git a/src/artifacts/apple/MachoLoadCommandReader.test.ts b/src/artifacts/apple/MachoLoadCommandReader.test.ts index c2008e599c..870e8aa082 100644 --- a/src/artifacts/apple/MachoLoadCommandReader.test.ts +++ b/src/artifacts/apple/MachoLoadCommandReader.test.ts @@ -142,6 +142,107 @@ it("uses legacy simulator and device platform facts when resolving dylibs", asyn ); }); +it("selects the ARMv7s FAT dylib slice before ARMv7 independent of FAT order", async () => { + const workspace = await createTestTempDirectory("rea-macho-arm-fat-order-"); + const armv7 = { type: 12, subtype: 9 }; + const armv7s = { type: 12, subtype: 11 }; + const app = machoImage({ + cpu: armv7s, + wide: false, + commands: [ + versionMinCommand(LC.VERSION_MIN_IPHONEOS), + dylibCommand(LC.LOAD_DYLIB, "@loader_path/lib.dylib"), + ], + }); + const generic = machoImage({ + cpu: armv7, + wide: false, + fileType: FILE_TYPE.dylib, + commands: [ + versionMinCommand(LC.VERSION_MIN_IPHONEOS), + dylibCommand(LC.LOAD_DYLIB, "@loader_path/generic-child.dylib"), + ], + }); + const specialized = machoImage({ + cpu: armv7s, + wide: false, + fileType: FILE_TYPE.dylib, + commands: [ + versionMinCommand(LC.VERSION_MIN_IPHONEOS), + dylibCommand(LC.LOAD_DYLIB, "@loader_path/specialized-child.dylib"), + ], + }); + const child = machoImage({ + cpu: armv7, + wide: false, + fileType: FILE_TYPE.dylib, + commands: [versionMinCommand(LC.VERSION_MIN_IPHONEOS)], + }); + const traceWithSlices = async ( + name: string, + slices: readonly { cpu: typeof armv7; bytes: Uint8Array }[], + ) => { + const root = join(workspace, name); + await mkdir(root, { recursive: true }); + await writeFile(join(root, "App"), app); + await writeFile(join(root, "lib.dylib"), fatImage(slices)); + await writeFile(join(root, "generic-child.dylib"), child); + await writeFile(join(root, "specialized-child.dylib"), child); + return traceDylibResolution({ + rootPath: root, + targetPath: join(root, "App"), + targetSha256: createHash("sha256").update(app).digest("hex"), + enumerateRoots: false, + parameters: { roots: ["App"] }, + }); + }; + + for (const [name, slices] of [ + [ + "generic-first", + [ + { cpu: armv7, bytes: generic }, + { cpu: armv7s, bytes: specialized }, + ], + ], + [ + "specialized-first", + [ + { cpu: armv7s, bytes: specialized }, + { cpu: armv7, bytes: generic }, + ], + ], + ] as const) { + const trace = await traceWithSlices(name, slices); + expect( + trace.images.find(({ path }) => path === "App")?.slices[0], + ).toMatchObject({ + architecture: "arm", + cpu_type: armv7s.type, + cpu_subtype: armv7s.subtype, + }); + expect(trace.edges.find(({ loader }) => loader === "App")).toMatchObject({ + resolution: { status: "resolved", image: "lib.dylib" }, + }); + expect( + trace.edges.find(({ loader }) => loader === "lib.dylib"), + ).toMatchObject({ + install_name: "@loader_path/specialized-child.dylib", + resolution: { status: "resolved", image: "specialized-child.dylib" }, + }); + } + + const fallback = await traceWithSlices("armv7-fallback", [ + { cpu: armv7, bytes: generic }, + ]); + expect( + fallback.edges.find(({ loader }) => loader === "lib.dylib"), + ).toMatchObject({ + install_name: "@loader_path/generic-child.dylib", + resolution: { status: "resolved", image: "generic-child.dylib" }, + }); +}); + it.each([false, true] as const)( "applies macOS dyld's implicit platform to a platform-less dylib and traces its required child present=%s", async (childPresent) => { diff --git a/src/browser/WebSourceMapFetcher.ts b/src/browser/WebSourceMapFetcher.ts index 67cdef3a79..f2f61782cd 100644 --- a/src/browser/WebSourceMapFetcher.ts +++ b/src/browser/WebSourceMapFetcher.ts @@ -929,12 +929,15 @@ const preflightResolvedSourceBytes = ( if (source === null) continue; const sourceKey = source; let resolved = sourcesForRoot.get(sourceKey); - if (resolved === undefined) - resolved = resolveJavaScriptSourceMapUrl( + if (resolved === undefined) { + const candidate = resolveJavaScriptSourceMapUrl( source, map.sourceRoot, mapUrl, ); + if (candidate === null) continue; + resolved = candidate; + } const bytes = Buffer.byteLength(resolved, "utf8"); if (bytes > SOURCE_MAP_RESPONSE_BYTES - resolvedBytes) throw resolvedSourceLimitFailure(); diff --git a/src/domain/analysisView/environmentStateRules.ts b/src/domain/analysisView/environmentStateRules.ts index 1c1d98d52a..3f53ffcc9d 100644 --- a/src/domain/analysisView/environmentStateRules.ts +++ b/src/domain/analysisView/environmentStateRules.ts @@ -207,8 +207,12 @@ export const IDENTITY_NAMES = [ ]; const VERSION_SEGMENT = /^v\d+(?:\.\d+)*$/u; -export const segmentsOf = (path: string): string[] => - path.split("/").filter((segment) => segment.length > 0); +export const segmentsOf = (path: string): string[] => { + const segments = path.split("/"); + if (segments[0] === "") segments.shift(); + if (segments.at(-1) === "") segments.pop(); + return segments; +}; export const isParam = (segment: string): boolean => segment.startsWith("{") && segment.endsWith("}"); export const isObject = (value: JsonValue | undefined): value is JsonObject => @@ -296,7 +300,7 @@ export const selectedCollection = ( export const lastSegmentValue = (path: string): string => decodedSegment(segmentsOf(path).at(-1) ?? ""); -/** A path's non-empty segments, so a trailing slash names the same collection. */ +/** A path's segments, preserving repeated separators; one trailing slash names the same collection. */ export const normalizedPath = (path: string): string => `/${segmentsOf(path).join("/")}`; diff --git a/src/domain/analysisView/environmentStateView.test.ts b/src/domain/analysisView/environmentStateView.test.ts index f010cb1a27..04cbcdf9a3 100644 --- a/src/domain/analysisView/environmentStateView.test.ts +++ b/src/domain/analysisView/environmentStateView.test.ts @@ -148,6 +148,18 @@ it("extracts the state the Grafana-shaped capture's reads observed", () => { ).toEqual([]); }); +it("keeps collections with repeated path separators distinct", () => { + const state = stateOf( + captureOf([ + ["GET", "/api/books", [{ id: 1, title: "Single separator" }]], + ["GET", "/api//books", [{ id: 2, title: "Repeated separator" }]], + ]), + ); + expect(state.types.map((type) => type.collection_template)).toEqual( + expect.arrayContaining(["/api/books", "/api//books"]), + ); +}); + it("ends a type's state at the first write under its root, not at unrelated writes", () => { const state = stateOf( captureOf([ diff --git a/src/domain/apple/dyldPaths.ts b/src/domain/apple/dyldPaths.ts index f834077479..a9976ae1c2 100644 --- a/src/domain/apple/dyldPaths.ts +++ b/src/domain/apple/dyldPaths.ts @@ -117,10 +117,44 @@ const COMPATIBLE_ARCHITECTURES: Readonly> = { x86_64h: ["x86_64h", "x86_64"], }; +/** ARM32 subtype preference rows from dyld's CPU_SUBTYPE_ARM compatibility table. */ +const ARM32_CPU_TYPE = 12; +const ARM32_SUBTYPE_PREFERENCES: Readonly> = { + 0: [0], + 5: [5, 0], + 6: [6, 7, 5, 0], + 7: [7, 5, 0], + 8: [8, 7, 5, 0], + 9: [9, 6, 7, 5, 0], + 10: [10, 9, 6, 7, 5, 0], + 11: [11, 9, 10, 6, 7, 5, 0], + 12: [12], +}; + export const compatibleSlice = ( slices: readonly MachoSlice[], process: Pick, -): MachoSlice | undefined => { +): + | { readonly kind: "selected"; readonly slice: MachoSlice } + | { readonly kind: "missing" } + | { readonly kind: "unknown" } => { + if (process.cpu_type === ARM32_CPU_TYPE) { + const subtype = machoCpuSubtypeBase(process.cpu_subtype); + const preferences = ARM32_SUBTYPE_PREFERENCES[subtype]; + if (preferences === undefined) + return slices.some((item) => item.cpu_type === ARM32_CPU_TYPE) + ? { kind: "unknown" } + : { kind: "missing" }; + for (const preferredSubtype of preferences) { + const slice = slices.find( + (item) => + item.cpu_type === ARM32_CPU_TYPE && + machoCpuSubtypeBase(item.cpu_subtype) === preferredSubtype, + ); + if (slice !== undefined) return { kind: "selected", slice }; + } + return { kind: "missing" }; + } for (const architecture of COMPATIBLE_ARCHITECTURES[process.architecture] ?? [ process.architecture, ]) { @@ -132,7 +166,7 @@ export const compatibleSlice = ( machoCpuSubtypeBase(item.cpu_subtype) === machoCpuSubtypeBase(process.cpu_subtype))), ); - if (slice !== undefined) return slice; + if (slice !== undefined) return { kind: "selected", slice }; } - return undefined; + return { kind: "missing" }; }; diff --git a/src/domain/apple/dylibResolution.test.ts b/src/domain/apple/dylibResolution.test.ts index aab085b6ad..385144990a 100644 --- a/src/domain/apple/dylibResolution.test.ts +++ b/src/domain/apple/dylibResolution.test.ts @@ -867,6 +867,34 @@ describe("loader-relative dylib resolution", () => { }); describe("dyld slice compatibility and coverage", () => { + it("keeps an unmodeled ARM32 subtype uncertain when matching slices exist", async () => { + const trace = await traceDylibLoading( + memoryView({ + [MAIN]: executable({ + architecture: "arm", + cpu_type: 12, + cpu_subtype: 42, + dependencies: [dependency("@executable_path/libarm.dylib")], + }), + "Contents/MacOS/libarm.dylib": parsed( + slice({ architecture: "arm", cpu_type: 12, cpu_subtype: 42 }), + slice({ architecture: "arm", cpu_type: 12, cpu_subtype: 9 }), + ), + }), + { roots: [MAIN] }, + ); + + expect(trace.edges[0]).toMatchObject({ + candidates: [ + { + outcome: "undetermined", + resolved_path: "Contents/MacOS/libarm.dylib", + }, + ], + resolution: { status: "undetermined", image: null }, + }); + }); + it("lets an x86_64h process load generic x86_64 but not arm64e load arm64", async () => { const files = { [MAIN]: parsed( diff --git a/src/domain/apple/dylibResolution.ts b/src/domain/apple/dylibResolution.ts index 3d93ae8314..e9a5155965 100644 --- a/src/domain/apple/dylibResolution.ts +++ b/src/domain/apple/dylibResolution.ts @@ -350,12 +350,14 @@ const evaluateCandidate = async ( let outcome: Candidate["outcome"]; if (facts.status !== "parsed") outcome = facts.status; else { - const slice = compatibleSlice(facts.slices, context.processIdentity); - if (slice === undefined) outcome = "architecture-missing"; + const selection = compatibleSlice(facts.slices, context.processIdentity); + if (selection.kind === "unknown") outcome = "undetermined"; + else if (selection.kind === "missing") outcome = "architecture-missing"; // Dependency commands require a dylib; generic dyld image loadability // also admits main executables and bundles for different operations. - else if (slice.file_type_code !== 6) outcome = "not-loadable"; + else if (selection.slice.file_type_code !== 6) outcome = "not-loadable"; else { + const { slice } = selection; const platformResult = platformLoadability(context, slice); outcome = platformResult.outcome === "compatible" @@ -544,10 +546,11 @@ const resolveDependency = async ( : searched; const image = resolved.image === null ? undefined : context.images.get(resolved.image); - const slice = + const selection = image?.status === "parsed" ? compatibleSlice(image.slices, context.processIdentity) : undefined; + const slice = selection?.kind === "selected" ? selection.slice : undefined; return { root: context.root, architecture: context.processIdentity.architecture, @@ -602,11 +605,12 @@ const traceProcess = async ( const image = edge.resolution.image; if (image === null || context.loaded.has(image)) continue; const facts = context.images.get(image); - const slice = + const selection = facts?.status === "parsed" ? compatibleSlice(facts.slices, context.processIdentity) : undefined; - if (slice === undefined) continue; + if (selection?.kind !== "selected") continue; + const { slice } = selection; context.loaded.set(image, { slice, chain: [...current.chain, image], diff --git a/src/domain/javascript/javascriptSemanticAnalysis.dataflow.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.dataflow.test.ts index f74a32e4ee..342353396d 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.dataflow.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.dataflow.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from "vitest"; -import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; +import { + analyzeJavaScriptSemantics, + analyzeParsedJavaScriptSemantics, +} from "./javascriptSemanticAnalysis.js"; +import { parseJavaScriptSource } from "./javascriptSourceParser.js"; import { onlyCallable, topLevelBinding, @@ -411,6 +415,69 @@ describe("JavaScript semantic analysis: dataflow 2", () => { }); }); +describe("JavaScript semantic analysis: mapped arguments writes", () => { + const commonJs = (source: string) => { + const parsed = parseJavaScriptSource(source, "mapped-arguments.cjs"); + if (parsed === null) throw new Error("Expected CommonJS fixture to parse"); + return analyzeParsedJavaScriptSemantics(parsed); + }; + const returnField = (source: string) => { + const value = onlyCallable(commonJs(source), "read").returnSites[0]?.value; + if (value?.status !== "object") + throw new Error("Expected an object return value"); + const field = value.properties.find(({ name }) => name === "value"); + if (field === undefined) throw new Error("Expected returned value field"); + return field.value; + }; + + it.each([ + ["direct arguments write", "arguments[0] = 2;"], + ["stable arguments alias write", "const args = arguments; args[0] = 2;"], + ["array assignment pattern write", "[arguments[0]] = [2];"], + [ + "nested object assignment pattern write", + "({ nested: arguments[0] } = { nested: 2 });", + ], + ])("does not retain a stale literal after a %s", (_name, write) => { + expect( + returnField( + `function read(value) { value = 1; ${write} return { value }; }`, + ).status, + ).not.toBe("literal"); + }); + + it.each([ + [ + "strict function", + 'function read(value) { "use strict"; value = 1; arguments[0] = 2; return { value }; }', + ], + [ + "default parameter", + "function read(value = 0) { value = 1; arguments[0] = 2; return { value }; }", + ], + [ + "no arguments write", + "function read(value) { value = 1; return { value }; }", + ], + ])("keeps the independent parameter value for a %s", (_name, fn) => { + expect(returnField(fn)).toEqual({ status: "literal", value: 1 }); + }); + + it("retains uncertainty when source mode cannot establish sloppy semantics", () => { + const parsed = parseJavaScriptSource( + "function read(value) { value = 1; arguments[0] = 2; return { value }; }", + ); + if (parsed === null) throw new Error("Expected fixture to parse"); + const value = onlyCallable(analyzeParsedJavaScriptSemantics(parsed), "read") + .returnSites[0]?.value; + if (value?.status !== "object") + throw new Error("Expected an object return value"); + expect( + value.properties.find(({ name }) => name === "value")?.value.status, + ).not.toBe("literal"); + }); +}); + describe("primitive addition values", () => { it.each([ ["true + 1", 2], diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index 80d6fcf9f2..b22047c67f 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -97,6 +97,7 @@ import { } from "./javascriptSemanticCoverage.js"; import { semanticResourceLimitReason } from "./javascriptSemanticResourceLimits.js"; import { + javaScriptRuntimeContext, parseJavaScriptSource, type ParsedJavaScriptSource, } from "./javascriptSourceParser.js"; @@ -1484,6 +1485,7 @@ const bindAssignment = ( ): void => { if (t.isAssignmentExpression(node)) { const left = unwrapJavaScriptExpression(node.left).node; + bindMappedArgumentWrite(left, node, state, conditionallySkipped); const selectedPattern = node.operator === "=" && (t.isObjectPattern(left) || t.isArrayPattern(left)); @@ -1524,6 +1526,7 @@ const bindAssignment = ( } } else if (t.isUpdateExpression(node)) { const argument = unwrapJavaScriptExpression(node.argument).node; + bindMappedArgumentWrite(argument, node, state, conditionallySkipped); if (t.isIdentifier(argument)) addAssignment(argument, node, scope, state, conditionallySkipped); } else if (t.isForOfStatement(node) || t.isForInStatement(node)) { @@ -1562,6 +1565,151 @@ const bindAssignment = ( } }; +type MappedArgumentsOwner = { + readonly scope: JavaScriptSemanticScopeState; + readonly parameters: ReadonlyMap; + readonly strictness: "sloppy" | "strict" | "unknown"; +}; + +/** Find an implicit arguments object through stable local aliases. */ +const mappedArgumentsOwner = ( + expression: t.Node, + state: JavaScriptSemanticAnalysisState, + seen: Set = new Set(), +): MappedArgumentsOwner | undefined => { + let value = unwrapJavaScriptExpression(expression).node; + while (t.isIdentifier(value) && value.name !== "arguments") { + const binding = resolveSemanticBindingState(state, value, value.name); + if (binding === undefined || seen.has(binding.bindingId)) return undefined; + seen.add(binding.bindingId); + const initializer = semanticStableInitializer(binding, state); + if (initializer === undefined) return undefined; + value = unwrapJavaScriptExpression(initializer.node).node; + } + if (!t.isIdentifier(value)) return undefined; + + if (semanticBindingLookupAt(state, value, "arguments").kind !== "ambient") + return undefined; + let owner: t.Function | undefined; + for ( + let current: t.Node | undefined = value; + current !== undefined; + current = state.parentsByNode.get(current) + ) { + if (t.isFunction(current) && !t.isArrowFunctionExpression(current)) { + owner = current; + break; + } + } + if (owner === undefined) return undefined; + const scope = state.scopeByNode.get(owner); + if (scope === undefined) return undefined; + + const parameters = runtimeFunctionParameters(owner); + if ( + parameters.some((parameter) => { + const node = t.isTSParameterProperty(parameter) + ? parameter.parameter + : parameter; + return !t.isIdentifier(node); + }) + ) + return undefined; + + // Duplicate formals map only their last occurrence to the shared binding. + const mapped = new Map(); + const names = new Set(); + for (let index = parameters.length - 1; index >= 0; index -= 1) { + const parameter = parameters[index]; + if (parameter === undefined) continue; + const node = t.isTSParameterProperty(parameter) + ? parameter.parameter + : parameter; + if (!t.isIdentifier(node) || names.has(node.name)) continue; + names.add(node.name); + mapped.set(index, node); + } + if (mapped.size === 0) return undefined; + + return { + scope, + parameters: mapped, + strictness: mappedArgumentsStrictness(owner, state), + }; +}; + +const mappedArgumentsStrictness = ( + owner: t.Function, + state: JavaScriptSemanticAnalysisState, +): MappedArgumentsOwner["strictness"] => { + for ( + let current: t.Node | undefined = owner; + current !== undefined; + current = state.parentsByNode.get(current) + ) { + if (t.isClass(current)) return "strict"; + if ( + t.isFunction(current) && + t.isBlockStatement(current.body) && + current.body.directives.some(({ value }) => value.value === "use strict") + ) + return "strict"; + if (t.isProgram(current)) { + if (current.sourceType === "module") return "strict"; + if (current.directives.some(({ value }) => value.value === "use strict")) + return "strict"; + const runtimeContext = javaScriptRuntimeContext(current); + if (runtimeContext === "module") return "strict"; + if (runtimeContext === "commonjs") return "sloppy"; + return "unknown"; + } + } + return "unknown"; +}; + +/** A mapped arguments property write is another write to its formal binding. */ +const bindMappedArgumentWrite = ( + target: t.Node, + writeSite: t.Node, + state: JavaScriptSemanticAnalysisState, + conditionallySkipped: boolean, +): void => { + for (const writeTarget of semanticWriteTargets(target)) { + if ( + !t.isMemberExpression(writeTarget) && + !t.isOptionalMemberExpression(writeTarget) + ) + continue; + const root = semanticReferenceRoot(writeTarget); + if (root.path.length !== 1) continue; + const owner = mappedArgumentsOwner(root.root, state); + if (owner === undefined || owner.strictness === "strict") continue; + const key = root.path[0]; + const index = + typeof key === "number" + ? key + : typeof key === "string" && /^(?:0|[1-9][0-9]*)$/u.test(key) + ? Number(key) + : undefined; + const parameters = + key === null + ? [...owner.parameters.values()] + : index !== undefined && owner.parameters.has(index) + ? [owner.parameters.get(index)!] + : []; + for (const parameter of parameters) { + addAssignment( + parameter, + writeTarget, + owner.scope, + state, + conditionallySkipped, + writeSite, + ); + } + } +}; + const nestedScope = ( node: t.Node, parent: t.Node | null, diff --git a/src/domain/javascript/javascriptSourceMapPaths.ts b/src/domain/javascript/javascriptSourceMapPaths.ts index 5966f89e6b..137150a2f4 100644 --- a/src/domain/javascript/javascriptSourceMapPaths.ts +++ b/src/domain/javascript/javascriptSourceMapPaths.ts @@ -54,35 +54,44 @@ export const resolveJavaScriptSourceMapReferences = ( }); }; -/** Resolve a source declaration using the same pinned format implementation as map consumers. */ +/** Resolve one source declaration with ECMA-426's WHATWG URL parsing rule. */ export const resolveJavaScriptSourceMapUrl = ( source: string | null, sourceRoot: string | null | undefined, mapUrl: string, +): string | null => { + if (source === null) return null; + return ( + URL.parse(sourceMapReference(source, sourceRoot), mapUrl)?.href ?? null + ); +}; + +const sourceMapReference = ( + source: string, + sourceRoot: string | null | undefined, ): string => { - const resolved = resolveSourceMapUrls([source ?? ""], sourceRoot, mapUrl)[0]; - if (resolved === undefined) - throw new TypeError( - "TraceMap did not resolve its single source declaration", - ); - return resolved; + if (sourceRoot == null) return source; + return `${sourceRoot}${sourceRoot.endsWith("/") ? "" : "/"}${source}`; }; const resolveSourceMapUrls = ( sources: string[], sourceRoot: string | null | undefined, mapUrl: string, -): readonly string[] => - new TraceMap( +): readonly string[] => { + // ECMA-426 prefixes each source when sourceRoot is non-null, adding `/` + // only when the root does not already end with one. Do this before handing + // the references to TraceMap so its URL resolver cannot duplicate a slash. + return new TraceMap( { version: 3, names: [], - sources, - ...(sourceRoot == null ? {} : { sourceRoot }), + sources: sources.map((source) => sourceMapReference(source, sourceRoot)), mappings: "", }, mapUrl, ).resolvedSources; +}; /** Resolve a source-map URL reference without changing its recorded identity. */ const resolveJavaScriptSourceMapPath = ( diff --git a/src/gpu/AmdGpuTarget.ts b/src/gpu/AmdGpuTarget.ts new file mode 100644 index 0000000000..1f372dd9ca --- /dev/null +++ b/src/gpu/AmdGpuTarget.ts @@ -0,0 +1,28 @@ +const processor = String.raw`gfx[0-9a-z]+(?:-[0-9a-z]+)*`; +const targetIdSource = `${processor}(?:(?::[a-z0-9_-]+[+-])|(?:\\+[a-z0-9_-]+))*`; +const targetId = new RegExp(`^${targetIdSource}$`, "iu"); +const bundleTriple = + /^(?:hip|hipv4)-(?:amdgcn|amdgpu(?:[0-9]+(?:\.[0-9a-f]+)?)?)-amd-amdhsa-/iu; +const bundleTarget = new RegExp(`(?:--|-)(${targetIdSource})$`, "iu"); +const noteTarget = new RegExp( + `^amdgcn-amd-amdhsa(?:--|-unknown-)(${targetIdSource})$`, + "iu", +); + +/** Target IDs retain the processor spelling and any declared feature modifiers. */ +export const parseAmdGpuTargetId = (value: string): string | null => + targetId.test(value) ? value : null; + +/** Clang HIP bundle IDs carry an AMD triple followed by an optional target ID. */ +export const isAmdGpuBundleId = (name: string): boolean => + bundleTriple.test(name); + +export const readAmdGpuBundleTarget = (name: string): string | null => { + if (!isAmdGpuBundleId(name)) return null; + const target = bundleTarget.exec(name)?.[1]; + return target === undefined ? null : parseAmdGpuTargetId(target); +}; + +/** Extract the processor identity from the two LLVM AMDGPU note spellings. */ +export const readAmdGpuNoteTargetKey = (target: string): string | null => + noteTarget.exec(target)?.[1] ?? null; diff --git a/src/gpu/ClangOffloadBundle.ts b/src/gpu/ClangOffloadBundle.ts index 8e9da156e1..fc2615f99d 100644 --- a/src/gpu/ClangOffloadBundle.ts +++ b/src/gpu/ClangOffloadBundle.ts @@ -7,11 +7,13 @@ import { import { assertArtifactReadActive } from "../artifacts/ArtifactFileOperation.js"; import { gpuCount, + GPU_OBJECT_BYTES, GPU_REPORT_BYTES, rejectGpu, type GpuSource, } from "./GpuSource.js"; import { gpuObject } from "./GpuObject.js"; +import { isAmdGpuBundleId, readAmdGpuBundleTarget } from "./AmdGpuTarget.js"; export const CLANG_BUNDLE_MAGIC = Buffer.from("__CLANG_OFFLOAD_BUNDLE__"); export const readClangBundleDirectory = async ( source: GpuSource, @@ -62,16 +64,13 @@ export const readClangBundleDirectory = async ( size, kind: name.startsWith("host-") ? "host" - : /^(?:hip|hipv4)-amdgcn-/u.test(name) + : isAmdGpuBundleId(name) ? "amdgpu" : "unknown", metadata: { offset: position, size: 24 + nameSize }, }); row.bundle_id = name; - row.target = - /-(?:-)?(gfx[a-zA-Z0-9]+(?:(?::[^\s]+)|(?:\+[a-zA-Z0-9_-]+))*)$/u.exec( - name, - )?.[1] ?? null; + row.target = readAmdGpuBundleTarget(name); objects.push(row); position += 24 + nameSize; } @@ -115,3 +114,31 @@ export const readCcobHeader = (bytes: Buffer, remaining: number) => { rejectGpu("CCOB declared extent is invalid"); return { version, method, width, total, decoded, hash }; }; + +/** Legacy v1 has no size field; LLVM finds the next CCOB magic as its boundary. */ +export const readLegacyCcobTotal = async ( + source: GpuSource, + base: number, +): Promise => { + const searchEnd = Math.min(source.size, base + GPU_OBJECT_BYTES); + let position = base + 20; + let tail: Buffer = Buffer.alloc(0); + while (position < searchEnd) { + const bytes = await source.read( + position, + Math.min(64 * 1024, searchEnd - position), + ); + const haystack = tail.length === 0 ? bytes : Buffer.concat([tail, bytes]); + const index = haystack.indexOf("CCOB"); + if (index >= 0) return position - tail.length + index - base; + tail = haystack.subarray(Math.max(0, haystack.length - 3)); + position += bytes.length; + } + if (searchEnd < source.size) + throw new BinaryResourceFailure( + "structure", + GPU_OBJECT_BYTES, + "Legacy CCOB v1 extent exceeds the 128 MiB decoding budget", + ); + return source.size - base; +}; diff --git a/src/gpu/CudaFatbinary.ts b/src/gpu/CudaFatbinary.ts index 1747e973b1..8089507dbb 100644 --- a/src/gpu/CudaFatbinary.ts +++ b/src/gpu/CudaFatbinary.ts @@ -40,7 +40,7 @@ export async function* readCudaFatbinary( encoded > padded ) rejectGpu("CUDA entry extent is invalid"); - const compressed = (flags & 0xa000n) !== 0n; + const compressed = (flags & 0xb000n) !== 0n; const row = gpuObject(source, { offset: position + width, size: compressed ? encoded : padded, diff --git a/src/gpu/GpuElfKernels.ts b/src/gpu/GpuElfKernels.ts index 5592a83713..8aa8f69883 100644 --- a/src/gpu/GpuElfKernels.ts +++ b/src/gpu/GpuElfKernels.ts @@ -1,4 +1,5 @@ import { readCudaTarget } from "./CudaTarget.js"; +import { readAmdGpuNoteTargetKey } from "./AmdGpuTarget.js"; import type { GpuPayloadTarget } from "./GpuObject.js"; import { err, ok, type Result } from "../domain/result.js"; import { @@ -378,13 +379,7 @@ export const readAmdElfMetadata = async ( const observed = (): AmdElfMetadata => { const target = targets.size === 1 ? [...targets][0]! : null; // LLVM notes use both the empty and explicit `unknown` environment forms. - const targetMatch = - target === null - ? null - : /^amdgcn-amd-amdhsa-(?:-(gfx[a-zA-Z0-9]+(?:(?::[^\s]+)|(?:\+[a-zA-Z0-9_-]+))*)|unknown-(gfx[a-zA-Z0-9]+(?:(?::[^\s]+)|(?:\+[a-zA-Z0-9_-]+))*))$/u.exec( - target, - ); - const target_key = targetMatch?.[1] ?? targetMatch?.[2] ?? null; + const target_key = target === null ? null : readAmdGpuNoteTargetKey(target); return { target, target_key, limitations, notes }; }; try { diff --git a/src/gpu/GpuInventory.ts b/src/gpu/GpuInventory.ts index 63946c5818..5b8441437b 100644 --- a/src/gpu/GpuInventory.ts +++ b/src/gpu/GpuInventory.ts @@ -1,5 +1,6 @@ import { CLANG_BUNDLE_MAGIC, + readLegacyCcobTotal, readClangBundleDirectory, readCcobHeader, } from "./ClangOffloadBundle.js"; @@ -544,7 +545,11 @@ export class GpuInventory { } if (prefix.subarray(0, 4).toString("ascii") !== "CCOB") rejectGpu("Unexpected bytes between Clang offload bundles"); - const header = readCcobHeader(prefix, source.size - base), + const legacyTotal = + prefix.length >= 6 && prefix.readUInt16LE(4) === 1 + ? await readLegacyCcobTotal(source, base) + : source.size - base; + const header = readCcobHeader(prefix, legacyTotal), { width, total, decoded, hash, version, method } = header; const row = gpuObject(source, { offset: base + width, @@ -569,6 +574,7 @@ export class GpuInventory { row.compression, decoded, this.signal, + version === 1, ); if ( createHash("md5").update(data).digest().readBigUInt64LE(0) !== hash diff --git a/src/gpu/GpuPayloadCodec.ts b/src/gpu/GpuPayloadCodec.ts index e3bdb6596a..d08697ad16 100644 --- a/src/gpu/GpuPayloadCodec.ts +++ b/src/gpu/GpuPayloadCodec.ts @@ -10,6 +10,7 @@ export const decodeGpuPayload = async ( compression: "zlib" | "zstd" | "lz4", expected: number, signal?: AbortSignal, + allowZeroPadding = false, ): Promise => { assertArtifactReadActive(signal); if (compression === "lz4") return decodeLz4Block(bytes, expected, signal); @@ -45,7 +46,14 @@ export const decodeGpuPayload = async ( "Built-in GPU decoder omitted its buffer or consumed-input accounting", ); const decoded = result.buffer; - if (result.engine.bytesWritten !== bytes.length) + const consumed = result.engine.bytesWritten; + if ( + consumed !== bytes.length && + (!allowZeroPadding || + consumed < 0 || + consumed > bytes.length || + !bytes.subarray(consumed).every((byte) => byte === 0)) + ) rejectGpu( "GPU compressed payload contains bytes outside its complete codec stream", ); diff --git a/src/il2cpp/Il2cppBinaryTypes.ts b/src/il2cpp/Il2cppBinaryTypes.ts index 3d8b56166b..4fa69ee93d 100644 --- a/src/il2cpp/Il2cppBinaryTypes.ts +++ b/src/il2cpp/Il2cppBinaryTypes.ts @@ -87,6 +87,59 @@ export const createTypeNamer = ( const name = join(parts.reverse(), "+"); return namespace === "" ? name : join([namespace, name], "."); } + function* genericDefinitionName( + index: number, + args: readonly string[], + ): Generator { + if (!Number.isInteger(index) || index < 0 || index >= metadata.types.length) + fail(`type definition ${index} is out of range`); + const chain: number[] = []; + const seen = new Set(); + for (let at = index; at !== -1;) { + if (seen.has(at)) fail(`cyclic declaring type at ${at}`); + if (++work % 256 === 0) yield; + const type = metadata.types[at]; + if (type === undefined) + return fail(`type definition ${at} is out of range`); + seen.add(at); + chain.push(at); + at = type.declaring_type; + } + + let previousArgumentCount = 0; + const parts: string[] = []; + for (const typeIndex of chain.reverse()) { + if (++work % 256 === 0) yield; + const type = metadata.types[typeIndex]!; + const argumentCount = facts.genericTypeArgumentCounts[typeIndex]; + if ( + argumentCount === undefined || + argumentCount < previousArgumentCount || + argumentCount > args.length + ) + return fail( + `generic argument count does not match nested type definition ${typeIndex}`, + ); + const componentArguments = args.slice( + previousArgumentCount, + argumentCount, + ); + previousArgumentCount = argumentCount; + const name = type.name.replace(/`\d+$/u, ""); + parts.push( + componentArguments.length === 0 + ? name + : join([name, "<", join(componentArguments, ", "), ">"]), + ); + } + if (previousArgumentCount !== args.length) + fail( + `generic argument count ${args.length} does not match nested type definition ${index}`, + ); + const name = join(parts, "+"); + const namespace = metadata.types[chain[0]!]!.namespace; + return namespace === "" ? name : join([namespace, name], "."); + } const byAddress = new Map(); const active = new Set(); const byrefBit = version >= 29 ? 29 : 30; @@ -145,10 +198,22 @@ export const createTypeNamer = ( } } function* generic(data: bigint, depth: number): Generator { - const definition = - version >= 27 - ? yield* name(pointer(data, "generic class type"), depth + 1) - : yield* definitionName(u32(data, "generic class definition")); + let definitionIndex: number; + if (version >= 27) { + const definitionType = pointer(data, "generic class type"); + const definitionData = pointer( + definitionType, + "generic class definition", + ); + const definitionKind = + (u32(definitionType + 8n, "generic class type attributes") >>> 16) & + 0xff; + if (definitionKind !== 0x11 && definitionKind !== 0x12) + fail("generic class type is not a class or value type definition"); + definitionIndex = Number( + BigInt.asIntN(32, definitionData & 0xffff_ffffn), + ); + } else definitionIndex = u32(data, "generic class definition"); const instance = pointer(data + 8n, "generic class instance"); const count = u32(instance, "generic argument count"); const argv = pointer(instance + 8n, "generic arguments"); @@ -167,7 +232,7 @@ export const createTypeNamer = ( guard(bytes); args.push(argument); } - return join([definition.replace(/`\d+$/u, ""), "<", join(args, ", "), ">"]); + return yield* genericDefinitionName(definitionIndex, args); } return name; }; diff --git a/src/il2cpp/Il2cppMetadataFile.ts b/src/il2cpp/Il2cppMetadataFile.ts index 730b4adf00..ea4858ce7b 100644 --- a/src/il2cpp/Il2cppMetadataFile.ts +++ b/src/il2cpp/Il2cppMetadataFile.ts @@ -526,6 +526,10 @@ function* readTypes(tables: MetadataTables): Generator< properties: [] as MetadataOwnerRange[], events: [] as MetadataOwnerRange[], }; + const genericContainerCount = + tables.linkFacts === undefined + ? 0 + : entriesIn(section("genericContainers"), GENERIC_CONTAINER_RECORD_BYTES); for (let index = 0; index < counts.typeDefinitions; index++) { if (index % 256 === 0) yield; const at = typeDefinitions.offset + index * sizes.typeDefinitions; @@ -536,6 +540,33 @@ function* readTypes(tables: MetadataTables): Generator< 12 + typeDefinitionTypeIndices * widths.type + widths.generic_container; + if (tables.linkFacts !== undefined) { + const genericContainerIndex = readIndex( + bytes, + at + 8 + typeDefinitionTypeIndices * widths.type, + widths.generic_container, + ); + if ( + genericContainerIndex < -1 || + genericContainerIndex >= genericContainerCount + ) + throw new Il2cppMetadataFormatFailure( + "index-out-of-range", + `type ${index} generic container ${genericContainerIndex} is outside the ${genericContainerCount} available rows`, + ); + if (genericContainerIndex !== -1) { + const genericContainerAt = + section("genericContainers").offset + + genericContainerIndex * GENERIC_CONTAINER_RECORD_BYTES; + const argumentCount = bytes.readInt32LE(genericContainerAt + 4); + if (argumentCount < 0) + throw new Il2cppMetadataFormatFailure( + "index-out-of-range", + `type ${index} generic container ${genericContainerIndex} has negative argument count ${argumentCount}`, + ); + tables.linkFacts.genericTypeArgumentCounts[index] = argumentCount; + } + } const countsAt = starts + 32; const type = { name: readString(bytes.readInt32LE(at), `type ${index} name`), @@ -730,6 +761,8 @@ export interface Il2cppMetadataLinkFacts { readonly returnTypes: Int32Array; readonly fieldTypes: Int32Array; readonly parameterTypes: Int32Array; + /** Total class generic argument count per TypeDefinition, including enclosing types. */ + readonly genericTypeArgumentCounts: Int32Array; /** Demand-read names from this stable metadata snapshot, by GenericParameterIndex. */ readonly genericParameterName: (index: number) => string | undefined; } @@ -835,6 +868,9 @@ function* readMetadataCoreSteps( returnTypes: new Int32Array(tables.counts.methods), fieldTypes: new Int32Array(tables.members.counts.fields), parameterTypes: new Int32Array(tables.members.counts.parameters), + genericTypeArgumentCounts: new Int32Array( + tables.counts.typeDefinitions, + ), genericParameterName: createGenericParameterReader(tables), }, }; diff --git a/src/il2cpp/Il2cppMetadataProvider.ts b/src/il2cpp/Il2cppMetadataProvider.ts index 9017ae3e2a..9e7d61d4cb 100644 --- a/src/il2cpp/Il2cppMetadataProvider.ts +++ b/src/il2cpp/Il2cppMetadataProvider.ts @@ -534,7 +534,7 @@ export class Il2cppMetadataProvider implements Il2cppMetadataPort { ? "Field, parameter and return types are not recovered without binary_path; type indices refer to the native binary." : decoded.metadata_version === 27 ? "Field, parameter and return type names are null for metadata version 27: versions 27.0 to 27.2 share that header number but place the Il2CppType byref flag differently, so a name could gain or lose & without warning; type indices are still reported." - : "Field, parameter and return type names are rendered from the binary's Il2CppType table in a C#-like form (keywords for primitives, Namespace.Outer+Inner, generic parameters by metadata name); custom modifiers, array bounds and attributes are not reported.", + : "Field, parameter and return type names are rendered from the binary's Il2CppType table in a C#-like form (keywords for primitives, Namespace.Outer+Inner, generic parameters by metadata name); nested generic arguments stay with the type component that declares them. Custom modifiers, array bounds and attributes are not reported.", ...(linked.binary !== null && linked.binary.code_gen_modules_field === null ? [ diff --git a/src/javascript/sourceMaps/TraceSourceMap.test.ts b/src/javascript/sourceMaps/TraceSourceMap.test.ts index 51377e7ec5..64fcde4ed4 100644 --- a/src/javascript/sourceMaps/TraceSourceMap.test.ts +++ b/src/javascript/sourceMaps/TraceSourceMap.test.ts @@ -142,6 +142,23 @@ it("keeps ignore-list precedence and section-local indexes for duplicate sources }); describe("upstream source-map point evidence", () => { + it("keeps repeated separators in distinct resolved source identities", () => { + const report = trace({ + ...leaf, + sources: ["dir/original.ts", "dir//original.ts"], + sourcesContent: ["single", "repeated"], + mappings: "AAAA,ACAA", + }); + expect( + report.matches.map((match) => + match.state === "mapped" ? match.resolved_url : null, + ), + ).toEqual([ + "https://app.test/maps/dir/original.ts", + "https://app.test/maps/dir//original.ts", + ]); + }); + it("preserves every duplicate-column/source-URL candidate and its own embedded bytes", () => { const report = trace({ ...leaf, diff --git a/src/javascript/sourceMaps/TraceSourceMap.ts b/src/javascript/sourceMaps/TraceSourceMap.ts index 56c188e95d..f71279cb78 100644 --- a/src/javascript/sourceMaps/TraceSourceMap.ts +++ b/src/javascript/sourceMaps/TraceSourceMap.ts @@ -11,6 +11,7 @@ import { z } from "zod"; import { javaScriptRuntime } from "../../domain/runtimeVersion.js"; import { javaScriptWorkerMemoryLimitBytes } from "../../process/JavaScriptWorkerMemoryLimit.js"; import { webSourceOffset } from "../../domain/webSourcePosition.js"; +import { resolveJavaScriptSourceMapUrl } from "../../domain/javascript/javascriptSourceMapPaths.js"; import { webSourceMapReportSchema, type WebSourceMapReport, @@ -68,7 +69,7 @@ export const traceSourceMap = ( let bytes = 0; const matches: WebSourceMapReport["matches"] = []; for (const segment of segments) { - const match = originalMatch(segment, map, inspected.declarations); + const match = originalMatch(segment, map, url, inspected.declarations); bytes += Buffer.byteLength(JSON.stringify(match)) + 1; if (bytes > WEB_SOURCE_MAP_LIMITS.outputBytes) throw new SourceMapFormatFailure( @@ -158,6 +159,7 @@ const greatestPosition = ( const originalMatch = ( segment: Readonly, map: TraceMap, + mapUrl: string, declarations: readonly SourceMapSourceDeclaration[], ): WebSourceMapReport["matches"][number] => { if (segment.length === 1) return { state: "generated-only" }; @@ -179,10 +181,11 @@ const originalMatch = ( source_index: declaration.sourceIndex, reported_source: declaration.source, reported_source_root: declaration.root, - resolved_url: - declaration.source === null - ? null - : (map.resolvedSources[segment[1]] ?? null), + resolved_url: resolveJavaScriptSourceMapUrl( + declaration.source, + declaration.root, + mapUrl, + ), ignored: declaration.ignored, content: declaration.content === null diff --git a/src/native/elf/ElfNoteParser.test.ts b/src/native/elf/ElfNoteParser.test.ts index 730fae6234..8f104bb110 100644 --- a/src/native/elf/ElfNoteParser.test.ts +++ b/src/native/elf/ElfNoteParser.test.ts @@ -4,6 +4,7 @@ import { expect, it } from "vitest"; import { inspectElfNotesInputSchema } from "../../domain/native/elfNotes.js"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; import { + compressedElfNoteFixture, elfNote, elfNoteFixture, elfNoteTablePastEnd, @@ -97,6 +98,51 @@ it("reads the same notes from an SHT_NOTE section including its name", () => { ).toBe(true); }); +it.each([ + [32, true], + [64, false], +] as const)( + "decompresses a zlib section-name table for ELF%i %s-endian notes", + (bits, little) => { + const report = parse(compressedElfNoteFixture(bits, little)); + expect(report.tables).toEqual([ + expect.objectContaining({ kind: "section", name: ".note.gnu.build-id" }), + ]); + expect(report.entries[0]?.gnu?.build_id_hex).toBe( + "0123456789abcdef0123456789abcdef01234567", + ); + }, +); + +it("admits a decoded section name larger than its compressed source", () => { + const name = "n".repeat(512); + const bytes = compressedElfNoteFixture(64, true, 1, name); + expect(bytes.length).toBeLessThan(name.length); + const report = parse(bytes); + expect(report.tables[0]?.name).toBe(name); +}); + +it("keeps note facts but marks section names unavailable for unknown compression", () => { + const report = parse(compressedElfNoteFixture(64, true, 2)); + expect(report.tables[0]?.name).toBeNull(); + expect(report.entries[0]?.gnu?.build_id_hex).toBe( + "0123456789abcdef0123456789abcdef01234567", + ); + expect(report.limitations).toContain( + "ELF section names are unavailable because compression type 2 is unsupported.", + ); +}); + +it("bounds compressed section-name expansion with the source bytes", () => { + const bytes = compressedElfNoteFixture(); + const input = inspectElfNotesInputSchema.parse({ + path: defaults.path, + max_file_bytes: bytes.length + 1, + }); + expect(() => parse(bytes, input)).toThrow(ArtifactReaderFailure); + expect(() => parse(bytes, input)).toThrow(/memory budget/u); +}); + it("reports independent PT_NOTE and SHT_NOTE declarations of the same bytes", () => { const report = parse(elfNoteFixture({ ownership: "both" })); expect(report.tables.map((table) => table.kind).sort()).toEqual([ diff --git a/src/native/elf/ElfNoteParser.ts b/src/native/elf/ElfNoteParser.ts index 272d13a2ba..37753f5cfa 100644 --- a/src/native/elf/ElfNoteParser.ts +++ b/src/native/elf/ElfNoteParser.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { inflateSync } from "node:zlib"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; import { readElfFixedHeader } from "../../domain/elfFixedHeader.js"; import { @@ -17,10 +18,12 @@ import { const PT_NOTE = 4; const SHT_NOTE = 7; const SHT_STRTAB = 3; +const SHF_COMPRESSED = 0x800; +const ELFCOMPRESS_ZLIB = 1; const HEX_DESCRIPTOR_LIMIT = 64; interface TextBudget { - readonly limit: number; + limit: number; used: number; hashed: number; scanned: number; @@ -323,6 +326,7 @@ export const parseElfNotes = ( scans: new Map(), }; let shstrtab: Buffer | null = null; + let sectionNameLimitation: string | undefined; if (shnum !== 0 && shstrndx !== 0) { if (shstrndx >= shnum) return fail("ELF section name index is out of range."); @@ -332,6 +336,38 @@ export const parseElfNotes = ( const strOff = reader.word(namesHeader + (wide ? 24 : 16)); const strSize = reader.word(namesHeader + (wide ? 32 : 20)); shstrtab = reader.slice(strOff, strSize); + if ((reader.word(namesHeader + 8) & SHF_COMPRESSED) !== 0) { + const compressionHeaderBytes = wide ? 24 : 12; + if (strSize < compressionHeaderBytes) + return fail("ELF compressed section name table header is truncated."); + const compressionType = reader.u32(strOff); + if (compressionType !== ELFCOMPRESS_ZLIB) { + shstrtab = null; + sectionNameLimitation = `ELF section names are unavailable because compression type ${String(compressionType)} is unsupported.`; + } else { + const uncompressedSize = reader.word(strOff + (wide ? 8 : 4)); + if (uncompressedSize > input.max_file_bytes - bytes.length) + throw new ArtifactReaderFailure( + "limit", + `ELF source and compressed section names exceed the ${String(input.max_file_bytes)}-byte memory budget.`, + { boundary: "memory" }, + ); + try { + shstrtab = inflateSync(shstrtab.subarray(compressionHeaderBytes), { + maxOutputLength: Math.max(1, uncompressedSize), + }); + } catch (cause: unknown) { + return fail( + `ELF compressed section name table could not be decompressed: ${cause instanceof Error ? cause.message : "invalid zlib data"}.`, + ); + } + if (shstrtab.length !== uncompressedSize) + return fail( + "ELF compressed section name table size is inconsistent.", + ); + text.limit = bytes.length + shstrtab.length; + } + } } const tables: ElfNotes["tables"] = []; const parsed: ReturnType[] = []; @@ -409,7 +445,10 @@ export const parseElfNotes = ( examined_tables: tables.length, examined_entries: examined.count, }, - limitations: [...(absent ? ABSENT_LIMITATIONS : LIMITATIONS)], + limitations: [ + ...(absent ? ABSENT_LIMITATIONS : LIMITATIONS), + ...(sectionNameLimitation === undefined ? [] : [sectionNameLimitation]), + ], }; }; diff --git a/src/native/elf/ElfNotes.fixture.ts b/src/native/elf/ElfNotes.fixture.ts index a7ac803800..48e08cdf3c 100644 --- a/src/native/elf/ElfNotes.fixture.ts +++ b/src/native/elf/ElfNotes.fixture.ts @@ -1,3 +1,5 @@ +import { deflateSync } from "node:zlib"; + const concat = (...parts: Buffer[]): Buffer => Buffer.concat(parts); const alignTo = (value: number, alignment: number): number => @@ -207,6 +209,84 @@ export const elfNoteFixture = (options: ElfNoteImageOptions = {}): Buffer => { return bytes; }; +/** ELF note fixture with a zlib-compressed section-name string table. */ +export const compressedElfNoteFixture = ( + bits: 32 | 64 = 64, + little = true, + compressionType = 1, + sectionName = ".note.gnu.build-id", +): Buffer => { + const image = elfNoteFixture({ + bits, + little, + ownership: "section", + sectionName, + }); + const wide = bits === 64; + const u16 = (offset: number) => + little ? image.readUInt16LE(offset) : image.readUInt16BE(offset); + const word = (offset: number) => + wide + ? Number( + little + ? image.readBigUInt64LE(offset) + : image.readBigUInt64BE(offset), + ) + : little + ? image.readUInt32LE(offset) + : image.readUInt32BE(offset); + const shoff = word(wide ? 40 : 32); + const shentsize = u16(wide ? 58 : 46); + const namesIndex = u16(wide ? 62 : 50); + const namesHeader = shoff + namesIndex * shentsize; + const namesOffset = word(namesHeader + (wide ? 24 : 16)); + const namesSize = word(namesHeader + (wide ? 32 : 20)); + const names = image.subarray(namesOffset, namesOffset + namesSize); + const compressed = deflateSync(names); + const compressionHeaderBytes = wide ? 24 : 12; + const replacement = Buffer.alloc(compressionHeaderBytes); + if (little) replacement.writeUInt32LE(compressionType, 0); + else replacement.writeUInt32BE(compressionType, 0); + if (wide) { + if (little) { + replacement.writeBigUInt64LE(BigInt(names.length), 8); + replacement.writeBigUInt64LE(1n, 16); + } else { + replacement.writeBigUInt64BE(BigInt(names.length), 8); + replacement.writeBigUInt64BE(1n, 16); + } + } else { + if (little) { + replacement.writeUInt32LE(names.length, 4); + replacement.writeUInt32LE(1, 8); + } else { + replacement.writeUInt32BE(names.length, 4); + replacement.writeUInt32BE(1, 8); + } + } + const result = concat( + image.subarray(0, namesOffset), + replacement, + compressed, + image.subarray(namesOffset + namesSize), + ); + const writeWord = (offset: number, value: number) => { + if (wide) { + if (little) result.writeBigUInt64LE(BigInt(value), offset); + else result.writeBigUInt64BE(BigInt(value), offset); + } else if (little) result.writeUInt32LE(value, offset); + else result.writeUInt32BE(value, offset); + }; + const flagsAt = namesHeader + 8; + writeWord(flagsAt, word(flagsAt) | 0x800); + writeWord( + namesHeader + (wide ? 32 : 20), + compressionHeaderBytes + compressed.length, + ); + writeWord(namesHeader + (wide ? 48 : 32), 1); + return result; +}; + /** * ELF64 whose only PT_NOTE declares 20 bytes. The file holds the 12-byte * header, 4-byte name, and 1-byte descriptor; the 3 alignment bytes are absent. diff --git a/src/native/elf/ElfNotesProvider.ts b/src/native/elf/ElfNotesProvider.ts index 766148a92e..8f6bf46170 100644 --- a/src/native/elf/ElfNotesProvider.ts +++ b/src/native/elf/ElfNotesProvider.ts @@ -84,7 +84,7 @@ export class ElfNotesProvider implements ElfNotesPort { } const NOTE_TEXT_BUDGET = - "The ELF note text, hash, and scan budget tied to the admitted image length"; + "The ELF note text, hash, scan, and decompression budget bounded by max_file_bytes"; const inputBudgetRemediation = ( boundary: "file-size" | "capacity", diff --git a/src/native/exports/NativeExportParser.test.ts b/src/native/exports/NativeExportParser.test.ts index fc17048652..b2409d4513 100644 --- a/src/native/exports/NativeExportParser.test.ts +++ b/src/native/exports/NativeExportParser.test.ts @@ -15,6 +15,75 @@ const artifact = { path: "/fixture/lib", sha256: "a".repeat(64), bytes: 1 }; const parse = (bytes: Buffer) => nativeExportsSchema.parse(parseNativeExports(bytes, artifact)); +const peExportNameAcrossSections = (overlapAtRva?: number): Buffer => { + const bytes = Buffer.concat([ + peExportImage({ + functions: [{ rva: 0x2000 }], + names: [{ name: "BoundaryName", index: 0 }], + }), + Buffer.alloc(overlapAtRva === undefined ? 0x400 : 0x800), + ]); + const pe = 128; + const optional = pe + 24; + const section = optional + bytes.readUInt16LE(pe + 20); + const second = section + 40; + bytes.writeUInt16LE(2, pe + 6); + bytes.write(".data", second, "ascii"); + bytes.writeUInt32LE(0x400, second + 8); + bytes.writeUInt32LE(0x1400, second + 12); + bytes.writeUInt32LE(0x400, second + 16); + bytes.writeUInt32LE(0x800, second + 20); + bytes.writeUInt32LE(0x40000040, second + 36); + bytes.writeUInt32LE(0x13fc, 0x480); + bytes.write("BoundaryName\0", 0x7fc, "latin1"); + if (overlapAtRva !== undefined) { + const third = second + 40; + bytes.writeUInt16LE(3, pe + 6); + bytes.write(".alias", third, "ascii"); + bytes.writeUInt32LE(0x100, third + 8); + bytes.writeUInt32LE(overlapAtRva, third + 12); + bytes.writeUInt32LE(0x100, third + 16); + bytes.writeUInt32LE(0xc00, third + 20); + bytes.writeUInt32LE(0x40000040, third + 36); + bytes.fill(0x41, 0xc00, 0xd00); + } + return bytes; +}; + +describe("PE export name mapping coverage", () => { + it("reads a bounded export name across adjacent section mappings", () => { + const result = parse(peExportNameAcrossSections()); + expect(result).toMatchObject({ + exports: [ + { + name: "BoundaryName", + ordinal: 1, + address: "0x180002000", + forwarder: null, + }, + ], + coverage: { status: "complete", examined_entries: 1, exports: 1 }, + issues: [], + }); + }); + + it("ignores conflicting mappings after the export name terminator", () => { + const result = parse(peExportNameAcrossSections(0x1409)); + + expect(result.exports).toMatchObject([{ name: "BoundaryName" }]); + expect(result.issues).toEqual([]); + expect(result.coverage.status).toBe("complete"); + }); + + it("rejects conflicting mappings that overlap the export name", () => { + const result = parse(peExportNameAcrossSections(0x1404)); + + expect(result.exports).toEqual([]); + expect(result.issues).toMatchObject([{ code: "name_unreadable" }]); + expect(result.coverage.status).toBe("partial"); + }); +}); + describe("PE export directories", () => { it.each([32, 64] as const)( "reports named, aliased, ordinal-only and forwarded PE%d exports", diff --git a/src/native/exports/NativeExportParser.ts b/src/native/exports/NativeExportParser.ts index 528ea8c1c9..c663e98fcd 100644 --- a/src/native/exports/NativeExportParser.ts +++ b/src/native/exports/NativeExportParser.ts @@ -120,18 +120,69 @@ const parsePeExports = (bytes: Buffer, artifact: Artifact): NativeExports => { const map = (rva: number, size: number): number => mappedFileOffset(mappings, base + BigInt(rva), size, image.zero_fills); const cString = (rva: number): string => { - const start = map(rva, 1); - const end = bytes - .subarray(start, start + MAXIMUM_NAME_BYTES + 1) - .indexOf(0); - if (end <= 0) - throw new BinaryFormatFailure( - "malformed", - `Export name at RVA ${hex(rva)} is empty, unterminated or longer than ${String(MAXIMUM_NAME_BYTES)} bytes`, + const chunks: Buffer[] = []; + let length = 0; + while (length <= MAXIMUM_NAME_BYTES) { + const address = base + BigInt(rva + length); + let sourceMapping: (typeof mappings)[number] | undefined; + for (const mapping of mappings) { + const end = mapping.address + BigInt(mapping.size); + if ( + mapping.address <= address && + address < end && + (sourceMapping === undefined || + end > sourceMapping.address + BigInt(sourceMapping.size)) + ) + sourceMapping = mapping; + } + if (sourceMapping === undefined) { + map(rva + length, 1); + throw new BinaryFormatFailure( + "malformed", + "PE export string mapping is inconsistent", + ); + } + const size = Math.min( + MAXIMUM_NAME_BYTES + 1 - length, + Number(sourceMapping.address + BigInt(sourceMapping.size) - address), ); - // The whole string must stay inside one file-backed mapping. - map(rva, end + 1); - return bytes.toString("latin1", start, start + end); + const offset = + sourceMapping.offset + Number(address - sourceMapping.address); + const chunk = bytes.subarray(offset, offset + size); + const terminator = chunk.indexOf(0); + if (terminator >= 0) { + const nameLength = length + terminator; + if (nameLength === 0) + throw new BinaryFormatFailure( + "malformed", + `Export name at RVA ${hex(rva)} is empty, unterminated or longer than ${String(MAXIMUM_NAME_BYTES)} bytes`, + ); + const consumedOffset = mappedFileOffset( + mappings, + address, + terminator + 1, + image.zero_fills, + ); + if (length === 0) { + // Preserve the common single-mapping path without retaining a + // temporary chunk array for every export name. + return bytes.toString( + "latin1", + consumedOffset, + consumedOffset + terminator, + ); + } + chunks.push(chunk.subarray(0, terminator)); + return Buffer.concat(chunks, nameLength).toString("latin1"); + } + mappedFileOffset(mappings, address, size, image.zero_fills); + chunks.push(chunk); + length += size; + } + throw new BinaryFormatFailure( + "malformed", + `Export name at RVA ${hex(rva)} is empty, unterminated or longer than ${String(MAXIMUM_NAME_BYTES)} bytes`, + ); }; const header = map(directory.virtual_address, PE_EXPORT_DIRECTORY_BYTES); diff --git a/src/native/pe/PeCertificateAsn1.ts b/src/native/pe/PeCertificateAsn1.ts index 43c78446a8..b104ccc7b4 100644 --- a/src/native/pe/PeCertificateAsn1.ts +++ b/src/native/pe/PeCertificateAsn1.ts @@ -74,6 +74,7 @@ export const MAX_DECODED_NAME_BYTES = 65_536; interface DecodedNameBudget { remaining: number; + teletexStringCompatibilityUsed: boolean; } const chargeDecodedName = (budget: DecodedNameBudget, bytes: number): void => { @@ -178,6 +179,11 @@ const decodeDirectoryString = ( case 0x17: case 0x18: return decodeUtf8(node.content, budget); + case 0x14: + chargeDecodedName(budget, node.content.length); + budget.teletexStringCompatibilityUsed = true; + // OpenSSL also assumes ISO-8859-1 when converting legacy T61String. + return node.content.toString("latin1"); default: return fail( `Unsupported certificate name encoding (DER tag 0x${node.tag.toString(16).padStart(2, "0")}).`, @@ -339,7 +345,10 @@ const finishTbsCertificate = ( /** Decode one X.509 Certificate SEQUENCE at a known file-relative DER node. */ export const parseCertificate = ( node: Asn1Node, - names: DecodedNameBudget = { remaining: MAX_DECODED_NAME_BYTES }, + names: DecodedNameBudget = { + remaining: MAX_DECODED_NAME_BYTES, + teletexStringCompatibilityUsed: false, + }, ): ParsedCertificate => { requireDefiniteEncoding(node, "X.509 Certificate"); const children = sequence(node, "Certificate"); @@ -510,6 +519,7 @@ export interface ParsedSignedData { readonly digest_algorithm_oids: readonly string[]; readonly certificates: readonly ParsedCertificate[]; readonly signers: readonly ParsedSigner[]; + readonly teletexStringCompatibilityUsed: boolean; } const parseSignedData = ( @@ -554,6 +564,7 @@ const parseSignedData = ( digest_algorithm_oids, certificates, signers, + teletexStringCompatibilityUsed: names.teletexStringCompatibilityUsed, }; }; @@ -579,5 +590,6 @@ export const parsePkcs7SignedData = (bytes: Buffer): ParsedSignedData => { return fail("signedData content has unexpected trailing fields."); return parseSignedData(signedData, { remaining: MAX_DECODED_NAME_BYTES, + teletexStringCompatibilityUsed: false, }); }; diff --git a/src/native/pe/PeCertificateParser.test.ts b/src/native/pe/PeCertificateParser.test.ts index fe401dfac6..4f95700723 100644 --- a/src/native/pe/PeCertificateParser.test.ts +++ b/src/native/pe/PeCertificateParser.test.ts @@ -94,6 +94,32 @@ it("reads BER indefinite constructed ContentInfo and preserves certificate bytes ); }); +it("retains certificate and signer facts for ASCII TeletexString names", () => { + const payload = pkcs7SignedDataFixture("Plain Name", "Plain Org"); + const root = readAsn1(payload); + const markTeletexNames = (node: ReturnType): void => { + if (node.tag === 0x0c) payload[node.start] = 0x14; + for (const child of node.children) markTeletexNames(child); + }; + markTeletexNames(root); + + const report = parse(peCertificateFixture(false, payload)); + expect(report.coverage).toMatchObject({ + status: "complete", + certificates: 1, + }); + expect(report.entries[0]?.pkcs7?.certificates[0]).toMatchObject({ + issuer: { text: "commonName=Plain Name, organizationName=Plain Org" }, + subject: { text: "commonName=Plain Name, organizationName=Plain Org" }, + }); + expect(report.entries[0]?.pkcs7?.signers[0]?.issuer?.text).toBe( + "commonName=Plain Name, organizationName=Plain Org", + ); + expect(report.limitations).toContain( + "TeletexString values use OpenSSL's ISO-8859-1 compatibility mapping; this is not a full T.61 decoder.", + ); +}); + it.each([ ["primitive", 0x04], ["constructed", 0x24], @@ -519,7 +545,6 @@ it("rejects a UTF8String name that is not valid UTF-8", () => { }); it.each([ - ["TeletexString", 0x14, Buffer.from([0xc2, 0x65])], ["GeneralString", 0x1b, Buffer.from("name")], ["OCTET STRING", 0x04, Buffer.from("name")], ] as const)( diff --git a/src/native/pe/PeCertificateParser.ts b/src/native/pe/PeCertificateParser.ts index bc12b46191..5f8cbb1e21 100644 --- a/src/native/pe/PeCertificateParser.ts +++ b/src/native/pe/PeCertificateParser.ts @@ -39,6 +39,9 @@ const LIMITATIONS = [ "PKCS#7 fields are decoded structurally; cryptographic validity remains unobserved.", ] as const; +const TELETEX_LIMITATION = + "TeletexString values use OpenSSL's ISO-8859-1 compatibility mapping; this is not a full T.61 decoder."; + const ABSENT_LIMITATIONS = [ "IMAGE_DIRECTORY_ENTRY_SECURITY is absent. That is not proof the image is unsigned in every catalog or overlay sense.", "This workflow does not verify signatures, certificate chains, timestamps, or Authenticode digest coverage of image bytes.", @@ -88,6 +91,7 @@ export const parsePeCertificates = ( const entries: PeCertificates["entries"] = []; let at = 0; let examined = 0; + let teletexStringCompatibilityUsed = false; while (at < table.length) { // Alignment padding sits inside the gap before the next 8-byte boundary. // Bytes that remain after that boundary are a short header, including @@ -112,6 +116,8 @@ export const parsePeCertificates = ( if (peCertificateTypeName(certificateType) === "pkcs7") { try { const parsed = parsePkcs7SignedData(payload); + teletexStringCompatibilityUsed ||= + parsed.teletexStringCompatibilityUsed; pkcs7 = { content_type_oid: parsed.content_type_oid, content_type_name: oidName(parsed.content_type_oid), @@ -199,6 +205,8 @@ export const parsePeCertificates = ( examined_entries: examined, certificates, }, - limitations: [...LIMITATIONS], + limitations: teletexStringCompatibilityUsed + ? [...LIMITATIONS, TELETEX_LIMITATION] + : [...LIMITATIONS], }; }; diff --git a/tests/acceptance/elfNotesCli.test.ts b/tests/acceptance/elfNotesCli.test.ts index cc797ec2c1..48aeab98c1 100644 --- a/tests/acceptance/elfNotesCli.test.ts +++ b/tests/acceptance/elfNotesCli.test.ts @@ -2,7 +2,10 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; import { toolContract } from "../../src/contracts/toolContracts.js"; -import { elfNoteFixture } from "../../src/native/elf/ElfNotes.fixture.js"; +import { + compressedElfNoteFixture, + elfNoteFixture, +} from "../../src/native/elf/ElfNotes.fixture.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { cliTest } from "../support/cli/cliFixture.js"; @@ -24,6 +27,28 @@ cliTest("reports a GNU build-id through the public CLI", async ({ cli }) => { ).toBe("0123456789abcdef0123456789abcdef01234567"); }); +cliTest( + "resolves section note names from a compressed string table through the CLI", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-elf-notes-compressed-cli-"); + const path = join(root, "program"); + await writeFile(path, compressedElfNoteFixture()); + const result = await cli.run({ + arguments: ["inspect-elf-notes", path, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const evidence = toolContract("inspect_elf_notes").outputSchema.parse( + result.json, + ); + expect(evidence.normalized_result.tables[0]?.name).toBe( + ".note.gnu.build-id", + ); + expect(evidence.normalized_result.entries[0]?.gnu?.build_id_hex).toBe( + "0123456789abcdef0123456789abcdef01234567", + ); + }, +); + cliTest( "resolves a relative ELF path against the working directory", async ({ cli }) => { diff --git a/tests/acceptance/il2cppMetadataCli.test.ts b/tests/acceptance/il2cppMetadataCli.test.ts index 9abd9d86c8..d8dbe610db 100644 --- a/tests/acceptance/il2cppMetadataCli.test.ts +++ b/tests/acceptance/il2cppMetadataCli.test.ts @@ -25,6 +25,7 @@ import { jsonObjectSchema } from "../../src/domain/jsonValue.js"; import { il2cppMetadataSchema } from "../../src/domain/il2cpp/il2cppMetadata.js"; import { createIl2cppMetadataFixture, + createLinkedIl2cppMetadataFixture, FIXTURE_IMAGES, FIXTURE_METHODS, FIXTURE_TYPES, @@ -432,7 +433,7 @@ const assertRetainedNativeSelections = async ( it("queries compact retained metadata through the real SDK with exact row identities and fields", async () => { const root = await createTestTempDirectory("rea-il2cpp-sdk-selection-"); const path = join(root, "global-metadata.dat"); - const fixture = createIl2cppMetadataFixture({ version: 31 }); + const fixture = createLinkedIl2cppMetadataFixture({ version: 31 }); await writeFile(path, fixture.bytes); const binaryPath = join(root, "GameAssembly.dylib"); const binaryBytes = createIl2cppBinaryFixture(); @@ -765,7 +766,7 @@ cliTest( const root = await createTestTempDirectory("rea-il2cpp-binary-"); const path = join(root, "global-metadata.dat"); const binaryPath = join(root, "GameAssembly.dylib"); - await writeFile(path, createIl2cppMetadataFixture().bytes); + await writeFile(path, createLinkedIl2cppMetadataFixture().bytes); await writeFile( binaryPath, createUniversalFixture([ @@ -851,13 +852,96 @@ cliTest( 30000, ); +cliTest( + "keeps nested IL2CPP generic arguments on their declaring type components", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-il2cpp-nested-generics-"); + const cases = [ + { + name: "top-level", + metadata: createLinkedIl2cppMetadataFixture(), + binary: createIl2cppBinaryFixture(), + expected: "System.Größe[]", + }, + { + name: "nested-own-parameter", + metadata: createLinkedIl2cppMetadataFixture({ + typeNames: { 0: "Outer`1", 1: "Inner`2" }, + typeNamespaces: { 0: "Game", 1: "" }, + genericTypeArgumentCounts: { 0: 1, 1: 2 }, + }), + binary: createIl2cppBinaryFixture({ + genericDefinitionTypeIndex: 1, + genericArgumentTypeIndices: [3, 3], + }), + expected: "Game.Outer+Inner[]", + }, + { + name: "nested-inherited-only", + metadata: createLinkedIl2cppMetadataFixture({ + typeNames: { 0: "Outer`1", 1: "Leaf`1" }, + typeNamespaces: { 0: "Game", 1: "" }, + genericTypeArgumentCounts: { 0: 1, 1: 1 }, + }), + binary: createIl2cppBinaryFixture({ + genericDefinitionTypeIndex: 1, + genericArgumentTypeIndices: [3], + }), + expected: "Game.Outer+Leaf[]", + }, + ] as const; + const { client, call } = await connectLocalToolsMcp(); + for (const entry of cases) { + const path = join(root, `${entry.name}.dat`); + const binaryPath = join(root, `${entry.name}.dylib`); + await writeFile(path, entry.metadata.bytes); + await writeFile(binaryPath, entry.binary); + const cliResult = await runFileCli(cli, { + arguments: [ + "inspect-il2cpp-metadata", + path, + "--binary", + binaryPath, + "--json", + ], + }); + expect(cliResult.exitCode, cliResult.stderr).toBe(0); + const cliReport = parseCliMetadata(cliResult.json); + expect(cliReport.return_types[0]?.name).toBe(entry.expected); + expect( + cliReport.field_types.some((row) => row.name === entry.expected), + ).toBe(true); + expect( + cliReport.parameter_types.some((row) => row.name === entry.expected), + ).toBe(true); + + const response = await call("inspect_il2cpp_metadata", { + path, + binary_path: binaryPath, + detail: "complete", + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const mcpReport = parseCliMetadata(response.structuredContent); + expect(mcpReport.return_types[0]?.name).toBe(entry.expected); + expect( + mcpReport.field_types.some((row) => row.name === entry.expected), + ).toBe(true); + expect( + mcpReport.parameter_types.some((row) => row.name === entry.expected), + ).toBe(true); + } + await client.ping(); + }, + 30000, +); + cliTest( "marks a module array found through an x86-64 instruction as derived", async ({ cli }) => { const root = await createTestTempDirectory("rea-il2cpp-inlined-"); const path = join(root, "global-metadata.dat"); const binaryPath = join(root, "GameAssembly.so"); - await writeFile(path, createIl2cppMetadataFixture().bytes); + await writeFile(path, createLinkedIl2cppMetadataFixture().bytes); await writeFile( binaryPath, createElfFixture( @@ -897,7 +981,7 @@ cliTest( const binaryPath = join(root, "libil2cpp.so"); const seeds = join(root, "seeds.tsv"); const shifted = join(root, "shifted.tsv"); - await writeFile(path, createIl2cppMetadataFixture().bytes); + await writeFile(path, createLinkedIl2cppMetadataFixture().bytes); await writeFile(binaryPath, createElfFixture(createIl2cppBinaryFixture())); const run = (output: string, extra: readonly string[] = []) => runFileCli(cli, { @@ -985,7 +1069,10 @@ cliTest( const root = await createTestTempDirectory("rea-il2cpp-array-kind-"); const path = join(root, "global-metadata.dat"); const binaryPath = join(root, "GameAssembly.dylib"); - await writeFile(path, createIl2cppMetadataFixture({ version: 31 }).bytes); + await writeFile( + path, + createLinkedIl2cppMetadataFixture({ version: 31 }).bytes, + ); const makeBinary = (kind: "vector" | "array", rank = 1) => { const bytes = createIl2cppBinaryFixture(); @@ -1050,7 +1137,7 @@ cliTest.skipIf(process.platform === "win32")( const path = join(root, "global-metadata.dat"); const binaryPath = join(root, "libil2cpp.so"); const parent = join(root, "regular-file"); - await writeFile(path, createIl2cppMetadataFixture().bytes); + await writeFile(path, createLinkedIl2cppMetadataFixture().bytes); await writeFile(binaryPath, createElfFixture(createIl2cppBinaryFixture())); await writeFile(parent, "unchanged"); const result = await runFileCli(cli, { diff --git a/tests/boundary/filesystem/sourceToBundlePaths.test.ts b/tests/boundary/filesystem/sourceToBundlePaths.test.ts index cdc65261a2..cc00f2b148 100644 --- a/tests/boundary/filesystem/sourceToBundlePaths.test.ts +++ b/tests/boundary/filesystem/sourceToBundlePaths.test.ts @@ -97,6 +97,38 @@ it("matches source-map originals named relative to the map", async () => { }); it.each([ + { root: "../src/", expected: "https://example.test/src/index.js" }, + { root: "", expected: "https://example.test/index.js" }, + { root: null, expected: "https://example.test/maps/index.js" }, +])("resolves sourceRoot $root according to ECMA-426", ({ root, expected }) => { + expect( + resolveJavaScriptSourceMapUrl( + "index.js", + root, + "https://example.test/maps/app.js.map", + ), + ).toBe(expected); +}); + +it.each([ + { + mapPath: "maps/app.js.map", + sourceRoot: "../src/", + source: "index.js", + expected: "src/index.js", + }, + { + mapPath: "maps/app.js.map", + sourceRoot: "", + source: "index.js", + expected: "index.js", + }, + { + mapPath: "maps/app.js.map", + sourceRoot: null, + source: "index.js", + expected: "maps/index.js", + }, { mapPath: "pkg/dist/main.js.map", source: "..\\src\\a.js", @@ -123,6 +155,7 @@ it.each([ sourceRoot: "../src", source: "/vendor/a.js", expected: "pkg/src/vendor/a.js", + expectedUrlPath: "pkg/src//vendor/a.js", decoy: "vendor/a.js", }, { @@ -182,6 +215,7 @@ it.each([ source, sourceRoot, expected, + expectedUrlPath, decoy: decoyPath = "src/a.js", }) => { if (sourceRoot !== undefined) { @@ -190,8 +224,10 @@ it.each([ sourceRoot, `https://example.test/${mapPath}`, ); + if (resolved === null) + throw new Error("Expected a valid resolved source URL"); expect(decodeURIComponent(new URL(resolved).pathname)).toBe( - `/${expected}`, + `/${expectedUrlPath ?? expected}`, ); } const root = await createTestTempDirectory("rea-nested-map-"); diff --git a/tests/boundary/gpu/codeObjects.test.ts b/tests/boundary/gpu/codeObjects.test.ts index 8536a7574f..053326c875 100644 --- a/tests/boundary/gpu/codeObjects.test.ts +++ b/tests/boundary/gpu/codeObjects.test.ts @@ -1,6 +1,7 @@ import { expect, it, onTestFinished } from "vitest"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { deflateSync } from "node:zlib"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { gpuCodeObjectsPartialSchema } from "../../../src/domain/gpu/gpuCodeObjects.js"; import { GpuCodeObjectsProvider } from "../../../src/gpu/GpuCodeObjectsProvider.js"; @@ -65,6 +66,26 @@ it.each(["none", "zstd"] as const)( ]); }, ); +it("retains the unsupported legacy CUDA compression declaration and extents", async () => { + const result = await inspect( + cudaFixture("legacy", [ + { kind: 1, target: 80, bytes: Buffer.from(ptxFixture), flags: 4 }, + ]), + ); + if (!result.ok) throw result.error; + expect(result.value.objects[0]).toMatchObject({ + compression: "unsupported", + compression_id: expect.stringContaining("cuda-flags:0x1015"), + encoded_size: deflateSync(Buffer.from(ptxFixture)).length, + decoded_size: Buffer.byteLength(ptxFixture), + inspection: "unsupported", + }); + expect(result.value.ptx_artifacts).toEqual([]); + expect(result.value.kernels).toEqual([]); + expect(result.value.objects[0]!.limitations.join(" ")).toContain( + "compression/version is not supported", + ); +}); it.each([1, 2, 3] as const)( "reads CCOB v%d zlib/Zstd and retains exact decoded provenance", async (version) => { @@ -274,6 +295,7 @@ it.each([ "hipv4-amdgcn-amd-amdhsa--gfx900", "hipv4-amdgcn-amd-amdhsa-gfx900", "hipv4-amdgcn-amd-amdhsa-opaque", + "hipv4-amdgpu9.00-amd-amdhsa--gfx900", ])("keeps discovered HIP targets selectable for %s", async (targetId) => { const bytes = hipFixture(amdFixture(), targetId); const inventory = await inspect(bytes, { analysis: "inventory" }); @@ -294,6 +316,44 @@ it.each([ inspection: "complete", }); }); +it.each(["gfx11-generic", "gfx9-4-generic"] as const)( + "preserves and exactly selects the generic AMDGPU processor %s", + async (processor) => { + const noteTarget = `amdgcn-amd-amdhsa--${processor}`; + const bytes = hipFixture( + amdFixture(false, undefined, noteTarget), + `hipv4-amdgcn-amd-amdhsa--${processor}`, + ); + const result = await inspect(bytes, { targets: [processor] }); + if (!result.ok) throw result.error; + expect( + result.value.objects.find(({ kind }) => kind === "amdgpu"), + ).toMatchObject({ + kind: "amdgpu", + target: processor, + payload_target: noteTarget, + target_consistency: "consistent", + inspection: "complete", + }); + expect(result.value.kernels).toHaveLength(1); + }, +); +it("recognizes a modern AMDGPU triple inside compressed CCOB v3", async () => { + const targetId = "hipv4-amdgpu9.00-amd-amdhsa--gfx900"; + const bytes = ccobFixture(3, "zstd", hipFixture(amdFixture(), targetId)); + const result = await inspect(bytes); + if (!result.ok) throw result.error; + expect( + result.value.objects.find(({ bundle_id }) => bundle_id === targetId), + ).toMatchObject({ + kind: "amdgpu", + target: "gfx900", + payload_target: "amdgcn-amd-amdhsa--gfx900", + target_consistency: "consistent", + inspection: "complete", + }); + expect(result.value.kernels).toHaveLength(1); +}); it("bounds standalone AMD note discovery before retaining metadata", async () => { const result = await inspect( elfFixture(224, [ @@ -646,6 +706,42 @@ it("iterates mixed plain/CCOB bundles without counting flat records as nesting", expect(result.value.kernels).toHaveLength(20); expect(result.value.coverage.discovery).toBe("complete"); }); +it("uses each legacy CCOB v1 stream boundary across alignment padding", async () => { + const first = ccobFixture(1, "zlib"), + second = ccobFixture(2, "zlib"), + padding = Buffer.alloc((8 - (first.length % 8)) % 8); + const result = await inspect(Buffer.concat([first, padding, second])); + if (!result.ok) throw result.error; + const bundles = result.value.objects.filter( + ({ kind }) => kind === "hip-bundle", + ); + expect(bundles).toHaveLength(2); + expect( + bundles.map(({ inspection, decoded_size }) => [inspection, decoded_size]), + ).toEqual([ + ["complete", expect.any(Number)], + ["complete", expect.any(Number)], + ]); + expect(bundles[0]!.sha256).toBe(bundles[1]!.sha256); + expect(result.value.kernels).toHaveLength(2); + expect(result.value.coverage.discovery).toBe("complete"); +}); +it("keeps a later CCOB bundle after a truncated legacy v1 stream", async () => { + const truncated = ccobFixture(1, "zlib").subarray(0, -1), + later = ccobFixture(2, "zlib"); + const result = await inspect(Buffer.concat([truncated, later])); + if (!result.ok) throw result.error; + const bundles = result.value.objects.filter( + ({ kind }) => kind === "hip-bundle", + ); + expect(bundles).toHaveLength(2); + expect(bundles.map(({ inspection }) => inspection)).toEqual([ + "malformed", + "complete", + ]); + expect(result.value.kernels).toHaveLength(1); + expect(result.value.coverage.discovery).toBe("partial"); +}); it("continues at the declared CCOB boundary after an unsupported codec", async () => { const unknown = ccobFixture(); unknown.writeUInt16LE(99, 6); diff --git a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts index a8c0738241..8d8f8c47bb 100644 --- a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts +++ b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts @@ -15,10 +15,13 @@ import { FIXTURE_RETURN_TYPE_NAME, type Il2cppBinaryFixtureOptions, } from "../../fixtures/il2cpp/binary.js"; -import { createIl2cppMetadataFixture } from "../../fixtures/il2cpp/metadata.js"; +import { + createIl2cppMetadataFixture, + createLinkedIl2cppMetadataFixture, +} from "../../fixtures/il2cpp/metadata.js"; const { metadata, link: facts } = readIl2cppMetadataForLinking( - createIl2cppMetadataFixture().bytes, + createLinkedIl2cppMetadataFixture().bytes, ); const expectedAddress = (value: bigint | "no-body" | "unknown") => typeof value === "bigint" @@ -233,6 +236,38 @@ it("reports type indices without names for metadata version 27", () => { ); }); +it("renders nested generic arguments on the components that declare them", () => { + const renderedName = ( + typeNames: Readonly>, + genericTypeArgumentCounts: Readonly>, + genericArgumentTypeIndices: readonly number[], + ) => { + const { metadata: nestedMetadata, link: nestedFacts } = + readIl2cppMetadataForLinking( + createLinkedIl2cppMetadataFixture({ + typeNames, + typeNamespaces: { 0: "Game", 1: "" }, + genericTypeArgumentCounts, + }).bytes, + ); + return linkIl2cppBinary( + createIl2cppBinaryFixture({ + genericDefinitionTypeIndex: 1, + genericArgumentTypeIndices, + }), + nestedMetadata, + nestedFacts, + ).return_types[0]?.name; + }; + + expect( + renderedName({ 0: "Outer`1", 1: "Inner`2" }, { 0: 1, 1: 2 }, [3, 3]), + ).toBe("Game.Outer+Inner[]"); + expect(renderedName({ 0: "Outer`1", 1: "Leaf`1" }, { 0: 1, 1: 1 }, [3])).toBe( + "Game.Outer+Leaf[]", + ); +}); + it("does not search for an empty image name", () => { const unnamed = { ...metadata, @@ -486,6 +521,11 @@ it("rejects a recursive binary type graph without overflowing the JavaScript sta }); it("guards a shared generic expansion before concatenating unchecked text", () => { + const expanded = readIl2cppMetadataForLinking( + createLinkedIl2cppMetadataFixture({ + genericTypeArgumentCounts: { 3: 2 }, + }).bytes, + ); const bytes = Buffer.alloc(0x4000); createIl2cppBinaryFixture().copy(bytes); bytes.writeBigUInt64LE(BigInt(bytes.length), 64); @@ -506,11 +546,17 @@ it("guards a shared generic expansion before concatenating unchecked text", () = } bytes.writeBigUInt64LE(prior, 0x900); const stopped = new Error("type text admission"); - const steps = linkIl2cppBinarySteps(bytes, metadata, facts, undefined, { - beforeText: (size) => { - if (size > 1024) throw stopped; + const steps = linkIl2cppBinarySteps( + bytes, + expanded.metadata, + expanded.link, + undefined, + { + beforeText: (size) => { + if (size > 1024) throw stopped; + }, }, - }); + ); expect(() => { for (;;) { if (steps.next().done) break; diff --git a/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts b/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts index 915f3ebc42..26612cd2e4 100644 --- a/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts +++ b/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts @@ -18,7 +18,10 @@ import { createElfFixture, createIl2cppBinaryFixture, } from "../../fixtures/il2cpp/binary.js"; -import { createIl2cppMetadataFixture } from "../../fixtures/il2cpp/metadata.js"; +import { + createIl2cppMetadataFixture, + createLinkedIl2cppMetadataFixture, +} from "../../fixtures/il2cpp/metadata.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; afterEach(() => { @@ -487,7 +490,7 @@ it("keeps the metadata source observation when the selected binary does not link const root = await createTestTempDirectory("rea-il2cpp-binary-failure-"); const path = join(root, "global-metadata.dat"); const binaryPath = join(root, "GameAssembly.dll"); - const fixture = createIl2cppMetadataFixture(); + const fixture = createLinkedIl2cppMetadataFixture(); await writeFile(path, fixture.bytes); await writeFile(binaryPath, Buffer.from("not a native image")); const provider = new Il2cppMetadataProvider(); @@ -515,7 +518,7 @@ it("accepts cancellation between binary linking batches and keeps the metadata s const root = await createTestTempDirectory("rea-il2cpp-link-cancel-"); const path = join(root, "global-metadata.dat"); const binaryPath = join(root, "libil2cpp.so"); - const fixture = createIl2cppMetadataFixture(); + const fixture = createLinkedIl2cppMetadataFixture(); await writeFile(path, fixture.bytes); await writeFile(binaryPath, createElfFixture(createIl2cppBinaryFixture())); // Yields taken by the metadata decode alone; later ones belong to linking. @@ -560,7 +563,7 @@ it("refuses duplicate MethodDef tokens before publishing linked Evidence or Ghid const binaryPath = join(root, "GameAssembly.dylib"); const duplicateSeeds = join(root, "duplicate.tsv"); const foldedSeeds = join(root, "folded.tsv"); - const fixture = createIl2cppMetadataFixture({ version: 31 }); + const fixture = createLinkedIl2cppMetadataFixture({ version: 31 }); const duplicate = Buffer.from(fixture.bytes); const firstTokenAt = fixture.record("methods", 0) + 24; const secondTokenAt = fixture.record("methods", 1) + 24; @@ -616,7 +619,7 @@ it("retains both observed file identities and the selected architecture when nat const path = join(root, "global-metadata.dat"), binary_path = join(root, "GameAssembly.dylib"); const bytes = createIl2cppBinaryFixture({ codeRegistration: false }); - await writeFile(path, createIl2cppMetadataFixture().bytes); + await writeFile(path, createLinkedIl2cppMetadataFixture().bytes); await writeFile(binary_path, bytes); const provider = new Il2cppMetadataProvider(); try { @@ -643,7 +646,7 @@ it("names the metadata, not the binary, when linking reads a malformed metadata const root = await createTestTempDirectory("rea-il2cpp-link-metadata-name-"); const path = join(root, "global-metadata.dat"), binaryPath = join(root, "GameAssembly.dylib"); - const fixture = createIl2cppMetadataFixture({ version: 39 }); + const fixture = createLinkedIl2cppMetadataFixture({ version: 39 }); // One generic parameter whose name index points past the string table; // only native linking reads it, through a field typed as that parameter. const parameterAt = fixture.bytes.length; @@ -652,10 +655,10 @@ it("names the metadata, not the binary, when linking reads a malformed metadata metadata.writeInt32LE(parameterAt, 8 + 12 * 12); metadata.writeInt32LE(13, 8 + 12 * 12 + 4); metadata.writeInt32LE(1, 8 + 12 * 12 + 8); - metadata.writeUInt8(2, fixture.section("fields").offset + 4); + metadata.writeUInt8(3, fixture.section("fields").offset + 4); const binary = Buffer.from(createIl2cppBinaryFixture()); - binary.writeBigUInt64LE(0n, 0x980 + 32); - binary.writeUInt32LE(0x13 << 16, 0x980 + 32 + 8); + binary.writeBigUInt64LE(0n, 0x980 + 48); + binary.writeUInt32LE(0x13 << 16, 0x980 + 48 + 8); await writeFile(path, metadata); await writeFile(binaryPath, binary); const provider = new Il2cppMetadataProvider(); diff --git a/tests/boundary/mcp/environmentServeMcp.test.ts b/tests/boundary/mcp/environmentServeMcp.test.ts index cf4f06e508..4215346a7a 100644 --- a/tests/boundary/mcp/environmentServeMcp.test.ts +++ b/tests/boundary/mcp/environmentServeMcp.test.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { timedCaptureEvidence } from "../../fixtures/analysisView.js"; import { Client, @@ -11,6 +12,7 @@ import { environmentServeSource, environmentStateServeFixture, writeEnvironmentServeFiles, + type EnvironmentServeFixture, } from "../../fixtures/environmentServe.js"; import { startListeningProcess } from "../../support/mcp/streamableHttpProcess.js"; import { @@ -20,17 +22,18 @@ import { const HTTP_BOUNDARY_TIMEOUT_MS = 90_000; const CARD_PATH = "/.well-known/agent-env.json"; -/** SHA-256 of the 12,818-byte card served for the fixture named "renamed", its descriptions naming each tool's recorded responses; pure replay must serve these bytes. */ +/** SHA-256 of the 13,882-byte card served for the fixture named "renamed", its descriptions naming each tool's recorded responses; pure replay must serve these bytes. */ const REPLAY_CARD_SHA256 = - "7c65ab63f0c00ce79b570024c4611c225729677b44fc49bbde32b2d65f868961"; + "27370fe0a89556dcd0dd1bc6ddd0439d77b628349829afd56e4af4ee393d0a64"; /** Serve the fixture pair as pure replay through the built CLI. */ const serveReplayOnly = async ( processes: TestProcesses, environment: Readonly> = {}, + fixture?: EnvironmentServeFixture, ) => { const root = await createTestTempDirectory("rea-environment-serve-mcp-"); - const files = await writeEnvironmentServeFiles(root); + const files = await writeEnvironmentServeFiles(root, fixture); const started = await startListeningProcess( processes, [ @@ -54,6 +57,50 @@ const connect = async (url: URL) => { return client; }; +processTest( + "validates repeated query arrays through the advertised MCP contract", + async ({ processes }) => { + const fixture = environmentStateServeFixture( + timedCaptureEvidence([ + ["GET", "/api/items?tag=a&tag=b", { selected: "b" }], + ["GET", "/api/items?tag=a&tag=c", { selected: "c" }], + ]), + ); + const source = environmentServeSource(fixture); + const tool = source.manifest.tools.find( + ({ path_template }) => path_template === "/api/items", + ); + if (tool === undefined) + throw new Error("Missing repeated-query fixture tool"); + const started = await serveReplayOnly(processes, {}, fixture); + const client = await connect(started.url); + try { + const { tools } = await client.listTools(); + expect( + tools.find(({ name }) => name === tool.name)?.inputSchema.properties + ?.tag, + ).toMatchObject({ + oneOf: [ + { type: "string" }, + { type: "array", items: { type: "string" } }, + ], + }); + const response = await client.callTool({ + name: tool.name, + arguments: { tag: ["a", "c"] }, + }); + expect(response.isError).toBeFalsy(); + expect(response.structuredContent).toMatchObject({ + body: { selected: "c" }, + replay: { exact_match: true, recorded_query: "?tag=a&tag=c" }, + }); + } finally { + await client.close(); + } + }, + HTTP_BOUNDARY_TIMEOUT_MS, +); + describe("rea environment-serve --replay-only", () => { processTest( "lists one tool per manifest tool and replays exact and best-effort records as untrusted target content", @@ -130,9 +177,10 @@ describe("rea environment-serve --replay-only", () => { const card = await fetch(new URL(CARD_PATH, origin)); expect(card.status).toBe(200); const text = await card.text(); - expect(createHash("sha256").update(text).digest("hex")).toBe( - REPLAY_CARD_SHA256, - ); + expect( + createHash("sha256").update(text).digest("hex"), + `${Buffer.byteLength(text)}-byte card`, + ).toBe(REPLAY_CARD_SHA256); const body: unknown = JSON.parse(text); expect(body).toMatchObject({ name: "renamed", diff --git a/tests/fixtures/gpu/codeObjects.ts b/tests/fixtures/gpu/codeObjects.ts index 8f6a3f6134..151f945f9c 100644 --- a/tests/fixtures/gpu/codeObjects.ts +++ b/tests/fixtures/gpu/codeObjects.ts @@ -257,15 +257,26 @@ export const amdFixture = ( ]); }; export const cudaFixture = ( - compression: "none" | "zstd" = "none", + compression: "none" | "zstd" | "legacy" = "none", entries: { kind: number; target: number; bytes: Buffer; flags?: number }[] = [ { kind: 1, target: 80, bytes: Buffer.from(ptxFixture) }, { kind: 2, target: 90, bytes: cubinFixture({ architecture: 90 }) }, ], ): Buffer => { const records = entries.map(({ kind, target, bytes, flags: entryFlags }) => { - const compressed = compression === "zstd", - data = compressed ? zstdCompressSync(bytes) : bytes; + const compressed = compression !== "none", + data = + compression === "zstd" + ? zstdCompressSync(bytes) + : compression === "legacy" + ? deflateSync(bytes) + : bytes, + compressionFlag = + compression === "zstd" + ? 0x8000n + : compression === "legacy" + ? 0x1000n + : 0n; const result = Buffer.alloc(64 + Math.ceil(data.length / 8) * 8); result.writeUInt16LE(kind, 0); result.writeUInt16LE(0x101, 2); @@ -274,7 +285,7 @@ export const cudaFixture = ( result.writeUInt32LE(compressed ? data.length : 0, 16); result.writeUInt32LE(target, 28); result.writeBigUInt64LE( - (compressed ? 0x8011n : 0x11n) | BigInt(entryFlags ?? 0), + 0x11n | compressionFlag | BigInt(entryFlags ?? 0), 40, ); result.writeBigUInt64LE(compressed ? BigInt(bytes.length) : 0n, 56); diff --git a/tests/fixtures/il2cpp/binary.ts b/tests/fixtures/il2cpp/binary.ts index 3b8283f6c7..84cdfad95c 100644 --- a/tests/fixtures/il2cpp/binary.ts +++ b/tests/fixtures/il2cpp/binary.ts @@ -37,6 +37,10 @@ export interface Il2cppBinaryFixtureOptions { readonly namePadding?: number; /** Text written directly before mscorlib.dll, making it the tail of a longer name. */ readonly namePrefix?: string; + /** TypeDefinitionIndex used by the fixture's generic instance. */ + readonly genericDefinitionTypeIndex?: number; + /** Native type-table indices serialized in its generic argument vector. */ + readonly genericArgumentTypeIndices?: readonly number[]; } /** Every fixture method returns type 0, an array of a generic instance. */ @@ -138,7 +142,7 @@ export const createIl2cppBinaryFixture = ( const types: readonly [kind: number, data: bigint][] = [ [0x1d, address(TYPE_RECORDS + 16)], [0x15, address(GENERIC_CLASS)], - [0x12, 3n], + [0x12, BigInt(options.genericDefinitionTypeIndex ?? 3)], [0x08, 0n], ]; types.forEach(([kind, data], index) => { @@ -150,9 +154,15 @@ export const createIl2cppBinaryFixture = ( // Version 29+ Il2CppGenericClass: definition type, then class_inst. bytes.writeBigUInt64LE(address(TYPE_RECORDS + 32), GENERIC_CLASS); bytes.writeBigUInt64LE(address(GENERIC_INSTANCE), GENERIC_CLASS + 8); - bytes.writeUInt32LE(1, GENERIC_INSTANCE); + const genericArguments = options.genericArgumentTypeIndices ?? [3]; + bytes.writeUInt32LE(genericArguments.length, GENERIC_INSTANCE); bytes.writeBigUInt64LE(address(GENERIC_ARGUMENTS), GENERIC_INSTANCE + 8); - bytes.writeBigUInt64LE(address(TYPE_RECORDS + 48), GENERIC_ARGUMENTS); + genericArguments.forEach((typeIndex, index) => + bytes.writeBigUInt64LE( + address(TYPE_RECORDS + typeIndex * 16), + GENERIC_ARGUMENTS + index * 8, + ), + ); if (options.metadataRegistration !== false) { // typesCount and types, then fieldOffsets and typeDefinitionsSizes, // whose counts equal the 4 metadata type definitions. diff --git a/tests/fixtures/il2cpp/metadata.ts b/tests/fixtures/il2cpp/metadata.ts index 56b46236b6..0a14669f3c 100644 --- a/tests/fixtures/il2cpp/metadata.ts +++ b/tests/fixtures/il2cpp/metadata.ts @@ -20,6 +20,11 @@ export interface Il2cppMetadataFixtureOptions { readonly emptyImage?: boolean; /** Nonempty property/event tables with deliberately different owner ranges. */ readonly propertiesAndEvents?: boolean; + /** Override type names while preserving the fixture's row ownership. */ + readonly typeNames?: Readonly>; + readonly typeNamespaces?: Readonly>; + /** Total generic argument count per type, including enclosing parameters. */ + readonly genericTypeArgumentCounts?: Readonly>; /** Metadata version; 29 and 31 use fixed 4-byte indices and no header counts. */ readonly version?: 24 | 27 | 29 | 31 | 39; /** @@ -337,7 +342,11 @@ export const fixtureRows = (options: Il2cppMetadataFixtureOptions = {}) => { const fillers = options.wide?.fillerTypes ?? 0; const last = FIXTURE_TYPES.length + fillers - 1; const types = [ - ...FIXTURE_TYPES, + ...FIXTURE_TYPES.map((type, index) => ({ + ...type, + name: options.typeNames?.[index] ?? type.name, + namespace: options.typeNamespaces?.[index] ?? type.namespace, + })), ...Array.from({ length: fillers }, (_, index) => ({ name: "Filler", namespace: "", @@ -383,6 +392,7 @@ const encodeTypes = ({ G, typeIndexFields, named, + genericContainerIndices, intern, }: { readonly typeRows: ReturnType["types"]; @@ -394,6 +404,7 @@ const encodeTypes = ({ readonly properties: readonly { declaring_type: number }[]; readonly events: readonly { declaring_type: number }[]; }; + readonly genericContainerIndices: ReadonlyMap; readonly intern: (value: string) => number; }) => { const types = Buffer.alloc(typeRows.length * recordSize); @@ -406,7 +417,7 @@ const encodeTypes = ({ // unrelated to nesting; the reader must use nestedTypes instead. for (let field = 0; field < typeIndexFields; field += 1, at += T) writeIndex(types, at, T, 0x2a); - writeIndex(types, at, G, -1); + writeIndex(types, at, G, genericContainerIndices.get(index) ?? -1); at += G + 4; const fieldStart = FIXTURE_FIELDS.findIndex( (field) => field.declaring_type === index, @@ -448,6 +459,113 @@ const encodeTypes = ({ return types; }; +const genericContainerLayout = ( + options: Il2cppMetadataFixtureOptions, + fixed: boolean, +) => { + const definitions = Object.entries(options.genericTypeArgumentCounts ?? {}) + .map(([type, argumentCount]) => ({ type: Number(type), argumentCount })) + .sort((left, right) => left.type - right.type); + const count = Math.max( + options.wide?.genericContainers ?? 0, + definitions.length, + ); + return { + definitions, + indices: new Map(definitions.map(({ type }, index) => [type, index])), + count, + width: fixed ? 4 : widthFor(count), + }; +}; + +const encodeGenericContainerTables = ( + definitions: readonly { type: number; argumentCount: number }[], + containerCount: number, + width: Width, + intern: (value: string) => number, +) => { + const parameters = definitions.flatMap(({ argumentCount }, container) => + Array.from({ length: argumentCount }, (_, num) => ({ + container, + num, + name: `T${String(num)}`, + })), + ); + for (const parameter of parameters) intern(parameter.name); + + const containers = Buffer.alloc(containerCount * 16); + let parameterStart = 0; + definitions.forEach(({ type, argumentCount }, index) => { + const at = index * 16; + containers.writeInt32LE(type, at); + containers.writeInt32LE(argumentCount, at + 4); + containers.writeInt32LE(0, at + 8); + containers.writeInt32LE(parameterStart, at + 12); + parameterStart += argumentCount; + }); + + const parameterBytes = Buffer.alloc(parameters.length * (width + 12)); + parameters.forEach((parameter, index) => { + const at = index * (width + 12); + writeIndex(parameterBytes, at, width, parameter.container); + parameterBytes.writeInt32LE(intern(parameter.name), at + width); + parameterBytes.writeInt16LE(-1, at + width + 4); + parameterBytes.writeUInt16LE(0, at + width + 6); + parameterBytes.writeUInt16LE(parameter.num, at + width + 8); + parameterBytes.writeUInt16LE(0, at + width + 10); + }); + return { + containers, + parameters: parameterBytes, + parameterCount: parameters.length, + }; +}; + +const encodeImageRows = ( + images: readonly ReturnType[number][], + recordSize: number, + typeDefinitionWidth: Width, + intern: (value: string) => number, +) => { + const bytes = Buffer.alloc(images.length * recordSize); + images.forEach((image, index) => { + const at = index * recordSize; + bytes.writeInt32LE(intern(image.name), at); + bytes.writeInt32LE(index, at + 4); + writeIndex(bytes, at + 8, typeDefinitionWidth, image.type_start); + bytes.writeUInt32LE(image.type_count, at + 8 + typeDefinitionWidth); + writeIndex(bytes, at + 12 + typeDefinitionWidth, typeDefinitionWidth, -1); + bytes.writeUInt32LE(0, at + 12 + 2 * typeDefinitionWidth); + bytes.writeInt32LE(-1, at + 16 + 2 * typeDefinitionWidth); + }); + return bytes; +}; + +const createStringTable = () => { + const parts: Buffer[] = []; + const offsets = new Map(); + let size = 0; + const intern = (value: string) => { + const known = offsets.get(value); + if (known !== undefined) return known; + const encoded = Buffer.from(`${value}\0`, "utf8"); + offsets.set(value, size); + parts.push(encoded); + size += encoded.length; + return offsets.get(value)!; + }; + return { + intern, + bytes: () => Buffer.concat(parts), + count: () => offsets.size, + offset: (value: string) => { + const index = offsets.get(value); + if (index === undefined) throw new Error(`Unknown string ${value}`); + return index; + }, + }; +}; + /** Lay sections out in header order; empty sections share the next offset. */ export const createIl2cppMetadataFixture = ( options: Il2cppMetadataFixtureOptions = {}, @@ -460,13 +578,13 @@ export const createIl2cppMetadataFixture = ( properties: options.propertiesAndEvents ? FIXTURE_PROPERTIES : [], events: options.propertiesAndEvents ? FIXTURE_EVENTS : [], }; - const genericContainers = options.wide?.genericContainers ?? 0; + const genericLayout = genericContainerLayout(options, fixed); const parameterCount = Math.max( options.wide?.parameters ?? 0, FIXTURE_PARAMETERS.length, ); const D: Width = fixed ? 4 : widthFor(rows.types.length); - const G: Width = fixed ? 4 : widthFor(genericContainers); + const G = genericLayout.width; const P: Width = fixed ? 4 : widthFor(parameterCount); // Versions 29 and 31 also store elementTypeIndex in type records. const typeIndexFields = version === 24 ? 5 : fixed ? 4 : 3; @@ -484,18 +602,8 @@ export const createIl2cppMetadataFixture = ( methods: 16 + returnToken + D + T + P + G, }; - const stringParts: Buffer[] = []; - const stringIndex = new Map(); - let stringBytes = 0; - const intern = (value: string) => { - const known = stringIndex.get(value); - if (known !== undefined) return known; - const encoded = Buffer.from(`${value}\0`, "utf8"); - stringIndex.set(value, stringBytes); - stringParts.push(encoded); - stringBytes += encoded.length; - return stringIndex.get(value)!; - }; + const stringTable = createStringTable(); + const { intern } = stringTable; const imageRows = fixtureImages(options); for (const image of imageRows) intern(image.name); for (const type of FIXTURE_TYPES) { @@ -505,6 +613,12 @@ export const createIl2cppMetadataFixture = ( for (const row of [...rows.types, ...rows.methods]) intern(row.name); for (const row of [...FIXTURE_FIELDS, ...FIXTURE_PARAMETERS]) intern(row.name); + const genericRows = encodeGenericContainerTables( + genericLayout.definitions, + genericLayout.count, + G, + intern, + ); const types = encodeTypes({ typeRows, @@ -513,20 +627,11 @@ export const createIl2cppMetadataFixture = ( G, typeIndexFields, named, + genericContainerIndices: genericLayout.indices, intern, }); - const images = Buffer.alloc(imageRows.length * recordBytes.images); - imageRows.forEach((image, index) => { - const at = index * recordBytes.images; - images.writeInt32LE(intern(image.name), at); - images.writeInt32LE(index, at + 4); - writeIndex(images, at + 8, D, image.type_start); - images.writeUInt32LE(image.type_count, at + 8 + D); - writeIndex(images, at + 12 + D, D, -1); - images.writeUInt32LE(0, at + 12 + 2 * D); - images.writeInt32LE(-1, at + 16 + 2 * D); - }); + const images = encodeImageRows(imageRows, recordBytes.images, D, intern); const methods = encodeMethods( rows.methods, @@ -560,14 +665,32 @@ export const createIl2cppMetadataFixture = ( const contents = new Map([ ["stringLiterals", { bytes: literals, count: literalRecords }], ["stringLiteralData", { bytes: literalData, count: 2 }], - ["strings", { bytes: Buffer.concat(stringParts), count: stringIndex.size }], + ["strings", { bytes: stringTable.bytes(), count: stringTable.count() }], // Wide-index padding rows are zero-filled and owned by no method. ["parameters", paddedRows(parameterRows, parameterCount, 8 + T)], + ...(genericRows.parameterCount === 0 + ? [] + : [ + [ + "genericParameters", + { + bytes: genericRows.parameters, + count: genericRows.parameterCount, + }, + ] as [SectionName, SectionContent], + ]), ["fields", { bytes: fields, count: FIXTURE_FIELDS.length }], ["properties", { bytes: properties, count: named.properties.length }], ["events", { bytes: events, count: named.events.length }], ["methods", { bytes: methods, count: rows.methods.length }], - ...zeroFilledSection("genericContainers", genericContainers, 16), + ...(genericLayout.count === 0 + ? [] + : [ + [ + "genericContainers", + { bytes: genericRows.containers, count: genericLayout.count }, + ] as [SectionName, SectionContent], + ]), ["nestedTypes", { bytes: nested, count: nestedTypeEntries.length }], ["typeDefinitions", { bytes: types, count: typeRows.length }], ["images", { bytes: images, count: imageRows.length }], @@ -589,10 +712,16 @@ export const createIl2cppMetadataFixture = ( header: headerField, section, record: (name, index) => section(name).offset + index * recordBytes[name], - stringOffset: (value) => { - const index = stringIndex.get(value); - if (index === undefined) throw new Error(`Unknown string ${value}`); - return section("strings").offset + index; - }, + stringOffset: (value) => + section("strings").offset + stringTable.offset(value), }; }; + +/** Fixture variant whose generic instance type arguments have metadata owners. */ +export const createLinkedIl2cppMetadataFixture = ( + options: Il2cppMetadataFixtureOptions = {}, +): Il2cppMetadataFixture => + createIl2cppMetadataFixture({ + ...options, + genericTypeArgumentCounts: options.genericTypeArgumentCounts ?? { 3: 1 }, + });