From 570abe95f2a35b5c580a5ec2fca90c33d4114bdc Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:53:22 +0800 Subject: [PATCH] feat(android): link native declarations to static JNI candidates (#7472) Link original DEX native declarations to per-ABI function exports and relocated static JNI table candidates through shared CLI/MCP workflows. Preserve global attribution ambiguity, exact provenance, partial coverage and retained Evidence; default MCP delivery to bounded summaries with filtered/batch pages. Use one immutable ELF relocation decoder with streaming IL2CPP materialization, cooperative bounded export/version parsing and bounded JNI state. Reuse validated session inventories without loading target libraries or executing JNI code. Validation: relevant type/lint/build and parser/domain/public CLI/MCP regressions; genuine four-ABI NDK fixtures and independent library/slot oracles; actual JDK/JADX Conscrypt workflows with retained-source removal. Full measurements and limitations are recorded in docs/verification/android-jni-bindings-2026-10-27.md. Closes #6403. --- .../references/android-applications.md | 11 + bridge/android/ReaJadxBridge.java | 218 +++- docs/android-analysis.md | 105 ++ .../android-jni-bindings-2026-10-27.md | 320 +++++ scripts/verify/android/metadata.mjs | 8 + src/android/AndroidJniLibraries.ts | 309 +++++ src/android/AndroidNativeInventory.test.ts | 99 ++ src/android/AndroidNativeInventory.ts | 42 + src/android/JadxAnalysis.ts | 59 +- src/android/JadxProtocol.ts | 4 + src/android/JadxProvider.ts | 2 + src/android/JadxRelease.ts | 4 +- src/android/JadxSession.ts | 135 +- .../android/AndroidAnalysisService.ts | 5 +- .../android/AndroidJniInspection.ts | 42 + src/cli/androidCommands.ts | 79 ++ src/cliCommandNames.ts | 1 + src/contracts/android/androidToolContracts.ts | 30 + src/contracts/toolEffects.test.ts | 2 + src/contracts/toolEffects.ts | 5 + src/domain/analysisView/analysisView.ts | 30 + src/domain/analysisView/androidJniView.ts | 271 ++++ src/domain/android/androidAnalysis.ts | 20 + src/domain/android/androidJniBindings.test.ts | 478 +++++++ src/domain/android/androidJniBindings.ts | 526 ++++++++ src/domain/android/androidJniLibraryFacts.ts | 81 ++ src/domain/android/androidJniQuery.ts | 80 ++ .../android/androidNativeDeclarations.ts | 40 + .../android/androidPartialObservation.ts | 66 +- src/il2cpp/ElfRelocations.ts | 343 ++--- src/native/android/AndroidJniLibrary.ts | 524 ++++++++ src/native/exports/NativeExportParser.test.ts | 54 +- src/native/exports/NativeExportParser.ts | 124 +- src/native/metadata/BinaryContainer.test.ts | 19 + src/native/metadata/BinaryContainer.ts | 17 +- src/native/metadata/ElfPointerRelocations.ts | 599 +++++++++ src/server/registerAndroidTools.ts | 22 + .../android/androidJniLibraries.test.ts | 519 ++++++++ .../android/androidJniLibrary.test.ts | 431 +++++++ .../boundary/il2cpp/il2cppBinaryFile.test.ts | 14 + .../mcp/androidPackageSummaryMcp.test.ts | 204 +++ .../boundary/process/jadxIntegration.test.ts | 3 + .../android/ReaJadxMetadataRegression.java | 182 ++- tests/fixtures/android/jadx-mcp.mjs | 38 + tests/fixtures/android/jni-bindings/README.md | 142 +++ tests/fixtures/android/jni-bindings/build.py | 157 +++ .../android/jni-bindings/native/bindings.c | 78 ++ .../android/jni-bindings/native/duplicate.c | 8 + tests/fixtures/android/jni-bindings/oracle.py | 233 ++++ .../jni-bindings/source-expectations.json | 127 ++ .../src/fixture/jni/Bindings.java | 17 + .../jni-bindings/src/fixture/jni/Twin.java | 8 + .../jni-bindings/src/fixture/jni/ๅŽŸๅ๐€.java | 8 + tests/fixtures/android/jniElf.ts | 332 +++++ .../NativeDeclarations.smali | 19 + .../binaryDiagnostics/nativeExports.ts | 56 + tests/fixtures/il2cpp/binary.ts | 46 + .../analysis-efficiency/android-jni-costs.py | 88 ++ .../android-jni-library.mjs | 156 +++ .../analysis-efficiency/android-jni.mjs | 1104 +++++++++++++++++ 60 files changed, 8414 insertions(+), 330 deletions(-) create mode 100644 docs/verification/android-jni-bindings-2026-10-27.md create mode 100644 src/android/AndroidJniLibraries.ts create mode 100644 src/android/AndroidNativeInventory.test.ts create mode 100644 src/android/AndroidNativeInventory.ts create mode 100644 src/application/android/AndroidJniInspection.ts create mode 100644 src/domain/analysisView/androidJniView.ts create mode 100644 src/domain/android/androidJniBindings.test.ts create mode 100644 src/domain/android/androidJniBindings.ts create mode 100644 src/domain/android/androidJniLibraryFacts.ts create mode 100644 src/domain/android/androidJniQuery.ts create mode 100644 src/domain/android/androidNativeDeclarations.ts create mode 100644 src/native/android/AndroidJniLibrary.ts create mode 100644 src/native/metadata/ElfPointerRelocations.ts create mode 100644 tests/boundary/android/androidJniLibraries.test.ts create mode 100644 tests/boundary/android/androidJniLibrary.test.ts create mode 100644 tests/fixtures/android/jni-bindings/README.md create mode 100644 tests/fixtures/android/jni-bindings/build.py create mode 100644 tests/fixtures/android/jni-bindings/native/bindings.c create mode 100644 tests/fixtures/android/jni-bindings/native/duplicate.c create mode 100644 tests/fixtures/android/jni-bindings/oracle.py create mode 100644 tests/fixtures/android/jni-bindings/source-expectations.json create mode 100644 tests/fixtures/android/jni-bindings/src/fixture/jni/Bindings.java create mode 100644 tests/fixtures/android/jni-bindings/src/fixture/jni/Twin.java create mode 100644 tests/fixtures/android/jni-bindings/src/fixture/jni/ๅŽŸๅ๐€.java create mode 100644 tests/fixtures/android/jniElf.ts create mode 100644 tests/fixtures/android/static-initializers/NativeDeclarations.smali create mode 100644 tests/support/analysis-efficiency/android-jni-costs.py create mode 100644 tests/support/analysis-efficiency/android-jni-library.mjs create mode 100644 tests/support/analysis-efficiency/android-jni.mjs diff --git a/.agents/skills/reverse-engineer-anything/references/android-applications.md b/.agents/skills/reverse-engineer-anything/references/android-applications.md index 3f44c52229..cc51e22afe 100644 --- a/.agents/skills/reverse-engineer-anything/references/android-applications.md +++ b/.agents/skills/reverse-engineer-anything/references/android-applications.md @@ -13,6 +13,17 @@ own APK path and explicit selectors; do not open a native session first. Choose an `overload_index` from the class inventory when methods are ambiguous instead of guessing. +For JNI implementations, discover `inspect_android_jni_bindings`. Select raw +DEX class/method identities and optional descriptors, ABI or logical library +paths. Prefer `detail: "summary"`; complete Evidence remains retained and +`android-jni` views page declarations, libraries, candidates or binding rows +with exact fields. Global native-pair ambiguity survives class selection. +Preserve slot/relocation provenance and source coverage: matched means observed +candidate presence; partial scans do not establish uniqueness or absence. +This reads DEX/ELF bytes without loading the APK into the decompiler or executing +target code. Runtime registration and loading remain unknown. See the +[JNI guide](https://github.com/morluto/rea/blob/main/docs/android-analysis.md#link-native-declarations-to-jni-candidates). + For an R8/ProGuard release build with its `mapping.txt`, pass the absolute `mapping_path` to `inspect_android_class` or `inspect_android_method`. Prefer `detail: "summary"` to receive normalized symbols or the selected body without diff --git a/bridge/android/ReaJadxBridge.java b/bridge/android/ReaJadxBridge.java index 3564de4fdf..09d0d6004e 100644 --- a/bridge/android/ReaJadxBridge.java +++ b/bridge/android/ReaJadxBridge.java @@ -1,5 +1,9 @@ import com.atxx.jhmcp.*; import com.google.gson.*; +import com.android.tools.smali.dexlib2.Opcodes; +import com.android.tools.smali.dexlib2.dexbacked.DexBackedDexFile; +import com.android.tools.smali.dexlib2.dexbacked.DexBackedClassDef; +import com.android.tools.smali.dexlib2.dexbacked.DexBackedMethod; import io.modelcontextprotocol.kotlin.sdk.server.*; import io.modelcontextprotocol.kotlin.sdk.types.*; import jadx.api.*; @@ -14,10 +18,16 @@ import jadx.core.dex.info.MethodInfo; import jadx.core.dex.nodes.*; import java.io.PrintStream; import java.io.StringReader; +import java.io.InputStream; +import java.io.IOException; +import java.nio.file.Path; import java.nio.charset.StandardCharsets; import java.util.*; import java.util.concurrent.CountDownLatch; import java.util.stream.Collectors; +import java.util.zip.CRC32; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; import kotlin.Unit; import kotlin.ResultKt; import kotlin.coroutines.Continuation; @@ -179,6 +189,205 @@ class ReaJadxBridge { manifestSnapshot = null; } + // One bounded DEX at a time, beside the already admitted analysis JVM. The + // response allowance accounts for worst-case JSON escaping before retention. + private static final int NATIVE_DEX_BYTES = 64 * 1024 * 1024; + private static final long NATIVE_TOTAL_DEX_BYTES = 256L * 1024 * 1024; + // This JSON is escaped again inside an 8 MiB MCP frame. + private static final long NATIVE_JSON_BYTES = 4L * 1024 * 1024; + + /** The bundled decoder allocates its declared UTF-16 length before validating bytes. */ + private static final class NativeDex extends DexBackedDexFile { + private OptionalIndexedSection boundedStrings; + NativeDex(byte[] bytes) { super(Opcodes.getDefault(), bytes); } + + @Override public OptionalIndexedSection getStringSection() { + if (boundedStrings == null) { + var source = super.getStringSection(); + boundedStrings = new OptionalIndexedSection<>() { + @Override public int size() { return source.size(); } + @Override public int getOffset(int index) { return source.getOffset(index); } + @Override public String getOptional(int index) { return index == -1 ? null : get(index); } + @Override public String get(int index) { + int offset = getBuffer().readSmallUint(source.getOffset(index)); + int units = getDataBuffer().readerAt(offset).readSmallUleb128(); + if (units > NATIVE_JSON_BYTES / 6 + || units > getDataBuffer().getBuf().length - offset) + throw new IllegalArgumentException("DEX string exceeds the native decoding allowance"); + return source.get(index); + } + }; + } + return boundedStrings; + } + } + + private static void checkNativeInterrupted() { + if (Thread.currentThread().isInterrupted()) + throw new java.util.concurrent.CancellationException("Native declaration inventory interrupted"); + } + + private static final class NativeInventory { + final List> methods = new ArrayList<>(); + final List unknowns = new ArrayList<>(); + long retainedBytes = 1024; + int dexEntries; + int examinedDexEntries; + int examinedClasses; + int examinedMethods; + boolean partial; + boolean diagnosticLimit; + + void unknown(String reason) { + partial = true; + long bytes = 64L + reason.length() * 6L; + if (unknowns.size() < 16 && retainedBytes + bytes <= NATIVE_JSON_BYTES - 4096) { + unknowns.add(reason); + retainedBytes += bytes; + } else if (!diagnosticLimit) { + diagnosticLimit = true; + unknowns.add("Additional native-inventory limitations exceed the retained diagnostic allowance."); + } + } + + boolean retain(Map method, long bytes) { + if (retainedBytes + bytes > NATIVE_JSON_BYTES - 4096) { + unknown("Native declarations exceed the 4 MiB retained JSON allowance; remaining declarations are unknown."); + return false; + } + retainedBytes += bytes; + methods.add(method); + return true; + } + + Map result() { + return Map.of("scope", "apk-root-dex-declarations", "methods", List.copyOf(methods), + "coverage", Map.of("status", partial ? "partial" : "complete", + "dex_entries", dexEntries, "examined_dex_entries", examinedDexEntries, + "examined_classes", examinedClasses, "examined_methods", examinedMethods), + "unknowns", List.copyOf(unknowns)); + } + } + + /** Match Android's standard root multidex carriers, without a basename guess. */ + private static boolean nativeDexEntry(String name) { + return name.equals("classes.dex") || name.matches("classes[0-9]+\\.dex"); + } + + private static byte[] readNativeDex(ZipFile archive, ZipEntry entry) throws IOException { + long size = entry.getSize(); + if (size < 0 || size > NATIVE_DEX_BYTES) + throw new IllegalArgumentException("DEX exceeds the 64 MiB single-member read allowance"); + byte[] bytes = new byte[(int) size]; + CRC32 crc = new CRC32(); + try (InputStream input = archive.getInputStream(entry)) { + int offset = 0; + while (offset < bytes.length) { + checkNativeInterrupted(); + int read = input.read(bytes, offset, Math.min(65536, bytes.length - offset)); + if (read < 0) throw new IOException("DEX is shorter than its ZIP declaration"); + if (read == 0) continue; + crc.update(bytes, offset, read); + offset += read; + } + if (input.read() != -1) throw new IOException("DEX exceeds its ZIP declaration"); + } + if (entry.getCrc() < 0 || entry.getCrc() != crc.getValue()) + throw new IOException("DEX does not match its ZIP CRC32 declaration"); + return bytes; + } + + /** Raw DEX ownership precedes JADX's duplicate-class selection and aliases. */ + static Map nativeDeclarations(Path apk) { + NativeInventory inventory = new NativeInventory(); + try (ZipFile archive = new ZipFile(apk.toFile())) { + Map names = new HashMap<>(); + var directory = archive.entries(); + while (directory.hasMoreElements()) { + checkNativeInterrupted(); + ZipEntry entry = directory.nextElement(); + if (!entry.isDirectory() && nativeDexEntry(entry.getName())) { + inventory.dexEntries++; + names.merge(entry.getName(), 1, Integer::sum); + } + } + long totalBytes = 0; + int entryIndex = -1; + directory = archive.entries(); + while (directory.hasMoreElements()) { + checkNativeInterrupted(); + ZipEntry entry = directory.nextElement(); + entryIndex++; + String path = entry.getName(); + if (entry.isDirectory() || !nativeDexEntry(path)) continue; + if (names.get(path) != 1) { + inventory.unknown("Duplicate root DEX entry name cannot be selected uniquely by the ZIP reader: " + path); + continue; + } + if (entry.getSize() < 0 || entry.getSize() > NATIVE_DEX_BYTES + || entry.getSize() > NATIVE_TOTAL_DEX_BYTES - totalBytes) { + inventory.unknown("Root DEX member exceeds the 64 MiB member or 256 MiB total read allowance: " + path); + continue; + } + totalBytes += entry.getSize(); + try { + DexBackedDexFile dex = new NativeDex(readNativeDex(archive, entry)); + var classes = dex.getClassSection(); + for (int classIndex = 0; classIndex < classes.size(); classIndex++) { + checkNativeInterrupted(); + DexBackedClassDef cls = classes.get(classIndex); + String type = cls.getType(); + if (!type.startsWith("L") || !type.endsWith(";")) + throw new IllegalArgumentException("Class definition has no reference type"); + String className = type.substring(1, type.length() - 1).replace('/', '.'); + // Passing false retains duplicate encoded method references too. + for (Iterable group : List.of( + cls.getDirectMethods(false), cls.getVirtualMethods(false))) { + for (DexBackedMethod method : group) { + checkNativeInterrupted(); + inventory.examinedMethods++; + if ((method.getAccessFlags() & 0x100) == 0) continue; + String methodName = method.getName(); + String returnType = method.getReturnType(); + long characters = 2L + returnType.length(); + for (CharSequence parameter : method.getParameterTypes()) { + characters += parameter.length(); + if (characters * 6L > NATIVE_JSON_BYTES) break; + } + long bytes = 512L + 6L * (className.length() + methodName.length() + + path.length() + characters); + if (inventory.retainedBytes + bytes > NATIVE_JSON_BYTES - 4096) { + inventory.unknown("Native declarations exceed the 4 MiB retained JSON allowance; remaining declarations are unknown."); + return inventory.result(); + } + StringBuilder descriptor = new StringBuilder((int) characters).append('('); + for (CharSequence parameter : method.getParameterTypes()) descriptor.append(parameter); + descriptor.append(')').append(returnType); + Map row = Map.of("source_index", inventory.methods.size(), + "class_name", className, "method_name", methodName, "descriptor", descriptor.toString(), + "dex_entry", path, "dex_entry_index", entryIndex, "class_index", classIndex, + "method_index", method.getMethodIndex()); + if (!inventory.retain(row, bytes)) return inventory.result(); + } + } + inventory.examinedClasses++; + } + inventory.examinedDexEntries++; + } catch (IOException | RuntimeException failure) { + if (failure instanceof java.util.concurrent.CancellationException) throw (RuntimeException) failure; + inventory.unknown("Root DEX native declarations could not be fully read from " + path + ": " + + failure.getClass().getSimpleName() + + (failure.getMessage() == null ? "" : " (" + failure.getMessage() + ")")); + } + } + } catch (IOException failure) { + // A failed archive close must fail the operation so the provider retires + // its owned JVM; do not cache a result with lost file-resource ownership. + throw new IllegalArgumentException("APK ZIP native declaration inventory could not finish its reader", failure); + } + return inventory.result(); + } + /** Original archive identity, never a deobfuscated basename or name suffix. */ private ManifestSnapshot manifest(JadxSession current) { if (manifestSession != current || manifestSnapshot == null) { @@ -475,6 +684,8 @@ class ReaJadxBridge { "available_processors", Runtime.getRuntime().availableProcessors(), "java_version", System.getProperty("java.version"), "metadata_scope", "parsed_members")); + if (name.equals("rea_native_declarations")) + return json(nativeDeclarations(Path.of(string(args, "path")))); JadxSession current = session(); if (name.equals("get_app_info")) return json(manifest(current).summary()); if (name.equals("get_android_manifest")) @@ -586,7 +797,7 @@ class ReaJadxBridge { SessionHolder holder = new SessionHolder(defaults); ServerCapabilities capabilities = new ServerCapabilities( new ServerCapabilities.Tools(null), null, null, null, null, null, null, Map.of()); - Server server = new Server(new Implementation("rea-jadx-bridge", "8", null, null, null), + Server server = new Server(new Implementation("rea-jadx-bridge", "9", null, null, null), new ServerOptions(capabilities), "", ignored -> Unit.INSTANCE); SessionToolsKt.registerSessionTools(server, holder); ClassToolsKt.registerClassTools(server, holder); @@ -606,6 +817,11 @@ class ReaJadxBridge { new ToolSchema(null, new kotlinx.serialization.json.JsonObject(Map.of()), List.of(), null), "Observed worker JVM and metadata scope", null, null, null, null, null, null); server.addTool(diagnostics, (connection, request, continuation) -> bridge.handle("rea_jvm_status", request.getArguments())); + registerBridgeTool(server, bridge, "rea_native_declarations", + new ToolSchema(null, new kotlinx.serialization.json.JsonObject(Map.of("path", + new kotlinx.serialization.json.JsonObject(Map.of("type", + kotlinx.serialization.json.JsonElementKt.JsonPrimitive("string"))))), List.of("path"), null), + "Original ACC_NATIVE declarations from the admitted APK's standard root DEX members, without loading or decompiling the APK."); for (String name : List.of("get_app_info", "get_android_manifest")) { String description = name.equals("get_app_info") ? "Namespace-aware summary of the exact root AndroidManifest.xml resource." diff --git a/docs/android-analysis.md b/docs/android-analysis.md index 047e9b7173..50b2c57bba 100644 --- a/docs/android-analysis.md +++ b/docs/android-analysis.md @@ -350,6 +350,98 @@ JADX is used. Requests read their mappings one at a time, in the same queue as the JADX work and under the same operation deadline. A mapping failure therefore never starts or retires the JADX session, and a different mapping reuses the loaded APK. +## Link native declarations to JNI candidates + +When advertised, `inspect_android_jni_bindings` links original `ACC_NATIVE` +declarations from standard root `classes*.dex` members to bundled ELF libraries. +The supplied JAR's DEX reader preserves duplicate declarations before decompiler +deduplication. This operation does not load the APK into the decompiler. +MCP defaults to retained summary delivery when `detail` is omitted; select +`complete` explicitly to receive the full report inline. + +```json +{ + "path": "/absolute/Example.apk", + "class_name": "org.conscrypt.NativeCrypto", + "method_names": ["EVP_PKEY_new_RSA"], + "abis": ["arm64-v8a"], + "detail": "summary" +} +``` + +Class/method names and optional `descriptors` select raw DEX identities. +`library_paths` selects exact logical APK paths such as +`lib/arm64-v8a/libconscrypt_jni.so`; each library's `source_path` preserves its +original ZIP spelling. Omitted filters include all scoped facts; `[]` matches +none. Class selection never narrows the global inventory used for table-class +attribution. + +Independent `declarations`, `libraries`, `candidates` and `bindings` collections +have stable IDs and source ordinals. Candidates retain APK/library digests, +ABI, function addresses and file offsets. Table candidates also retain section, +table and entry identity plus all three name/signature/function pointer slots +and their relocation sources. JNI exports follow the +[UTF-16 escaping and short-before-long rules](https://docs.oracle.com/en/java/javase/21/docs/specs/jni/design.html#resolving-native-method-names); +failed escaping does not acquire an invented export match. Tables require +NUL-terminated [Modified UTF-8 strings](https://docs.oracle.com/en/java/javase/21/docs/specs/jni/types.html#modified-utf-8-strings), +valid descriptors and executable load mappings. REL, RELA and RELR retain +separate provenance; unsupported packed relocations and unresolved pointer +spans stay explicit unknowns. + +A static `JNINativeMethod` entry names no class. Attribute it uniquely only +when exactly one declaration in the complete recorded native inventory has +that `(name, descriptor)` pair. Multiple classes remain ambiguous after a +class filter; a partial inventory cannot establish unique attribution. + +Each selected declaration has a row per selected/observed ABI. `matched` means +candidate presence was observed; `ambiguous` means multiple observed targets +or class associations remain. Binding `coverage` is separate: a matched +partial row does not establish complete uniqueness, and an empty partial scan +is `unknown`, never `unmatched`. ELF/ABI label mismatches preserve facts without +asserting a binding. No result establishes library loading, runtime registration +or method invocation. + +Summary returns the first 64 selected binding rows, their declaration/candidate +closure and every selected library's identity, coverage and failure reason; +counts and page coverage describe the full selected binding set. Selected +declaration IDs have separate preview coverage, so native declarations remain +visible even when the APK has no bundled ABI libraries. Complete +Evidence stays retained. Read exact fields or further pages without reopening +files or rerunning analysis: + +```json +{ + "source": { "kind": "retained-evidence", "evidence_id": "ev_" }, + "view": { + "kind": "android-jni", + "collection": "bindings", + "offset": 64, + "limit": 64, + "fields": ["declaration_id", "abi", "status", "coverage", "candidate_ids"] + } +} +``` + +Collections also include `declarations`, `candidates` and `libraries`, with +exact filters and batch lookup by declaration/candidate/library IDs. `row_index` +is the original parent collection index; DEX and +ZIP source ordinals stay separate. Complete detail returns the full report. +CLI uses the same selection and delivery contracts. Its common one-shot +default remains complete; summary delivery requires an explicit Evidence path: + +```sh +rea inspect-android-jni-bindings /absolute/Example.apk \ + --query '{"class_name":"org.conscrypt.NativeCrypto","method_names":["EVP_PKEY_new_RSA"],"abis":["arm64-v8a"]}' \ + --detail summary --evidence_output /absolute/fresh-jni-evidence.json +``` + +Optional `--view` accepts the same JSON page and requires summary delivery. +Evidence bundles never overwrite files. Complete declarations and one library +selection are cached within the immutable APK session. Changing ABI/library +selection replaces the scan; class/method selection reuses it. Fresh analysis +still validates current caller APK bytes. Idle/close releases the caches; +retained Evidence remains historical and independently readable. + ## Interpret the result - Package inspection retains decoded manifest XML, summary declarations, @@ -480,6 +572,19 @@ target/configuration changes, failure, timeout, cancellation, MCP disconnect or server shutdown. One-shot CLI commands always join cleanup before returning. REA never writes to the original APK or launches target code. +JNI scans reuse the analyzer process without loading the APK into the +decompiler. They admit at most 64 MiB per DEX member and 256 MiB of DEX bytes, +with a 4 MiB native-declaration JSON allowance before MCP string escaping. +Native libraries are processed sequentially: one 64 MiB buffer, 256 MiB total +selected read work, and a 16 MiB parsed-library-facts allowance. ZIP inventory +and library counts also have explicit resource bounds. Unexamined occurrences +remain unknown. A complete relationship report has a separate 16 MiB serialized +allowance; summaries/pages allow 1 MiB including their provenance envelope. +These are resource admissions, not claims about maximum valid Android formats. +Select fewer ABI/library occurrences or smaller retained pages when needed. +Completed parsed library facts survive failed reader finalization in typed +partial observations; the exact reader owner remains available for cleanup retry. + On Windows, stdin writes preserve stream backpressure and run outside the Node event loop. Cancellation terminates the retained job, including descendants, before workspace cleanup. On Windows, a detached REA helper owns the private diff --git a/docs/verification/android-jni-bindings-2026-10-27.md b/docs/verification/android-jni-bindings-2026-10-27.md new file mode 100644 index 0000000000..d0adeb0d76 --- /dev/null +++ b/docs/verification/android-jni-bindings-2026-10-27.md @@ -0,0 +1,320 @@ +# Static Android JNI verification, 2026-10-27 + +Issue #6403 asks which bundled native functions could implement original DEX +native declarations. The implementation reads immutable APK bytes, preserving +declarations, library occurrences, exported/table candidates and selected +per-ABI bindings as independently queryable facts. No target, library, +`JNI_OnLoad` or `RegisterNatives` code was executed. + +## Revisions and independent oracles + +The full qualification and alternating delivery measurements below use +implementation `4a1839de6f091ade19e82557874c358ed34b8e36`, based on main +`03f17cf41d3c844f9383d7792d9b1a14330d0bb2`. Subsequent merge +`e8a8a50725cd384ae38ce66a94bafdaba1138108` incorporates main +`d6f88becd531cabafca1d968b89cf7fc77a80041`; its browser input-presentation and +SQLite test changes are separate from JNI analysis. Post-merge confirmation is +recorded below. The native parser's built SHA-256 is +`3bf0a2c342f1c404f5861b7c913f7f1b931f740a8c808b518474fc8de20821d5`. + +Linux x86-64, Node 24.22.1/npm 11.16.0 and JDK 21.0.12.1 qualified the workflows. +JADX MCP 0.7.1 remains the supplied unmodified JAR, SHA-256 +`6e5eacf500b64292bfb73c49797c1958f6ee44646e43e868039ae7feb573ff75`. +The REA metadata bridge is version 9. JNI declarations use the JAR's existing +dexlib2 reader before decompiler deduplication, with no `load_apk` call. + +The genuine Android NDK r27d package is revision 27.3.13750724, clang/LLD 18.0.4. +Its official Linux ZIP is 663,956,036 bytes, publisher SHA-1 +`22105e410cf29afcf163760cc95522b9fb981121`; observed SHA-256 is +`601246087a682d1944e1e16dd85bc6e49560fe8b6d61255be2829178c8ed15d9`. +Pins come from the [official repository manifest](https://dl.google.com/android/repository/repository2-3.xml). +The package was streamed, hashed and selectively extracted into a private task +cache, without installing or changing global tools. The selected compiler, +sysroot and four additional official libunwind archives total 319,933,537 bytes. +An initial link failure exposed those omitted archives; the corrected fixture +build passed in 4.606 seconds, CPU 6.026 seconds, peak family RSS 110.4 MiB. + +[Committed source fixtures and reproduction](https://github.com/morluto/rea/blob/main/tests/fixtures/android/jni-bindings/README.md) +build native declarations, javac headers and two libraries for arm64-v8a, +armeabi-v7a, x86 and x86_64. Ordinary REL/RELA, RELR and Android packed APS2 +variants include overloads, BMP/supplementary names, explicit Modified UTF-8, +`.data` and `.data.rel.ro` tables, repeated exports in two libraries, an +unmatched declaration and two classes sharing one registered pair. Source, +`javac -h`, independent DEX flag/index reads, readelf and a separate byte/relocation +oracle establish expected facts. Oracle SHA-256 is +`fa634493bc8559a80eda3b70f01d44550c2f06e176cfc2a7cc7e10794b1eb793`. + +| Static APK | SHA-256 | +| -------------------- | ------------------------------------------------------------------ | +| Ordinary | `c4eb50ef8bdecc6f56c454294c0c8f198fa51a4174d101dc720a849976546649` | +| RELR | `5a2ce56191f633575e87bd239702f988d0d1ad82795a203e13a4f907f8624658` | +| APS2 | `d2f0d613bc8f22ed1a6435f13b5ae1f088e904216aa8a04dd70c27c5cb606778` | +| Conscrypt derivative | `5985c40d5a6b14b64c202c80335d851adb7e8a987e6154a1488a194253856385` | + +The Conscrypt derivative packages the public Android 2.8.0 AAR's original +library bytes with its class files transformed by D8 8.0.40. It is a static +test container, not a vendor APK or a claimed launchable application. The AAR +SHA-256 is `752044c289bc98e2dcabc00a1f88ca158b3aa3f505b38f124552bf4b2a0902b1`. +The original-name DEX is used, not an R8-renamed DEX paired with unchanged JNI +strings. D8's missing platform-adapter dependency warning remains a qualification +limit; no full application linkage or signature verification is claimed. + +## Executed correctness and Evidence checks + +The built pure parser passed independent checks over 25 real libraries: +24 genuine NDK outputs plus Conscrypt arm64. All 33 golden table entries match +their strings, slot addresses/file offsets, relocated values, encodings, +function addresses and code offsets. Conscrypt yields 367 structural entries; +the requested `EVP_PKEY_new_RSA ([B[B[B[B[B[B[B[B)J` candidate is `0xfa11c`. +Its name/signature/function slots are `0x2ada68`, `0x2ada70`, `0x2ada78`, +with relocated values `0x749a8`, `0x7affa`, `0xfa11c`. This built-artifact +qualification took 0.684 seconds, CPU 0.736 seconds, peak family RSS 99.5 MiB. + +Real SDK/stdio MCP and CLI checks independently validate: + +- All 12 source native declarations across four ABIs, exact library digests, + overloads, UTF-16 names and per-library alternatives. Ordinary/RELR return + 32 matched, 12 ambiguous and four unknown rows. The source-known unmatched + method remains unknown because symbolic writes/NOBITS leave scanning partial. +- APS2 preserves 20 observed export matches, four ambiguous export rows and + 24 unknown rows. It explicitly reports unsupported table relocation encoding, + retaining each selected library's reason even without a positive candidate. +- Class selection never hides the other class sharing a registered pair. + All selected native declarations survive independently of available ABI rows; + the no-library selection case is covered by a domain regression, not claimed + as a separate real SDK artifact qualification. +- All four retained collections support exact fields, empty filters and batch + ID retrieval. Parent-array `row_index` remains separate from raw DEX/ZIP + source ordinals. Advertised converted schemas validate with the real SDK/Ajv. +- Complete portable parents preserve the same normalized facts through summary + and CLI delivery. Ordinary and each Conscrypt cohort copy the source to an + exclusively created owned file, inspect it, remove only that copy, then read + retained summary, exact table slots, binding fields and a full export. + Original fixture bytes remain unchanged. Post-removal success is observed; + hidden internal engine activity is not directly instrumented by this harness. + +The first three four-ABI cohorts execute 107 SDK calls and three cold CLI parity +checks. Each Conscrypt cohort adds 28 investigation calls, five separately +accounted retained-removal calls and one CLI parity check. Four alternating +Conscrypt cohorts preserve the same normalized fact digest: +`7e4738d2dd9e518277d5268ce5c334d9d25728198f66b2102cdc3411865f2d46`. + +Raw parent transcripts show one native inventory RPC on the first investigation, +then only JVM-status RPCs on later main queries; normalized native facts remain +present. No JNI parent contains `load_apk`. Library reuse is owned by the same +immutable APK session; ABI/library selection replaces one scan, while class +and method selection reuses it. Fresh analysis admission still hashes current +caller APK bytes. Historical views never reopen them. + +Type/lint/build and 396 tests across ten exact files pass together: native/domain +projection, producer parsing, real ZIP occurrence/cleanup boundaries, unchanged +JADX session/idle/cancellation behavior, public contracts and existing IL2CPP +decoding/ownership. The gate took 215.198 seconds, CPU 257.470 seconds, peak +family RSS 4396.1 MiB. A full real JDK/JADX metadata run additionally passed +native reads before any APK load, malformed/duplicate DEX and CRC negatives, +raw aliases, constants, absence facts, manifest identity and downstream CLI/MCP +bodies/direct-invoke parity from both DEX entries: 154.890 seconds, CPU 233.275 +seconds, peak family RSS 984.8 MiB. + +Finalization regressions retain already parsed library facts/digests in typed +partial observations before ZIP close. Caller/deadline-first cancellation, +failed close, blocked admission and retry preserve their first cause and exact +resource lease. Interrupted later libraries preserve earlier completed facts. +Stable table candidate identity uses physical pointer-slot address/file offset, +not inferred run grouping. Known ZIP size claims stay separate from verified +read size/hash. + +## Completed-task costs + +Earlier main cannot decode/register JNI tables with equivalent coverage, so +manual export lookup is not an equivalent before workflow. The controlled +comparison is complete versus summary delivery of the same new investigation, +same bytes, query, selected pages and proof export. Conscrypt trials alternate +summary/complete/summary/complete, each with one cold and one warm task. + +Each fixture task has four investigations, 12 selected collection pages and a +proof export (17 calls). Each Conscrypt task has one investigation, the same +12 page checks and proof export (14 calls). Task time includes public calls and +local oracle checks; one-time startup/catalog, separate retained-removal +qualification, final cleanup and a cold CLI parity run are included in the +whole-process columns, not hidden in task timings. Ordinary includes removal +qualification; RELR/APS2 do not, so their whole-process costs are not a paired +performance comparison. All four Conscrypt cohorts use the same phase flags. + +| Cohort | Cold task s | Warm task s | Whole s | CPU s | Peak family MiB | Warm text tokens | Warm SDK tokens | +| -------------------- | ----------: | ----------: | ------: | -----: | --------------: | ---------------: | --------------: | +| Ordinary | 11.058 | 0.606 | 32.658 | 45.124 | 1215.7 | 58,319 | 401,681 | +| RELR | 12.140 | 0.797 | 29.122 | 38.375 | 1266.2 | 58,231 | 401,502 | +| APS2 | 12.578 | 0.917 | 30.892 | 41.071 | 1268.0 | 50,900 | 386,634 | +| Conscrypt summary | 11.237 | 0.444 | 33.999 | 46.953 | 1282.0 | 32,265 | 349,128 | +| Conscrypt complete | 10.846 | 0.456 | 32.907 | 45.948 | 1398.3 | 224,954 | 737,723 | +| Conscrypt summary 2 | 19.205 | 1.200 | 57.545 | 77.644 | 1456.2 | 32,223 | 349,055 | +| Conscrypt complete 2 | 20.978 | 0.925 | 64.513 | 81.218 | 1502.0 | 224,526 | 736,950 | + +Warm text estimates are about 85.7% lower for summary delivery. Cold text +estimates are 32,087/32,202 versus 247,593/247,912. SDK-inclusive warm estimates +are about 52.6% lower. Latency/CPU/peak-memory samples vary substantially and do +not establish a general improvement. Cold costs include the first generic +view-schema compilation in the client; subsequent tasks reuse it. Startup/ +catalog alone ranges 2.692โ€“4.955 seconds across these Conscrypt cohorts. + +Token estimates use tiktoken 0.14.0/o200k_base. Every task counts its analyst +question, discovered input catalog once, arguments and all text replies, +including pages and export receipts. SDK totals additionally count output +schemas and structured-content duplication. Qualifications and CLI response +costs are reported separately by the cost script. Exported file contents are +verified locally and are not counted as model context unless actually returned +inline. Actual billing, hidden prompts, client rendering/pruning and OS cache +state are unmeasured. + +### Latest-main confirmation + +After merging main `d6f88becd` into `e8a8a5072`, type/lint/build and the eight +input-presentation/public-contract cases pass. The first direct schema-suite +invocation could not load its generated test catalog; `test:focused` supplied +the required artifacts and all 16 schema tests then passed (63.310 seconds, +CPU 72.830 seconds, peak family RSS 1943.5 MiB). No schema assertion was relaxed. + +Fresh full Conscrypt SDK/CLI cohorts use the same proof, 12 pages and owned-copy +removal flags. Both retain the identical fact digest above: + +| Delivery | Cold task s | Warm task s | Whole s | CPU s | Peak family MiB | Warm text tokens | Warm SDK tokens | +| -------- | ----------: | ----------: | ------: | -----: | --------------: | ---------------: | --------------: | +| Summary | 11.739 | 0.680 | 36.712 | 51.209 | 1395.1 | 31,932 | 348,522 | +| Complete | 11.161 | 0.459 | 34.692 | 48.319 | 1385.2 | 224,954 | 737,723 | + +Summary text estimates remain about 85.8% lower. This pair does not show a +latency or peak-memory improvement. Each adds 33 SDK calls and one cold CLI +parity check, bringing the executed public qualification to 305 SDK calls and +nine CLI checks across the pinned cohorts. + +`docs:check`, `format:check` and `docs:build` pass together: 44.365 seconds, +CPU 56.935 seconds, peak sampled family RSS 1300.6 MiB. Generated catalogs, +schemas, packaged skills and provider binaries remain uncommitted outputs. + +All expensive checks use CPU affinity 0,1, bounded heaps and one active process +family, with an 8 GiB RSS guard and 4 GiB host-availability guard. Peak memory is +the sum of family RSS sampled every 100 ms: shared pages can be double-counted +and short peaks can be missed. No guard stop or OOM occurred. + +### Default-summary delivery confirmation + +The new MCP operation now defaults to retained `summary` when `detail` is omitted. +Explicit `complete` remains supported; the CLI retains its common explicit-detail +policy. The actual SDK regression verifies both modes, caller selectors and +reading an unselected declaration from the complete parent after source removal. +Type/lint/build and all five cases in that MCP boundary file pass: 80.012 seconds, +CPU 110.240 seconds, peak sampled family RSS 4747.6 MiB. An earlier attempt stopped +at the test-function length lint rule; the fixture helper was extracted before +this successful gate. + +A fresh genuine Conscrypt workflow omits `detail` on JNI investigation calls and +passes 33 SDK calls, all 12 retained pages, proof export, one cold CLI parity +check and the separate owned-copy removal qualification. The same investigation +fact digest above is retained. Cold/warm task times are 15.608/0.481 seconds; +whole process-family time is 45.239 seconds, CPU 61.038 seconds, peak sampled RSS +1431.2 MiB. Warm text/SDK token estimates are 31,928/348,516 using the same cost +method. This adds a tenth CLI check and brings the successful public qualification +total to 338 SDK calls. Explicit-complete removal qualification remains separate +from the investigation task; default summaries do not discard the full parent. + +Final `docs:check`, `format:check` and `docs:build` also pass: 77.269 seconds, +CPU 70.506 seconds, peak sampled family RSS 1191.1 MiB. + +The updated harness adds `--omit-detail` (with `--detail summary`), SHA-256 +`c2a5d090f32429c852e3c974083bb9c21a0dc6a608407687031f155a722b53d7`. +The earlier cohort tables use the original frozen harness below. Default-run +summary/catalog/call receipt SHA-256 values are respectively +`c10e4d702b9694b0ccc69926a4e1f688fa8dea33a3de2e2837ea15190a706b37`, +`0c36244f5ff788fabd5491144b0711e99ad211ce3c09e1812d1d4da5753624ac`, and +`e1f021f0a7a92f101a037a1467a3428b4c620482eb556d81501a5ea64226ab82`. + +### Review corrections and final qualification + +Four review findings were reproduced or confirmed at their producing boundaries: + +- The shared relocation reader had charged every file-backed write against a + sparse retention limit. Its single immutable decoder now streams writes; + IL2CPP applies them to its existing private copy and retains only bounded + unresolved/zero-fill state and a file-offset bitmap. JNI keeps its bounded + sparse provenance index. The two actual IL2CPP registration-read regressions + each include 65,537 extra file-backed relative writes (AArch64 and x86-64) + and verify the original bytes remain unchanged. +- GNU version headers could replay shared auxiliary chains and names. The shared + export owner now bounds total record visits from section bytes and the existing + export-row allowance, caches decoded/failed string offsets within 16 MiB, and + yields checkpoints through the same generator used by synchronous callers. + Refusal retains exports with partial coverage and unknown version names. + Both GNU table variants exercise 10,000 headers replaying 65,535 auxiliary + records and a 65,535-byte name, verifying one decode and cooperative yields. +- A named mapped-string read-limit failure now distinguishes budget exhaustion + from invalid text. JNI preserves later candidates, marks tables partial and + projects absent candidates as unknown. Both long names and descriptors are + covered. An additional actual JNI-consumer regression queues cancellation + after GNU name decoding begins and stops before export-symbol decoding. +- The static verifier now requires the Conscrypt oracle only when expectations + advertise it. Executed controls pass with 25 libraries/33 entries when present, + and 24 libraries/32 entries for NDK-only expectations; an advertised missing + oracle rejects with ENOENT and writes no success receipt. + +Type/lint/build and 330 cases in 13 related domain, parser, archive, IL2CPP and +actual CLI/stdio MCP files pass together: 192.437 seconds, CPU 224.376 seconds, +peak sampled family RSS 4190.2 MiB. After adding the during-version cancellation +case, type/lint and all 38 JNI parser cases pass: 39.842 seconds, CPU 67.581 +seconds, peak RSS 3928.0 MiB. These overlap earlier gates; counts are not summed +as unique tests. + +The final built parser again passes the independent 25-library/33-entry oracle: +0.975 seconds, CPU 1.009 seconds, peak RSS 103.9 MiB. Built parser SHA-256 is +`7c0638751427b2b282b8e55bb21c95f6c3f05c246b0ea66a31d0280694f9ffeb`; +its receipt also fingerprints every shared parsing dependency. The fixed oracle +SHA-256 above is unchanged. Large-capacity IL2CPP verification uses valid producer +fixtures, not a newly qualified real IL2CPP application. + +A fresh Conscrypt investigation qualification again passes 33 SDK calls, +12 pages, proof, a cold CLI check and owned-copy removal. Investigation facts +retain the same digest above. Cold/warm task times are 23.529/1.015 seconds; +whole time 66.284 seconds, CPU 77.940 seconds, peak RSS 1402.6 MiB. Warm text/SDK +estimates are 31,955/348,569. Host/time variation does not establish a general +latency improvement. Cumulative successful public qualification is now 371 SDK +calls and eleven CLI parity checks, across the explicitly separated revisions. +The final cohort summary/calls SHA-256 values are +`84df4f4ed97f6d1a931010db8e16420e5f06df371940ed9225eb99a7a7c01094` and +`2c22b89e5ec4b26f15f0e9c5f8fdfd4af8583ad0926ad32edb359ecd991cd7fd`; +its catalog hash matches the default-summary cohort. No resource guard or OOM +occurred. Final `docs:check`, `format:check` and `docs:build` pass in +38.602 seconds, CPU 49.448 seconds, peak sampled family RSS 1286.2 MiB. + +## Reproduction and limitations + +Build the runtime, generate the source fixtures using selected existing JDK, +NDK and D8 paths, then run the maintained +[pure parser verifier](https://github.com/morluto/rea/blob/main/tests/support/analysis-efficiency/android-jni-library.mjs) +against the built parser. The +[public workflow harness](https://github.com/morluto/rea/blob/main/tests/support/analysis-efficiency/android-jni.mjs) +accepts `--variant ordinary|relr|aps2|conscrypt`, `--detail complete|summary`, +`--proof --pages --cli` and optional `--retained-removal`, explicit selected +repository/artifact/JDK/JAR/oracle paths and a fresh output directory. Frozen +harness SHA-256: +`14836895659de0f161c479b2b013ec39d754d172fd9935898872ab9cb5937035`. +The [cost script](https://github.com/morluto/rea/blob/main/tests/support/analysis-efficiency/android-jni-costs.py) +consumes every case's catalog, calls and summary receipts, fingerprints them +and keeps qualification costs separate. + +Initial source integration type/lint errors were corrected before qualification. +A first CLI verifier attempt incorrectly treated its summary wrapper as flat +Evidence; the CLI had written its valid bundle. The corrected harness saves raw +stdout/stderr before assertions and validates the unwrapped named contract. +One driver used the wrong Conscrypt derivative filename and failed before +analysis; corrected cohorts use the pinned file above. A direct schema-suite +invocation lacked generated test catalog metadata; artifact-aware verification +is recorded separately. These attempts are not counted as successful runs. + +Only Linux/JDK21/JADX0.7.1 and the fixed genuine NDK variants/public Conscrypt +arm64 sample are qualified. Cross-ABI ELF fixtures do not establish real +Windows/macOS provider lifecycle, device/emulator operation or runtime JNI +registration. Existing IL2CPP regressions cover the shared relocation refactor; +no new real IL2CPP application qualification is claimed. Runtime-built tables, +obfuscated strings, unsupported encodings and missing split/runtime-loaded code +remain unknown. Byte digests identify artifacts, not authenticity or build +ownership. diff --git a/scripts/verify/android/metadata.mjs b/scripts/verify/android/metadata.mjs index 1f9be7b9a1..aa4fb1dbdb 100644 --- a/scripts/verify/android/metadata.mjs +++ b/scripts/verify/android/metadata.mjs @@ -93,6 +93,14 @@ export const verifyAndroidMetadata = async ({ result.stdout, /NO_BODY_REGRESSION_PASS native=true abstract=true frozen=true/u, ); + assert.match( + result.stdout, + /NATIVE_DECLARATION_REGRESSION_PASS raw=true native_only=true unicode=true overloads=true duplicate_dex=true partial=true no_source=true/u, + ); + assert.match( + result.stdout, + /NATIVE_DECLARATION_COLD_PASS no_apk_load=true raw=true/u, + ); assert.match( result.stdout, /MERGED_METHOD_REGRESSION_PASS exact=true overloaded=true handlers=true/u, diff --git a/src/android/AndroidJniLibraries.ts b/src/android/AndroidJniLibraries.ts new file mode 100644 index 0000000000..9ab519130f --- /dev/null +++ b/src/android/AndroidJniLibraries.ts @@ -0,0 +1,309 @@ +import { createHash } from "node:crypto"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { ZipArtifactReader } from "../artifacts/ZipArtifactReader.js"; +import { streamChunkToBuffer } from "../artifacts/StreamBytes.js"; +import type { + AndroidJniLibraryObservation, + AndroidJniReport, + AndroidJniSelection, + AndroidJniLibraryCheckpoint, +} from "../domain/android/androidJniBindings.js"; +import { compileAndroidJniLibrarySelection } from "../domain/android/androidJniBindings.js"; +import { jsonValueByteLength } from "../domain/jsonValueByteLength.js"; +import type { JsonValue } from "../domain/jsonValue.js"; +import { inspectAndroidJniLibrary } from "../native/android/AndroidJniLibrary.js"; +import { + JadxInterruption, + throwIfJadxInterrupted, +} from "./JadxInterruption.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; +import { analysisClientFailureForOperation } from "../providers/AnalysisClientFailure.js"; + +/** Bounds cover the live library buffer, completed scan work and retained JSON separately. */ +export const ANDROID_JNI_LIBRARY_LIMITS = Object.freeze({ + libraryBytes: 64 * 1024 * 1024, + totalBytes: 256 * 1024 * 1024, + factsBytes: 16 * 1024 * 1024, + entries: 100_000, + libraries: 2048, +}); +export interface AndroidJniLibraries { + readonly libraries: readonly AndroidJniLibraryObservation[]; + readonly inventory: AndroidJniReport["library_inventory"]; +} + +/** Read each exact ZIP occurrence once from the provider's immutable admitted APK. */ +export const readAndroidJniLibraries = async (input: { + readonly snapshot: string; + readonly apkSha256: string; + readonly selection: AndroidJniSelection; + readonly resources: ArtifactResourceScope; + readonly signal: AbortSignal; + readonly interruption: (reason: unknown) => AnalysisError; + readonly onObservation: (checkpoint: AndroidJniLibraryCheckpoint) => void; +}): Promise => + input.resources + .run(async () => { + const reader = new ZipArtifactReader( + input.snapshot, + "apk", + ANDROID_JNI_LIBRARY_LIMITS.factsBytes, + ); + const owner = { + kind: "reader" as const, + resource: input.snapshot, + reader, + }; + const libraries: AndroidJniLibraryObservation[] = []; + const inventory: AndroidJniReport["library_inventory"] = { + status: "complete", + examined_entries: 0, + selected_libraries: 0, + unknowns: [], + }; + let totalBytes = 0; + let factsBytes = 0; + let primary: unknown; + let failed = false; + let inProgress: AndroidJniLibraryObservation | undefined; + const selectLibrary = compileAndroidJniLibrarySelection(input.selection); + try { + for await (const entry of reader.entries(input.signal)) { + throwIfJadxInterrupted(input.signal); + const source_index = inventory.examined_entries++; + if (inventory.examined_entries > ANDROID_JNI_LIBRARY_LIMITS.entries) { + inventory.status = "partial"; + inventory.unknowns.push( + "APK entry inventory exceeded the bounded entry allowance; remaining libraries are unknown.", + ); + break; + } + const match = /^lib\/([^/]+)\/([^/]+\.so)$/u.exec(entry.path); + if (match === null || entry.kind === "directory") continue; + const abi = match[1]!; + if (!selectLibrary({ abi, path: entry.path })) continue; + if (libraries.length >= ANDROID_JNI_LIBRARY_LIMITS.libraries) { + inventory.status = "partial"; + inventory.unknowns.push( + "Selected native-library occurrence inventory exceeded its retained-state allowance.", + ); + break; + } + const identity = { + library_id: `jni_library_${createHash("sha256") + .update(JSON.stringify([input.apkSha256, entry.adapterKey])) + .digest("hex")}`, + source_index, + source_key: entry.adapterKey, + path: entry.path, + source_path: entry.sourcePath ?? entry.path, + abi, + sha256: null, + size: null, + declared_size: entry.declaredSize, + compressed_size: entry.compressedSize, + }; + inProgress = { + ...identity, + facts: null, + failure: { + kind: "incomplete", + message: "Native-library inspection did not complete", + }, + }; + let observation: AndroidJniLibraryObservation; + const limit = + entry.declaredSize === null || + !Number.isSafeInteger(entry.declaredSize) || + entry.declaredSize < 0 || + entry.declaredSize > ANDROID_JNI_LIBRARY_LIMITS.libraryBytes || + entry.declaredSize > + ANDROID_JNI_LIBRARY_LIMITS.totalBytes - totalBytes; + if (entry.kind !== "file" || entry.encrypted || limit) { + observation = { + ...identity, + facts: null, + failure: { + kind: entry.encrypted + ? "encrypted" + : entry.kind !== "file" + ? "unsupported" + : "capacity", + message: entry.encrypted + ? "Selected native library is encrypted." + : entry.kind !== "file" + ? "Selected native library is not a regular ZIP member." + : "Selected native library exceeds the bounded library or cumulative scan-byte allowance.", + }, + }; + } else { + totalBytes += entry.declaredSize!; + try { + const bytes = Buffer.alloc(entry.declaredSize!); + let offset = 0; + const stream = await reader.open(entry, input.signal); + try { + for await (const chunk of stream) { + throwIfJadxInterrupted(input.signal); + const data = streamChunkToBuffer(chunk); + if (data.length > bytes.length - offset) + throw new ArtifactReaderFailure( + "integrity", + "Native library exceeds its declared ZIP size", + ); + data.copy(bytes, offset); + offset += data.length; + } + } finally { + stream.destroy(); + } + if (offset !== bytes.length) + throw new ArtifactReaderFailure( + "integrity", + "Native library is shorter than its declared ZIP size", + ); + throwIfJadxInterrupted(input.signal); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + inProgress = { + ...identity, + sha256, + size: bytes.length, + facts: null, + failure: { + kind: "incomplete", + message: "Native-library parsing did not complete", + }, + }; + const parsed = await inspectAndroidJniLibrary(bytes, { + signal: input.signal, + }); + throwIfJadxInterrupted(input.signal); + observation = { + ...identity, + sha256, + size: bytes.length, + facts: parsed.ok ? parsed.value : null, + failure: parsed.ok ? null : parsed.error, + }; + } catch (cause: unknown) { + if (cause instanceof JadxInterruption) throw cause; + if ( + cause instanceof ArtifactReaderFailure && + cause.reason === "cancelled" + ) + throw input.interruption(input.signal.reason); + observation = { + ...identity, + facts: null, + failure: { + kind: + cause instanceof ArtifactReaderFailure + ? cause.reason + : "io", + message: + cause instanceof Error ? cause.message : String(cause), + }, + }; + } + } + factsBytes += jsonValueByteLength( + observation as unknown as JsonValue, + ANDROID_JNI_LIBRARY_LIMITS.factsBytes, + ); + if (factsBytes > ANDROID_JNI_LIBRARY_LIMITS.factsBytes) { + inventory.status = "partial"; + inventory.unknowns.push( + "Selected native-library facts exceed the retained-facts allowance; remaining occurrences are unknown.", + ); + break; + } + libraries.push(observation); + inProgress = undefined; + } + inventory.selected_libraries = libraries.length; + throwIfJadxInterrupted(input.signal); + } catch (cause: unknown) { + failed = true; + primary = + cause instanceof JadxInterruption + ? input.interruption(cause.reason) + : cause instanceof ArtifactReaderFailure && + cause.reason === "cancelled" + ? input.interruption(input.signal.reason) + : cause; + } + if (failed) { + inventory.status = "partial"; + inventory.unknowns.push( + "Native-library scan did not exhaust APK inventory; unexamined selected occurrences remain unknown.", + ); + } + if (inProgress !== undefined) { + // An interrupted member still has an observed carrier identity; retain + // its digest only if those bytes actually completed authentication. + libraries.push(inProgress); + } + inventory.selected_libraries = libraries.length; + const checkpoint: AndroidJniLibraryCheckpoint = { + scan_state: + failed || inventory.status !== "complete" + ? "interrupted" + : "completed", + libraries: [...libraries], + inventory: { ...inventory, unknowns: [...inventory.unknowns] }, + }; + try { + input.onObservation(checkpoint); + } catch (cause: unknown) { + if (!failed) { + failed = true; + primary = cause; + } + } + const released = await input.resources.release(owner); + if (released.kind === "failed") { + const operation = "inspect_android_jni_bindings"; + const cleanup = new ProviderCleanupError( + "jadx", + [owner.resource], + { + reason: + released.cause instanceof Error + ? released.cause.message + : String(released.cause), + }, + { operation, cause: released.cause }, + ); + if (!failed && input.signal.aborted) { + failed = true; + primary = input.interruption(input.signal.reason); + } + const failure = failed + ? analysisErrorWithCleanupFailure( + primary instanceof AnalysisError + ? primary + : analysisClientFailureForOperation(primary, "jadx", operation), + cleanup, + operation, + ) + : cleanup; + released.lease.bindCleanupError(failure); + throw failure; + } + if (failed) throw primary; + throwIfJadxInterrupted(input.signal); + return { libraries, inventory }; + }) + .catch((cause: unknown) => { + // A prior owned close can reject scope admission before this callback runs. + if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined) + throw analysisClientFailureForOperation( + cause, + "jadx", + "inspect_android_jni_bindings", + ); + throw cause; + }); diff --git a/src/android/AndroidNativeInventory.test.ts b/src/android/AndroidNativeInventory.test.ts new file mode 100644 index 0000000000..18445e19c6 --- /dev/null +++ b/src/android/AndroidNativeInventory.test.ts @@ -0,0 +1,99 @@ +import { expect, it } from "vitest"; +import { readAndroidNativeInventory } from "./AndroidNativeInventory.js"; +import type { JadxToolPort } from "./JadxAnalysis.js"; +import type { JsonValue } from "../domain/jsonValue.js"; + +const SNAPSHOT = "/admitted/immutable target.apk"; + +const row = { + source_index: 0, + class_name: "fixture.Native", + method_name: "read_ๅŽŸ", + descriptor: "([B)J", + dex_entry: "classes.dex", + dex_entry_index: 0, + class_index: 2, + method_index: 5, +}; +const inventory = () => ({ + scope: "apk-root-dex-declarations", + methods: [row], + coverage: { + status: "complete", + dex_entries: 1, + examined_dex_entries: 1, + examined_classes: 3, + examined_methods: 9, + }, + unknowns: [] as string[], +}); +const tools = (value: unknown): JadxToolPort => ({ + json: async (name, input) => { + expect(name).toBe("rea_native_declarations"); + expect(input).toEqual({ path: SNAPSHOT }); + return value as JsonValue; + }, + text: async () => { + throw new Error("Native inventory must not request source"); + }, + listClasses: async () => { + throw new Error("Native inventory must not page display classes"); + }, +}); + +it("preserves duplicate native identities from distinct original DEX members", async () => { + const source = inventory(); + source.methods.push({ + ...row, + source_index: 1, + dex_entry: "classes2.dex", + dex_entry_index: 2, + }); + source.coverage.dex_entries = 2; + source.coverage.examined_dex_entries = 2; + const actual = await readAndroidNativeInventory(tools(source), SNAPSHOT); + expect(actual.methods).toEqual(source.methods); + expect(actual.coverage.status).toBe("complete"); +}); + +it("keeps usable facts and explicit incomplete DEX coverage", async () => { + const source = inventory(); + source.coverage.status = "partial"; + source.coverage.dex_entries = 2; + source.unknowns.push("classes2.dex cannot be read"); + expect(await readAndroidNativeInventory(tools(source), SNAPSHOT)).toEqual( + source, + ); +}); + +it.each([ + (source: ReturnType) => { + source.methods[0] = { ...row, descriptor: "not a prototype" }; + }, + (source: ReturnType) => { + source.methods[0] = { ...row, source_index: 7 }; + }, + (source: ReturnType) => { + source.coverage.examined_dex_entries = 2; + }, + (source: ReturnType) => { + source.coverage.dex_entries = 2; + }, + (source: ReturnType) => { + source.coverage.examined_methods = 0; + }, + (source: ReturnType) => { + source.unknowns.push("hidden incomplete input"); + }, + (source: ReturnType) => { + source.methods.push({ ...row, source_index: 1, dex_entry: "other.dex" }); + }, +])("rejects malformed or inconsistent producer facts (%#)", async (mutate) => { + const source = inventory(); + mutate(source); + await expect( + readAndroidNativeInventory(tools(source), SNAPSHOT), + ).rejects.toMatchObject({ + _tag: "AnalysisOutputError", + }); +}); diff --git a/src/android/AndroidNativeInventory.ts b/src/android/AndroidNativeInventory.ts new file mode 100644 index 0000000000..1000c37e23 --- /dev/null +++ b/src/android/AndroidNativeInventory.ts @@ -0,0 +1,42 @@ +import type { JadxToolPort } from "./JadxAnalysis.js"; +import type { AndroidNativeDeclarations } from "../domain/android/androidNativeDeclarations.js"; +import { jadxNativeDeclarationsSchema } from "./JadxProtocol.js"; +import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; + +/** Validate the native-only batch at its producing boundary, preserving raw facts. */ +export const readAndroidNativeInventory = async ( + tools: JadxToolPort, + snapshot: string, +): Promise => { + const raw = await tools.json("rea_native_declarations", { path: snapshot }); + const parsed = jadxNativeDeclarationsSchema.safeParse(raw); + if (!parsed.success) + throw new AnalysisOutputError( + "inspect_android_jni_bindings", + "JADX native declaration inventory was malformed", + { cause: parsed.error }, + ); + const inventory = parsed.data; + const origins = new Map(); + const inconsistent = + inventory.coverage.examined_dex_entries > inventory.coverage.dex_entries || + inventory.methods.length > inventory.coverage.examined_methods || + (inventory.coverage.status === "complete" && + (inventory.coverage.examined_dex_entries !== + inventory.coverage.dex_entries || + inventory.unknowns.length !== 0)) || + inventory.methods.some((method, index) => { + const previous = origins.get(method.dex_entry_index); + origins.set(method.dex_entry_index, method.dex_entry); + return ( + method.source_index !== index || + (previous !== undefined && previous !== method.dex_entry) + ); + }); + if (inconsistent) + throw new AnalysisOutputError( + "inspect_android_jni_bindings", + "JADX native declaration counts or source identities were inconsistent", + ); + return inventory; +}; diff --git a/src/android/JadxAnalysis.ts b/src/android/JadxAnalysis.ts index 427f22ad3b..83d20e5324 100644 --- a/src/android/JadxAnalysis.ts +++ b/src/android/JadxAnalysis.ts @@ -13,6 +13,13 @@ import { AnalysisOutputError, } from "../domain/analysisErrorCore.js"; import type { JsonValue } from "../domain/jsonValue.js"; +import type { AndroidNativeDeclarations } from "../domain/android/androidNativeDeclarations.js"; +import type { AndroidJniLibraries } from "./AndroidJniLibraries.js"; +import { + ANDROID_JNI_REPORT_BYTES, + projectAndroidJniBindings, +} from "../domain/android/androidJniBindings.js"; +import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; import { jadxAppSchema, type JadxClassPage, @@ -46,9 +53,14 @@ type Engine = z.infer< type ClassSummary = z.infer; /** Session facts shared by every request, and the caller's optional mapping. */ export interface JadxAnalysisContext { - readonly loaded: z.infer; + readonly loaded: z.infer | undefined; readonly engine: Engine; readonly mapping?: AndroidMappingSource | undefined; + readonly jni?: { + readonly apkSha256: string; + readonly inventory: () => Promise; + readonly libraries: () => Promise; + }; } const observedPackageSchema = @@ -129,7 +141,12 @@ const classSummary = async ( tools: JadxToolPort, request: Exclude< AndroidRequest, - { operation: "inspect_android_package" | "search_android_classes" } + { + operation: + | "inspect_android_package" + | "search_android_classes" + | "inspect_android_jni_bindings"; + } >, ): Promise => { // Only class inspection reports DEX static values; method and reference @@ -355,6 +372,44 @@ export const analyzeJadxRequest = async ( context: JadxAnalysisContext, ): Promise => { const { loaded, engine } = context; + if (request.operation === "inspect_android_jni_bindings") { + if (context.jni === undefined) + throw new AnalysisOutputError( + request.operation, + "Native declaration and immutable APK library readers are unavailable.", + ); + const inventory = await context.jni.inventory(); + const libraries = await context.jni.libraries(); + const { path, ...selection } = request.input; + const projected = projectAndroidJniBindings({ + apk: { path, sha256: context.jni.apkSha256 }, + selection, + inventory, + libraries: libraries.libraries, + libraryInventory: libraries.inventory, + }); + if (!projected.ok) + throw new AnalysisResourceConstraintError( + request.operation, + "memory", + "JNI relationship facts exceed the bounded retained report allowance.", + { limit_bytes: ANDROID_JNI_REPORT_BYTES }, + { + retryable: false, + remediationAction: + "Select fewer ABI or library occurrences; raw declaration uniqueness remains global.", + }, + ); + return androidResultSchemas.inspect_android_jni_bindings.parse({ + ...projected.value, + engine, + }); + } + if (loaded === undefined) + throw new AnalysisOutputError( + request.operation, + "Decompiler operation requires the admitted APK to be loaded.", + ); if (request.operation === "inspect_android_package") return inspectPackage(tools, loaded, engine); if (request.operation === "search_android_classes") diff --git a/src/android/JadxProtocol.ts b/src/android/JadxProtocol.ts index c032ef37ce..6e4729c1a5 100644 --- a/src/android/JadxProtocol.ts +++ b/src/android/JadxProtocol.ts @@ -7,6 +7,10 @@ import { dexDescriptorSchema, dexTypeSchema, } from "../domain/android/dexTypes.js"; +import { androidNativeDeclarationsSchema } from "../domain/android/androidNativeDeclarations.js"; + +/** Native-only raw DEX facts authored before decompiler deduplication. */ +export const jadxNativeDeclarationsSchema = androidNativeDeclarationsSchema; const count = z.int().nonnegative(); /** Typed allocation failures authored before upstream erases the JVM exception. */ diff --git a/src/android/JadxProvider.ts b/src/android/JadxProvider.ts index 2d2709b6a8..b17e5a9d3d 100644 --- a/src/android/JadxProvider.ts +++ b/src/android/JadxProvider.ts @@ -703,6 +703,8 @@ export class JadxProvider implements AndroidAnalysisPort { signal: activeSignal, timeoutMs: timeoutMs.value, mapping: mapping.value, + resources: this.#prerequisites, + deadline: timeout, }), ); throwIfJadxInterrupted(activeSignal); diff --git a/src/android/JadxRelease.ts b/src/android/JadxRelease.ts index 287e4df85f..b72bede0f4 100644 --- a/src/android/JadxRelease.ts +++ b/src/android/JadxRelease.ts @@ -10,7 +10,7 @@ export const JADX_RELEASE = Object.freeze({ /** Identity of REA's metadata bridge, distinct from the producing engine. */ export const JADX_BRIDGE_IDENTITY = Object.freeze({ name: "rea-jadx-bridge", - version: "8", + version: "9", }); /** Remediation when `REA_JADX_MCP_JAR` is missing or not an absolute path. */ @@ -49,5 +49,5 @@ export const JADX_LIMITATIONS = [ "Method signatures use provider display types; DEX prototypes come from frozen JADX metadata. Display names may be aliases, and overload indices are local to this engine and artifact.", "Class inventories use parsed metadata before code generation, including synthetic members that may be omitted from decompiled source. Source positions remain unknown.", "Incoming reference totals count JADX using nodes. Direct invoke entries within a using method report raw DEX code-unit offsets and method indexes, but coverage is limited to those nodes and directly matched invokes; it does not establish every possible dynamic or reflective call site.", - "Manifest summaries are parsed once by REA's bridge from the exact root AndroidManifest.xml resource, using namespace-aware XML attributes; decoded XML is retained. APK signatures, split APKs, native libraries and Android runtime capture are not verified by this provider.", + "Manifest summaries are parsed once by REA's bridge from the exact root AndroidManifest.xml resource, using namespace-aware XML attributes; decoded XML is retained. These summaries do not verify APK signatures, split APKs or Android runtime capture.", ] as const; diff --git a/src/android/JadxSession.ts b/src/android/JadxSession.ts index d49747e1ea..5d2dbe53c4 100644 --- a/src/android/JadxSession.ts +++ b/src/android/JadxSession.ts @@ -13,6 +13,8 @@ import { AnalysisInputError, AnalysisOutputError, AnalysisResourceConstraintError, + AnalysisCancelledError, + AnalysisTimeoutError, } from "../domain/analysisErrorCore.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { @@ -23,6 +25,14 @@ import { import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import type { OwnedProviderProcessSpawnOptions } from "../process/ProviderProcess.js"; import { analyzeJadxRequest, type JadxToolPort } from "./JadxAnalysis.js"; +import { readAndroidNativeInventory } from "./AndroidNativeInventory.js"; +import type { AndroidNativeDeclarations } from "../domain/android/androidNativeDeclarations.js"; +import { + readAndroidJniLibraries, + type AndroidJniLibraries, +} from "./AndroidJniLibraries.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { jadxOperationTimeoutGuidance } from "./JadxOperationTimeout.js"; import { JadxMcpTransport, type JadxLauncher } from "./JadxMcpTransport.js"; import { parseJadxClassPage, @@ -68,6 +78,11 @@ export class JadxSession { #serverName: string | null = null; #serverVersion: string | null = null; #engineVersion: string | null = null; + #nativeInventory: AndroidNativeDeclarations | undefined; + #nativeLibraries: + | { readonly key: string; readonly value: AndroidJniLibraries } + | undefined; + #jniObservation: NonNullable | undefined; constructor( options: Omit, @@ -88,6 +103,8 @@ export class JadxSession { readonly timeoutMs: number; /** Caller-selected mapping, read before this request reached the queue. */ readonly mapping?: AndroidMappingSource | undefined; + readonly resources: ArtifactResourceScope; + readonly deadline: AbortSignal; }): Promise { const { request, target, snapshot, jarHash, signal, timeoutMs, mapping } = context; @@ -99,6 +116,29 @@ export class JadxSession { bridgeHash: context.bridgeHash, }; this.#observedCalls = []; + this.#jniObservation = undefined; + if (request.operation === "inspect_android_jni_bindings") { + const selected = request.input; + this.#jniObservation = { + selection: { + ...(selected.class_name === undefined + ? {} + : { class_name: selected.class_name }), + ...(selected.method_names === undefined + ? {} + : { method_names: selected.method_names }), + ...(selected.descriptors === undefined + ? {} + : { descriptors: selected.descriptors }), + ...(selected.abis === undefined ? {} : { abis: selected.abis }), + ...(selected.library_paths === undefined + ? {} + : { library_paths: selected.library_paths }), + }, + native_inventory: null, + libraries: null, + }; + } this.#serverName = null; this.#serverVersion = null; this.#engineVersion = null; @@ -138,20 +178,22 @@ export class JadxSession { `Expected engine ${JADX_RELEASE.version}; JAR reported ${runtime.engine_reported_version}.`, ); const loaded = - this.#loaded ?? - jadxLoadSchema.parse( - await tools.json("load_apk", { - path: snapshot, - threads: 1, - resources: "full", - }), - ); - if (loaded.apk_path !== snapshot) + request.operation === "inspect_android_jni_bindings" + ? undefined + : (this.#loaded ?? + jadxLoadSchema.parse( + await tools.json("load_apk", { + path: snapshot, + threads: 1, + resources: "full", + }), + )); + if (loaded !== undefined && loaded.apk_path !== snapshot) throw new AnalysisOutputError( request.operation, "JADX loaded a different APK path than the admitted snapshot", ); - this.#loaded = loaded; + if (loaded !== undefined) this.#loaded = loaded; const engine = { name: "jadx-headless-mcp", version: runtime.engine_reported_version, @@ -166,12 +208,75 @@ export class JadxSession { loaded, engine, mapping, + ...(request.operation !== "inspect_android_jni_bindings" + ? {} + : { + jni: { + apkSha256: target.sha256, + inventory: async () => { + const inventory = + this.#nativeInventory ?? + (await readAndroidNativeInventory(tools, snapshot)); + if (this.#jniObservation !== undefined) + this.#jniObservation.native_inventory = inventory; + if (inventory.coverage.status === "complete") + this.#nativeInventory = inventory; + return inventory; + }, + libraries: async () => { + const key = JSON.stringify({ + abis: request.input.abis, + library_paths: request.input.library_paths, + }); + if (this.#nativeLibraries?.key === key) { + if (this.#jniObservation !== undefined) + this.#jniObservation.libraries = { + scan_state: "completed", + libraries: [...this.#nativeLibraries.value.libraries], + inventory: this.#nativeLibraries.value.inventory, + }; + return this.#nativeLibraries.value; + } + const value = await readAndroidJniLibraries({ + snapshot, + apkSha256: target.sha256, + selection: request.input, + resources: context.resources, + signal, + onObservation: (checkpoint) => { + if (this.#jniObservation !== undefined) + this.#jniObservation.libraries = checkpoint; + }, + interruption: (reason) => + context.deadline.aborted && + reason === context.deadline.reason + ? new AnalysisTimeoutError( + request.operation, + timeoutMs, + { + cause: reason, + ...jadxOperationTimeoutGuidance(timeoutMs), + }, + ) + : new AnalysisCancelledError(request.operation, { + cause: reason, + }), + }); + if ( + value.inventory.status === "complete" && + value.libraries.every((library) => library.failure === null) + ) + this.#nativeLibraries = { key, value }; + return value; + }, + }, + }), }); const raw = jsonObjectSchema.parse({ server, bridge_sha256: context.bridgeHash, jar_sha256: jarHash, - loaded, + ...(loaded === undefined ? {} : { loaded }), }); const execution = createAnalysisExecution( result, @@ -183,8 +288,9 @@ export class JadxSession { }, subject: target, limitations: [ - ...JADX_LIMITATIONS, - ...androidMappingLimitations(mapping), + ...(request.operation === "inspect_android_jni_bindings" + ? [] + : [...JADX_LIMITATIONS, ...androidMappingLimitations(mapping)]), ], locations: request.operation === "inspect_android_package" @@ -231,6 +337,9 @@ export class JadxSession { engine_version: this.#engineVersion, }, calls: this.#observedCalls.map(({ complete }) => complete), + ...(this.#jniObservation === undefined + ? {} + : { jni: this.#jniObservation }), }; } diff --git a/src/application/android/AndroidAnalysisService.ts b/src/application/android/AndroidAnalysisService.ts index 12433b3311..e1c086f6d5 100644 --- a/src/application/android/AndroidAnalysisService.ts +++ b/src/application/android/AndroidAnalysisService.ts @@ -67,7 +67,10 @@ export class AndroidAnalysisService { limitations: execution.limitations, locations: execution.locations, confidence: - operation === "inspect_android_method" ? "derived" : "observed", + operation === "inspect_android_method" || + operation === "inspect_android_jni_bindings" + ? "derived" + : "observed", }, ), options?.signal, diff --git a/src/application/android/AndroidJniInspection.ts b/src/application/android/AndroidJniInspection.ts new file mode 100644 index 0000000000..bb1d8b0094 --- /dev/null +++ b/src/application/android/AndroidJniInspection.ts @@ -0,0 +1,42 @@ +import type { ExecutionOptions } from "../AnalysisProvider.js"; +import type { AndroidAnalysisService } from "./AndroidAnalysisService.js"; +import { + requireAnalysisViewRetention, + projectRetainedAnalysis, + type AnalysisViewRetention, +} from "../analysisView/AnalysisViewService.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { androidJniInspectionInputSchema } from "../../domain/android/androidAnalysis.js"; +import type { Evidence } from "../../domain/evidence.js"; +import type { Result } from "../../domain/result.js"; +import { parseAnalysisInput } from "../AnalysisInput.js"; + +/** Keep global attribution facts retained while delivering only the requested view. */ +export const inspectAndroidJniBindings = async ( + service: AndroidAnalysisService, + rawInput: unknown, + retention: AnalysisViewRetention, + options?: ExecutionOptions, +): Promise> => { + const operation = "inspect_android_jni_bindings"; + const parsed = parseAnalysisInput( + operation, + rawInput, + androidJniInspectionInputSchema, + options?.signal, + ); + if (!parsed.ok) return parsed; + const { detail, view, ...input } = parsed.value; + if (detail === "summary") { + const ready = requireAnalysisViewRetention(operation, retention); + if (!ready.ok) return ready; + } + const executed = await service.execute(operation, input, options); + return !executed.ok || detail !== "summary" + ? executed + : projectRetainedAnalysis( + executed.value, + retention, + view ?? { kind: "summary" }, + ); +}; diff --git a/src/cli/androidCommands.ts b/src/cli/androidCommands.ts index d0c7c1a642..20cee4c67d 100644 --- a/src/cli/androidCommands.ts +++ b/src/cli/androidCommands.ts @@ -16,6 +16,10 @@ import { } from "./analysisEvidenceDelivery.js"; import { androidPackageDeliverySchema } from "../domain/android/androidPackageQuery.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { AnalysisInputError } from "../domain/analysisErrorCore.js"; +import { parseAnalysisInput } from "../application/AnalysisInput.js"; +import { androidJniSelectionSchema } from "../domain/android/androidJniBindings.js"; +import { androidJniDeliverySchema } from "../domain/android/androidJniQuery.js"; /** Expose the same explicit APK requests through one-shot CLI commands. */ export const registerAndroidCommands = ( @@ -67,6 +71,81 @@ export const registerAndroidCommands = ( ); const mappingInput = (path: string | undefined) => path === undefined ? {} : { mapping_path: path }; + cli.command(CLI_COMMANDS.inspectAndroidJniBindings, { + description: + "Link raw DEX native declarations to static JNI export/table candidates without executing the APK", + args: z.object({ path }), + options: z.object({ + ...analysisEvidenceDeliveryOptions, + query: z + .string() + .optional() + .describe( + "JSON selection: class_name, method_names, descriptors, abis, library_paths; arrays preserve exact names", + ), + view: z + .string() + .optional() + .describe( + "JSON android-jni collection page; requires --detail summary and --evidence_output", + ), + }), + run: ({ args, options }) => + withCommandCancellation((signal) => + logCliCommand( + logger, + CLI_COMMANDS.inspectAndroidJniBindings, + async () => { + const operation = "inspect_android_jni_bindings"; + const delivery = parseCliAnalysisEvidenceDelivery( + operation, + options, + androidJniDeliverySchema, + ); + if (!delivery.ok) return projectAnalysisError(delivery.error); + let query: unknown = {}; + try { + if (options.query !== undefined) + query = JSON.parse(options.query); + } catch (cause: unknown) { + return projectAnalysisError( + new AnalysisInputError(operation, { cause }, [ + { + path: ["query"], + reason: "invalid_format", + message: "query must be a JSON selection object.", + }, + ]), + ); + } + const selected = parseAnalysisInput( + operation, + query, + androidJniSelectionSchema, + signal, + ); + if (!selected.ok) return projectAnalysisError(selected.error); + return deliverCliAnalysisEvidence({ + run: () => + service.execute( + operation, + { ...selected.value, path: args.path }, + { signal }, + ), + service: { close: () => provider.close() }, + providerId: "jadx", + operation, + signal, + detail: delivery.value.detail, + evidenceOutput: delivery.value.evidenceOutput, + ...(delivery.value.view === undefined + ? {} + : { view: delivery.value.view }), + }); + }, + ), + ), + }); cli.command(CLI_COMMANDS.inspectAndroidPackage, { description: "Inspect APK declarations and decoded manifest without executing it", diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index b0d397d019..3144735c09 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -119,6 +119,7 @@ export const CLI_COMMANDS = Object.freeze({ searchAndroidClasses: "search-android-classes", inspectAndroidClass: "inspect-android-class", inspectAndroidMethod: "inspect-android-method", + inspectAndroidJniBindings: "inspect-android-jni-bindings", traceAndroidReferences: "trace-android-references", inspectJebClient: "inspect-jeb-client", openJebProject: "open-jeb-project", diff --git a/src/contracts/android/androidToolContracts.ts b/src/contracts/android/androidToolContracts.ts index 153ed1ea3f..4cc9a6ddf1 100644 --- a/src/contracts/android/androidToolContracts.ts +++ b/src/contracts/android/androidToolContracts.ts @@ -4,6 +4,7 @@ import { androidPackageInspectionInputSchema, androidSymbolInspectionInputSchemas, androidResultSchemas, + androidJniInspectionInputSchema, } from "../../domain/android/androidAnalysis.js"; import { androidPackageSummaryViewResultSchema, @@ -12,6 +13,8 @@ import { androidComponentSummaryViewResultSchema, androidClassSummaryViewResultSchema, androidMethodSummaryViewResultSchema, + androidJniSummaryViewResultSchema, + androidJniPageViewResultSchema, } from "../../domain/analysisView/analysisView.js"; import type { ToolContract } from "../toolContractTypes.js"; import { evidenceResultOf } from "../toolOutputSchemaPrimitives.js"; @@ -19,6 +22,33 @@ import { toolContractMetadata } from "../toolEffects.js"; /** Static APK operations backed by a separately supplied headless engine. */ export const ANDROID_TOOL_CONTRACTS = [ + { + name: "inspect_android_jni_bindings", + ...toolContractMetadata("inspect_android_jni_bindings"), + description: + 'Link raw APK DEX ACC_NATIVE declarations to observed Java_* function exports and relocated static JNINativeMethod table candidates, per ABI and exact library occurrence. Optional class_name/method_names/descriptors narrow bindings; abis/library_paths narrow parsed libraries. RegisterNatives class attribution always uses the global native declaration inventory before these filters. Matched records establish observed candidate presence; partial coverage does not establish uniqueness, and empty partial scans remain unknown. Includes APK/library digests, stable relation IDs and table slot/relocation provenance. No target execution, library load or runtime registration claim. Omitted detail defaults to summary: a bounded relation preview retaining complete Evidence. Optional view:{kind:"android-jni",collection:"bindings"|"declarations"|"candidates"|"libraries",offset:0,limit:64,fields:[...]} selects exact columns/pages; [] matches none. Read later pages with inspect_analysis_view and the returned parent_evidence_id without reopening files or rerunning analysis. Explicit complete detail returns full retained facts inline. Requires caller-supplied JADX MCP JAR and JDK.', + kind: "android-provider", + inputSchema: androidJniInspectionInputSchema, + outputSchema: evidenceResultOf( + z.union([ + androidResultSchemas.inspect_android_jni_bindings, + androidJniSummaryViewResultSchema, + androidJniPageViewResultSchema, + ]), + ), + examples: [ + { + title: "Find one Conscrypt native implementation", + input: { + path: "/tmp/Example.apk", + class_name: "org.conscrypt.NativeCrypto", + method_names: ["EVP_PKEY_new_RSA"], + abis: ["arm64-v8a"], + detail: "summary", + }, + }, + ], + }, { name: "inspect_android_package", ...toolContractMetadata("inspect_android_package"), diff --git a/src/contracts/toolEffects.test.ts b/src/contracts/toolEffects.test.ts index 20a02fbf5d..fe103c2d2f 100644 --- a/src/contracts/toolEffects.test.ts +++ b/src/contracts/toolEffects.test.ts @@ -72,6 +72,8 @@ const EXECUTION_WORDING_EXCEPTIONS: Readonly> = { project_managed_application_graph: "states it does not execute managed code", inspect_android_package: "states it does not execute the APK", search_android_classes: "states it does not execute the APK", + inspect_android_jni_bindings: + "reads DEX and ELF bytes without executing target code", inspect_apktool_client: "inspects the operator-selected apktool launcher; never installs it", inspect_pcap_client: diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index 6e700ff16c..67a3ed159f 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -488,6 +488,11 @@ export const TOOL_EFFECTS: Readonly> = { launchesProcess: true, writesFilesystem: true, }), + inspect_android_jni_bindings: effects({ + mutatesSession: true, + launchesProcess: true, + writesFilesystem: true, + }), trace_android_references: effects({ mutatesSession: true, launchesProcess: true, diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 0c811994a1..609b243fed 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -316,6 +316,14 @@ import { androidClassSummarySchema, androidMethodSummarySchema, } from "./androidSymbolView.js"; +import { + projectAndroidJniView, + androidJniSummarySchema, +} from "./androidJniView.js"; +import { + androidJniViewRequestSchema, + androidJniViewItemsSchema, +} from "../android/androidJniQuery.js"; import { projectDartAotView, dartAotSummarySchema, @@ -446,6 +454,7 @@ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ managedViewRequestSchema, managedMembersViewRequestSchema, androidViewRequestSchema, + androidJniViewRequestSchema, apiSurfaceViewSchema, environmentManifestViewRequestSchema, environmentStateViewRequestSchema, @@ -566,6 +575,7 @@ const viewResultBase = { "inspect_android_package", "inspect_android_class", "inspect_android_method", + "inspect_android_jni_bindings", "inspect_pcap_capture", "inspect_dart_aot", "inspect_il2cpp_metadata", @@ -870,6 +880,23 @@ export const androidPackageSummaryViewResultSchema = z.strictObject({ parent_operation: z.literal("inspect_android_package"), summary: androidPackageSummarySchema, }); +export const androidJniSummaryViewResultSchema = z.strictObject({ + ...summaryViewResultBase, + parent_operation: z.literal("inspect_android_jni_bindings"), + summary: androidJniSummarySchema, +}); +export const androidJniPageViewResultSchema = z.strictObject({ + ...viewResultBase, + kind: z.literal("android-jni"), + parent_operation: z.literal("inspect_android_jni_bindings"), + view: androidJniViewRequestSchema, + items: androidJniViewItemsSchema, + source: androidJniSummarySchema.pick({ + native_inventory: true, + library_inventory: true, + selection: true, + }), +}); /** Exact class facts with complete raw protocol Evidence retained separately. */ export const androidClassSummaryViewResultSchema = z.strictObject({ @@ -1276,6 +1303,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ webNetworkCaptureSummaryViewResultSchema, browserScenarioSummaryViewResultSchema, androidPackageSummaryViewResultSchema, + androidJniSummaryViewResultSchema, androidClassSummaryViewResultSchema, androidMethodSummaryViewResultSchema, pcapCaptureSummaryViewResultSchema, @@ -1452,6 +1480,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ parent_operation: z.literal("inspect_managed_members"), }), androidPackageViewResultSchema, + androidJniPageViewResultSchema, z.strictObject({ kind: z.literal("dart-aot"), view: dartAotViewRequestSchema, @@ -1599,6 +1628,7 @@ const retainedViewProjectors = new Map([ ["inspect_android_package", projectAndroidPackageView], ["inspect_android_class", projectAndroidSymbolSummary], ["inspect_android_method", projectAndroidSymbolSummary], + ["inspect_android_jni_bindings", projectAndroidJniView], ["capture_browser_scenario", projectBrowserScenarioView], ["inspect_dart_aot", projectDartAotView], ["inspect_il2cpp_metadata", projectIl2cppMetadataView], diff --git a/src/domain/analysisView/androidJniView.ts b/src/domain/analysisView/androidJniView.ts new file mode 100644 index 0000000000..4718d689cf --- /dev/null +++ b/src/domain/analysisView/androidJniView.ts @@ -0,0 +1,271 @@ +import { z } from "zod"; +import { androidResultSchemas } from "../android/androidAnalysis.js"; +import { + androidJniBindingSchema, + androidJniCandidateSchema, + androidJniDeclarationSchema, + androidJniLibrarySchema, +} from "../android/androidJniBindings.js"; +import { androidJniViewItemsSchema } from "../android/androidJniQuery.js"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import { AnalysisResourceConstraintError } from "../analysisErrorCore.js"; +import type { JsonValue } from "../jsonValue.js"; +import { jsonValueByteLength } from "../jsonValueByteLength.js"; +import { err, ok, type Result } from "../result.js"; +import type { + AnalysisViewParent, + AnalysisViewRequest, + UnsignedAnalysisView, +} from "./analysisView.js"; +import { analysisViewInputError } from "./analysisViewErrors.js"; +import { createAnalysisViewResultReader } from "./analysisViewParent.js"; +import { + boundedViewMessages, + pageViewCoverage, + analysisViewCoverageSchema, +} from "./analysisViewCoverage.js"; +import { selectViewRows } from "./selectViewRows.js"; + +const reportSchema = androidResultSchemas.inspect_android_jni_bindings; +const readReport = createAnalysisViewResultReader(reportSchema); +/** A bounded relation closure; preview coverage counts selected binding rows only. */ +export const androidJniSummarySchema = reportSchema + .pick({ + engine: true, + apk: true, + selection: true, + native_inventory: true, + library_inventory: true, + }) + .extend({ + counts: z.strictObject({ + declarations: z.int().nonnegative(), + selected_declarations: z.int().nonnegative(), + libraries: z.int().nonnegative(), + candidates: z.int().nonnegative(), + bindings: z.int().nonnegative(), + matched: z.int().nonnegative(), + unmatched: z.int().nonnegative(), + ambiguous: z.int().nonnegative(), + unknown: z.int().nonnegative(), + }), + bindings: z.array(androidJniBindingSchema), + declarations: z.array(androidJniDeclarationSchema), + candidates: z.array(androidJniCandidateSchema), + libraries: z.array(androidJniLibrarySchema), + selected_declaration_ids: z.array(z.string()), + declaration_coverage: analysisViewCoverageSchema, + }); + +/** Source facts are already parsed; views never reopen the APK or rerun either reader. */ +export const projectAndroidJniView = ( + parent: AnalysisViewParent, + view: AnalysisViewRequest, +): Result => { + if ( + parent.operation !== "inspect_android_jni_bindings" || + (view.kind !== "summary" && view.kind !== "android-jni") + ) + return err( + analysisViewInputError([ + { + path: ["view"], + reason: "invalid_value", + message: + "JNI Evidence supports summary or android-jni collection pages.", + }, + ]), + ); + const parsed = readReport(parent); + if (!parsed.ok) return parsed; + const report = parsed.value; + const shared = { + parent_operation: "inspect_android_jni_bindings" as const, + parent_evidence_id: parent.evidenceId, + parent_digest: parent.evidenceId.slice(3), + artifact: report.apk, + limitations: boundedViewMessages( + parent.limitations, + parent.evidenceId, + "limitations", + ), + unknowns: boundedViewMessages( + report.unknowns, + parent.evidenceId, + "unknowns", + ), + }; + let result: UnsignedAnalysisView; + if (view.kind === "summary") { + const bindings = report.bindings.slice(0, 64); + const candidateIds = new Set( + bindings.flatMap((binding) => binding.candidate_ids), + ); + const candidates = report.candidates.filter((candidate) => + candidateIds.has(candidate.candidate_id), + ); + const declarationIds = new Set([ + ...report.selected_declaration_ids.slice(0, 64), + ...bindings.map((binding) => binding.declaration_id), + ...candidates.flatMap((candidate) => candidate.declaration_ids), + ]); + const counts = { + declarations: report.declarations.length, + selected_declarations: report.selected_declaration_ids.length, + libraries: report.libraries.length, + candidates: report.candidates.length, + bindings: report.bindings.length, + matched: 0, + unmatched: 0, + ambiguous: 0, + unknown: 0, + }; + for (const binding of report.bindings) counts[binding.status]++; + result = { + ...shared, + kind: "summary", + view, + coverage: pageViewCoverage(0, bindings.length, report.bindings.length), + summary: { + engine: report.engine, + apk: report.apk, + selection: report.selection, + native_inventory: report.native_inventory, + library_inventory: report.library_inventory, + counts, + bindings, + candidates, + declarations: report.declarations.filter((method) => + declarationIds.has(method.declaration_id), + ), + // Absence, unsupported encoding and read failures have no positive + // candidate edge. Their selected library provenance is still essential. + libraries: report.libraries, + selected_declaration_ids: report.selected_declaration_ids.slice(0, 64), + declaration_coverage: pageViewCoverage( + 0, + Math.min(64, report.selected_declaration_ids.length), + report.selected_declaration_ids.length, + ), + }, + }; + } else { + const indices = + view.indices === undefined ? undefined : new Set(view.indices); + const fields = + view.fields === undefined ? undefined : new Set(view.fields); + const project = (row: Row, index: number) => + fields === undefined + ? { ...row, row_index: index } + : { + ...Object.fromEntries( + Object.entries(row).filter(([key]) => fields.has(key)), + ), + row_index: index, + }; + const pick = ( + rows: readonly Row[], + matches: (row: Row) => boolean, + ) => + selectViewRows( + rows, + view, + (row, index) => + (indices === undefined || indices.has(index)) && matches(row), + project, + ); + const set = (values: readonly string[] | undefined) => + values === undefined ? undefined : new Set(values); + let selected: { rows: object[]; total: number }; + switch (view.collection) { + case "declarations": { + const classes = set(view.class_names), + names = set(view.method_names), + ids = set(view.declaration_ids), + descriptors = set(view.descriptors); + selected = pick( + report.declarations, + (row) => + (classes === undefined || classes.has(row.class_name)) && + (names === undefined || names.has(row.method_name)) && + (ids === undefined || ids.has(row.declaration_id)) && + (descriptors === undefined || descriptors.has(row.descriptor)), + ); + break; + } + case "libraries": { + const ids = set(view.library_ids), + abis = set(view.abis); + selected = pick( + report.libraries, + (row) => + (ids === undefined || ids.has(row.library_id)) && + (abis === undefined || abis.has(row.abi)), + ); + break; + } + case "candidates": { + const ids = set(view.declaration_ids), + libs = set(view.library_ids), + candidates = set(view.candidate_ids); + selected = pick( + report.candidates, + (row) => + (ids === undefined || + row.declaration_ids.some((id) => ids.has(id))) && + (libs === undefined || libs.has(row.library_id)) && + (candidates === undefined || candidates.has(row.candidate_id)), + ); + break; + } + case "bindings": { + const ids = set(view.declaration_ids), + abis = set(view.abis), + statuses = set(view.statuses); + selected = pick( + report.bindings, + (row) => + (ids === undefined || ids.has(row.declaration_id)) && + (abis === undefined || abis.has(row.abi)) && + (statuses === undefined || statuses.has(row.status)), + ); + break; + } + } + const source = { + native_inventory: report.native_inventory, + library_inventory: report.library_inventory, + selection: report.selection, + }; + result = { + ...shared, + kind: "android-jni", + view, + source, + items: androidJniViewItemsSchema.parse(selected.rows), + coverage: pageViewCoverage( + view.offset, + selected.rows.length, + selected.total, + ), + }; + } + if ( + jsonValueByteLength(result as unknown as JsonValue, 1024 * 1024 - 128) > + 1024 * 1024 - 128 + ) + return err( + new AnalysisResourceConstraintError( + "inspect_analysis_view", + "transport", + "JNI view exceeds its 1 MiB serialized allowance.", + { parent_evidence_id: parent.evidenceId, limit_bytes: 1024 * 1024 }, + { + retryable: false, + remediationAction: + "Select a smaller collection page and requested fields, or read the complete retained Evidence.", + }, + ), + ); + return ok(result); +}; diff --git a/src/domain/android/androidAnalysis.ts b/src/domain/android/androidAnalysis.ts index 4977dabfc3..8daec79d58 100644 --- a/src/domain/android/androidAnalysis.ts +++ b/src/domain/android/androidAnalysis.ts @@ -10,6 +10,11 @@ import { } from "./androidPackageQuery.js"; import { androidMappingReportSchema } from "./androidMapping.js"; import { dexDescriptorSchema } from "./dexTypes.js"; +import { + androidJniSelectionSchema, + androidJniReportSchema, +} from "./androidJniBindings.js"; +import { androidJniViewRequestSchema } from "./androidJniQuery.js"; const target = { path: localPathStringSchema.describe( @@ -31,6 +36,7 @@ const mappingPath = { /** Caller intent for static Android inspection, independent of its engine. */ export const androidInputSchemas = { inspect_android_package: z.strictObject(target), + inspect_android_jni_bindings: androidJniSelectionSchema.extend(target), search_android_classes: z.strictObject({ ...target, query: z @@ -88,6 +94,15 @@ export const androidSymbolInspectionInputSchemas = { }), } as const; +/** JNI selection changes analysis; delivery only selects retained views. */ +export const androidJniInspectionInputSchema = + androidInputSchemas.inspect_android_jni_bindings + .extend({ + detail: z.enum(["complete", "summary"]).default("summary"), + view: androidJniViewRequestSchema.optional(), + }) + .superRefine(validateAndroidPackageDelivery); + /** Supported static Android operations. */ export type AndroidOperation = keyof typeof androidInputSchemas; /** Validated Android requests carry their selected operation. */ @@ -100,6 +115,10 @@ export type AndroidRequest = { /** Validate operation and input together so their types remain correlated. */ export const androidRequestSchema = z.discriminatedUnion("operation", [ + z.strictObject({ + operation: z.literal("inspect_android_jni_bindings"), + input: androidInputSchemas.inspect_android_jni_bindings, + }), z.strictObject({ operation: z.literal("inspect_android_package"), input: androidInputSchemas.inspect_android_package, @@ -192,6 +211,7 @@ const common = { engine }; /** Portable observations with explicit decompiler and identity limitations. */ export const androidResultSchemas = { + inspect_android_jni_bindings: androidJniReportSchema.extend(common), inspect_android_package: z.strictObject({ ...common, package_name: z.string().nullable(), diff --git a/src/domain/android/androidJniBindings.test.ts b/src/domain/android/androidJniBindings.test.ts new file mode 100644 index 0000000000..49a034955e --- /dev/null +++ b/src/domain/android/androidJniBindings.test.ts @@ -0,0 +1,478 @@ +import { expect, it } from "vitest"; +import { + androidJniReportSchema, + expectedJniNames, + projectAndroidJniBindings, + type AndroidJniLibraryObservation, + type AndroidJniSelection, +} from "./androidJniBindings.js"; +import type { + AndroidNativeDeclaration, + AndroidNativeDeclarations, +} from "./androidNativeDeclarations.js"; +import type { AndroidJniLibraryFacts } from "./androidJniLibraryFacts.js"; + +const declaration = ( + changes: Partial = {}, +): AndroidNativeDeclaration => ({ + source_index: 0, + class_name: "fixture.Native", + method_name: "readNative", + descriptor: "(I)I", + dex_entry: "classes.dex", + dex_entry_index: 0, + class_index: 0, + method_index: 1, + ...changes, +}); + +const inventory = (methods = [declaration()]): AndroidNativeDeclarations => ({ + scope: "apk-root-dex-declarations", + methods, + coverage: { + status: "complete", + dex_entries: 1, + examined_dex_entries: 1, + examined_classes: methods.length, + examined_methods: methods.length, + }, + unknowns: [], +}); + +const exportRow = ( + source_index: number, + name = "Java_fixture_Native_readNative", + address = "0x1000", +) => ({ + source_index, + name, + address, + file_offset: 512, +}); +const table = (): AndroidJniLibraryFacts["tables"][number] => ({ + section_index: 12, + section_name: ".data", + table_address: "0x2000", + entries: [ + { + entry_index: 0, + name: "readNative", + descriptor: "(I)I", + function_address: "0x1000", + code_address: "0x1000", + function_file_offset: 512, + slots: [ + { + role: "name", + address: "0x2000", + file_offset: 1536, + value: "0x3000", + relocation: { encoding: "rela", record_file_offset: 256, kind: 1027 }, + }, + { + role: "signature", + address: "0x2008", + file_offset: 1544, + value: "0x3010", + relocation: { encoding: "rela", record_file_offset: 280, kind: 1027 }, + }, + { + role: "function", + address: "0x2010", + file_offset: 1552, + value: "0x1000", + relocation: { encoding: "rela", record_file_offset: 304, kind: 1027 }, + }, + ], + }, + ], +}); +const facts = ( + changes: Partial = {}, +): AndroidJniLibraryFacts => ({ + architecture: "arm64", + bits: 64, + byte_order: "little", + exports: [], + jni_on_load: false, + tables: [], + coverage: { + status: "complete", + exports_status: "complete", + tables_status: "complete", + relocations_status: "complete", + examined_sections: 1, + examined_pointer_slots: 3, + }, + unknowns: [], + ...changes, +}); +const library = ( + source_index: number, + abi = "arm64-v8a", + observed = facts(), +): AndroidJniLibraryObservation => ({ + library_id: `library-${source_index}`, + source_index, + source_key: `zip-entry-${source_index}`, + path: `lib/${abi}/lib${source_index}.so`, + source_path: `lib/${abi}/lib${source_index}.so`, + abi, + sha256: "b".repeat(64), + size: 8192, + declared_size: 8192, + compressed_size: 4096, + facts: observed, + failure: null, +}); +const project = ( + observed = [library(0)], + declarations = inventory(), + selection: AndroidJniSelection = {}, +) => { + const result = projectAndroidJniBindings({ + apk: { path: "/selected/app.apk", sha256: "a".repeat(64) }, + selection, + inventory: declarations, + libraries: observed, + libraryInventory: { + status: "complete", + examined_entries: observed.length, + selected_libraries: observed.length, + unknowns: [], + }, + }); + if (!result.ok) throw new Error(JSON.stringify(result.error)); + expect(androidJniReportSchema.safeParse(result.value).success).toBe(true); + return result.value; +}; + +it.each([ + [ + "p.q.Native", + "read", + "(I)V", + "Java_p_q_Native_read", + "Java_p_q_Native_read__I", + ], + [ + "p.Native", + "read_buffer", + "([Ljava/lang/String;[I)J", + "Java_p_Native_read_1buffer", + "Java_p_Native_read_1buffer___3Ljava_lang_String_2_3I", + ], + [ + "fixture.Native_ๅŽŸ", + "read_ๅŽŸ", + "([B)J", + "Java_fixture_Native_1_0539f_read_1_0539f", + "Java_fixture_Native_1_0539f_read_1_0539f___3B", + ], + [ + "p.๐€$Inner", + "method", + "()V", + "Java_p__0d801_0dc00_00024Inner_method", + "Java_p__0d801_0dc00_00024Inner_method__", + ], + [ + "p.Native", + "f_รฉ", + "(Lp/ๅŽŸ;)V", + "Java_p_Native_f_1_000e9", + "Java_p_Native_f_1_000e9__Lp__0539f_2", + ], +])( + "projects exact UTF-16 JNI names for %s.%s", + (className, method, descriptor, short, long) => { + expect(expectedJniNames(className, method, descriptor)).toEqual({ + ok: true, + value: { short, long }, + }); + }, +); + +it.each(["0", "1", "2", "3"])( + "reports source-digit %s escape failure at every JNI precursor boundary", + (digit) => { + for (const [className, method, descriptor] of [ + [`${digit}Native`, "read", "()V"], + [`p.${digit}Native`, "read", "()V"], + ["p.Native", `${digit}read`, "()V"], + ["p.Native", "read", `(Lp/${digit}Argument;)V`], + ]) + expect(expectedJniNames(className!, method!, descriptor!)).toEqual({ + ok: false, + error: { kind: "escape-failure" }, + }); + }, +); + +it("does not confuse literal underscores with package separators or digits 4โ€“9", () => { + expect(expectedJniNames("p.Native_1", "_3read", "()V")).toMatchObject({ + ok: true, + value: { short: "Java_p_Native_11__13read" }, + }); + expect(expectedJniNames("p.4Native", "4read", "()V")).toMatchObject({ + ok: true, + }); +}); + +it("uses the global native pair inventory before narrowing to a class", () => { + const global = inventory([ + declaration(), + declaration({ + source_index: 1, + class_name: "fixture.Other", + class_index: 1, + method_index: 2, + }), + ]); + const report = project( + [library(0, "arm64-v8a", facts({ tables: [table()] }))], + global, + { class_name: "fixture.Native" }, + ); + expect(report.declarations).toHaveLength(2); + expect(report.bindings).toHaveLength(1); + expect(report.bindings[0]?.status).toBe("ambiguous"); + expect(report.candidates[0]).toMatchObject({ + class_attribution: "ambiguous-native-pair", + declaration_ids: report.declarations.map((row) => row.declaration_id), + }); +}); + +it("keeps duplicate encoded method occurrences independently identifiable", () => { + const duplicate = inventory([ + declaration(), + declaration({ source_index: 1 }), + ]); + const report = project( + [library(0, "arm64-v8a", facts({ tables: [table()] }))], + duplicate, + ); + expect( + new Set(report.declarations.map((method) => method.declaration_id)).size, + ).toBe(2); + expect(report.candidates[0]?.declaration_ids).toEqual( + report.declarations.map((method) => method.declaration_id), + ); + expect(report.bindings.map((binding) => binding.status)).toEqual([ + "ambiguous", + "ambiguous", + ]); +}); + +it("keeps libraries within their ABI and preserves same-ABI alternatives", () => { + const report = project([ + library(0, "arm64-v8a", facts({ exports: [exportRow(0)] })), + library(1, "arm64-v8a", facts({ exports: [exportRow(0)] })), + library( + 2, + "x86_64", + facts({ architecture: "amd64", exports: [exportRow(0)] }), + ), + library(3, "x86", facts({ architecture: "386", bits: 32 })), + ]); + expect(report.bindings.map(({ abi, status }) => ({ abi, status }))).toEqual([ + { abi: "arm64-v8a", status: "ambiguous" }, + { abi: "x86_64", status: "matched" }, + { abi: "x86", status: "unmatched" }, + ]); + expect(report.bindings[0]?.candidate_ids).toHaveLength(2); +}); + +it("preserves physical export rows without inventing ambiguity at one target address", () => { + const report = project([ + library(0, "arm64-v8a", facts({ exports: [exportRow(1), exportRow(2)] })), + ]); + expect(report.candidates).toHaveLength(2); + expect( + new Set(report.candidates.map((candidate) => candidate.candidate_id)).size, + ).toBe(2); + expect(report.bindings[0]).toMatchObject({ + status: "matched", + candidate_ids: report.candidates.map((candidate) => candidate.candidate_id), + }); +}); + +it("retains the long export while respecting the VM's short-name-first lookup", () => { + const report = project([ + library( + 0, + "arm64-v8a", + facts({ + exports: [ + exportRow(0), + exportRow(1, "Java_fixture_Native_readNative__I", "0x1100"), + ], + }), + ), + ]); + expect(report.candidates).toHaveLength(2); + expect(report.candidates[1]?.declaration_ids).toEqual([]); + expect(report.bindings[0]).toMatchObject({ + status: "matched", + candidate_ids: [report.candidates[0]?.candidate_id], + }); +}); + +it("looks up the long name when no short export exists", () => { + const report = project([ + library( + 0, + "arm64-v8a", + facts({ exports: [exportRow(0, "Java_fixture_Native_readNative__I")] }), + ), + ]); + expect(report.bindings[0]?.status).toBe("matched"); +}); + +it.each([{ method_names: [] }, { descriptors: [] }, { abis: [] }])( + "honors an explicitly empty selection %j", + (selection) => { + const report = project( + [library(0, "arm64-v8a", facts({ tables: [table()] }))], + inventory(), + selection, + ); + expect(report.bindings).toEqual([]); + expect(report.declarations).toHaveLength(1); + }, +); + +it("keeps an explicit empty library selection without introducing ABI candidates", () => { + const report = project([], inventory(), { + library_paths: [], + abis: ["arm64-v8a"], + }); + expect(report.libraries).toEqual([]); + expect(report.candidates).toEqual([]); + expect(report.bindings[0]).toMatchObject({ + abi: "arm64-v8a", + status: "unmatched", + candidate_ids: [], + }); +}); + +it("preserves selected native declarations when no library establishes an ABI", () => { + const report = project([], inventory()); + expect(report.bindings).toEqual([]); + expect(report.libraries).toEqual([]); + expect(report.declarations).toHaveLength(1); + expect(report.selected_declaration_ids).toEqual([ + report.declarations[0]?.declaration_id, + ]); + expect(report.native_inventory.coverage.status).toBe("complete"); +}); + +it("applies exact library-path selection at the shared join boundary", () => { + const first = library(0, "arm64-v8a", facts({ exports: [exportRow(0)] })); + const selected = library(1, "arm64-v8a", facts({ exports: [exportRow(0)] })); + const report = project([first, selected], inventory(), { + library_paths: [selected.path], + }); + expect(report.libraries.map((observation) => observation.library_id)).toEqual( + [selected.library_id], + ); + expect(report.library_inventory.selected_libraries).toBe(1); + expect(report.bindings[0]?.status).toBe("matched"); + expect(report.candidates.map((candidate) => candidate.library_id)).toEqual([ + selected.library_id, + ]); +}); + +it("keeps table attribution unknown for an incomplete global native inventory", () => { + const partial = inventory(); + partial.coverage.status = "partial"; + partial.coverage.dex_entries = 2; + partial.unknowns.push("classes2.dex is unreadable"); + const report = project( + [library(0, "arm64-v8a", facts({ tables: [table()] }))], + partial, + ); + expect(report.candidates[0]?.class_attribution).toBe( + "unknown-native-inventory", + ); + expect(report.bindings[0]?.status).toBe("unknown"); + expect(report.native_inventory.unknowns).toEqual(partial.unknowns); +}); + +it("preserves every table slot and raw Thumb pointer separately from its code lookup", () => { + const observed = table(); + observed.entries[0]!.function_address = "0x1001"; + observed.entries[0]!.slots[2]!.value = "0x1001"; + const report = project([ + library( + 0, + "armeabi-v7a", + facts({ architecture: "arm", bits: 32, tables: [observed] }), + ), + ]); + expect(report.candidates[0]).toMatchObject({ + address: "0x1001", + file_offset: 512, + table: { + section_index: 12, + section_name: ".data", + table_address: "0x2000", + entry: observed.entries[0], + }, + }); + expect(report.candidates[0]?.table?.entry.code_address).toBe("0x1000"); +}); + +it("does not turn a library failure into an unmatched result", () => { + const failed = library(0); + const report = project([ + { + ...failed, + facts: null, + failure: { kind: "malformed", message: "selected ELF is truncated" }, + }, + ]); + expect(report.bindings[0]?.status).toBe("unknown"); + expect(report.libraries[0]?.failure).toEqual({ + kind: "malformed", + message: "selected ELF is truncated", + }); +}); + +it("preserves a candidate in a mismatched ABI directory with an unknown binding", () => { + const report = project([ + library( + 0, + "x86_64", + facts({ architecture: "arm64", exports: [exportRow(0)] }), + ), + ]); + expect(report.candidates).toHaveLength(1); + expect(report.libraries[0]).toMatchObject({ + abi: "x86_64", + architecture: "arm64", + abi_compatibility: "mismatch", + }); + expect(report.bindings[0]).toMatchObject({ + status: "unknown", + candidate_ids: [], + }); +}); + +it("preserves an observed export with explicit incomplete uniqueness coverage", () => { + const observed = facts({ exports: [exportRow(0)] }); + observed.coverage.status = "partial"; + observed.coverage.relocations_status = "unknown"; + observed.coverage.tables_status = "unknown"; + observed.unknowns.push( + "Android packed APS2 relocation encoding is unsupported", + ); + const report = project([library(0, "arm64-v8a", observed)]); + expect(report.candidates[0]?.basis).toBe("jni-export"); + expect(report.bindings[0]).toMatchObject({ + status: "matched", + coverage: "partial", + candidate_ids: [report.candidates[0]?.candidate_id], + }); + expect(report.libraries[0]?.unknowns).toEqual(observed.unknowns); +}); diff --git a/src/domain/android/androidJniBindings.ts b/src/domain/android/androidJniBindings.ts new file mode 100644 index 0000000000..858ed3608d --- /dev/null +++ b/src/domain/android/androidJniBindings.ts @@ -0,0 +1,526 @@ +import { createHash } from "node:crypto"; +import { z } from "zod"; +import { + androidNativeDeclarationSchema, + androidNativeDeclarationsSchema, + type AndroidNativeDeclarations, +} from "./androidNativeDeclarations.js"; +import { + androidJniLibraryFactsSchema, + androidJniTableEntrySchema, +} from "./androidJniLibraryFacts.js"; +import { jsonValueByteLength } from "../jsonValueByteLength.js"; +import type { JsonValue } from "../jsonValue.js"; +import { err, ok, type Result } from "../result.js"; +import { dexDescriptorSchema } from "./dexTypes.js"; + +const count = z.int().nonnegative(); +const digest = z.string().regex(/^[a-f0-9]{64}$/u); +const address = z.string().regex(/^0x[0-9a-f]+$/u); +const status = z.enum(["complete", "partial", "unknown"]); +const ABI_LAYOUTS = new Map([ + ["arm64-v8a", ["arm64", 64]], + ["armeabi-v7a", ["arm", 32]], + ["armeabi", ["arm", 32]], + ["x86", ["386", 32]], + ["x86_64", ["amd64", 64]], +]); +export const androidJniSelectionSchema = z.strictObject({ + class_name: z.string().min(1).exactOptional(), + method_names: z.array(z.string().min(1)).exactOptional(), + descriptors: z.array(dexDescriptorSchema).exactOptional(), + abis: z.array(z.string().min(1)).exactOptional(), + library_paths: z.array(z.string().min(1)).exactOptional(), +}); +export type AndroidJniSelection = z.infer; +/** One selection owner supports early archive admission and in-memory composition. */ +export const compileAndroidJniLibrarySelection = ( + selection: AndroidJniSelection, +): ((library: { readonly abi: string; readonly path: string }) => boolean) => { + const abis = + selection.abis === undefined ? undefined : new Set(selection.abis); + const paths = + selection.library_paths === undefined + ? undefined + : new Set(selection.library_paths); + return (library) => + (abis === undefined || abis.has(library.abi)) && + (paths === undefined || paths.has(library.path)); +}; + +/** Physical member occurrence and authenticated bytes; paths are never keys alone. */ +export const androidJniLibraryIdentitySchema = z.strictObject({ + library_id: z.string(), + source_index: count, + source_key: z.string(), + path: z.string(), + source_path: z.string(), + abi: z.string(), + sha256: digest.nullable(), + size: count.nullable(), + declared_size: count.nullable(), + compressed_size: count.nullable(), +}); +export const androidJniLibrarySchema = androidJniLibraryIdentitySchema.extend({ + abi_compatibility: z.enum(["consistent", "mismatch", "unknown"]), + byte_order: z.enum(["little", "big"]).nullable(), + architecture: z.string().nullable(), + bits: z.union([z.literal(32), z.literal(64)]).nullable(), + jni_on_load: z.boolean().nullable(), + coverage: androidJniLibraryFactsSchema.shape.coverage.nullable(), + export_count: count, + table_count: count, + table_entry_count: count, + unknowns: z.array(z.string()), + failure: z.strictObject({ kind: z.string(), message: z.string() }).nullable(), +}); +export const androidJniLibraryObservationSchema = + androidJniLibraryIdentitySchema.extend({ + facts: androidJniLibraryFactsSchema.nullable(), + failure: z + .strictObject({ kind: z.string(), message: z.string() }) + .nullable(), + }); +export type AndroidJniLibraryObservation = z.infer< + typeof androidJniLibraryObservationSchema +>; +export const androidJniLibraryInventorySchema = z.strictObject({ + status, + examined_entries: count, + selected_libraries: count, + unknowns: z.array(z.string()), +}); +/** Parsed, JSON-only checkpoint captured before a live ZIP reader is finalized. */ +export const androidJniLibraryCheckpointSchema = z.strictObject({ + scan_state: z.enum(["completed", "interrupted"]), + libraries: z.array(androidJniLibraryObservationSchema), + inventory: androidJniLibraryInventorySchema, +}); +export type AndroidJniLibraryCheckpoint = z.infer< + typeof androidJniLibraryCheckpointSchema +>; +export const androidJniPartialFactsSchema = z.strictObject({ + selection: androidJniSelectionSchema, + native_inventory: androidNativeDeclarationsSchema.nullable(), + libraries: androidJniLibraryCheckpointSchema.nullable(), +}); +export const androidJniDeclarationSchema = + androidNativeDeclarationSchema.extend({ + declaration_id: z.string(), + }); +export const androidJniCandidateSchema = z.strictObject({ + candidate_id: z.string(), + library_id: z.string(), + basis: z.enum(["jni-export", "static-registration-table"]), + name: z.string(), + descriptor: z.string().nullable(), + address, + file_offset: count.nullable(), + declaration_ids: z.array(z.string()), + class_attribution: z.enum([ + "export-name", + "unique-native-pair", + "ambiguous-native-pair", + "unknown-native-inventory", + "no-native-declaration", + ]), + export_source_index: count.nullable(), + table: z + .strictObject({ + section_index: count, + section_name: z.string().nullable(), + table_address: address, + entry: androidJniTableEntrySchema, + }) + .nullable(), +}); +export const androidJniBindingSchema = z.strictObject({ + declaration_id: z.string(), + abi: z.string(), + status: z.enum(["matched", "unmatched", "ambiguous", "unknown"]), + /** Status describes observed candidates; partial coverage never establishes uniqueness or absence. */ + coverage: status, + candidate_ids: z.array(z.string()), + unknowns: z.array(z.string()), +}); +export const androidJniReportSchema = z.strictObject({ + apk: z.strictObject({ path: z.string(), sha256: digest }), + selection: androidJniSelectionSchema, + native_inventory: androidNativeDeclarationsSchema.omit({ methods: true }), + library_inventory: androidJniLibraryInventorySchema, + declarations: z.array(androidJniDeclarationSchema), + selected_declaration_ids: z.array(z.string()), + libraries: z.array(androidJniLibrarySchema), + candidates: z.array(androidJniCandidateSchema), + bindings: z.array(androidJniBindingSchema), + unknowns: z.array(z.string()), +}); +export type AndroidJniReport = z.infer; +export const ANDROID_JNI_REPORT_BYTES = 16 * 1024 * 1024; +const id = (kind: string, parts: readonly unknown[]): string => + `${kind}_${createHash("sha256").update(JSON.stringify(parts)).digest("hex")}`; + +/** JNI escaping is over UTF-16 code units; it can fail for non-Java-source names. */ +export const expectedJniNames = ( + className: string, + method: string, + descriptor: string, +): Result< + { short: string; long: string }, + { kind: "escape-failure" | "capacity" } +> => { + const precursor = [ + className.replaceAll(".", "/"), + method, + descriptor.slice(1, descriptor.indexOf(")")), + ]; + let bytes = 16; + for (const value of precursor) { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if ( + code >= 48 && + code <= 51 && + (index === 0 || value.charCodeAt(index - 1) === 47) + ) + return err({ kind: "escape-failure" }); + bytes += + code === 47 || + (code >= 48 && code <= 57) || + (code >= 65 && code <= 90) || + (code >= 97 && code <= 122) + ? 2 + : code === 95 || code === 59 || code === 91 + ? 4 + : 12; + if (bytes > ANDROID_JNI_REPORT_BYTES) return err({ kind: "capacity" }); + } + } + const escape = (value: string): string => { + let result = ""; + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + result += + code === 47 + ? "_" + : code === 95 + ? "_1" + : code === 59 + ? "_2" + : code === 91 + ? "_3" + : (code >= 48 && code <= 57) || + (code >= 65 && code <= 90) || + (code >= 97 && code <= 122) + ? value[index] + : `_0${code.toString(16).padStart(4, "0")}`; + } + return result; + }; + const short = `Java_${escape(precursor[0]!)}_${escape(method)}`; + return ok({ short, long: `${short}__${escape(precursor[2]!)}` }); +}; + +type ProjectionFailure = { readonly kind: "capacity" }; +/** Join once against the COMPLETE declaration inventory before applying selection. */ +export const projectAndroidJniBindings = (input: { + readonly apk: AndroidJniReport["apk"]; + readonly selection: AndroidJniSelection; + readonly inventory: AndroidNativeDeclarations; + readonly libraries: readonly AndroidJniLibraryObservation[]; + readonly libraryInventory: AndroidJniReport["library_inventory"]; +}): Result => { + const declarations = input.inventory.methods.map((method) => ({ + ...method, + declaration_id: id("jni_method", [ + input.apk.sha256, + method.dex_entry_index, + method.class_index, + method.method_index, + method.source_index, + ]), + })); + const byPair = new Map(); + const byExport = new Map(); + const names = new Map(); + let retained = jsonValueByteLength( + declarations as unknown as JsonValue, + ANDROID_JNI_REPORT_BYTES, + ); + if (retained > ANDROID_JNI_REPORT_BYTES) return err({ kind: "capacity" }); + const charge = (value: unknown) => { + retained += jsonValueByteLength( + value as JsonValue, + ANDROID_JNI_REPORT_BYTES, + ); + return retained <= ANDROID_JNI_REPORT_BYTES; + }; + const append = (map: Map, key: string, value: string) => { + const old = map.get(key); + if (old === undefined) map.set(key, [value]); + else old.push(value); + }; + for (const method of declarations) { + append( + byPair, + JSON.stringify([method.method_name, method.descriptor]), + method.declaration_id, + ); + const expected = expectedJniNames( + method.class_name, + method.method_name, + method.descriptor, + ); + if (!expected.ok && expected.error.kind === "capacity") + return err({ kind: "capacity" }); + names.set(method.declaration_id, expected.ok ? expected.value : null); + if (expected.ok) { + if (!charge(expected.value)) return err({ kind: "capacity" }); + append(byExport, expected.value.short, method.declaration_id); + append(byExport, expected.value.long, method.declaration_id); + } + } + const libraries: AndroidJniReport["libraries"] = []; + const selectLibrary = compileAndroidJniLibrarySelection(input.selection); + const candidates: AndroidJniReport["candidates"] = []; + const matches = new Map(); + const addCandidate = ( + candidate: AndroidJniReport["candidates"][number], + ): boolean => { + if (!charge(candidate)) return false; + candidates.push(candidate); + for (const method of candidate.declaration_ids) { + const key = JSON.stringify([method, candidate.library_id]); + const previous = matches.get(key); + if (previous === undefined) matches.set(key, [candidate]); + else previous.push(candidate); + } + return true; + }; + for (const observation of input.libraries) { + if (!selectLibrary(observation)) continue; + const { facts, failure, ...identity } = observation; + const expected = ABI_LAYOUTS.get(identity.abi); + const abiCompatibility = + facts === null || expected === undefined + ? "unknown" + : facts.architecture === expected[0] && + facts.bits === expected[1] && + facts.byte_order === "little" + ? "consistent" + : "mismatch"; + const library: AndroidJniReport["libraries"][number] = { + ...identity, + abi_compatibility: abiCompatibility, + byte_order: facts?.byte_order ?? null, + architecture: facts?.architecture ?? null, + bits: facts?.bits ?? null, + jni_on_load: facts?.jni_on_load ?? null, + coverage: facts?.coverage ?? null, + export_count: facts?.exports.length ?? 0, + table_count: facts?.tables.length ?? 0, + table_entry_count: + facts?.tables.reduce((sum, table) => sum + table.entries.length, 0) ?? + 0, + unknowns: facts?.unknowns ?? [], + failure, + }; + if (!charge(library)) return err({ kind: "capacity" }); + libraries.push(library); + if (facts === null) continue; + const shortNames = new Set(facts.exports.map((symbol) => symbol.name)); + for (const symbol of facts.exports) { + if (!symbol.name.startsWith("Java_")) continue; + const methods = (byExport.get(symbol.name) ?? []).filter((method) => { + const expected = names.get(method); + return ( + expected != null && + (symbol.name === expected.short || !shortNames.has(expected.short)) + ); + }); + if ( + !addCandidate({ + candidate_id: id("jni_candidate", [ + identity.library_id, + "export", + symbol.source_index, + ]), + library_id: identity.library_id, + basis: "jni-export", + name: symbol.name, + descriptor: null, + address: symbol.address, + file_offset: symbol.file_offset, + declaration_ids: methods, + class_attribution: "export-name", + export_source_index: symbol.source_index, + table: null, + }) + ) + return err({ kind: "capacity" }); + } + for (const table of facts.tables) + for (const entry of table.entries) { + const methods = + byPair.get(JSON.stringify([entry.name, entry.descriptor])) ?? []; + const attribution = + input.inventory.coverage.status !== "complete" + ? "unknown-native-inventory" + : methods.length === 0 + ? "no-native-declaration" + : methods.length === 1 + ? "unique-native-pair" + : "ambiguous-native-pair"; + if ( + !addCandidate({ + candidate_id: id("jni_candidate", [ + identity.library_id, + "table", + entry.slots[0]?.address, + entry.slots[0]?.file_offset, + ]), + library_id: identity.library_id, + basis: "static-registration-table", + name: entry.name, + descriptor: entry.descriptor, + address: entry.function_address, + file_offset: entry.function_file_offset, + declaration_ids: methods, + class_attribution: attribution, + export_source_index: null, + table: { + section_index: table.section_index, + section_name: table.section_name, + table_address: table.table_address, + entry, + }, + }) + ) + return err({ kind: "capacity" }); + } + } + const methodNames = + input.selection.method_names === undefined + ? undefined + : new Set(input.selection.method_names); + const descriptors = + input.selection.descriptors === undefined + ? undefined + : new Set(input.selection.descriptors); + const selectedMethods = declarations.filter( + (method) => + (input.selection.class_name === undefined || + method.class_name === input.selection.class_name) && + (methodNames === undefined || methodNames.has(method.method_name)) && + (descriptors === undefined || descriptors.has(method.descriptor)), + ); + const selectedDeclarationIds = selectedMethods.map( + (method) => method.declaration_id, + ); + if (!charge(selectedDeclarationIds)) return err({ kind: "capacity" }); + const librariesByAbi = new Map(); + for (const library of libraries) { + const previous = librariesByAbi.get(library.abi); + if (previous === undefined) librariesByAbi.set(library.abi, [library]); + else previous.push(library); + } + const abis = [...new Set(input.selection.abis ?? librariesByAbi.keys())]; + const bindings: AndroidJniReport["bindings"] = []; + for (const method of selectedMethods) + for (const abi of abis) { + const abiLibraries = librariesByAbi.get(abi) ?? []; + const selected = abiLibraries + .filter((library) => library.abi_compatibility === "consistent") + .flatMap( + (library) => + matches.get( + JSON.stringify([method.declaration_id, library.library_id]), + ) ?? [], + ); + const physical = new Set( + selected.map((candidate) => + JSON.stringify([candidate.library_id, candidate.address]), + ), + ); + const ambiguous = selected.some( + (candidate) => candidate.class_attribution === "ambiguous-native-pair", + ); + const incomplete = + input.libraryInventory.status !== "complete" || + abiLibraries.some( + (library) => + library.failure !== null || + library.coverage?.status !== "complete" || + library.abi_compatibility !== "consistent", + ); + const unknown = selected.some( + (candidate) => + candidate.class_attribution === "unknown-native-inventory", + ); + const binding: AndroidJniReport["bindings"][number] = { + declaration_id: method.declaration_id, + abi, + status: + physical.size > 1 || ambiguous + ? "ambiguous" + : unknown + ? "unknown" + : physical.size === 1 + ? "matched" + : incomplete + ? "unknown" + : "unmatched", + coverage: incomplete || unknown ? "partial" : "complete", + candidate_ids: selected.map((candidate) => candidate.candidate_id), + unknowns: [ + ...(incomplete + ? [ + "Selected library coverage is incomplete; additional static candidates may exist. Matched means observed candidate presence, not complete uniqueness.", + ] + : []), + ...(unknown + ? [ + "The full native declaration inventory is incomplete; table class attribution is unknown.", + ] + : []), + ...(names.get(method.declaration_id) === null + ? [ + "JNI name escaping failed; no VM export lookup name is asserted. RegisterNatives candidates remain applicable.", + ] + : []), + ], + }; + if (!charge(binding)) return err({ kind: "capacity" }); + bindings.push(binding); + } + const native_inventory = { + scope: input.inventory.scope, + coverage: input.inventory.coverage, + unknowns: input.inventory.unknowns, + }; + const report: AndroidJniReport = { + apk: input.apk, + selection: input.selection, + native_inventory, + library_inventory: { + ...input.libraryInventory, + selected_libraries: libraries.length, + }, + declarations, + selected_declaration_ids: selectedDeclarationIds, + libraries, + candidates, + bindings, + unknowns: [ + "Static candidates do not establish library loading, RegisterNatives execution or method invocation.", + "Runtime-built tables, encrypted/obfuscated strings and runtime-loaded code are not recovered.", + "Uniqueness is scoped to the complete recorded root DEX native inventory and selected library occurrences.", + ], + }; + return jsonValueByteLength( + report as unknown as JsonValue, + ANDROID_JNI_REPORT_BYTES, + ) > ANDROID_JNI_REPORT_BYTES + ? err({ kind: "capacity" }) + : ok(report); +}; diff --git a/src/domain/android/androidJniLibraryFacts.ts b/src/domain/android/androidJniLibraryFacts.ts new file mode 100644 index 0000000000..e0ba8d6192 --- /dev/null +++ b/src/domain/android/androidJniLibraryFacts.ts @@ -0,0 +1,81 @@ +import { z } from "zod"; +import { dexDescriptorSchema } from "./dexTypes.js"; + +const address = z.string().regex(/^0x[0-9a-f]+$/u); +const index = z.number().int().nonnegative(); +const status = z.enum(["complete", "partial", "unknown"]); + +/** The exact source record that established a link-time pointer value. */ +export const androidJniRelocationSourceSchema = z.strictObject({ + encoding: z.enum(["rel", "rela", "relr"]), + record_file_offset: index, + kind: index.nullable(), +}); + +/** A file-backed JNINativeMethod word, before any runtime load bias. */ +export const androidJniTableSlotSchema = z.strictObject({ + role: z.enum(["name", "signature", "function"]), + address, + file_offset: index, + value: address, + relocation: androidJniRelocationSourceSchema.nullable(), +}); + +/** Structural table evidence does not establish RegisterNatives execution. */ +export const androidJniTableEntrySchema = z.strictObject({ + entry_index: index, + name: z.string().min(1), + descriptor: dexDescriptorSchema, + function_address: address, + /** ARM Thumb's low bit is retained above and cleared only for this code lookup. */ + code_address: address, + function_file_offset: index.nullable(), + slots: z.array(androidJniTableSlotSchema).length(3), +}); + +/** Facts from one immutable ELF library, before APK/DEX attribution. */ +export const androidJniLibraryFactsSchema = z.strictObject({ + architecture: z.string(), + bits: z.union([z.literal(32), z.literal(64)]), + byte_order: z.enum(["little", "big"]), + exports: z.array( + z.strictObject({ + /** Index in the shared export parser's ordered defined-export inventory. */ + source_index: index, + name: z.string(), + address, + file_offset: index.nullable(), + }), + ), + jni_on_load: z.boolean(), + tables: z.array( + z.strictObject({ + section_index: index, + section_name: z.string().nullable(), + table_address: address, + entries: z.array(androidJniTableEntrySchema).min(1), + }), + ), + coverage: z.strictObject({ + status, + exports_status: status, + tables_status: status, + relocations_status: status, + examined_sections: index, + examined_pointer_slots: index, + }), + unknowns: z.array(z.string()), +}); + +export type AndroidJniLibraryFacts = z.infer< + typeof androidJniLibraryFactsSchema +>; +export type AndroidJniTableEntry = z.infer; +export type AndroidJniTableSlot = z.infer; +export type AndroidJniRelocationSource = z.infer< + typeof androidJniRelocationSourceSchema +>; +export type AndroidJniLibraryFailure = { + readonly kind: "malformed" | "unsupported" | "capacity" | "cancelled"; + readonly message: string; +}; diff --git a/src/domain/android/androidJniQuery.ts b/src/domain/android/androidJniQuery.ts new file mode 100644 index 0000000000..908d6ef94b --- /dev/null +++ b/src/domain/android/androidJniQuery.ts @@ -0,0 +1,80 @@ +import { z } from "zod"; +import { + androidJniBindingSchema, + androidJniCandidateSchema, + androidJniDeclarationSchema, + androidJniLibrarySchema, +} from "./androidJniBindings.js"; +import { validateAndroidPackageDelivery } from "./androidPackageQuery.js"; +const page = { + kind: z.literal("android-jni"), + offset: z.int().nonnegative().default(0), + limit: z.int().positive().default(64), + indices: z.array(z.int().nonnegative()).optional(), +}; +/** Pages reference original parent collection indices, independently of source ordinals. */ +export const androidJniViewRequestSchema = z.discriminatedUnion("collection", [ + z.strictObject({ + ...page, + collection: z.literal("declarations"), + declaration_ids: z.array(z.string()).optional(), + descriptors: z + .array(androidJniDeclarationSchema.shape.descriptor) + .optional(), + class_names: z.array(z.string()).optional(), + method_names: z.array(z.string()).optional(), + fields: z.array(androidJniDeclarationSchema.keyof()).optional(), + }), + z.strictObject({ + ...page, + collection: z.literal("libraries"), + library_ids: z.array(z.string()).optional(), + abis: z.array(z.string()).optional(), + fields: z.array(androidJniLibrarySchema.keyof()).optional(), + }), + z.strictObject({ + ...page, + collection: z.literal("candidates"), + candidate_ids: z.array(z.string()).optional(), + declaration_ids: z.array(z.string()).optional(), + library_ids: z.array(z.string()).optional(), + fields: z.array(androidJniCandidateSchema.keyof()).optional(), + }), + z.strictObject({ + ...page, + collection: z.literal("bindings"), + declaration_ids: z.array(z.string()).optional(), + abis: z.array(z.string()).optional(), + statuses: z.array(androidJniBindingSchema.shape.status).optional(), + fields: z.array(androidJniBindingSchema.keyof()).optional(), + }), +]); +export const androidJniViewItemsSchema = z.union([ + z.array( + androidJniDeclarationSchema + .partial() + .extend({ row_index: z.int().nonnegative() }), + ), + z.array( + androidJniLibrarySchema + .partial() + .extend({ row_index: z.int().nonnegative() }), + ), + z.array( + androidJniCandidateSchema + .partial() + .extend({ row_index: z.int().nonnegative() }), + ), + z.array( + androidJniBindingSchema + .partial() + .extend({ row_index: z.int().nonnegative() }), + ), +]); +/** CLI and MCP share the same delivery admission before analysis starts. */ +export const androidJniDeliverySchema = z + .strictObject({ + detail: z.enum(["complete", "summary"]).default("complete"), + view: androidJniViewRequestSchema.optional(), + }) + .superRefine(validateAndroidPackageDelivery); diff --git a/src/domain/android/androidNativeDeclarations.ts b/src/domain/android/androidNativeDeclarations.ts new file mode 100644 index 0000000000..7d1b5d4a45 --- /dev/null +++ b/src/domain/android/androidNativeDeclarations.ts @@ -0,0 +1,40 @@ +import { z } from "zod"; +import { dexDescriptorSchema } from "./dexTypes.js"; + +const count = z.int().nonnegative().safe(); + +/** One original ACC_NATIVE declaration, before decompiler class deduplication. */ +export const androidNativeDeclarationSchema = z.strictObject({ + source_index: count, + class_name: z.string().min(1), + method_name: z.string().min(1), + descriptor: dexDescriptorSchema, + dex_entry: z.string().min(1), + /** Central-directory ordinal; duplicate names are explicitly unsupported. */ + dex_entry_index: count, + /** Class-definition ordinal within this DEX. */ + class_index: count, + /** Method-reference index in the originating DEX. */ + method_index: count, +}); + +/** Scope is standard root classes*.dex members, never runtime-loaded code. */ +export const androidNativeDeclarationsSchema = z.strictObject({ + scope: z.literal("apk-root-dex-declarations"), + methods: z.array(androidNativeDeclarationSchema), + coverage: z.strictObject({ + status: z.enum(["complete", "partial", "unknown"]), + dex_entries: count, + examined_dex_entries: count, + examined_classes: count, + examined_methods: count, + }), + unknowns: z.array(z.string()), +}); + +export type AndroidNativeDeclaration = z.infer< + typeof androidNativeDeclarationSchema +>; +export type AndroidNativeDeclarations = z.infer< + typeof androidNativeDeclarationsSchema +>; diff --git a/src/domain/android/androidPartialObservation.ts b/src/domain/android/androidPartialObservation.ts index 6a0cbf9474..bef800ae1d 100644 --- a/src/domain/android/androidPartialObservation.ts +++ b/src/domain/android/androidPartialObservation.ts @@ -3,9 +3,11 @@ import { z } from "zod"; import { digestSchema } from "../digests.js"; import { jsonObjectSchema, jsonValueSchema } from "../jsonValue.js"; import { localPathStringSchema } from "../localPath.js"; +import { androidJniPartialFactsSchema } from "./androidJniBindings.js"; const operationSchema = z.enum([ "inspect_android_package", + "inspect_android_jni_bindings", "search_android_classes", "inspect_android_class", "inspect_android_method", @@ -13,32 +15,46 @@ const operationSchema = z.enum([ ]); /** JSON-safe Android provider facts collected before an analysis failure. */ -export const androidPartialObservationSchema = z.strictObject({ - provider_id: z.literal("jadx"), - operation: operationSchema, - target: z.strictObject({ - selected_path: localPathStringSchema, - path: localPathStringSchema, - sha256: digestSchema, - format: z.literal("apk"), - }), - provider_facts: z.strictObject({ - jar_sha256: digestSchema.nullable(), - bridge_sha256: digestSchema.nullable(), - bridge_name: z.string().min(1), - bridge_version: z.string().min(1), - server_name: z.string().nullable(), - server_version: z.string().nullable(), - engine_version: z.string().min(1).nullable(), - }), - calls: z.array( - z.strictObject({ - operation: z.string().min(1), - input: jsonObjectSchema, - response: jsonValueSchema, +export const androidPartialObservationSchema = z + .strictObject({ + provider_id: z.literal("jadx"), + operation: operationSchema, + target: z.strictObject({ + selected_path: localPathStringSchema, + path: localPathStringSchema, + sha256: digestSchema, + format: z.literal("apk"), }), - ), -}); + provider_facts: z.strictObject({ + jar_sha256: digestSchema.nullable(), + bridge_sha256: digestSchema.nullable(), + bridge_name: z.string().min(1), + bridge_version: z.string().min(1), + server_name: z.string().nullable(), + server_version: z.string().nullable(), + engine_version: z.string().min(1).nullable(), + }), + calls: z.array( + z.strictObject({ + operation: z.string().min(1), + input: jsonObjectSchema, + response: jsonValueSchema, + }), + ), + jni: androidJniPartialFactsSchema.optional(), + }) + .superRefine((observation, context) => { + if ( + observation.jni !== undefined && + observation.operation !== "inspect_android_jni_bindings" + ) + context.addIssue({ + code: "custom", + path: ["jni"], + message: + "Parsed JNI facts belong only to inspect_android_jni_bindings observations", + }); + }); /** JSON-safe Android provider facts retained when analysis fails. */ export type AndroidPartialObservation = z.infer< diff --git a/src/il2cpp/ElfRelocations.ts b/src/il2cpp/ElfRelocations.ts index b3d7624443..461d115061 100644 --- a/src/il2cpp/ElfRelocations.ts +++ b/src/il2cpp/ElfRelocations.ts @@ -1,263 +1,102 @@ -import { readElfRelocationEntry } from "../domain/elfRelocation.js"; -import { readElfDynamicEntry } from "../domain/elfDynamicEntry.js"; import { - BinaryMappingFailure, - mappedFileOffset, - isZeroFilledSpan, - type BinaryContainer, - type ElfDynamicSegment, - type FileMapping, - type ZeroFillMapping, + readElfRelocationWritesSteps, + ElfPointerRelocationFailure, +} from "../native/metadata/ElfPointerRelocations.js"; +import type { + BinaryContainer, + FileMapping, + ZeroFillMapping, } from "../native/metadata/BinaryContainer.js"; import { Il2cppBinaryFormatFailure } from "./Il2cppBinaryFormat.js"; -const DT_RELA = 7n, - DT_RELASZ = 8n, - DT_RELAENT = 9n; -const DT_PLTRELSZ = 2n, - DT_PLTREL = 20n, - DT_JMPREL = 23n; -const MAX_OVERLAY_ENTRIES = 65536; -const malformed = (message: string) => - new Il2cppBinaryFormatFailure("malformed-container", message); -const unsupported = (message: string) => - new Il2cppBinaryFormatFailure("unsupported-pointer-encoding", message); - -interface RelocatedImage { - readonly bytes: Buffer; - readonly unresolved: Map; - readonly zeroFillPointers: Map; -} -interface RelocationSource { - readonly image: Buffer; - readonly mappings: readonly FileMapping[]; - readonly zeroFills: readonly ZeroFillMapping[]; - readonly relative: number; - readonly unknownPointerWidths: ReadonlyMap; -} - -/** Standard dynamic relocations whose destination spans at most eight bytes. - * Other encodings (including COPY and TLS descriptors) need their own width - * semantics; do not silently apply them or turn their stored zero into absence. - */ -const RELOCATION_KINDS = new Map([ - [ - 183, - { - relative: 1027, - unknownPointerWidths: new Map([ - [257, 8], - [258, 4], - [259, 2], - [260, 8], - [261, 4], - [262, 2], - [1025, 8], - [1026, 8], - [1028, 8], - [1029, 8], - [1030, 8], - [1032, 8], - ]), - }, - ], - [ - 62, - { - relative: 8, - unknownPointerWidths: new Map([ - [1, 8], - [2, 4], - [6, 8], - [7, 8], - [10, 4], - [11, 4], - [16, 8], - [17, 8], - [18, 8], - [24, 8], - [37, 8], - ]), - }, - ], -]); - -function* dynamicTags( - image: Buffer, - declarations: readonly ElfDynamicSegment[], -): Generator> { - yield; - if (declarations.length > 1) throw malformed("multiple PT_DYNAMIC segments"); - const declaration = declarations[0]; - if (declaration === undefined) - throw new Il2cppBinaryFormatFailure( - "unsupported-container", - "ELF image has no PT_DYNAMIC segment", - ); - if ( - declaration.size % 16n !== 0n || - declaration.offset > BigInt(image.length) || - declaration.size > BigInt(image.length) - declaration.offset - ) - throw malformed("PT_DYNAMIC is truncated or outside the file"); - const dynamic = { - offset: Number(declaration.offset), - size: Number(declaration.size), - }; - const tags = new Map(); - const unique = new Set([ - DT_RELA, - DT_RELASZ, - DT_RELAENT, - DT_JMPREL, - DT_PLTRELSZ, - DT_PLTREL, - ]); - for (let at = dynamic.offset; at < dynamic.offset + dynamic.size; at += 16) { - if ((at - dynamic.offset) % 65536 === 0) yield; - const parsedEntry = readElfDynamicEntry( - image.subarray(at, at + 16), - 64, - true, - ); - if (!parsedEntry.ok) throw malformed(parsedEntry.error.message); - const { tag, value } = parsedEntry.value; - if (tag === 0n) return tags; - if (tags.has(tag) && unique.has(tag)) - throw malformed(`duplicate dynamic tag ${tag}`); - tags.set(tag, value); - } - throw malformed("PT_DYNAMIC has no DT_NULL terminator"); -} - -const fileOffset = ( - source: RelocationSource, - address: bigint, - size: number, -): number | null => { - try { - return mappedFileOffset(source.mappings, address, size, source.zeroFills); - } catch (cause) { - if (cause instanceof BinaryMappingFailure && cause.reason === "unmapped") - return null; - throw cause; - } -}; - -const admitOverlay = (output: RelocatedImage) => { - if ( - output.unresolved.size + output.zeroFillPointers.size >= - MAX_OVERLAY_ENTRIES - ) - throw new Il2cppBinaryFormatFailure( - "native-index-capacity", - `ELF relocation overlay exceeds ${MAX_OVERLAY_ENTRIES} entries`, - ); -}; - -/** Each admitted record is interpreted once from the original immutable bytes. */ -function* applyRelaTable( - source: RelocationSource, - output: RelocatedImage, - address: bigint, - size: bigint | undefined, - entry: bigint, -): Generator { - if ( - size === undefined || - size < 0n || - entry !== 24n || - size % entry !== 0n || - size > BigInt(source.image.length) - ) - throw malformed("invalid DT_RELASZ or DT_RELAENT"); - const start = fileOffset(source, address, Number(size)); - if (start === null) throw malformed("RELA table is not file-backed"); - for (let at = start; at < start + Number(size); at += 24) { - if ((at - start) % (24 * 4096) === 0) yield; - const parsedEntry = readElfRelocationEntry( - source.image.subarray(at, at + 24), - 64, - true, - "rela", - ); - if (!parsedEntry.ok) throw malformed(parsedEntry.error.message); - const { - type: kind, - offset: destination, - symbolIndex, - addend, - } = parsedEntry.value; - if (kind === 0) continue; - if (kind !== source.relative || symbolIndex !== 0) { - const width = source.unknownPointerWidths.get(kind); - if (width === undefined) - throw unsupported(`ELF relocation kind ${kind} is not decoded`); - const priorWidth = output.unresolved.get(destination); - if (priorWidth === undefined) admitOverlay(output); - output.unresolved.set(destination, Math.max(priorWidth ?? 0, width)); - // Registration scans read this private image directly. Clear unknown - // fields so their stored bytes cannot masquerade as resolved pointers. - const target = fileOffset(source, destination, width); - if (target !== null) output.bytes.fill(0, target, target + width); - continue; - } - const target = fileOffset(source, destination, 8); - const value = BigInt.asUintN(64, addend); - if (target !== null) output.bytes.writeBigUInt64LE(value, target); - else if ( - isZeroFilledSpan(source.mappings, source.zeroFills, destination, 8) - ) { - if (!output.zeroFillPointers.has(destination)) admitOverlay(output); - output.zeroFillPointers.set(destination, value); - } else - throw malformed( - "RELATIVE relocation target is outside unambiguous loaded memory", - ); - } -} - -/** Materialize relative pointers in a private copy, preserving unknown and .bss facts. */ +/** Materialize the shared loader facts only for IL2CPP's existing private reader. */ export function* applyElfRelativeRelocationsSteps( image: Buffer, elf: Extract["elf"], mappings: readonly FileMapping[], zeroFills: readonly ZeroFillMapping[], -): Generator { - const kinds = RELOCATION_KINDS.get(elf.machine); - if (kinds === undefined) - throw new Il2cppBinaryFormatFailure( - "unsupported-container", - "ELF relocations require AArch64 or x86-64", - ); - const tags = yield* dynamicTags(image, elf.dynamicSegments); - if (tags.has(0x6000000fn) || tags.has(0x60000011n) || tags.has(17n)) - throw unsupported( - "Android packed (APS2) or REL relocations are not decoded", - ); - yield; - const output: RelocatedImage = { - bytes: Buffer.from(image), - unresolved: new Map(), - zeroFillPointers: new Map(), - }; - const source = { image, mappings, zeroFills, ...kinds }; - const rela = tags.get(DT_RELA); - if (rela !== undefined) - yield* applyRelaTable( - source, - output, - rela, - tags.get(DT_RELASZ), - tags.get(DT_RELAENT) ?? 24n, - ); - else if ((tags.get(DT_RELASZ) ?? 0n) !== 0n) - throw malformed("DT_RELASZ has no DT_RELA"); - const plt = tags.get(DT_JMPREL); - if (plt !== undefined) { - if (tags.get(DT_PLTREL) !== DT_RELA) - throw unsupported("PLT relocations require RELA encoding"); - yield* applyRelaTable(source, output, plt, tags.get(DT_PLTRELSZ), 24n); +): Generator< + void, + { + readonly bytes: Buffer; + readonly unresolved: ReadonlyMap; + readonly zeroFillPointers: ReadonlyMap; + } +> { + try { + const steps = readElfRelocationWritesSteps( + image, + { + bits: 64, + little: true, + machine: elf.machine, + dynamicSegments: elf.dynamicSegments, + mappings, + zeroFills, + }, + { requireDynamic: true }, + ); + const bytes = Buffer.from(image), + zeroFillPointers = new Map(), + unresolved = new Map(); + // One bit per original file offset replaces per-relocation objects. Reads + // of implicit addends continue using image, never this mutable private copy. + const starts = Buffer.alloc(Math.ceil(image.length / 8)); + const wasWritten = (offset: number) => + offset >= 0 && + offset < image.length && + ((starts[Math.floor(offset / 8)] ?? 0) & (1 << (offset % 8))) !== 0; + const admitState = () => { + if (unresolved.size + zeroFillPointers.size >= 65_536) + throw new Il2cppBinaryFormatFailure( + "native-index-capacity", + "ELF unresolved and zero-fill overlay exceeds 65536 entries", + ); + }; + const unknown = (address: bigint, width: number) => { + if (!unresolved.has(address)) admitState(); + unresolved.set(address, Math.max(unresolved.get(address) ?? 0, width)); + }; + let next = steps.next(); + while (!next.done) { + const write = next.value; + if (write === undefined) { + yield; + next = steps.next(); + continue; + } + const at = write.file_offset; + if (write.value === null) { + unknown(write.address, write.width); + if (at !== null) bytes.fill(0, at, at + write.width); + } else if (at === null) { + for (let delta = -7; delta <= 7; delta++) { + const previous = zeroFillPointers.get(write.address + BigInt(delta)); + if ( + previous !== undefined && + (delta !== 0 || previous !== write.value) + ) + unknown(write.address, 8); + } + if (!zeroFillPointers.has(write.address)) admitState(); + zeroFillPointers.set(write.address, write.value); + } else { + for (let delta = -7; delta <= 7; delta++) + if ( + wasWritten(at + delta) && + (delta !== 0 || bytes.readBigUInt64LE(at) !== write.value) + ) + unknown(write.address, 8); + const byte = Math.floor(at / 8); + starts[byte] = (starts[byte] ?? 0) | (1 << (at % 8)); + bytes.writeBigUInt64LE(write.value, at); + } + next = steps.next(); + } + return { bytes, unresolved, zeroFillPointers }; + } catch (cause) { + if (cause instanceof ElfPointerRelocationFailure) + throw new Il2cppBinaryFormatFailure(cause.reason, cause.message); + throw cause; } - return output; } diff --git a/src/native/android/AndroidJniLibrary.ts b/src/native/android/AndroidJniLibrary.ts new file mode 100644 index 0000000000..fe34399b0c --- /dev/null +++ b/src/native/android/AndroidJniLibrary.ts @@ -0,0 +1,524 @@ +import { createHash } from "node:crypto"; +import { setImmediate as immediate } from "node:timers/promises"; +import { err, ok, type Result } from "../../domain/result.js"; +import { dexDescriptorSchema } from "../../domain/android/dexTypes.js"; +import type { + AndroidJniLibraryFacts, + AndroidJniLibraryFailure, + AndroidJniTableEntry, + AndroidJniTableSlot, +} from "../../domain/android/androidJniLibraryFacts.js"; +import { parseNativeExportsSteps } from "../exports/NativeExportParser.js"; +import { readElfDirectoryFromBuffer } from "../metadata/NativeSectionDirectory.js"; +import { readElfImage } from "../metadata/ElfImage.js"; +import { + BinaryFormatFailure, + BinaryMappingFailure, + BinaryResourceFailure, + BinaryStringReadLimitFailure, + createFileMappingIndex, +} from "../metadata/BinaryContainer.js"; +import { + readElfPointerRelocationsSteps, + ElfPointerRelocationFailure, + type ElfPointerOverlay, +} from "../metadata/ElfPointerRelocations.js"; + +const MAXIMUM_STRING_BYTES = 65_535; +const MAXIMUM_STRING_WORK = 64 * 1024 * 1024; +const MAXIMUM_RETAINED_BYTES = 16 * 1024 * 1024; +const MAXIMUM_SCAN_BYTES = 64 * 1024 * 1024; +const MAXIMUM_TABLE_ENTRIES = 65_536; +const hex = (value: bigint): string => `0x${value.toString(16)}`; + +/** Strict Modified UTF-8: no four-byte UTF-8, overlong forms or replacements. */ +export const decodeJniModifiedUtf8 = ( + bytes: Uint8Array, +): string | undefined => { + const chunks: string[] = []; + let units: number[] = []; + for (let at = 0; at < bytes.length;) { + const first = bytes[at++] ?? 0; + let unit: number; + if (first >= 1 && first <= 0x7f) unit = first; + else if (first >= 0xc0 && first <= 0xdf) { + const second = bytes[at++]; + if (second === undefined || (second & 0xc0) !== 0x80) return undefined; + unit = ((first & 0x1f) << 6) | (second & 0x3f); + if (unit < 0x80 && !(first === 0xc0 && second === 0x80)) return undefined; + } else if (first >= 0xe0 && first <= 0xef) { + const second = bytes[at++], + third = bytes[at++]; + if ( + second === undefined || + third === undefined || + (second & 0xc0) !== 0x80 || + (third & 0xc0) !== 0x80 + ) + return undefined; + unit = ((first & 0xf) << 12) | ((second & 0x3f) << 6) | (third & 0x3f); + if (unit < 0x800) return undefined; + } else return undefined; + units.push(unit); + if (units.length === 4096) { + chunks.push(String.fromCharCode(...units)); + units = []; + } + } + if (units.length !== 0) chunks.push(String.fromCharCode(...units)); + return chunks.join(""); +}; + +const methodName = (name: string): boolean => + name !== "" && !/[.;\[/<>\0]/u.test(name); +const methodDescriptor = (descriptor: string): boolean => { + if ( + !dexDescriptorSchema.safeParse(descriptor).success || + /[.\s\0]/u.test(descriptor) + ) + return false; + for (const match of descriptor.matchAll(/L([^;]+);/gu)) { + const name = match[1]; + if ( + name === undefined || + name.includes("[") || + name.split("/").some((part) => part === "") + ) + return false; + } + return true; +}; + +/** Parse borrowed immutable ELF bytes; no filesystem, process or target execution. */ +export const inspectAndroidJniLibrary = async ( + bytes: Buffer, + options: { readonly signal?: AbortSignal } = {}, +): Promise> => { + const active = (): void => { + if (options.signal?.aborted === true) throw options.signal; + }; + try { + active(); + if (!bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) + return err({ + kind: "unsupported", + message: "JNI library inspection requires an ELF image", + }); + const directory = readElfDirectoryFromBuffer(bytes); + const image = readElfImage(bytes); + const mappings = createFileMappingIndex(image.mappings, image.zero_fills); + if (image.executable_mappings === undefined) + return err({ + kind: "unsupported", + message: "ELF executable PT_LOAD inventory is unavailable", + }); + const executable = createFileMappingIndex( + image.executable_mappings, + image.zero_fills, + ); + const unknowns = new Set(); + let retained = 0, + stringWork = 0; + const retain = (amount: number): void => { + retained += amount; + if (retained > MAXIMUM_RETAINED_BYTES) + throw new BinaryResourceFailure( + "structure", + MAXIMUM_RETAINED_BYTES, + "JNI library facts exceed the 16 MiB retained representation allowance", + ); + }; + const exportSteps = parseNativeExportsSteps(bytes, { + path: "[admitted immutable ELF bytes]", + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.length, + }); + let exportStep = exportSteps.next(); + while (!exportStep.done) { + await immediate(); + active(); + exportStep = exportSteps.next(); + } + const rawExports = exportStep.value; + if (rawExports.format !== "elf") + return err({ + kind: "unsupported", + message: "JNI library inspection requires ELF exports", + }); + const exportRows: AndroidJniLibraryFacts["exports"] = []; + let exportsStatus: AndroidJniLibraryFacts["coverage"]["exports_status"] = + rawExports.coverage.status === "complete" + ? "complete" + : rawExports.coverage.status === "unsupported" + ? "unknown" + : "partial"; + if (exportsStatus !== "complete") + unknowns.add( + rawExports.coverage.reason ?? "Dynamic export inventory is incomplete", + ); + for (const [sourceIndex, row] of rawExports.exports.entries()) { + if (row.name !== "JNI_OnLoad" && !row.name.startsWith("Java_")) continue; + if (row.type !== "function") { + exportsStatus = "partial"; + unknowns.add( + "JNI-named non-function or IFUNC exports are not direct implementation addresses", + ); + continue; + } + if ( + row.version !== null && + row.version !== "unknown" && + row.version.hidden + ) + continue; + if (row.version === "unknown") { + exportsStatus = "partial"; + unknowns.add( + "JNI export version applicability is unknown; unresolved versions are not emitted as available bindings", + ); + continue; + } + const address = BigInt(row.value); + const code = directory.machine === 40 ? address & ~1n : address; + let offset: number | null = null; + try { + offset = executable.offset(code, 1); + } catch (cause) { + if ( + cause instanceof BinaryMappingFailure && + cause.reason === "unmapped" + ) { + exportsStatus = "partial"; + unknowns.add( + "Some JNI function exports are outside executable file-backed PT_LOAD ranges", + ); + continue; + } else throw cause; + } + retain(192 + row.name.length * 2); + exportRows.push({ + source_index: sourceIndex, + name: row.name, + address: row.value, + file_offset: offset, + }); + } + await immediate(); + active(); + let overlay: ElfPointerOverlay; + try { + const steps = readElfPointerRelocationsSteps( + bytes, + { + bits: directory.bits, + little: directory.little, + machine: directory.machine, + dynamicSegments: image.elf.dynamicSegments, + mappings: image.mappings, + zeroFills: image.zero_fills, + }, + { unsupported: "record" }, + ); + let next = steps.next(); + while (!next.done) { + await immediate(); + active(); + next = steps.next(); + } + overlay = next.value; + } catch (cause) { + if ( + !(cause instanceof ElfPointerRelocationFailure) || + cause.reason !== "unsupported-container" + ) + throw cause; + return ok({ + architecture: image.architecture, + bits: directory.bits, + byte_order: image.byte_order, + exports: exportRows, + jni_on_load: rawExports.exports.some( + (row) => row.name === "JNI_OnLoad", + ), + tables: [], + coverage: { + status: "partial", + exports_status: exportsStatus, + tables_status: "unknown", + relocations_status: "unknown", + examined_sections: 0, + examined_pointer_slots: 0, + }, + unknowns: [ + ...unknowns, + cause.message, + "Runtime-built tables and obfuscated strings are not recovered; no loading or invocation is established", + ], + }); + } + for (const issue of overlay.issues) unknowns.add(issue); + if (overlay.unresolved.size !== 0) + unknowns.add( + `${overlay.unresolved.size} unresolved, conflicting or indirect relocation destinations have unknown values`, + ); + const relocationStatus = !overlay.pointer_coverage_known + ? "unknown" + : overlay.unresolved.size !== 0 + ? "partial" + : "complete"; + let tablesStatus: AndroidJniLibraryFacts["coverage"]["tables_status"] = + !overlay.pointer_coverage_known ? "unknown" : "complete"; + const tables: AndroidJniLibraryFacts["tables"] = []; + const word = directory.bits / 8; + const strings = new Map(); + const stringAt = (address: bigint): string | undefined => { + const cached = strings.get(address); + if (cached !== undefined) return cached ?? undefined; + let value: string | undefined; + const remaining = MAXIMUM_STRING_WORK - stringWork; + if (remaining <= 0) + throw new BinaryResourceFailure( + "search", + MAXIMUM_STRING_WORK, + "JNI string decoding exceeds the 64 MiB work allowance", + ); + const allowance = Math.min(MAXIMUM_STRING_BYTES, remaining - 1); + let scanned = false; + try { + mappings.offset(address, 1); + scanned = true; + const raw = mappings.readCString(bytes, address, allowance); + stringWork += raw.length + 1; + if (stringWork > MAXIMUM_STRING_WORK) + throw new BinaryResourceFailure( + "search", + MAXIMUM_STRING_WORK, + "JNI string decoding exceeds the 64 MiB work allowance", + ); + if (overlay.hasWrites(address, raw.length + 1)) { + tablesStatus = "partial"; + unknowns.add( + "Some candidate strings overlap relocation writes; their text is not established by stored bytes", + ); + } else value = decodeJniModifiedUtf8(raw); + } catch (cause) { + if (cause instanceof BinaryMappingFailure) { + if (cause.reason !== "unmapped") throw cause; + stringWork += scanned ? allowance + 1 : 1; + } else if (cause instanceof BinaryStringReadLimitFailure) { + stringWork += allowance + 1; + tablesStatus = "partial"; + unknowns.add( + `Some JNI candidate strings exceeded the ${cause.maximumBytes}-byte NUL-termination read allowance; omitted entries remain unknown`, + ); + } else if (cause instanceof BinaryFormatFailure) { + // Unterminated source bytes also consume their actual scan allowance. + stringWork += allowance + 1; + } else throw cause; + } + if (strings.size >= MAXIMUM_TABLE_ENTRIES) + throw new BinaryResourceFailure( + "structure", + MAXIMUM_RETAINED_BYTES, + "JNI string lookup index exceeds its 65536-entry allowance", + ); + retain(96 + (value?.length ?? 0) * 2); + strings.set(address, value ?? null); + return value; + }; + let scannedBytes = 0, + examinedSections = 0, + examinedSlots = 0, + entries = 0; + if (overlay.pointer_coverage_known) { + if (directory.sections.length === 0) { + tablesStatus = "unknown"; + unknowns.add( + "ELF section inventory is unavailable; writable-section table coverage is unknown", + ); + } + for (const section of directory.sections) { + if ((section.flags & 3n) !== 3n || section.size === 0n) continue; + active(); + if ( + (section.flags & 0x800n) !== 0n || + !section.file_backed || + section.type === 8 + ) { + tablesStatus = "partial"; + unknowns.add( + "Allocated writable sections without ordinary file bytes were not scanned; runtime-built tables remain unknown", + ); + continue; + } + if (section.size > BigInt(MAXIMUM_SCAN_BYTES - scannedBytes)) + throw new BinaryResourceFailure( + "search", + MAXIMUM_SCAN_BYTES, + "JNI writable-section scan exceeds the 64 MiB work allowance", + ); + try { + mappings.offset(section.address, Number(section.size)); + } catch (cause) { + if ( + cause instanceof BinaryMappingFailure && + cause.reason === "unmapped" + ) { + tablesStatus = "partial"; + unknowns.add( + "Some allocated writable section bytes are not file-backed in PT_LOAD; table coverage there is unknown", + ); + continue; + } + throw cause; + } + scannedBytes += Number(section.size); + examinedSections++; + let table: AndroidJniLibraryFacts["tables"][number] | undefined; + let previousEntry: bigint | undefined; + // C pointers have word alignment in virtual memory, including unnamed sections. + const padding = + (BigInt(word) - (section.address % BigInt(word))) % BigInt(word); + for ( + let offset = padding; + offset + BigInt(3 * word) <= section.size; + offset += BigInt(word) + ) { + if (examinedSlots % 4096 === 0) { + await immediate(); + active(); + } + examinedSlots++; + const address = section.address + offset; + const fn = overlay.pointerAt(address + BigInt(2 * word)); + if (fn === undefined || fn.value === 0n) continue; + const code = directory.machine === 40 ? fn.value & ~1n : fn.value; + let codeOffset: number; + try { + codeOffset = executable.offset(code, 1); + } catch (cause) { + if ( + cause instanceof BinaryMappingFailure && + cause.reason === "unmapped" + ) + continue; + throw cause; + } + const name = overlay.pointerAt(address), + signature = overlay.pointerAt(address + BigInt(word)); + if ( + name === undefined || + signature === undefined || + name.value === 0n || + signature.value === 0n + ) + continue; + const decodedName = stringAt(name.value); + if (decodedName === undefined || !methodName(decodedName)) continue; + const descriptor = stringAt(signature.value); + if (descriptor === undefined || !methodDescriptor(descriptor)) + continue; + if (++entries > MAXIMUM_TABLE_ENTRIES) + throw new BinaryResourceFailure( + "structure", + MAXIMUM_RETAINED_BYTES, + "JNI table inventory exceeds its 65536-entry allowance", + ); + retain(1024 + 2 * (decodedName.length + descriptor.length)); + if ( + table === undefined || + previousEntry === undefined || + address !== previousEntry + BigInt(3 * word) + ) { + table = { + section_index: section.index, + section_name: section.name, + table_address: hex(address), + entries: [], + }; + retain(256 + (section.name?.length ?? 0) * 2); + tables.push(table); + } + const slots: AndroidJniTableSlot[] = [name, signature, fn].map( + (pointer, index) => ({ + role: + index === 0 ? "name" : index === 1 ? "signature" : "function", + address: hex(address + BigInt(index * word)), + file_offset: mappings.offset( + address + BigInt(index * word), + word, + ), + value: hex(pointer.value), + relocation: pointer.relocation, + }), + ); + const entry: AndroidJniTableEntry = { + entry_index: table.entries.length, + name: decodedName, + descriptor, + function_address: hex(fn.value), + code_address: hex(code), + function_file_offset: codeOffset, + slots, + }; + table.entries.push(entry); + previousEntry = address; + } + } + } + if (overlay.unresolved.size !== 0 && tablesStatus === "complete") + tablesStatus = "partial"; + const onLoad = rawExports.exports.some((row) => row.name === "JNI_OnLoad"); + if (onLoad && tables.length === 0) + unknowns.add( + "JNI_OnLoad is exported but no static JNINativeMethod candidate was decoded", + ); + unknowns.add( + "Runtime-built tables and obfuscated strings are not recovered; no loading, registration or invocation is established", + ); + active(); + return ok({ + architecture: image.architecture, + bits: directory.bits, + byte_order: image.byte_order, + exports: exportRows, + jni_on_load: onLoad, + tables, + coverage: { + status: + exportsStatus === "complete" && + tablesStatus === "complete" && + relocationStatus === "complete" + ? "complete" + : "partial", + exports_status: exportsStatus, + tables_status: tablesStatus, + relocations_status: relocationStatus, + examined_sections: examinedSections, + examined_pointer_slots: examinedSlots, + }, + unknowns: [...unknowns], + }); + } catch (cause) { + if (cause === options.signal) + return err({ + kind: "cancelled", + message: "JNI library inspection was cancelled", + }); + if ( + cause instanceof BinaryResourceFailure || + (cause instanceof ElfPointerRelocationFailure && + cause.reason === "native-index-capacity") + ) + return err({ kind: "capacity", message: cause.message }); + if (cause instanceof BinaryFormatFailure) + return err({ kind: cause.kind, message: cause.message }); + if (cause instanceof ElfPointerRelocationFailure) + return err({ + kind: + cause.reason === "malformed-container" ? "malformed" : "unsupported", + message: cause.message, + }); + throw cause; + } +}; diff --git a/src/native/exports/NativeExportParser.test.ts b/src/native/exports/NativeExportParser.test.ts index 8be99f8e6f..7ee2d2d535 100644 --- a/src/native/exports/NativeExportParser.test.ts +++ b/src/native/exports/NativeExportParser.test.ts @@ -1,19 +1,65 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { elfExportImage, + elfAliasedVersionImage, peExportImage, } from "../../../tests/fixtures/binaryDiagnostics/nativeExports.js"; -import { nativeExportsSchema } from "../../domain/native/nativeExports.js"; +import * as nativeExportContracts from "../../domain/native/nativeExports.js"; import { BinaryFormatFailure, BinaryResourceFailure, } from "../metadata/BinaryContainer.js"; -import { parseNativeExports } from "./NativeExportParser.js"; +import { + parseNativeExports, + parseNativeExportsSteps, +} from "./NativeExportParser.js"; const artifact = { path: "/fixture/lib", sha256: "a".repeat(64), bytes: 1 }; const parse = (bytes: Buffer) => - nativeExportsSchema.parse(parseNativeExports(bytes, artifact)); + nativeExportContracts.nativeExportsSchema.parse( + parseNativeExports(bytes, artifact), + ); + +it.each(["requirements", "definitions"] as const)( + "bounds aliased GNU version %s work and keeps exports with unknown versions", + (kind) => { + const bytes = elfAliasedVersionImage(kind, 65_535); + const decoded = vi.spyOn(nativeExportContracts, "decodeElfExportName"); + try { + const steps = parseNativeExportsSteps(bytes, artifact); + let checkpoint = steps.next(), + yields = 0; + while (!checkpoint.done) { + yields++; + checkpoint = steps.next(); + } + const result = nativeExportContracts.nativeExportsSchema.parse( + checkpoint.value, + ); + expect(yields).toBeGreaterThan(2); + expect(result.coverage.status).toBe("partial"); + expect(result.exports).toMatchObject([ + { name: "versioned_export", version: { index: 2, name: null } }, + ]); + expect(result.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "version_unresolved", + message: expect.stringContaining("traversal work allowance"), + }), + ]), + ); + // The cost oracle observes the shared decoding boundary: a long aliased + // name must not be decoded for every visit despite tiny retained output. + expect( + decoded.mock.calls.filter(([name]) => name.length === 65_535), + ).toHaveLength(1); + } finally { + decoded.mockRestore(); + } + }, +); const peExportNameAcrossSections = (overlapAtRva?: number): Buffer => { const bytes = Buffer.concat([ diff --git a/src/native/exports/NativeExportParser.ts b/src/native/exports/NativeExportParser.ts index bf18d3cf8c..12d49aaf28 100644 --- a/src/native/exports/NativeExportParser.ts +++ b/src/native/exports/NativeExportParser.ts @@ -314,18 +314,80 @@ const versionSectionData = ( /** Version names by index, read from `.gnu.version_d` and `.gnu.version_r`. */ type DecodedElfString = ReturnType; -const readVersionNames = ( +class ElfVersionWorkFailure extends BinaryResourceFailure {} + +/** Charge both headers and auxiliary visits, including aliased record replays. */ +const versionRecordStep = (size: number, width: number): (() => boolean) => { + const allowance = Math.min(Math.floor(size / width), MAX_NATIVE_EXPORT_ROWS); + let visited = 0; + return () => { + if (++visited > allowance) + throw new ElfVersionWorkFailure( + "structure", + size, + `ELF version traversal work allowance of ${String(allowance)} record visits was exhausted; remaining version names are unknown`, + ); + return visited % 1024 === 0; + }; +}; + +/** One format owner decodes each selected string-table offset at most once. */ +const createVersionNameReader = () => { + const decodedNames = new Map< + string, + | { readonly value: DecodedElfString | null } + | { readonly failure: BinaryFormatFailure } + >(); + let cacheBytes = 0; + const chargeCache = (bytes: number): void => { + if (cacheBytes + bytes > MAXIMUM_RETAINED_TEXT_BYTES) + throw new ElfVersionWorkFailure( + "structure", + MAXIMUM_RETAINED_TEXT_BYTES, + "ELF version name decoding exceeds its 16 MiB cache allowance; remaining version names are unknown", + ); + cacheBytes += bytes; + }; + const readName = (names: Buffer, offset: number): DecodedElfString | null => { + const key = `${String(names.byteOffset)}:${String(names.length)}:${String(offset)}`; + const cached = decodedNames.get(key); + if (cached !== undefined) { + if ("failure" in cached) throw cached.failure; + return cached.value; + } + let raw: Buffer; + try { + raw = readElfStringBytes(names, offset); + } catch (cause) { + if (!(cause instanceof BinaryFormatFailure)) throw cause; + // Invalid offsets/unterminated aliases must not repeatedly scan strings + // either. Account for the cached error and its stack conservatively. + chargeCache(4096); + decodedNames.set(key, { failure: cause }); + throw cause; + } + chargeCache(512 + raw.length * 6); + const value = raw.length === 0 ? null : decodeElfExportName(raw); + decodedNames.set(key, { value }); + return value; + }; + return readName; +}; + +function* readVersionNames( reader: BinaryReader, sections: readonly NativeSection[], issues: NativeExportIssue[], -): Map => { +): Generator> { const namesByIndex = new Map(); + const readName = createVersionNameReader(); // This intermediate map owns its storage independently of repeated text in // output rows. Replacing an index releases its previous decoded name. let textBytes = 0; - const addName = (index: number, name: Buffer): void => { - const decoded = name.length === 0 ? null : decodeElfExportName(name); + const addName = (index: number, decoded: DecodedElfString | null): void => { const previous = namesByIndex.get(index); + // Replayed offsets reuse the exact parsed fact, including its accounted text. + if (previous === decoded) return; const nextTextBytes = textBytes - retainedTextBytes( @@ -349,6 +411,7 @@ const readVersionNames = ( verneed, "Version requirements", ); + const visit = versionRecordStep(table.size, 16); let offset = 0; for (let record = 0; record < verneed.info; record++) { if (offset + 16 > table.size) @@ -356,6 +419,7 @@ const readVersionNames = ( "malformed", "Version requirement chain leaves its section", ); + if (visit()) yield; const at = table.offset + offset; const auxiliaryCount = reader.u16(at + 2); const auxiliaryOffset = reader.u32(at + 8); @@ -372,15 +436,14 @@ const readVersionNames = ( "malformed", "Version requirement auxiliary chain leaves its section", ); + if (visit()) yield; const auxAt = table.offset + auxiliary; const versionIndex = reader.u16(auxAt + 6) & 0x7fff; const auxiliaryNext = reader.u32(auxAt + 12); try { - addName( - versionIndex, - readElfStringBytes(names, reader.u32(auxAt + 8)), - ); + addName(versionIndex, readName(names, reader.u32(auxAt + 8))); } catch (cause) { + if (cause instanceof ElfVersionWorkFailure) throw cause; issues.push({ code: "version_unresolved", location: hex(verneed.offset + BigInt(auxiliary)), @@ -406,10 +469,13 @@ const readVersionNames = ( } } } catch (cause) { + // Unvisited records may redefine an index seen in the prefix. Do not + // promote an interrupted name lookup to a complete version identity. + if (cause instanceof ElfVersionWorkFailure) namesByIndex.clear(); issues.push({ code: "version_unresolved", location: hex(verneed.offset), - message: `Version requirements: ${failureMessage(cause)}`, + message: `Version requirements: ${cause instanceof ElfVersionWorkFailure ? cause.message : failureMessage(cause)}`, }); } } @@ -423,6 +489,7 @@ const readVersionNames = ( verdef, "Version definitions", ); + const visit = versionRecordStep(table.size, 8); let offset = 0; for (let index = 0; index < verdef.info; index++) { if (offset + 20 > table.size) @@ -430,6 +497,7 @@ const readVersionNames = ( "malformed", "Version definition chain leaves its section", ); + if (visit()) yield; const at = table.offset + offset; const versionIndex = reader.u16(at + 4) & 0x7fff; const aux = reader.u32(at + 12); @@ -441,7 +509,7 @@ const readVersionNames = ( "Version definition name entry leaves its section", ); let auxiliary = offset + aux; - const versionName = readElfStringBytes( + const versionName = readName( names, reader.u32(table.offset + auxiliary), ); @@ -452,6 +520,7 @@ const readVersionNames = ( "malformed", "Version definition auxiliary chain leaves its section", ); + if (visit()) yield; if (item + 1 < auxiliaryCount) { const auxiliaryNext = reader.u32(table.offset + auxiliary + 4); if (auxiliaryNext < 8) @@ -472,19 +541,23 @@ const readVersionNames = ( } } } catch (cause) { + if (cause instanceof ElfVersionWorkFailure) namesByIndex.clear(); issues.push({ code: "version_unresolved", location: hex(verdef.offset), - message: `Version definitions: ${failureMessage(cause)}`, + message: `Version definitions: ${cause instanceof ElfVersionWorkFailure ? cause.message : failureMessage(cause)}`, }); } } return namesByIndex; -}; +} /** Read defined exported entries of `.dynsym` with their symbol versions. */ -const parseElfExports = (bytes: Buffer, artifact: Artifact): NativeExports => { +function* parseElfExports( + bytes: Buffer, + artifact: Artifact, +): Generator { const directory = readElfDirectoryFromBuffer(bytes); const reader = new BinaryReader(bytes, directory.little); const result = { @@ -578,11 +651,12 @@ const parseElfExports = (bytes: Buffer, artifact: Artifact): NativeExports => { }); } } - const versionNames = readVersionNames(reader, sections, issues); + const versionNames = yield* readVersionNames(reader, sections, issues); const unresolvedVersions = new Set(); const retain = createRetention(); const exports: ElfExport[] = []; for (let index = 1; index < count; index++) { + if (index % 1024 === 0) yield; const at = table + index * width; const parsedSymbol = readElfSymbol( bytes.subarray(at, at + width), @@ -678,19 +752,31 @@ const parseElfExports = (bytes: Buffer, artifact: Artifact): NativeExports => { issues, limitations: ELF_LIMITATIONS, }; -}; +} -/** Static export inventory of one PE or ELF image; other formats are unsupported. */ -export const parseNativeExports = ( +/** Shared bounded checkpoints; consumers may yield/cancel without another parser. */ +export function* parseNativeExportsSteps( bytes: Buffer, artifact: Artifact, -): NativeExports => { +): Generator { + yield; if (bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) - return parseElfExports(bytes, artifact); + return yield* parseElfExports(bytes, artifact); if (bytes.subarray(0, 2).equals(Buffer.from("MZ"))) return parsePeExports(bytes, artifact); throw new BinaryFormatFailure( "unsupported", "Only PE and ELF images are supported; inspect Mach-O exports with inspect_macho", ); +} + +/** Synchronous callers use exactly the same parsed facts and resource bounds. */ +export const parseNativeExports = ( + bytes: Buffer, + artifact: Artifact, +): NativeExports => { + const steps = parseNativeExportsSteps(bytes, artifact); + let step = steps.next(); + while (!step.done) step = steps.next(); + return step.value; }; diff --git a/src/native/metadata/BinaryContainer.test.ts b/src/native/metadata/BinaryContainer.test.ts index 8c32250893..ab6f8a98a5 100644 --- a/src/native/metadata/BinaryContainer.test.ts +++ b/src/native/metadata/BinaryContainer.test.ts @@ -3,6 +3,7 @@ import { expect, it } from "vitest"; import { BinaryFormatFailure, BinaryMappingFailure, + BinaryStringReadLimitFailure, createFileMappingIndex, mappedFileOffset, readMappedCString, @@ -11,6 +12,24 @@ import { type ZeroFillMapping, } from "./BinaryContainer.js"; +it("distinguishes a mapped string read allowance from malformed bytes", () => { + const bytes = Buffer.from("long-name\0"); + const index = createFileMappingIndex([ + { address: 100n, offset: 0, size: bytes.length }, + ]); + expect(() => index.readCString(bytes, 100n, 3)).toThrow( + BinaryStringReadLimitFailure, + ); + try { + index.readCString(bytes, 100n, 3); + } catch (cause) { + expect(cause).toMatchObject({ maximumBytes: 3 }); + } + expect(index.readCString(bytes, 100n, bytes.length).toString()).toBe( + "long-name", + ); +}); + const outcome = (translate: () => number): number | string => { try { return translate(); diff --git a/src/native/metadata/BinaryContainer.ts b/src/native/metadata/BinaryContainer.ts index c076a826cb..cc24e1b556 100644 --- a/src/native/metadata/BinaryContainer.ts +++ b/src/native/metadata/BinaryContainer.ts @@ -33,6 +33,18 @@ export class BinaryResourceFailure extends Error { } } +/** A bounded read could not establish termination; it does not prove invalid text. */ +export class BinaryStringReadLimitFailure extends BinaryFormatFailure { + constructor(readonly maximumBytes: number) { + // Preserve the existing broad failure contract for legacy callers while + // allowing coverage-aware consumers to distinguish a limit from corruption. + super( + "malformed", + `String is unterminated or longer than ${String(maximumBytes)} bytes`, + ); + } +} + /** One file-backed virtual range; addresses stay exact until mapped to file offsets. */ export interface FileMapping { readonly address: bigint; @@ -577,10 +589,7 @@ const readMappedString = ( chunks.push(chunk); length += size; } - throw new BinaryFormatFailure( - "malformed", - `String is unterminated or longer than ${String(maximumBytes)} bytes`, - ); + throw new BinaryStringReadLimitFailure(maximumBytes); }; /** Recognize a wholly zero-filled span while refusing even partial file aliases. */ diff --git a/src/native/metadata/ElfPointerRelocations.ts b/src/native/metadata/ElfPointerRelocations.ts new file mode 100644 index 0000000000..a59470dbf1 --- /dev/null +++ b/src/native/metadata/ElfPointerRelocations.ts @@ -0,0 +1,599 @@ +import { readElfDynamicEntry } from "../../domain/elfDynamicEntry.js"; +import { readElfRelocationEntry } from "../../domain/elfRelocation.js"; +import { + BinaryMappingFailure, + createFileMappingIndex, + isZeroFilledSpan, + type ElfDynamicSegment, + type FileMapping, + type ZeroFillMapping, +} from "./BinaryContainer.js"; + +/** Shared loader interpretation; language adapters choose their public errors. */ +export class ElfPointerRelocationFailure extends Error { + constructor( + readonly reason: + | "malformed-container" + | "unsupported-container" + | "unsupported-pointer-encoding" + | "native-index-capacity", + message: string, + ) { + super(message); + } +} +export interface ElfPointerRelocationSource { + readonly encoding: "rel" | "rela" | "relr"; + readonly record_file_offset: number; + readonly kind: number | null; +} +export interface ElfPointerWrite { + readonly address: bigint; + readonly width: number; + /** null: a symbolic/indirect relocation whose value is not established. */ + readonly value: bigint | null; + readonly file_offset: number | null; + readonly source: ElfPointerRelocationSource; +} +export interface ElfOverlayPointer { + readonly value: bigint; + readonly file_offset: number | null; + readonly relocation: ElfPointerRelocationSource | null; +} +export interface ElfPointerOverlay { + readonly writes: readonly ElfPointerWrite[]; + readonly unresolved: ReadonlyMap; + readonly issues: readonly string[]; + readonly dynamic_present: boolean; + readonly pointer_coverage_known: boolean; + readonly hasWrites: (address: bigint, size: number) => boolean; + /** Unknown/partially overwritten words are undefined, never their stored zero. */ + readonly pointerAt: (address: bigint) => ElfOverlayPointer | undefined; +} +export interface ElfPointerRelocationInput { + readonly bits: 32 | 64; + readonly little: boolean; + readonly machine: number; + readonly dynamicSegments: readonly ElfDynamicSegment[]; + readonly mappings: readonly FileMapping[]; + readonly zeroFills: readonly ZeroFillMapping[]; +} + +// JNI sparse consumers retain per-write provenance; streaming materializers do not. +const MAX_RELOCATION_RECORDS = 65_536; +const DT_RELA = 7n, + DT_RELASZ = 8n, + DT_RELAENT = 9n; +const DT_REL = 17n, + DT_RELSZ = 18n, + DT_RELENT = 19n; +const DT_PLTRELSZ = 2n, + DT_PLTREL = 20n, + DT_JMPREL = 23n; +const DT_RELRSZ = 35n, + DT_RELR = 36n, + DT_RELRENT = 37n; +const PACKED_TAGS = [0x6000000fn, 0x60000010n, 0x60000011n, 0x60000012n]; +const ANDROID_RELR_TAGS = [0x6fffe000n, 0x6fffe001n, 0x6fffe003n]; +const UNIQUE_TAGS = new Set([ + DT_RELA, + DT_RELASZ, + DT_RELAENT, + DT_REL, + DT_RELSZ, + DT_RELENT, + DT_RELR, + DT_RELRSZ, + DT_RELRENT, + DT_JMPREL, + DT_PLTRELSZ, + DT_PLTREL, + ...PACKED_TAGS, + ...ANDROID_RELR_TAGS, +]); +const malformed = (message: string): never => { + throw new ElfPointerRelocationFailure("malformed-container", message); +}; +const RELOCATION_KINDS = new Map([ + [ + 183, + { + bits: 64, + relative: 1027, + widths: new Map([ + [257, 8], + [258, 4], + [259, 2], + [260, 8], + [261, 4], + [262, 2], + [1025, 8], + [1026, 8], + [1028, 8], + [1029, 8], + [1030, 8], + [1032, 8], + ]), + }, + ], + [ + 62, + { + bits: 64, + relative: 8, + widths: new Map([ + [1, 8], + [2, 4], + [6, 8], + [7, 8], + [10, 4], + [11, 4], + [16, 8], + [17, 8], + [18, 8], + [24, 8], + [37, 8], + ]), + }, + ], + [ + 40, + { + bits: 32, + relative: 23, + widths: new Map([ + [2, 4], + [3, 4], + [17, 4], + [18, 4], + [19, 4], + [21, 4], + [22, 4], + [160, 4], + ]), + }, + ], + [ + 3, + { + bits: 32, + relative: 8, + widths: new Map([ + [1, 4], + [2, 4], + [3, 4], + [4, 4], + [6, 4], + [7, 4], + [9, 4], + [10, 4], + [11, 4], + [14, 4], + [15, 4], + [16, 4], + [17, 4], + [18, 4], + [19, 4], + [20, 2], + [21, 2], + [22, 1], + [23, 1], + [35, 4], + [36, 4], + [37, 4], + [38, 4], + [42, 4], + [43, 4], + ]), + }, + ], +]); + +function* dynamicTags( + bytes: Buffer, + input: ElfPointerRelocationInput, +): Generator> { + yield; + if (input.dynamicSegments.length > 1) + malformed("multiple PT_DYNAMIC segments"); + const declaration = input.dynamicSegments[0]; + if (declaration === undefined) return new Map(); + const width = input.bits / 4; + if (declaration.size > 16n * 1024n * 1024n) + throw new ElfPointerRelocationFailure( + "native-index-capacity", + "ELF dynamic directory exceeds the 16 MiB structural allowance", + ); + if ( + declaration.size % BigInt(width) !== 0n || + declaration.offset > BigInt(bytes.length) || + declaration.size > BigInt(bytes.length) - declaration.offset + ) + malformed("PT_DYNAMIC is truncated or outside the file"); + const tags = new Map(); + const start = Number(declaration.offset), + end = start + Number(declaration.size); + for (let at = start; at < end; at += width) { + if ((at - start) % 65_536 === 0) yield; + const parsed = readElfDynamicEntry( + bytes.subarray(at, at + width), + input.bits, + input.little, + ); + if (!parsed.ok) return malformed(parsed.error.message); + const { tag, value } = parsed.value; + if (tag === 0n) return tags; + if (UNIQUE_TAGS.has(tag) && tags.has(tag)) + malformed(`duplicate dynamic tag ${tag}`); + if (UNIQUE_TAGS.has(tag)) tags.set(tag, value); + } + return malformed("PT_DYNAMIC has no DT_NULL terminator"); +} + +export interface ElfRelocationStreamResult { + readonly issues: readonly string[]; + readonly dynamic_present: boolean; + readonly pointer_coverage_known: boolean; + readonly originalPointerAt: ( + address: bigint, + ) => ElfOverlayPointer | undefined; +} +type RelocationOptions = { + readonly unsupported?: "reject" | "record"; + readonly requireDynamic?: boolean; +}; + +/** Decode once from immutable bytes; consumers choose streaming or sparse retention. */ +export function* readElfRelocationWritesSteps( + bytes: Buffer, + input: ElfPointerRelocationInput, + options: RelocationOptions = {}, +): Generator { + const kinds = RELOCATION_KINDS.get(input.machine); + if (kinds === undefined || kinds.bits !== input.bits) + throw new ElfPointerRelocationFailure( + "unsupported-container", + "ELF relative pointers require a matching AArch64, x86-64, ARM or i386 image", + ); + const relative = kinds.relative; + const unknownWidths = kinds.widths; + if (options.requireDynamic === true && input.dynamicSegments.length === 0) + throw new ElfPointerRelocationFailure( + "unsupported-container", + "ELF image has no PT_DYNAMIC segment", + ); + const tags = yield* dynamicTags(bytes, input); + const word = input.bits / 8; + const index = createFileMappingIndex(input.mappings, input.zeroFills); + const fileOffset = (address: bigint, width: number): number | null => { + try { + return index.offset(address, width); + } catch (cause) { + if (cause instanceof BinaryMappingFailure && cause.reason === "unmapped") + return null; + throw cause; + } + }; + const rawWord = (offset: number): bigint => + input.bits === 64 + ? input.little + ? bytes.readBigUInt64LE(offset) + : bytes.readBigUInt64BE(offset) + : BigInt( + input.little + ? bytes.readUInt32LE(offset) + : bytes.readUInt32BE(offset), + ); + const target = (address: bigint, width: number): number | null => { + const at = fileOffset(address, width); + if ( + at === null && + !isZeroFilledSpan(input.mappings, input.zeroFills, address, width) + ) + malformed( + "RELATIVE relocation target is outside unambiguous loaded memory", + ); + return at; + }; + const issues = new Set(); + let known = true; + const unsupported = (message: string) => { + if (options.unsupported !== "record") + throw new ElfPointerRelocationFailure( + "unsupported-pointer-encoding", + message, + ); + if (issues.size < 16) issues.add(message); + else + issues.add( + "Additional unsupported relocation encodings were not individually listed", + ); + known = false; + }; + const write = ( + address: bigint, + width: number, + value: bigint | null, + source: ElfPointerRelocationSource, + ): ElfPointerWrite => ({ + address, + width, + value, + file_offset: + value === null ? fileOffset(address, width) : target(address, width), + source, + }); + if (PACKED_TAGS.some((tag) => tags.has(tag))) + unsupported( + "Android packed APS2 relocation write locations and addends are not decoded", + ); + if (ANDROID_RELR_TAGS.some((tag) => tags.has(tag))) + unsupported("Android-specific RELR dynamic tags are not decoded"); + + function* standardTable( + address: bigint, + size: bigint | undefined, + entry: bigint, + encoding: "rel" | "rela", + ): Generator { + const width = word * (encoding === "rela" ? 3 : 2); + if ( + size === undefined || + size < 0n || + entry !== BigInt(width) || + size % entry !== 0n || + size > BigInt(bytes.length) + ) + return malformed( + `invalid DT_${encoding.toUpperCase()}SZ or DT_${encoding.toUpperCase()}ENT`, + ); + if (size === 0n) return; + const start = fileOffset(address, Number(size)); + if (start === null) + return malformed(`${encoding.toUpperCase()} table is not file-backed`); + for (let at = start; at < start + Number(size); at += width) { + if ((at - start) % (width * 4096) === 0) yield; + const parsed = + encoding === "rela" + ? readElfRelocationEntry( + bytes.subarray(at, at + width), + input.bits, + input.little, + "rela", + ) + : readElfRelocationEntry( + bytes.subarray(at, at + width), + input.bits, + input.little, + "rel", + ); + if (!parsed.ok) return malformed(parsed.error.message); + const { type: kind, offset: destination, symbolIndex } = parsed.value; + if (kind === 0) continue; + const source = { encoding, record_file_offset: at, kind }; + if (kind !== relative || symbolIndex !== 0) { + const width = unknownWidths.get(kind); + if (width === undefined) { + unsupported(`ELF relocation kind ${kind} is not decoded`); + continue; + } + yield write(destination, width, null, source); + continue; + } + const slot = target(destination, word); + const explicitAddend = + "addend" in parsed.value && typeof parsed.value.addend === "bigint" + ? parsed.value.addend + : undefined; + if (encoding === "rela" && explicitAddend === undefined) + return malformed("RELA relocation has no decoded addend"); + const addend = explicitAddend ?? (slot === null ? 0n : rawWord(slot)); + yield write( + destination, + word, + BigInt.asUintN(input.bits, addend), + source, + ); + } + } + for (const [addressTag, sizeTag, entryTag, encoding] of [ + [DT_RELA, DT_RELASZ, DT_RELAENT, "rela"], + [DT_REL, DT_RELSZ, DT_RELENT, "rel"], + ] as const) { + const address = tags.get(addressTag); + if (address !== undefined) + yield* standardTable( + address, + tags.get(sizeTag), + tags.get(entryTag) ?? BigInt(word * (encoding === "rela" ? 3 : 2)), + encoding, + ); + else if ((tags.get(sizeTag) ?? 0n) !== 0n) + malformed( + `DT_${encoding.toUpperCase()}SZ has no DT_${encoding.toUpperCase()}`, + ); + } + const plt = tags.get(DT_JMPREL); + if (plt !== undefined) { + const encoding = + tags.get(DT_PLTREL) === DT_RELA + ? "rela" + : tags.get(DT_PLTREL) === DT_REL + ? "rel" + : undefined; + if (encoding === undefined) + unsupported("PLT relocations require a declared REL or RELA encoding"); + else + yield* standardTable( + plt, + tags.get(DT_PLTRELSZ), + BigInt(word * (encoding === "rela" ? 3 : 2)), + encoding, + ); + } else if ((tags.get(DT_PLTRELSZ) ?? 0n) !== 0n) + malformed("DT_PLTRELSZ has no DT_JMPREL"); + + const relr = tags.get(DT_RELR), + relrSize = tags.get(DT_RELRSZ) ?? 0n; + if (relr === undefined) { + if (relrSize !== 0n) malformed("DT_RELRSZ has no DT_RELR"); + } else { + if ( + relrSize % BigInt(word) !== 0n || + relrSize > BigInt(bytes.length) || + (tags.get(DT_RELRENT) ?? BigInt(word)) !== BigInt(word) + ) + malformed("invalid DT_RELRSZ or DT_RELRENT"); + const start = relrSize === 0n ? 0 : fileOffset(relr, Number(relrSize)); + if (start === null) return malformed("RELR table is not file-backed"); + let next: bigint | undefined; + for (let at = start; at < start + Number(relrSize); at += word) { + if ((at - start) % (word * 4096) === 0) yield; + const entry = rawWord(at); + const add = (address: bigint) => { + if (address > (1n << BigInt(input.bits)) - BigInt(word)) + malformed("RELR destination overflows the ELF address space"); + const slot = target(address, word); + return write(address, word, slot === null ? 0n : rawWord(slot), { + encoding: "relr", + record_file_offset: at, + kind: null, + }); + }; + if ((entry & 1n) === 0n) { + yield add(entry); + next = entry + BigInt(word); + } else { + if (next === undefined) + return malformed("RELR bitmap has no preceding address"); + for (let bit = 1; bit < input.bits; bit++) + if ((entry & (1n << BigInt(bit))) !== 0n) + yield add(next + BigInt((bit - 1) * word)); + next += BigInt((input.bits - 1) * word); + if (next > 1n << BigInt(input.bits)) + malformed("RELR bitmap overflows the ELF address space"); + } + } + } + + return { + issues: [...issues], + dynamic_present: input.dynamicSegments.length !== 0, + pointer_coverage_known: known, + originalPointerAt: (address) => { + const at = fileOffset(address, word); + return at === null + ? undefined + : { value: rawWord(at), file_offset: at, relocation: null }; + }, + }; +} + +/** Retain bounded sparse pointer facts for consumers that query the original image. */ +export function* readElfPointerRelocationsSteps( + bytes: Buffer, + input: ElfPointerRelocationInput, + options: RelocationOptions = {}, +): Generator { + const word = input.bits / 8; + const writes: ElfPointerWrite[] = []; + const unresolved = new Map(); + const values = new Map(); + const widths = new Map(); + const steps = readElfRelocationWritesSteps(bytes, input, options); + let next = steps.next(); + while (!next.done) { + const write = next.value; + if (write === undefined) yield; + else { + if (writes.length >= MAX_RELOCATION_RECORDS) + throw new ElfPointerRelocationFailure( + "native-index-capacity", + `ELF sparse relocation index exceeds ${MAX_RELOCATION_RECORDS} retained writes`, + ); + writes.push(write); + const { address, width, value } = write; + widths.set(address, Math.max(widths.get(address) ?? 0, width)); + if (value === null) + unresolved.set(address, Math.max(unresolved.get(address) ?? 0, width)); + else { + const previous = values.get(address); + if (previous !== undefined && previous.value !== value) + unresolved.set( + address, + Math.max(unresolved.get(address) ?? 0, width), + ); + values.set(address, write); + } + } + next = steps.next(); + } + const { + issues, + dynamic_present, + pointer_coverage_known: known, + originalPointerAt, + } = next.value; + // Index one span per destination. Reads cannot mistake bytes changed by a + // partial or unresolved relocation for an original valid pointer. + const ranges = [...widths] + .map(([address, width]) => ({ address, end: address + BigInt(width) })) + .sort((a, b) => + a.address < b.address ? -1 : a.address > b.address ? 1 : 0, + ); + const reach: bigint[] = []; + for (const range of ranges) + reach.push( + range.end > (reach.at(-1) ?? 0n) ? range.end : (reach.at(-1) ?? 0n), + ); + const pointerAt = (address: bigint): ElfOverlayPointer | undefined => { + if (!known) return undefined; + const end = address + BigInt(word); + let low = 0, + high = ranges.length; + while (low < high) { + const mid = (low + high) >>> 1; + if ((ranges[mid]?.address ?? end) < end) low = mid + 1; + else high = mid; + } + for (let at = low - 1; at >= 0 && (reach[at] ?? 0n) > address; at--) { + const range = ranges[at]; + if ( + range !== undefined && + range.end > address && + (range.address !== address || + unresolved.has(range.address) || + range.end !== end) + ) + return undefined; + } + const relocated = values.get(address); + if (relocated?.value !== undefined && relocated.value !== null) + return { + value: relocated.value, + file_offset: relocated.file_offset, + relocation: relocated.source, + }; + return originalPointerAt(address); + }; + const hasWrites = (address: bigint, size: number): boolean => { + const end = address + BigInt(size); + let low = 0, + high = ranges.length; + while (low < high) { + const mid = (low + high) >>> 1; + if ((ranges[mid]?.address ?? end) < end) low = mid + 1; + else high = mid; + } + return low > 0 && (reach[low - 1] ?? 0n) > address; + }; + return { + writes, + unresolved, + issues, + dynamic_present, + pointer_coverage_known: known, + pointerAt, + hasWrites, + }; +} diff --git a/src/server/registerAndroidTools.ts b/src/server/registerAndroidTools.ts index 2589b1be90..5b29f6e5a5 100644 --- a/src/server/registerAndroidTools.ts +++ b/src/server/registerAndroidTools.ts @@ -3,6 +3,7 @@ import type { EvidenceWriter } from "../application/investigation/InvestigationR import { AndroidAnalysisService } from "../application/android/AndroidAnalysisService.js"; import { inspectAndroidPackage } from "../application/android/AndroidPackageInspection.js"; import { inspectAndroidSymbol } from "../application/android/AndroidSymbolInspection.js"; +import { inspectAndroidJniBindings } from "../application/android/AndroidJniInspection.js"; import type { EvidenceLookup } from "../application/EvidenceInputResolver.js"; import { toolContract } from "../contracts/toolContracts.js"; import type { Logger } from "pino"; @@ -52,6 +53,27 @@ export const registerAndroidTools = ( packageHandler(packageContract), ); const searchContract = toolContract("search_android_classes"); + const jniContract = toolContract("inspect_android_jni_bindings"); + const jniService: ContractToolService<"inspect_android_jni_bindings"> = { + execute: (_operation, input, options) => + inspectAndroidJniBindings( + service, + input, + { recordEvidence, evidenceById }, + options, + ), + }; + server.registerTool( + jniContract.name, + toolRegistrationOptions(jniContract), + createContractToolHandler( + server, + jniService, + logger, + recordEvidence, + withAdmittedAnalysis, + )(jniContract), + ); server.registerTool( searchContract.name, toolRegistrationOptions(searchContract), diff --git a/tests/boundary/android/androidJniLibraries.test.ts b/tests/boundary/android/androidJniLibraries.test.ts new file mode 100644 index 0000000000..be9b28ddb0 --- /dev/null +++ b/tests/boundary/android/androidJniLibraries.test.ts @@ -0,0 +1,519 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { Uint8ArrayReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; +import { expect, it, onTestFinished, vi } from "vitest"; +import { + readAndroidJniLibraries, + ANDROID_JNI_LIBRARY_LIMITS, +} from "../../../src/android/AndroidJniLibraries.js"; +import { JadxInterruption } from "../../../src/android/JadxInterruption.js"; +import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; +import { ZipArtifactReader } from "../../../src/artifacts/ZipArtifactReader.js"; +import { + AnalysisCancelledError, + AnalysisTimeoutError, +} from "../../../src/domain/analysisErrorCore.js"; +import { AnalysisError } from "../../../src/domain/analysisErrorBase.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { ProviderCleanupError } from "../../../src/domain/providerCleanupError.js"; +import type { AndroidJniSelection } from "../../../src/domain/android/androidJniBindings.js"; +import { + androidJniLibraryCheckpointSchema, + type AndroidJniLibraryCheckpoint, +} from "../../../src/domain/android/androidJniBindings.js"; +import { androidPartialObservationSchema } from "../../../src/domain/android/androidPartialObservation.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { jniElfFixture } from "../../fixtures/android/jniElf.js"; + +const OPERATION = "inspect_android_jni_bindings"; +const LIBRARY = "lib/arm64-v8a/libfixture.so"; +const digest = (bytes: Uint8Array) => + createHash("sha256").update(bytes).digest("hex"); + +const fixture = async ( + members: readonly (readonly [string, Uint8Array])[] = [ + [LIBRARY, jniElfFixture().bytes], + ], + rename?: readonly [string, string], +) => { + const directory = await createTestTempDirectory("rea-android-jni-libraries-"); + const snapshot = join(directory, "selected app.apk"); + const resources = new ArtifactResourceScope(); + onTestFinished(async () => { + await resources.close(); + }); + const writer = new ZipWriter(new Uint8ArrayWriter(), { + useWebWorkers: false, + }); + for (const [path, bytes] of members) + await writer.add(path, new Uint8ArrayReader(bytes), { + level: 0, + lastModDate: new Date("2020-01-01T00:00:00Z"), + }); + const bytes = Buffer.from(await writer.close()); + if (rename !== undefined) { + const before = Buffer.from(rename[0]), + after = Buffer.from(rename[1]); + expect(before.length).toBe(after.length); + let changed = 0; + for ( + let at = bytes.indexOf(before); + at >= 0; + at = bytes.indexOf(before, at + after.length) + ) { + after.copy(bytes, at); + changed++; + } + // Only the independently authored local and central member-name fields. + expect(changed).toBe(2); + } + await writeFile(snapshot, bytes); + return { snapshot, apkSha256: digest(bytes), resources }; +}; +type Fixture = Awaited>; +const read = ( + source: Fixture, + selection: AndroidJniSelection = {}, + signal = new AbortController().signal, + interruption = (reason: unknown): AnalysisError => + new AnalysisCancelledError(OPERATION, { cause: reason }), + onObservation: (checkpoint: AndroidJniLibraryCheckpoint) => void = () => {}, +) => + readAndroidJniLibraries({ + ...source, + selection, + signal, + interruption, + onObservation, + }); + +const partial = (source: Fixture, checkpoint: AndroidJniLibraryCheckpoint) => ({ + provider_id: "jadx", + operation: OPERATION, + target: { + selected_path: source.snapshot, + path: source.snapshot, + sha256: source.apkSha256, + format: "apk", + }, + provider_facts: { + jar_sha256: null, + bridge_sha256: null, + bridge_name: "fixture-checkpoint", + bridge_version: "1", + server_name: null, + server_version: null, + engine_version: null, + }, + calls: [], + jni: { selection: {}, native_inventory: null, libraries: checkpoint }, +}); + +it("keeps duplicate library-name occurrences bound to their own opaque ZIP keys", async () => { + const bytes = jniElfFixture().bytes; + const first = "lib/arm64-v8a/libcopy1.so", + second = "lib/arm64-v8a/libcopy2.so"; + const source = await fixture( + [ + ["assets/ignored.so", Buffer.from("an unrelated member")], + [first, bytes], + [second, bytes], + ], + [second, first], + ); + const result = await read(source); + expect(result.inventory).toMatchObject({ + status: "complete", + examined_entries: 3, + selected_libraries: 2, + }); + expect(result.libraries.map((library) => library.path)).toEqual([ + first, + first, + ]); + expect(result.libraries.map((library) => library.source_path)).toEqual([ + first, + first, + ]); + expect(result.libraries.map((library) => library.source_index)).toEqual([ + 1, 2, + ]); + expect( + result.libraries.every( + (library) => library.sha256 === digest(bytes) && library.facts !== null, + ), + ).toBe(true); + expect( + new Set(result.libraries.map((library) => library.source_key)).size, + ).toBe(2); + expect( + new Set(result.libraries.map((library) => library.library_id)).size, + ).toBe(2); + const repeated = await read(source); + expect( + repeated.libraries.map(({ library_id, source_key }) => ({ + library_id, + source_key, + })), + ).toEqual( + result.libraries.map(({ library_id, source_key }) => ({ + library_id, + source_key, + })), + ); + expect(source.resources.hasPending).toBe(false); +}); + +it.each([{ abis: [] }, { library_paths: [] }])( + "applies explicit empty selection %j before opening library bytes", + async (selection) => { + const source = await fixture([ + [LIBRARY, Buffer.from("deliberately malformed ELF")], + ]); + const open = vi.spyOn(ZipArtifactReader.prototype, "open"); + try { + const result = await read(source, selection); + expect(result.libraries).toEqual([]); + expect(result.inventory).toMatchObject({ + status: "complete", + examined_entries: 1, + selected_libraries: 0, + }); + expect(open).not.toHaveBeenCalled(); + } finally { + open.mockRestore(); + } + }, +); + +it("retains a malformed selected library's exact carrier identity and digest", async () => { + const path = "lib/arm64-v8a/lib selected_ๅŽŸ#.so"; + const bytes = Buffer.from("not an ELF image\0caller evidence"); + const source = await fixture([[path, bytes]]); + const result = await read(source); + expect(result.libraries).toHaveLength(1); + expect(result.libraries[0]).toMatchObject({ + path, + source_path: path, + abi: "arm64-v8a", + sha256: digest(bytes), + size: bytes.length, + declared_size: bytes.length, + compressed_size: bytes.length, + facts: null, + }); + expect(result.libraries[0]?.failure).toMatchObject({ + kind: expect.stringMatching(/^(?:malformed|unsupported)$/u), + message: expect.any(String), + }); + expect(source.resources.hasPending).toBe(false); +}); + +it("preserves declared capacity metadata without presenting unread bytes as verified", async () => { + const source = await fixture(); + const originalEntries = ZipArtifactReader.prototype.entries; + const declared = ANDROID_JNI_LIBRARY_LIMITS.libraryBytes + 1; + const entries = vi + .spyOn(ZipArtifactReader.prototype, "entries") + .mockImplementation(async function* (this: ZipArtifactReader, signal) { + for await (const entry of originalEntries.call(this, signal)) + yield { ...entry, declaredSize: declared, compressedSize: 1234 }; + }); + const open = vi.spyOn(ZipArtifactReader.prototype, "open"); + try { + const result = await read(source); + expect(result.libraries[0]).toMatchObject({ + path: LIBRARY, + declared_size: declared, + compressed_size: 1234, + sha256: null, + size: null, + facts: null, + failure: { kind: "capacity" }, + }); + expect(open).not.toHaveBeenCalled(); + } finally { + entries.mockRestore(); + open.mockRestore(); + } +}); + +it("retains the live reader after failed close and gates the next read on its cleanup retry", async () => { + const source = await fixture(); + const originalClose = ZipArtifactReader.prototype.close; + const attempts: ZipArtifactReader[] = []; + let allowClose = false; + const closeCause = new Error("controlled live ZIP reader close failure"); + const close = vi + .spyOn(ZipArtifactReader.prototype, "close") + .mockImplementation(async function (this: ZipArtifactReader) { + attempts.push(this); + if (!allowClose) throw closeCause; + await originalClose.call(this); + }); + const open = vi.spyOn(ZipArtifactReader.prototype, "open"); + const observations: AndroidJniLibraryCheckpoint[] = []; + try { + const first = await read(source, {}, undefined, undefined, (checkpoint) => + observations.push(checkpoint), + ).catch((cause: unknown) => cause); + expect(first).toBeInstanceOf(ProviderCleanupError); + if (!(first instanceof ProviderCleanupError)) throw first; + expect(first.cleanupIncomplete).toBe(true); + expect(first.cleanupResources).toContain(source.snapshot); + expect(source.resources.hasPending).toBe(true); + const checkpoint = observations[0]; + expect(checkpoint).toMatchObject({ + scan_state: "completed", + inventory: { status: "complete", selected_libraries: 1 }, + }); + expect( + androidJniLibraryCheckpointSchema.safeParse(checkpoint).success, + ).toBe(true); + expect(checkpoint?.libraries[0]).toMatchObject({ + path: LIBRARY, + sha256: digest(jniElfFixture().bytes), + size: jniElfFixture().bytes.length, + facts: { tables: expect.any(Array) }, + }); + expect( + checkpoint?.libraries[0]?.facts?.tables.flatMap((table) => table.entries), + ).toHaveLength(3); + if (checkpoint === undefined) + throw new Error("Missing parsed library checkpoint"); + expect( + androidPartialObservationSchema.safeParse(partial(source, checkpoint)) + .success, + ).toBe(true); + expect( + androidPartialObservationSchema.safeParse({ + ...partial(source, checkpoint), + operation: "inspect_android_package", + }).success, + ).toBe(false); + const historical = structuredClone(checkpoint); + const owner = attempts[0]; + if (owner === undefined) throw new Error("No retained ZIP reader owner"); + // Close interception preceded the actual close: verify the real resource is + // still readable, rather than claiming a closed descriptor remains owned. + const retainedPaths: string[] = []; + for await (const entry of owner.entries()) retainedPaths.push(entry.path); + expect(retainedPaths).toEqual([LIBRARY]); + const blocked = await read(source).catch((cause: unknown) => cause); + expect(blocked).toBeInstanceOf(AnalysisError); + if (!(blocked instanceof AnalysisError)) throw blocked; + expect(blocked.cleanupIncomplete).toBe(true); + expect(attempts.length).toBeGreaterThanOrEqual(2); + expect(attempts.every((attempt) => attempt === owner)).toBe(true); + expect(open).toHaveBeenCalledTimes(1); + expect(source.resources.hasPending).toBe(true); + allowClose = true; + const recovered = await read(source); + expect(recovered.libraries[0]?.facts).not.toBeNull(); + expect(open).toHaveBeenCalledTimes(2); + expect(source.resources.hasPending).toBe(false); + expect(first.cleanupIncomplete).toBe(false); + expect(blocked.cleanupIncomplete).toBe(false); + expect(checkpoint).toEqual(historical); + expect(attempts.at(-1)).not.toBe(owner); + const closed = async () => { + for await (const entry of owner.entries()) + throw new Error(`Closed reader unexpectedly yielded ${entry.path}`); + }; + await expect(closed()).rejects.toMatchObject({ reason: "unavailable" }); + } finally { + allowClose = true; + try { + await source.resources.close(); + } finally { + close.mockRestore(); + open.mockRestore(); + } + } +}); + +for (const first of ["caller", "deadline"] as const) + for (const cleanupFails of [false, true]) + it(`preserves ${first}-first interruption during final ZIP close (cleanup fails: ${cleanupFails})`, async () => { + const source = await fixture(); + const caller = new AbortController(), + deadline = new AbortController(); + const callerReason = { caller: "exact selected caller cause" }; + const deadlineReason = { deadline: "exact selected deadline cause" }; + const signal = AbortSignal.any([caller.signal, deadline.signal]); + const reason = first === "caller" ? callerReason : deadlineReason; + const primary = + first === "caller" + ? new AnalysisCancelledError(OPERATION, { cause: reason }) + : new AnalysisTimeoutError(OPERATION, 25, { cause: reason }); + const interruption = vi.fn((observed: unknown) => { + expect(observed).toBe(reason); + return primary; + }); + const originalClose = ZipArtifactReader.prototype.close; + let checkpoint: AndroidJniLibraryCheckpoint | undefined; + let injected = false, + allowClose = !cleanupFails; + const close = vi + .spyOn(ZipArtifactReader.prototype, "close") + .mockImplementation(async function (this: ZipArtifactReader) { + if (!injected) { + injected = true; + if (first === "caller") { + caller.abort(callerReason); + deadline.abort(deadlineReason); + } else { + deadline.abort(deadlineReason); + caller.abort(callerReason); + } + } + if (!allowClose) + throw new Error("controlled finalization close failure"); + await originalClose.call(this); + }); + try { + const failure = await read( + source, + {}, + signal, + interruption, + (observation) => { + checkpoint = observation; + }, + ).catch((cause: unknown) => cause); + const expectedCode = + first === "caller" ? "cancelled" : "provider_timeout"; + expect(checkpoint).toMatchObject({ + scan_state: "completed", + inventory: { status: "complete", selected_libraries: 1 }, + }); + expect(checkpoint?.libraries[0]?.sha256).toBe( + digest(jniElfFixture().bytes), + ); + expect( + checkpoint?.libraries[0]?.facts?.tables.flatMap( + (table) => table.entries, + ), + ).toHaveLength(3); + if (checkpoint === undefined) + throw new Error("Missing final-close checkpoint"); + expect( + androidPartialObservationSchema.safeParse(partial(source, checkpoint)) + .success, + ).toBe(true); + const historical = structuredClone(checkpoint); + if (cleanupFails) { + expect(interruption).toHaveBeenCalledTimes(1); + expect(failure).toBeInstanceOf(AnalysisError); + if (!(failure instanceof AnalysisError)) throw failure; + expect(failure.cleanupPrimary).toBe(primary); + expect(primary.cause).toBe(reason); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "cleanup_incomplete", + details: { execution_failure: expectedCode }, + }); + expect(source.resources.hasPending).toBe(true); + allowClose = true; + await source.resources.retryCleanup(); + expect(source.resources.hasPending).toBe(false); + expect(failure.cleanupIncomplete).toBe(false); + expect(projectAnalysisError(failure).code).toBe(expectedCode); + expect(failure.cleanupPrimary?.cause).toBe(reason); + } else { + expect(failure).toBeInstanceOf(JadxInterruption); + if (!(failure instanceof JadxInterruption)) throw failure; + expect(failure.reason).toBe(reason); + expect(interruption).not.toHaveBeenCalled(); + expect(source.resources.hasPending).toBe(false); + } + expect(checkpoint).toEqual(historical); + } finally { + allowClose = true; + try { + await source.resources.close(); + } finally { + close.mockRestore(); + } + } + }); + +it("keeps completed facts and the interrupted carrier when a later library is cancelled", async () => { + const second = "lib/arm64-v8a/libsecond.so", + third = "lib/arm64-v8a/libunexamined.so"; + const bytes = jniElfFixture().bytes; + const source = await fixture([ + [LIBRARY, bytes], + [second, bytes], + [third, bytes], + ]); + const controller = new AbortController(), + reason = { caller: "stop before second member bytes" }; + const originalOpen = ZipArtifactReader.prototype.open; + const open = vi + .spyOn(ZipArtifactReader.prototype, "open") + .mockImplementation(async function ( + this: ZipArtifactReader, + entry, + signal, + ) { + if (entry.path === second) { + controller.abort(reason); + throw new JadxInterruption(reason); + } + return originalOpen.call(this, entry, signal); + }); + let checkpoint: AndroidJniLibraryCheckpoint | undefined; + try { + const failure = await read( + source, + {}, + controller.signal, + undefined, + (value) => { + checkpoint = value; + }, + ).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(AnalysisCancelledError); + if (!(failure instanceof AnalysisCancelledError)) throw failure; + expect(failure.cause).toBe(reason); + expect(checkpoint).toMatchObject({ + scan_state: "interrupted", + inventory: { + status: "partial", + examined_entries: 2, + selected_libraries: 2, + }, + }); + expect(checkpoint?.libraries.map((library) => library.path)).toEqual([ + LIBRARY, + second, + ]); + expect(checkpoint?.libraries[0]).toMatchObject({ + sha256: digest(bytes), + size: bytes.length, + facts: { tables: expect.any(Array) }, + }); + expect(checkpoint?.libraries[1]).toMatchObject({ + sha256: null, + size: null, + declared_size: bytes.length, + compressed_size: bytes.length, + facts: null, + failure: { kind: "incomplete" }, + }); + expect(checkpoint?.inventory.unknowns.join("\n")).toContain( + "unexamined selected occurrences remain unknown", + ); + if (checkpoint === undefined) + throw new Error("Missing interrupted scan checkpoint"); + expect( + androidPartialObservationSchema.safeParse(partial(source, checkpoint)) + .success, + ).toBe(true); + expect(JSON.parse(JSON.stringify(checkpoint))).toEqual(checkpoint); + expect(source.resources.hasPending).toBe(false); + } finally { + open.mockRestore(); + } +}); diff --git a/tests/boundary/android/androidJniLibrary.test.ts b/tests/boundary/android/androidJniLibrary.test.ts new file mode 100644 index 0000000000..7998e9ddea --- /dev/null +++ b/tests/boundary/android/androidJniLibrary.test.ts @@ -0,0 +1,431 @@ +import { expect, it, vi } from "vitest"; +import { + decodeJniModifiedUtf8, + inspectAndroidJniLibrary, +} from "../../../src/native/android/AndroidJniLibrary.js"; +import { androidJniLibraryFactsSchema } from "../../../src/domain/android/androidJniLibraryFacts.js"; +import { projectAndroidJniBindings } from "../../../src/domain/android/androidJniBindings.js"; +import { jniElfFixture } from "../../fixtures/android/jniElf.js"; +import { elfAliasedVersionImage } from "../../fixtures/binaryDiagnostics/nativeExports.js"; +import * as nativeExportNames from "../../../src/domain/native/nativeExports.js"; + +it.each(["name", "descriptor"] as const)( + "keeps a %s beyond the string read allowance unknown and preserves later valid entries", + async (facet) => { + const original = jniElfFixture(); + const name = facet === "name" ? "m".repeat(65_536) : "call_็Œซ"; + const descriptor = + facet === "descriptor" ? `(L${"a".repeat(65_536)};)V` : "()V"; + const text = facet === "name" ? name : descriptor; + const bytes = Buffer.concat([original.bytes, Buffer.from(text + "\0")]); + bytes.writeBigUInt64LE(BigInt(bytes.length - 0x400), 152); + bytes.writeBigUInt64LE(BigInt(bytes.length - 0x400), 160); + bytes.writeBigUInt64LE( + original.base + BigInt(original.bytes.length), + 0x800 + (facet === "name" ? 0 : 24) + 16, + ); + const parsed = await inspectAndroidJniLibrary(bytes); + if (!parsed.ok) throw new Error(parsed.error.message); + expect(parsed.value.coverage.tables_status).toBe("partial"); + expect(parsed.value.unknowns.join("\n")).toContain( + "NUL-termination read allowance", + ); + expect( + parsed.value.tables + .flatMap((table) => table.entries) + .map((entry) => entry.name), + ).toEqual(["overload", "๐€"]); + const report = projectAndroidJniBindings({ + apk: { path: "/selected/large-native.apk", sha256: "a".repeat(64) }, + selection: {}, + inventory: { + scope: "apk-root-dex-declarations", + methods: [ + { + source_index: 0, + class_name: "fixture.Owner", + method_name: name, + descriptor, + dex_entry: "classes.dex", + dex_entry_index: 0, + class_index: 0, + method_index: 0, + }, + ], + coverage: { + status: "complete", + dex_entries: 1, + examined_dex_entries: 1, + examined_classes: 1, + examined_methods: 1, + }, + unknowns: [], + }, + libraries: [ + { + library_id: "selected-library", + source_index: 0, + source_key: "zip-entry:0", + path: "lib/arm64-v8a/libfixture.so", + source_path: "lib/arm64-v8a/libfixture.so", + abi: "arm64-v8a", + sha256: "b".repeat(64), + size: bytes.length, + declared_size: bytes.length, + compressed_size: bytes.length, + facts: parsed.value, + failure: null, + }, + ], + libraryInventory: { + status: "complete", + examined_entries: 1, + selected_libraries: 1, + unknowns: [], + }, + }); + if (!report.ok) throw new Error(report.error.kind); + expect(report.value.bindings[0]).toMatchObject({ + status: "unknown", + coverage: "partial", + candidate_ids: [], + }); + }, +); + +it.each([ + [64, 183, "rela"], + [64, 62, "rela"], + [32, 40, "rel"], + [32, 3, "rel"], + [64, 183, "relr"], + [64, 62, "relr"], + [32, 40, "relr"], + [32, 3, "relr"], +] as const)( + "decodes %s-bit machine %s %s static tables with complete slot provenance", + async (bits, machine, encoding) => { + const fixture = jniElfFixture({ bits, machine, encoding }); + const original = Buffer.from(fixture.bytes); + const result = await inspectAndroidJniLibrary(fixture.bytes); + expect(result.ok).toBe(true); + if (!result.ok) throw new Error(result.error.message); + expect(androidJniLibraryFactsSchema.safeParse(result.value).success).toBe( + true, + ); + expect(result.value.exports.map((row) => row.name)).toEqual([ + "JNI_OnLoad", + "Java_fixture_Owner_exported", + ]); + expect(result.value.coverage).toMatchObject({ + status: "complete", + relocations_status: "complete", + tables_status: "complete", + }); + expect(result.value.tables.map((table) => table.section_name)).toEqual([ + ".data", + ".data.rel.ro", + ]); + expect( + result.value.tables + .flatMap((table) => table.entries) + .map((row) => [ + row.name, + row.descriptor, + row.function_address, + row.code_address, + row.function_file_offset, + ]), + ).toEqual([ + [ + "call_็Œซ", + "()V", + machine === 40 ? "0x10101" : "0x10100", + "0x10100", + 0x100, + ], + [ + "overload", + "(I)I", + machine === 40 ? "0x10111" : "0x10110", + "0x10110", + 0x110, + ], + [ + "๐€", + "([Ljava/lang/String;)J", + machine === 40 ? "0x10121" : "0x10120", + "0x10120", + 0x120, + ], + ]); + const first = result.value.tables[0]?.entries[0]; + expect( + first?.slots.map((slot) => [ + slot.role, + slot.address, + slot.file_offset, + slot.value, + slot.relocation?.encoding, + ]), + ).toEqual([ + ["name", "0x10500", 0x500, "0x10400", encoding], + [ + "signature", + `0x${(0x10500 + fixture.word).toString(16)}`, + 0x500 + fixture.word, + "0x10420", + encoding, + ], + [ + "function", + `0x${(0x10500 + 2 * fixture.word).toString(16)}`, + 0x500 + fixture.word * 2, + machine === 40 ? "0x10101" : "0x10100", + encoding, + ], + ]); + expect(fixture.bytes.equals(original)).toBe(true); + }, +); + +it("does not limit the writable scan to named or read-only relocation data", async () => { + const result = await inspectAndroidJniLibrary( + jniElfFixture({ unnamedData: true }).bytes, + ); + expect(result.ok && result.value.tables[0]).toMatchObject({ + section_index: 3, + section_name: "", + table_address: "0x10500", + }); +}); + +it.each([32, 64] as const)( + "keeps later valid entries after a %s-bit unresolved function relocation", + async (bits) => { + const result = await inspectAndroidJniLibrary( + jniElfFixture({ bits, symbolicFirstFunction: true }).bytes, + ); + if (!result.ok) throw new Error(result.error.message); + expect(result.value.coverage).toMatchObject({ + status: "partial", + relocations_status: "partial", + tables_status: "partial", + }); + expect( + result.value.tables + .flatMap((table) => table.entries) + .map((row) => row.name), + ).toEqual(["overload", "๐€"]); + }, +); + +it("keeps exports and explicit unknown table coverage for Android packed relocations", async () => { + const result = await inspectAndroidJniLibrary( + jniElfFixture({ packed: true }).bytes, + ); + if (!result.ok) throw new Error(result.error.message); + expect(result.value.exports).toHaveLength(2); + expect(result.value.tables).toEqual([]); + expect(result.value.coverage).toMatchObject({ + exports_status: "complete", + relocations_status: "unknown", + tables_status: "unknown", + }); + expect(result.value.unknowns.join("\n")).toContain("APS2"); +}); + +it("rejects a leading RELR bitmap rather than interpreting it as an address", async () => { + const { bytes } = jniElfFixture({ encoding: "relr" }); + bytes.writeBigUInt64LE(7n, 0x800); + expect(await inspectAndroidJniLibrary(bytes)).toMatchObject({ + ok: false, + error: { + kind: "malformed", + message: "RELR bitmap has no preceding address", + }, + }); +}); + +it("preserves later candidates after malformed Modified UTF-8 and a malformed JNI descriptor", async () => { + const { bytes } = jniElfFixture(); + bytes.set([0xf0, 0x90, 0x90, 0x80, 0], 0x400); + bytes.write("(Lbad.name;)V\0", 0x460, "utf8"); + const result = await inspectAndroidJniLibrary(bytes); + if (!result.ok) throw new Error(result.error.message); + expect( + result.value.tables + .flatMap((table) => table.entries) + .map((row) => row.name), + ).toEqual(["๐€"]); +}); + +it.each([ + [[], ""], + [[0xc0, 0x80], "\0"], + [[0xed, 0xa0, 0x81, 0xed, 0xb0, 0x80], "๐€"], + [[0xed, 0xa0, 0x80], "\ud800"], + [[0], undefined], + [[0xc0, 0x81], undefined], + [[0xc1, 0xbf], undefined], + [[0xe0, 0x80, 0x80], undefined], + [[0xf0, 0x90, 0x90, 0x80], undefined], + [[0xe7, 0x8c], undefined], + [[0x80], undefined], +] as const)("decodes exact Modified UTF-8 bytes %o", (bytes, value) => { + expect(decodeJniModifiedUtf8(Uint8Array.from(bytes))).toBe(value); +}); + +it("returns explicit cancellation before decoding and never runs a target", async () => { + const control = new AbortController(); + control.abort(new Error("selected cancellation")); + expect( + await inspectAndroidJniLibrary(jniElfFixture().bytes, { + signal: control.signal, + }), + ).toMatchObject({ ok: false, error: { kind: "cancelled" } }); +}); + +it.each([32, 64] as const)( + "preserves an observed %s-bit big-endian ELF rather than assigning an Android ABI", + async (bits) => { + const result = await inspectAndroidJniLibrary( + jniElfFixture({ bits, little: false }).bytes, + ); + if (!result.ok) throw new Error(result.error.message); + expect(result.value.byte_order).toBe("big"); + expect(result.value.tables.flatMap((table) => table.entries)).toHaveLength( + 3, + ); + expect(androidJniLibraryFactsSchema.safeParse(result.value).success).toBe( + true, + ); + }, +); + +it("cooperatively preserves cancellation at the first parser checkpoint", async () => { + const controller = new AbortController(); + const operation = inspectAndroidJniLibrary(jniElfFixture().bytes, { + signal: controller.signal, + }); + controller.abort(new Error("cancel at the first scheduled yield")); + expect(await operation).toMatchObject({ + ok: false, + error: { kind: "cancelled" }, + }); +}); + +it("observes cancellation during GNU version work before decoding export symbols", async () => { + const controller = new AbortController(); + const reason = new Error("cancel after version-name decoding began"); + const originalDecode = nativeExportNames.decodeElfExportName; + let enteredVersionWork = false; + const decode = vi + .spyOn(nativeExportNames, "decodeElfExportName") + .mockImplementation((bytes) => { + if (bytes.length === 65_535 && !enteredVersionWork) { + enteredVersionWork = true; + queueMicrotask(() => controller.abort(reason)); + } + return originalDecode(bytes); + }); + try { + const result = await inspectAndroidJniLibrary( + elfAliasedVersionImage("requirements", 65_535), + { signal: controller.signal }, + ); + expect(enteredVersionWork).toBe(true); + expect(controller.signal.reason).toBe(reason); + expect(result).toMatchObject({ ok: false, error: { kind: "cancelled" } }); + expect( + decode.mock.calls.filter(([bytes]) => bytes.length === 65_535), + ).toHaveLength(1); + expect( + decode.mock.calls.some( + ([bytes]) => bytes.toString("utf8") === "versioned_export", + ), + ).toBe(false); + } finally { + decode.mockRestore(); + } +}); + +const appendRelative = ( + bytes: Buffer, + destination: bigint, + value: bigint, + kind = 1027, +) => { + const at = 0x800 + 9 * 24; + bytes.writeBigUInt64LE(destination, at); + bytes.writeBigUInt64LE(BigInt(kind), at + 8); + bytes.writeBigUInt64LE(value, at + 16); + bytes.writeBigUInt64LE(10n * 24n, 0xc18); +}; + +it("keeps conflicting and partially overwritten pointers unknown while retaining later entries", async () => { + for (const conflict of [ + "different-relative-value", + "partial-symbolic-write", + ] as const) { + const { bytes } = jniElfFixture(); + appendRelative( + bytes, + conflict === "different-relative-value" ? 0x10510n : 0x10511n, + 0x10110n, + conflict === "different-relative-value" ? 1027 : 258, + ); + const result = await inspectAndroidJniLibrary(bytes); + if (!result.ok) throw new Error(result.error.message); + expect( + result.value.tables + .flatMap((table) => table.entries) + .map((entry) => entry.name), + ).toEqual(["overload", "๐€"]); + expect(result.value.coverage.relocations_status).toBe("partial"); + } +}); + +it("does not recover a method name from stored bytes changed by loader relocations", async () => { + const { bytes } = jniElfFixture(); + appendRelative(bytes, 0x10400n, 0n); + const result = await inspectAndroidJniLibrary(bytes); + if (!result.ok) throw new Error(result.error.message); + expect( + result.value.tables + .flatMap((table) => table.entries) + .map((entry) => entry.name), + ).toEqual(["overload", "๐€"]); + expect(result.value.coverage.tables_status).toBe("partial"); + expect(result.value.unknowns.join("\n")).toContain( + "candidate strings overlap relocation writes", + ); +}); + +it("reports unavailable section load mappings without hiding later writable sections", async () => { + const { bytes } = jniElfFixture(); + bytes.writeBigUInt64LE(0x900000n, 0x1400 + 3 * 64 + 16); + const result = await inspectAndroidJniLibrary(bytes); + if (!result.ok) throw new Error(result.error.message); + expect( + result.value.tables + .flatMap((table) => table.entries) + .map((entry) => entry.name), + ).toEqual(["๐€"]); + expect(result.value.coverage.tables_status).toBe("partial"); + expect(result.value.unknowns.join("\n")).toContain( + "not file-backed in PT_LOAD", + ); +}); + +it.each(["rel", "rela", "relr"] as const)( + "accepts empty %s tables without demanding a mapping for an empty span", + async (encoding) => { + const { bytes } = jniElfFixture({ encoding }); + bytes.writeBigUInt64LE(0n, 0xc08); + bytes.writeBigUInt64LE(0n, 0xc18); + const result = await inspectAndroidJniLibrary(bytes); + if (!result.ok) throw new Error(result.error.message); + expect(result.value.coverage.relocations_status).toBe("complete"); + }, +); diff --git a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts index c158bc44a0..81c9e97bde 100644 --- a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts +++ b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts @@ -18,6 +18,7 @@ import { } from "../../../src/il2cpp/Il2cppBinaryGenerics.js"; import { createElfFixture, + createLargeElfRelativeFixture, createIl2cppBinaryFixture, createIl2cppDyldBindFixture, createIl2cppRegistrationTailFixture, @@ -906,6 +907,19 @@ it("links an ELF shared object after applying its RELATIVE relocations", () => { expect(link(elf)).toEqual({ ...fromMacho, format: "elf" }); }); +it.each([183, 62])( + "streams more than 65536 file-backed relative writes without shrinking IL2CPP capacity (machine %s)", + (machine) => { + const source = createIl2cppBinaryFixture(); + const bytes = createLargeElfRelativeFixture(source, 65_537, machine); + const snapshot = Buffer.from(bytes); + expect(link(bytes).method_addresses).toEqual( + link(createElfFixture(source, { machine })).method_addresses, + ); + expect(bytes.equals(snapshot)).toBe(true); + }, +); + it.each([183, 62])( "links ELF relocations using the resolved PN_XNUM program count (machine %s)", (machine) => { diff --git a/tests/boundary/mcp/androidPackageSummaryMcp.test.ts b/tests/boundary/mcp/androidPackageSummaryMcp.test.ts index 0b4aba3861..831fbdb6b9 100644 --- a/tests/boundary/mcp/androidPackageSummaryMcp.test.ts +++ b/tests/boundary/mcp/androidPackageSummaryMcp.test.ts @@ -1,8 +1,11 @@ import { unlink } from "node:fs/promises"; +import { join } from "node:path"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { Ajv2020 } from "ajv/dist/2020.js"; import { expect, it, onTestFinished, vi } from "vitest"; import { AndroidAnalysisService } from "../../../src/application/android/AndroidAnalysisService.js"; +import type { AndroidAnalysisPort } from "../../../src/application/android/AndroidAnalysisPort.js"; +import { createAnalysisExecution } from "../../../src/application/AnalysisProvider.js"; import { inspectAndroidPackage } from "../../../src/application/android/AndroidPackageInspection.js"; import { inspectAnalysisView } from "../../../src/application/analysisView/AnalysisViewService.js"; import { androidResultSchemas } from "../../../src/domain/android/androidAnalysis.js"; @@ -11,6 +14,8 @@ import { androidManifestViewResultSchema, androidComponentsViewResultSchema, androidComponentSummaryViewResultSchema, + androidJniSummaryViewResultSchema, + androidJniPageViewResultSchema, analysisViewResultSchema, } from "../../../src/domain/analysisView/analysisView.js"; import { @@ -24,6 +29,8 @@ import { createServer } from "../../../src/server/createServer.js"; import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { writeOrderedZip } from "../../fixtures/artifactEntryOrder.js"; import { createJadxProtocolFixture, verifyJadxFixtureCleanup, @@ -68,6 +75,203 @@ async function assertAndroidPackageSelectedFacetContract(options: { } } +const createJniSummaryContractFixture = async () => { + const root = await createTestTempDirectory("rea-jni-default-summary-"); + const apk = join(root, "selected.apk"); + await writeOrderedZip(apk, ["AndroidManifest.xml", "classes.dex"]); + const selection = { + class_name: "fixture.Native", + method_names: ["readNative"], + abis: ["arm64-v8a"], + }; + const raw = { + diagnostic: "retained native producer bytes: " + "x".repeat(8192), + }; + const declarations = ["fixture.Native", "fixture.Other"].map( + (class_name, index) => ({ + declaration_id: `jni-declaration-${index}`, + source_index: index, + class_name, + method_name: index === 0 ? "readNative" : "notSelected", + descriptor: "(I)I", + dex_entry: "classes.dex", + dex_entry_index: 1, + class_index: index, + method_index: index, + }), + ); + const execute = vi.fn( + async (target, request) => { + expect(request).toEqual({ + operation: "inspect_android_jni_bindings", + input: { path: apk, ...selection }, + }); + const result = androidResultSchemas.inspect_android_jni_bindings.parse({ + engine: { + name: "contract fixture", + version: "1", + artifact_sha256: "b".repeat(64), + source_revision: null, + worker_count: 1, + heap_limit_mib: null, + }, + apk: { path: apk, sha256: target.sha256 }, + selection, + native_inventory: { + scope: "apk-root-dex-declarations", + coverage: { + status: "complete", + dex_entries: 1, + examined_dex_entries: 1, + examined_classes: 2, + examined_methods: 2, + }, + unknowns: [], + }, + library_inventory: { + status: "complete", + examined_entries: 2, + selected_libraries: 0, + unknowns: [], + }, + declarations, + selected_declaration_ids: [declarations[0]!.declaration_id], + libraries: [], + candidates: [], + bindings: [ + { + declaration_id: declarations[0]!.declaration_id, + abi: "arm64-v8a", + status: "unmatched", + coverage: "complete", + candidate_ids: [], + unknowns: [], + }, + ], + unknowns: ["Static fixture facts do not establish runtime execution."], + }); + return ok( + createAnalysisExecution( + result, + { + id: "jni-contract-fixture", + name: "JNI contract fixture", + version: "1", + }, + { rawResult: raw }, + ), + ); + }, + ); + const harness = await createApplicationMcpHarness({ + serverOptions: { + androidAnalysis: { + execute, + close: async () => {}, + inspectAvailability: vi.fn(), + }, + environment: { REA_MCP_TOOL_FAMILIES: "android,application,session" }, + }, + }); + return { ...harness, apk, selection, raw, declarations, execute }; +}; + +it("defaults JNI MCP inspection to a retained summary while preserving explicit complete delivery and selectors", async () => { + const { client, session, apk, selection, raw, declarations, execute } = + await createJniSummaryContractFixture(); + const catalog = (await client.listTools()).tools; + const definition = catalog.find( + (tool) => tool.name === "inspect_android_jni_bindings", + ); + if (definition?.outputSchema === undefined) + throw new Error("Missing JNI output schema"); + const outputSchema: Record = definition.outputSchema; + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const request = { path: apk, ...selection }; + expect( + ajv.validate(definition.inputSchema, request), + JSON.stringify(ajv.errors), + ).toBe(true); + const compact = await client.callTool({ + name: "inspect_android_jni_bindings", + arguments: request, + }); + expect(compact.isError).not.toBe(true); + expect( + ajv.validate(outputSchema, compact.structuredContent), + JSON.stringify(ajv.errors), + ).toBe(true); + const summaryEvidence = parseEvidence(compact.structuredContent); + const summary = androidJniSummaryViewResultSchema.parse( + summaryEvidence.normalized_result, + ); + expect(summary.summary.selection).toEqual(selection); + expect(summary.summary.counts).toMatchObject({ + declarations: 2, + selected_declarations: 1, + bindings: 1, + unmatched: 1, + }); + expect( + summary.summary.declarations.map((method) => method.class_name), + ).toEqual(["fixture.Native"]); + expect(summaryEvidence.raw_result).toBeNull(); + const parent = session.evidenceById(summary.parent_evidence_id); + if (parent === undefined) + throw new Error("Complete JNI parent Evidence was not retained"); + const fullReport = androidResultSchemas.inspect_android_jni_bindings.parse( + parent.normalized_result, + ); + expect(parent.raw_result).toEqual(raw); + expect(fullReport.declarations).toEqual(declarations); + expect(parent.parameters).toEqual(request); + expect(summaryEvidence.evidence_links).toEqual([parent.evidence_id]); + const complete = await client.callTool({ + name: "inspect_android_jni_bindings", + arguments: { ...request, detail: "complete" }, + }); + expect(complete.isError).not.toBe(true); + expect( + ajv.validate(outputSchema, complete.structuredContent), + JSON.stringify(ajv.errors), + ).toBe(true); + const completeEvidence = parseEvidence(complete.structuredContent); + expect(completeEvidence.normalized_result).toEqual(fullReport); + expect(completeEvidence.raw_result).toEqual(raw); + expect(JSON.stringify(compact.structuredContent).length).toBeLessThan( + JSON.stringify(complete.structuredContent).length, + ); + expect(execute).toHaveBeenCalledTimes(2); + await unlink(apk); + const retained = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { + kind: "android-jni", + collection: "declarations", + indices: [1], + fields: ["declaration_id", "class_name", "method_name"], + limit: 1, + }, + }, + }); + expect(retained.isError).not.toBe(true); + const page = androidJniPageViewResultSchema.parse( + parseEvidence(retained.structuredContent).normalized_result, + ); + expect(page.parent_evidence_id).toBe(parent.evidence_id); + expect(page.items).toEqual([ + { + row_index: 1, + declaration_id: declarations[1]!.declaration_id, + class_name: "fixture.Other", + method_name: "notSelected", + }, + ]); + expect(execute).toHaveBeenCalledTimes(2); +}); + it.skipIf(process.platform === "win32")( "retains complete APK Evidence and reads exact manifest without its source or another engine launch", async () => { diff --git a/tests/boundary/process/jadxIntegration.test.ts b/tests/boundary/process/jadxIntegration.test.ts index c37ad9fd77..ad89c8def1 100644 --- a/tests/boundary/process/jadxIntegration.test.ts +++ b/tests/boundary/process/jadxIntegration.test.ts @@ -1,4 +1,5 @@ import { SdkErrorCode } from "@modelcontextprotocol/client"; +import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; import { expect, it as test, onTestFinished, vi } from "vitest"; import { access, @@ -983,6 +984,8 @@ it("bounds each upstream call by the selected deadline instead of a fixed limit" bridgeHash: "0".repeat(64), signal: new AbortController().signal, timeoutMs: 300, + resources: new ArtifactResourceScope(), + deadline: new AbortController().signal, }), ).rejects.toMatchObject({ code: SdkErrorCode.RequestTimeout, diff --git a/tests/conformance/android/ReaJadxMetadataRegression.java b/tests/conformance/android/ReaJadxMetadataRegression.java index a6248211b2..678ee24825 100644 --- a/tests/conformance/android/ReaJadxMetadataRegression.java +++ b/tests/conformance/android/ReaJadxMetadataRegression.java @@ -1,6 +1,8 @@ import com.atxx.jhmcp.*; import com.android.tools.smali.smali.Smali; import com.android.tools.smali.smali.SmaliOptions; +import com.android.tools.smali.dexlib2.Opcodes; +import com.android.tools.smali.dexlib2.dexbacked.DexBackedDexFile; import com.google.gson.*; import io.modelcontextprotocol.kotlin.sdk.types.CallToolResult; import io.modelcontextprotocol.kotlin.sdk.types.TextContent; @@ -306,6 +308,7 @@ public class ReaJadxMetadataRegression { Path.of(sources, "MergedMethod.smali").toString(), Path.of(sources, "DefaultPackageSelector.smali").toString(), Path.of(sources, "ConstantFields.smali").toString(), + Path.of(sources, "NativeDeclarations.smali").toString(), Path.of(sources, "RenamedMembers.smali").toString(), Path.of(sources, "RenamedMembersInner.smali").toString())), "Initializer fixture assembly failed"); Path apk = directory.resolve("initializers.apk"); @@ -396,6 +399,161 @@ public class ReaJadxMetadataRegression { return value; } + private static List nativeIdentities(JsonObject result) { + List identities = new ArrayList<>(); + int source = 0; + for (JsonElement value : result.getAsJsonArray("methods")) { + JsonObject method = value.getAsJsonObject(); + require(method.get("source_index").getAsInt() == source++, "Native source ordinal changed"); + require(method.get("class_index").getAsInt() >= 0 && method.get("method_index").getAsInt() >= 0, + "Native declaration lost DEX indices"); + identities.add(method.get("class_name").getAsString() + "." + + method.get("method_name").getAsString() + method.get("descriptor").getAsString()); + } + return identities; + } + + private static JsonObject nativeArguments(Path apk) { + JsonObject args = new JsonObject(); + args.addProperty("path", apk.toString()); + return args; + } + + private static int replaceFixtureBytes(byte[] bytes, byte[] before, byte[] after) { + require(before.length == after.length, "Fixture replacement changes lengths"); + int replacements = 0; + for (int at = 0; at <= bytes.length - before.length; at++) { + if (Arrays.equals(bytes, at, at + before.length, before, 0, before.length)) { + System.arraycopy(after, 0, bytes, at, after.length); + replacements++; + at += before.length - 1; + } + } + return replacements; + } + + private static void verifyNativeDeclarations(Method handle, Object bridge, JadxSession session, Path apk) + throws Exception { + JadxDecompiler decompiler = session.getClasses().get(0).getClassNode().root().getDecompiler(); + IdentityHashMap states = new IdentityHashMap<>(); + for (ClassNode cls : decompiler.getRoot().getClasses()) states.put(cls, cls.getState()); + JsonObject result = call(handle, bridge, "rea_native_declarations", nativeArguments(apk)); + require(result.get("scope").getAsString().equals("apk-root-dex-declarations"), "Native scope changed"); + require(result.getAsJsonObject("coverage").get("status").getAsString().equals("complete"), + "Source-built native inventory was incomplete: " + result); + Set expected = Set.of( + "fixture.InitializationPolicy.originalNative()V", + "fixture.NativeOverload.zLastNative()V", + "fixture.NativeOverload.readBuffer(Ljava/nio/ByteBuffer;I)Ljava/lang/String;", + "sb.readNative(I)I", + "fixture.Native_ๅŽŸ.read_ๅŽŸ([B)J", + "fixture.Native_ๅŽŸ.overloaded(I)I", + "fixture.Native_ๅŽŸ.overloaded(Ljava/lang/String;)I", + "fixture.Native_ๅŽŸ.do()V"); + List actual = nativeIdentities(result); + require(actual.size() == expected.size() && new HashSet<>(actual).equals(expected), + "Native facts differ from smali declarations: " + actual); + for (JsonElement value : result.getAsJsonArray("methods")) + require(value.getAsJsonObject().get("dex_entry").getAsString().equals("classes.dex"), + "Native DEX origin changed"); + for (var state : states.entrySet()) require(state.getKey().getState() == state.getValue(), + "Native batch generated source: " + state.getKey().getRawName()); + require(result.equals(call(handle, bridge, "rea_native_declarations", nativeArguments(apk))), + "Repeated native batch changed frozen facts"); + + byte[] dex; + try (ZipFile archive = new ZipFile(apk.toFile())) { + try (var input = archive.getInputStream(archive.getEntry("classes.dex"))) { + dex = input.readAllBytes(); + } + } + Path duplicate = apk.resolveSibling("native-duplicate-classes.apk"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(duplicate))) { + for (String name : List.of("classes.dex", "classes2.dex")) { + zip.putNextEntry(new ZipEntry(name)); zip.write(dex); zip.closeEntry(); + } + } + Gson json = new Gson(); + JsonObject duplicated = json.toJsonTree(ReaJadxBridge.nativeDeclarations(duplicate)).getAsJsonObject(); + require(duplicated.getAsJsonObject("coverage").get("status").getAsString().equals("complete"), + "Distinct DEX entries were confused with duplicate ZIP names"); + List doubled = nativeIdentities(duplicated); + require(doubled.size() == expected.size() * 2, "Duplicate DEX classes were selected away"); + for (String identity : expected) require(Collections.frequency(doubled, identity) == 2, + "Duplicate declaration was deduplicated: " + identity); + boolean secondOrigin = false; + for (JsonElement value : duplicated.getAsJsonArray("methods")) + secondOrigin |= value.getAsJsonObject().get("dex_entry").getAsString().equals("classes2.dex"); + require(secondOrigin, "Second DEX source identity was omitted"); + + Path partial = apk.resolveSibling("native-malformed-second-dex.apk"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(partial))) { + zip.putNextEntry(new ZipEntry("classes.dex")); zip.write(dex); zip.closeEntry(); + zip.putNextEntry(new ZipEntry("classes2.dex")); zip.write(new byte[] { 1, 2, 3 }); zip.closeEntry(); + } + JsonObject malformed = json.toJsonTree(ReaJadxBridge.nativeDeclarations(partial)).getAsJsonObject(); + require(malformed.getAsJsonObject("coverage").get("status").getAsString().equals("partial") + && malformed.getAsJsonObject("coverage").get("examined_dex_entries").getAsInt() == 1 + && nativeIdentities(malformed).equals(actual) && !malformed.getAsJsonArray("unknowns").isEmpty(), + "Malformed later DEX erased facts or claimed complete coverage"); + Path duplicateName = apk.resolveSibling("native-duplicate-zip-name.apk"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(duplicateName))) { + for (String name : List.of("classes1.dex", "classes2.dex")) { + zip.putNextEntry(new ZipEntry(name)); zip.write(dex); zip.closeEntry(); + } + } + byte[] duplicateBytes = Files.readAllBytes(duplicateName); + require(replaceFixtureBytes(duplicateBytes, "classes2.dex".getBytes(StandardCharsets.UTF_8), + "classes1.dex".getBytes(StandardCharsets.UTF_8)) == 2, "ZIP fixture names were not patched exactly"); + Files.write(duplicateName, duplicateBytes); + JsonObject duplicateNames = json.toJsonTree(ReaJadxBridge.nativeDeclarations(duplicateName)).getAsJsonObject(); + require(duplicateNames.getAsJsonObject("coverage").get("status").getAsString().equals("partial") + && duplicateNames.getAsJsonObject("coverage").get("dex_entries").getAsInt() == 2 + && duplicateNames.getAsJsonArray("methods").isEmpty() + && duplicateNames.getAsJsonArray("unknowns").toString().contains("Duplicate root DEX"), + "Duplicate ZIP names acquired invented member identities"); + + Path wrongCrc = apk.resolveSibling("native-wrong-crc.apk"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(wrongCrc))) { + zip.putNextEntry(new ZipEntry("classes.dex")); zip.write(dex); zip.closeEntry(); + } + byte[] wrongCrcBytes = Files.readAllBytes(wrongCrc); + int changedCrc = 0; + for (int at = 0; at < wrongCrcBytes.length - 20; at++) { + if (wrongCrcBytes[at] == 'P' && wrongCrcBytes[at + 1] == 'K' + && wrongCrcBytes[at + 2] == 1 && wrongCrcBytes[at + 3] == 2) { + wrongCrcBytes[at + 16] ^= 1; changedCrc++; + } + } + require(changedCrc == 1, "CRC fixture did not alter exactly one central directory record"); + Files.write(wrongCrc, wrongCrcBytes); + JsonObject wrongCrcResult = json.toJsonTree(ReaJadxBridge.nativeDeclarations(wrongCrc)).getAsJsonObject(); + require(wrongCrcResult.getAsJsonObject("coverage").get("status").getAsString().equals("partial") + && wrongCrcResult.getAsJsonArray("methods").isEmpty() + && wrongCrcResult.getAsJsonArray("unknowns").toString().contains("CRC32"), + "A ZIP CRC mismatch acquired native declaration facts"); + + // The bundled decoder would allocate the declared UTF-16 char count first. + // Keep this bad declaration small enough for a failing guard regression to + // stay bounded; the producing diagnostic proves admission precedes decoding. + DexBackedDexFile original = new DexBackedDexFile(Opcodes.getDefault(), dex); + String firstClass = original.getClassSection().get(0).getType(); + int stringIndex = original.getStringSection().indexOf(firstClass); + int stringOffset = original.getBuffer().readSmallUint(original.getStringSection().getOffset(stringIndex)); + byte[] oversized = dex.clone(); + System.arraycopy(new byte[] { (byte) 0xc0, (byte) 0x84, 0x3d }, 0, oversized, stringOffset, 3); // 1,000,000 + Path oversizedString = apk.resolveSibling("native-oversized-string.apk"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(oversizedString))) { + zip.putNextEntry(new ZipEntry("classes.dex")); zip.write(oversized); zip.closeEntry(); + } + JsonObject oversizedResult = json.toJsonTree(ReaJadxBridge.nativeDeclarations(oversizedString)).getAsJsonObject(); + require(oversizedResult.getAsJsonObject("coverage").get("status").getAsString().equals("partial") + && oversizedResult.getAsJsonArray("unknowns").toString().contains("native decoding allowance"), + "Oversized declared DEX string was not rejected at admission"); + Files.delete(duplicate); Files.delete(partial); Files.delete(duplicateName); Files.delete(oversizedString); Files.delete(wrongCrc); + System.out.println("NATIVE_DECLARATION_REGRESSION_PASS raw=true native_only=true unicode=true overloads=true duplicate_dex=true partial=true no_source=true"); + } + private static void verifyInitializers(SessionHolder holder, SessionConfig config, Constructor constructor, Method handle, Path apk) throws Exception { BuildersKt.runBlocking(EmptyCoroutineContext.INSTANCE, @@ -403,6 +561,7 @@ public class ReaJadxMetadataRegression { BuildersKt.runBlocking(EmptyCoroutineContext.INSTANCE, (scope, continuation) -> holder.load(apk.toString(), config, continuation)); Object bridge = constructor.newInstance(holder); + verifyNativeDeclarations(handle, bridge, holder.current(), apk); verifyDefaultPackageSelectors(handle, bridge, holder.current()); String[] classNames = { "fixture.InitializationPolicy", "fixture.MixedInitialization" }; String[][] witnesses = { @@ -729,6 +888,21 @@ public class ReaJadxMetadataRegression { SessionConfig config = new SessionConfig(1048576, 0, 90000L, 1, List.of(), List.of(), 64, ResourceMode.FULL); SessionHolder holder = new SessionHolder(config); try { + Class bridgeType = Class.forName("ReaJadxBridge"); + Constructor constructor = bridgeType.getDeclaredConstructor(SessionHolder.class); + constructor.setAccessible(true); + Object bridge = constructor.newInstance(holder); + Method handle = bridgeType.getDeclaredMethod("handle", String.class, kotlinx.serialization.json.JsonObject.class); + handle.setAccessible(true); + Path initializerApk = initializerFixture(args[1], args[2]); + require(holder.current() == null, "Cold native inventory unexpectedly has a loaded APK"); + JsonObject coldNative = call(handle, bridge, "rea_native_declarations", nativeArguments(initializerApk)); + require(holder.current() == null, "Native declaration inventory loaded/decompiled the APK"); + require(coldNative.getAsJsonObject("coverage").get("status").getAsString().equals("complete") + && nativeIdentities(coldNative).size() == 8 + && nativeIdentities(coldNative).contains("fixture.Native_ๅŽŸ.do()V"), + "Cold native inventory lost raw source declarations"); + System.out.println("NATIVE_DECLARATION_COLD_PASS no_apk_load=true raw=true"); BuildersKt.runBlocking(EmptyCoroutineContext.INSTANCE, (scope, continuation) -> holder.load(args[0], config, continuation)); JadxSession session = holder.current(); @@ -739,12 +913,6 @@ public class ReaJadxMetadataRegression { require(node.getState() == ProcessState.NOT_LOADED, "Fixture unexpectedly generated source"); states.put(node, node.getState()); } - Class bridgeType = Class.forName("ReaJadxBridge"); - Constructor constructor = bridgeType.getDeclaredConstructor(SessionHolder.class); - constructor.setAccessible(true); - Object bridge = constructor.newInstance(holder); - Method handle = bridgeType.getDeclaredMethod("handle", String.class, kotlinx.serialization.json.JsonObject.class); - handle.setAccessible(true); verifyClassPages(handle, bridge, session.getClassFqns()); verifyRawClassSelectors(handle, bridge, decompiler); JsonObject apiArgs = arguments("io.appium.android.apis.ApiDemos"); @@ -807,7 +975,7 @@ public class ReaJadxMetadataRegression { require(apiSummary.equals(call(handle, bridge, "get_class_summary", apiArgs)), "Unrelated frozen summary changed"); System.out.println("METADATA_REGRESSION_PASS classes=" + states.size() + " overloads=" + signatures.size() + " reference_counts=" + referenceCounts); - verifyInitializers(holder, config, constructor, handle, initializerFixture(args[1], args[2])); + verifyInitializers(holder, config, constructor, handle, initializerApk); verifyDirectInvokes(holder, config, constructor, handle, Path.of(args[3]), Path.of(args[2])); verifyRootManifests(holder, config, constructor, handle, Path.of(args[2])); diff --git a/tests/fixtures/android/jadx-mcp.mjs b/tests/fixtures/android/jadx-mcp.mjs index 606afdfdfa..97c73740ab 100644 --- a/tests/fixtures/android/jadx-mcp.mjs +++ b/tests/fixtures/android/jadx-mcp.mjs @@ -102,6 +102,44 @@ register("load_apk", async ({ path }) => { resources: "full", }); }); +register("rea_native_declarations", ({ path }) => { + if (typeof path !== "string") + return { ...text("path must be a string"), isError: true }; + const methods = [ + { + source_index: 0, + class_name: "fixture.Target", + method_name: "readNative", + descriptor: "(I)I", + dex_entry: "classes.dex", + dex_entry_index: 0, + class_index: 0, + method_index: 1, + }, + ]; + if (mode === "duplicate-native-declaration") + methods.push({ + ...methods[0], + source_index: 1, + dex_entry: "classes2.dex", + dex_entry_index: 1, + }); + return json({ + scope: "apk-root-dex-declarations", + methods, + coverage: { + status: mode === "partial-native-inventory" ? "partial" : "complete", + dex_entries: mode === "partial-native-inventory" ? 2 : methods.length, + examined_dex_entries: methods.length, + examined_classes: methods.length, + examined_methods: methods.length, + }, + unknowns: + mode === "partial-native-inventory" + ? ["Controlled unreadable second DEX member"] + : [], + }); +}); register("get_app_info", () => { if (mode === "rpc-disconnect") process.exit(17); return json({ diff --git a/tests/fixtures/android/jni-bindings/README.md b/tests/fixtures/android/jni-bindings/README.md new file mode 100644 index 0000000000..870b1e07fc --- /dev/null +++ b/tests/fixtures/android/jni-bindings/README.md @@ -0,0 +1,142 @@ +# Static JNI fixture built with the Android NDK + +This fixture contains genuine Java `native` declarations, javac-generated JNI +headers, `Java_*` exports and actual `JNI_OnLoad`/`RegisterNatives` source. Its +DEX-only APK containers have no Android manifest, resources or signature. They +are never installed, loaded or executed; static matches do not prove runtime +registration, loading or invocation. + +`source-expectations.json` is handwritten from these sources. There are twelve +native declarations, including overloaded and supplementary-Unicode names, two +classes with the same `(ambiguous, (I)I)` pair, one export in two libraries per ABI, +and an intentionally unmatched declaration. `data_methods` has three entries in +`.data`; `relro_methods` has one in `.data.rel.ro`. A Modified UTF-8 table name +encodes U+10400 as the two UTF-16 surrogate units, unlike ordinary four-byte UTF-8. + +## Pinned existing tools + +- NDK **r27d / 27.3.13750724**, official Linux archive: + . + Size **663956036 bytes**; official SHA-1 + `22105e410cf29afcf163760cc95522b9fb981121`, from Google's + [repository manifest](https://dl.google.com/android/repository/repository2-3.xml). + Observed archive SHA-256: + `601246087a682d1944e1e16dd85bc6e49560fe8b6d61255be2829178c8ed15d9`. + The latter is an acquired-byte identity, not an independently published checksum. +- Observed bundled Android clang **18.0.4**, build **13691557**, LLVM source + `d8003a456d14a3deb8054cdaa529ffbf02d9b262`; genuine LLD/readelf **18.0.4**. +- Existing Temurin JDK **21.0.12.1+1**, Android **36** platform JAR, and D8 + **8.0.40** from the checksum-pinned R8 JAR described in the neighboring + [mapping fixture](../r8-mapping/README.md#compiler-source-and-integrity). + +Private acquisition streams the archive to disk and checks its byte count/SHA-1 +before extraction. The executed selection retained about 305 MiB, admitted below +512 MiB: clang/LLD/readelf and their archive-owned relative links, host shared +dependencies, Clang headers, four target builtins/**libunwind.a**, NDK sysroot +headers, and selected target API 21/23 libraries. It omitted unrelated debuggers, +samples and sanitizer runtimes. Preserve actual archive paths/modes/link targets; +the host JDK's `jni.h` or a host `ld.lld` wrapper cannot substitute for the NDK. +No global toolchain or configuration is changed. + +## Generate with explicit inputs + +Run serially with two CPUs and a process-family memory supervisor. Java commands +also cap their heap and active processors; LLD uses two threads. Select an output +directory that does not exist. `build.py` does not download or install tools. + +```sh +python3 tests/fixtures/android/jni-bindings/build.py \ + --ndk /absolute/private/android-ndk-r27d \ + --jdk /absolute/existing/jdk-21 \ + --r8-jar /absolute/verified/r8-8.0.40.jar \ + --android-jar /absolute/sdk/platforms/android-36/android.jar \ + --output /absolute/fresh-output-directory +``` + +The generator compiles three Java classes with `javac -h`, checks every expected +export against those generated declarations, runs D8, and independently checks +the resulting DEX `ACC_NATIVE` flags and exact names/prototypes. It then compiles +two libraries for each of arm64-v8a, armeabi-v7a, x86_64 and x86, with three packing +variants. All subprocess arguments and tool versions are recorded in the output. + +| APK variant | Observed table relocation encoding | Required interpretation | +| ------------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------ | +| `jni-ordinary-static.apk` | AArch64/x86-64 RELA; ARM/i686 REL implicit addends | Every source-known candidate and exact slot value survives | +| `jni-relr-static.apk` | Standard DT_RELR with encoded bitmaps on all four ABIs | Same facts, with RELR source evidence | +| `jni-aps2-static.apk` | Android packed APS2 REL/RELA on all four ABIs | Explicit unsupported/partial table coverage; preserve independently readable exports | + +Ordinary and RELR source expectations have, across four ABIs, 32 matched, +12 ambiguous and four unmatched bindings. These are source-known outcomes. +The report's matched status establishes observed candidate presence; separate +partial coverage does not promise complete uniqueness. A source-known unmatched +method can remain unknown when GLOB_DAT or NOBITS handling leaves incomplete +coverage. APS2's `expected_methods` retains source-known export facts and marks +unrecovered table declarations unknown. The report must retain readable export +candidates and must not claim that an undecoded table is absent. + +## Independent evidence and limits + +`oracle.py` imports no REA parser. For these small generated artifacts it reads +known ELF symbols and source-specified table slots, applies standard relative +REL/RELA/RELR records, checks the exact Modified UTF-8 bytes, and requires each +function pointer to equal the compiled function symbol inside executable PT_LOAD. +It is a known-slot oracle, not a general JNI table discovery implementation. +It deliberately does not decode APS2. The separate official `llvm-readelf` +transcripts establish actual sections, symbols, relocation tags and GNU_RELRO. + +Generated `full-expectations.json` contains every ABI/library digest, export and +expected declaration binding, including the independently recovered slot addresses, +file offsets, raw words, relocated words and function addresses. `oracle/dex.json`, +`headers/`, `javap/`, `readelf/`, `commands.json` and compiler logs retain the +independent proof. Generated APK hashes are per-run identities; ZIP metadata can +vary between runs. Keep all generated binaries, caches and receipts outside Git. + +The static library checker reads the Conscrypt oracle only when the generated +expectations declare that optional case. A declared but missing oracle fails +verification; NDK-only expectations require only their 24 generated libraries. + +Optional `--conscrypt-aar` and `--conscrypt-original-dex-zip` package the already +acquired Conscrypt Android **2.8.0** original-name D8 together with the exact four +native libraries from the public Apache-2.0 AAR. The source AAR SHA-256 is +`752044c289bc98e2dcabc00a1f88ca158b3aa3f505b38f124552bf4b2a0902b1`. +The independent arm64 oracle pins its library SHA-256 and validates slots +`0x2ada68`, `0x2ada70`, `0x2ada78`, resolving `EVP_PKEY_new_RSA` with descriptor +`([B[B[B[B[B[B[B[B)J` to `0xfa11c`. Its existing D8 warning about the missing +platform adapter `OpenSSLSocketImpl` remains a limitation. This is a static +container, and pairing an R8-minified DEX with unchanged original JNI strings +would be a different negative case. + +## Public workflow checker + +After building REA and selecting the existing JADX MCP JAR, the maintained +[public MCP checker](../../../support/analysis-efficiency/android-jni.mjs) accepts +the generated `full-expectations.json`. Run it under the same external resource +supervisor; it does not rebuild fixtures or run target code. + +```sh +node tests/support/analysis-efficiency/android-jni.mjs \ + --repository /absolute/rea-checkout \ + --target /absolute/fixture-output/jni-ordinary-static.apk \ + --expectations /absolute/fixture-output/full-expectations.json \ + --variant ordinary --detail summary --proof --pages --cli \ + --java-home /absolute/existing/jdk-21 --jar /absolute/existing/jadx-mcp.jar \ + --output /absolute/fresh-receipts-directory +``` + +Use `relr`, `aps2` or `conscrypt` with its corresponding APK for the other cases. +Two passes distinguish a cold provider from warm reuse. Add `--omit-detail` with +`--detail summary` to verify the MCP default through an actually omitted argument. `--proof` counts and +checks complete Evidence export; `--pages` checks all four collections, selected +fields, empty filters, native source ordinals, coverage and parent row indices; +`--cli` checks one CLI result after MCP cleanup. Optional `--retained-removal` +runs a separate qualification after the investigation receipts: it copies the APK +exclusively into the fresh output directory, analyzes that copy, removes only +that owned copy, then reads retained summary/slot/pages and exports unchanged +parent Evidence. It verifies the original APK digest remains unchanged. These +calls are recorded and costed separately from cold/warm investigation tasks. +The optional +[cost reader](../../../support/analysis-efficiency/android-jni-costs.py) counts +catalogs, all request/reply content and proof/page calls with an existing +`tiktoken` installation. These are estimates, not model billing. Earlier main has +no equivalent static RegisterNatives workflow, so it is not an equal-coverage +performance baseline. diff --git a/tests/fixtures/android/jni-bindings/build.py b/tests/fixtures/android/jni-bindings/build.py new file mode 100644 index 0000000000..a735e43657 --- /dev/null +++ b/tests/fixtures/android/jni-bindings/build.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Generate static-only verification artifacts with caller-selected existing tools.""" +import argparse +import hashlib +import json +import subprocess +import sys +import zipfile +from pathlib import Path + +# Keep generated Python artifacts out of this source-only fixture directory. +sys.dont_write_bytecode = True +from oracle import dex_oracle, elf_oracle + + +def digest(path): + result = hashlib.sha256() + with path.open("rb") as source: + while chunk := source.read(1024 * 1024): + result.update(chunk) + return result.hexdigest() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--ndk", required=True, type=Path) + parser.add_argument("--jdk", required=True, type=Path) + parser.add_argument("--r8-jar", required=True, type=Path) + parser.add_argument("--android-jar", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--conscrypt-aar", type=Path) + parser.add_argument("--conscrypt-original-dex-zip", type=Path) + args = parser.parse_args() + for path in (args.ndk, args.jdk, args.r8_jar, args.android_jar, args.output): + assert path.is_absolute(), "Keep every selected input/output path explicit" + assert "Pkg.Revision = 27.3.13750724" in (args.ndk / "source.properties").read_text() + assert digest(args.r8_jar) == "ab1379835c7d3e5f21f80347c3c81e2f762e0b9b02748ae5232c3afa14adf702" + root = Path(__file__).resolve().parent + expected = json.loads((root / "source-expectations.json").read_text()) + output = args.output + output.mkdir(parents=True, exist_ok=False) + for directory in ("classes", "headers", "oracle", "readelf", "javap"): + (output / directory).mkdir() + toolchain = args.ndk / "toolchains/llvm/prebuilt/linux-x86_64" + clang = toolchain / "bin/clang" + readelf = toolchain / "bin/llvm-readelf" + java = args.jdk / "bin/java" + javac = args.jdk / "bin/javac" + commands = [] + + def run(command, log): + command = [str(value) for value in command] + commands.append({"arguments": command, "log": str(log)}) + print(json.dumps({"command": command, "log": str(log)}), flush=True) + with log.open("w") as stream: + subprocess.run(command, check=True, stdout=stream, stderr=subprocess.STDOUT) + + versions = {} + for name, command in { + "clang": [clang, "--version"], + "lld": [toolchain / "bin/ld.lld", "--version"], + "readelf": [readelf, "--version"], + "java": [java, "-version"], + "d8": [java, "-Xmx256m", "-XX:ActiveProcessorCount=2", "-cp", args.r8_jar, "com.android.tools.r8.D8", "--version"], + }.items(): + versions[name] = subprocess.run([str(value) for value in command], check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True).stdout + sources = sorted((root / "src").rglob("*.java")) + run([javac, "-J-Xmx256m", "-J-XX:ActiveProcessorCount=2", "--release", "8", "-encoding", "UTF-8", "-h", output / "headers", "-d", output / "classes", *sources], output / "javac.log") + headers = "\n".join(path.read_text() for path in (output / "headers").glob("*.h")) + for row in expected["native_declarations"]: + if row.get("symbol"): + assert row["symbol"] in headers, ("javac -h disagrees", row) + for name in ("fixture.jni.Bindings", "fixture.jni.Twin", "fixture.jni.ๅŽŸๅ๐€"): + run([args.jdk / "bin/javap", "-s", "-p", "-classpath", output / "classes", name], output / "javap" / (name + ".txt")) + with zipfile.ZipFile(output / "input.jar", "w") as jar: + for path in sorted((output / "classes").rglob("*.class")): + jar.write(path, path.relative_to(output / "classes").as_posix()) + run([java, "-Xmx512m", "-XX:ActiveProcessorCount=2", "-cp", args.r8_jar, "com.android.tools.r8.D8", "--release", "--min-api", "21", "--lib", args.android_jar, "--output", output / "dex.zip", output / "input.jar"], output / "d8.log") + with zipfile.ZipFile(output / "dex.zip") as dex: + (output / "classes.dex").write_bytes(dex.read("classes.dex")) + dex_facts = dex_oracle(output / "classes.dex", expected) + (output / "oracle/dex.json").write_text(json.dumps(dex_facts, ensure_ascii=False, indent=2) + "\n") + triples = { + "arm64-v8a": "aarch64-linux-android21", + "x86_64": "x86_64-linux-android21", + "armeabi-v7a": "armv7a-linux-androideabi21", + "x86": "i686-linux-android21", + } + variants = {"ordinary": "none", "relr": "relr", "aps2": "android"} + receipts = {"static_only": True, "versions": versions, "ndk": str(args.ndk), "r8_sha256": digest(args.r8_jar), "source_expectations_sha256": digest(root / "source-expectations.json"), "dex": dex_facts, "variants": {}} + for variant, packing in variants.items(): + libraries = {} + for abi, triple in triples.items(): + for library, source in (("libfixture.so", "bindings.c"), ("libduplicate.so", "duplicate.c")): + destination = output / variant / "lib" / abi / library + destination.parent.mkdir(parents=True, exist_ok=True) + run([clang, "--target=" + triple, "-shared", "-fPIC", "-O0", "-g0", "-I", output / "headers", "-Wl,--threads=2", "-Wl,--pack-dyn-relocs=" + packing, "-Wl,-soname," + library, root / "native" / source, "-o", destination], output / (variant + "-" + abi + "-" + library + ".log")) + facts = elf_oracle(destination, expected if library == "libfixture.so" else {"tables": [], "native_declarations": []}, False) + required = expected["variants"]["ordinary"][abi] if variant == "ordinary" else variant.upper() if variant == "relr" else "APS2" + assert required in facts["relocation_encodings"], (variant, abi, facts["relocation_encodings"]) + for row in expected["native_declarations"]: + symbol = row.get("symbol") + if symbol and (library == "libfixture.so" or row["name"] == "sharedExport"): + assert symbol in facts["exports"], (destination, symbol) + libraries[f"lib/{abi}/{library}"] = facts + (output / "oracle" / (variant + "-" + abi + "-" + library + ".json")).write_text(json.dumps(facts, ensure_ascii=False, indent=2) + "\n") + run([readelf, "--file-header", "--program-headers", "--sections", "--symbols", "--relocations", "--dynamic", "--wide", destination], output / "readelf" / (variant + "-" + abi + "-" + library + ".txt")) + apk = output / ("jni-" + variant + "-static.apk") + with zipfile.ZipFile(apk, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.write(output / "classes.dex", "classes.dex") + for member, facts in libraries.items(): + archive.write(facts["path"], member) + rows = [] + for abi in triples: + for row in expected["native_declarations"]: + claim = {**row, "abi": abi, "implementations": []} + if variant == "aps2" and row["basis"] == "table": + claim["per_abi_status"] = "unknown" + claim["reason"] = "unsupported-aps2" + for member, facts in libraries.items(): + if member.split("/")[1] != abi: + continue + symbol = row.get("symbol") + entries = [entry for entry in facts["tables"] if entry["name"] == row["name"] and entry["descriptor"] == row["descriptor"]] + if symbol is not None and symbol in facts["exports"]: + claim["implementations"].append({"library": member, "library_sha256": facts["sha256"], "export": symbol, "address": facts["exports"][symbol]["address"]}) + for entry in entries: + claim["implementations"].append({"library": member, "library_sha256": facts["sha256"], **entry}) + rows.append(claim) + receipts["variants"][variant] = {"apk": str(apk), "sha256": digest(apk), "libraries": libraries, "expected_methods": rows} + if args.conscrypt_aar or args.conscrypt_original_dex_zip: + assert args.conscrypt_aar and args.conscrypt_original_dex_zip + assert digest(args.conscrypt_aar) == "752044c289bc98e2dcabc00a1f88ca158b3aa3f505b38f124552bf4b2a0902b1" + apk = output / "conscrypt-original-static.apk" + copied = [] + with zipfile.ZipFile(args.conscrypt_aar) as aar, zipfile.ZipFile(args.conscrypt_original_dex_zip) as original, zipfile.ZipFile(apk, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for entry in original.infolist(): + if entry.filename.endswith(".dex"): + archive.writestr(entry.filename, original.read(entry)) + for abi in triples: + member = f"lib/{abi}/libconscrypt_jni.so" + data = aar.read(f"jni/{abi}/libconscrypt_jni.so") + archive.writestr(member, data) + copied.append({"member": member, "sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data)}) + if abi == "arm64-v8a": + library = output / "conscrypt-arm64.so" + library.write_bytes(data) + facts = elf_oracle(library, {}, True) + (output / "oracle/conscrypt-arm64.json").write_text(json.dumps(facts, indent=2) + "\n") + receipts["conscrypt"] = {"apk": str(apk), "sha256": digest(apk), "aar_sha256": digest(args.conscrypt_aar), "original_dex_zip_sha256": digest(args.conscrypt_original_dex_zip), "libraries": copied, "limitation": "Original-name D8 from classes.jar, not R8-minified DEX; static APK only. Existing D8 platform-adapter missing OpenSSLSocketImpl warning remains."} + (output / "commands.json").write_text(json.dumps(commands, indent=2) + "\n") + (output / "full-expectations.json").write_text(json.dumps(receipts, ensure_ascii=False, indent=2) + "\n") + print(json.dumps({"output": str(output), "native_declarations": len(dex_facts["native_methods"]), "variants": list(variants), "expectations": str(output / "full-expectations.json")}), flush=True) + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/android/jni-bindings/native/bindings.c b/tests/fixtures/android/jni-bindings/native/bindings.c new file mode 100644 index 0000000000..4d0be1b55b --- /dev/null +++ b/tests/fixtures/android/jni-bindings/native/bindings.c @@ -0,0 +1,78 @@ +#include +#include "fixture_jni_Bindings.h" +#include "fixture_jni_ๅŽŸๅ๐€.h" + +JNIEXPORT jint JNICALL Java_fixture_jni_Bindings_plain_1export(JNIEnv *env, jclass cls) { + (void)env; (void)cls; + return 11; +} + +JNIEXPORT jint JNICALL Java_fixture_jni_Bindings_over_1load__I(JNIEnv *env, jclass cls, jint value) { + (void)env; (void)cls; + return value + 12; +} + +JNIEXPORT jint JNICALL Java_fixture_jni_Bindings_over_1load___3Ljava_lang_String_2(JNIEnv *env, jclass cls, jobjectArray values) { + (void)env; (void)cls; (void)values; + return 13; +} + +JNIEXPORT jint JNICALL Java_fixture_jni_Bindings__065b9_06cd5_0d801_0dc00(JNIEnv *env, jclass cls, jintArray values) { + (void)env; (void)cls; (void)values; + return 14; +} + +JNIEXPORT jint JNICALL Java_fixture_jni__0539f_0540d_0d801_0dc00__05165_053e3_1_065b9_06cd5(JNIEnv *env, jclass cls) { + (void)env; (void)cls; + return 15; +} + +JNIEXPORT jint JNICALL Java_fixture_jni_Bindings_sharedExport(JNIEnv *env, jclass cls) { + (void)env; (void)cls; + return 16; +} + +static jint data_impl(JNIEnv *env, jclass cls, jint value) { + (void)env; (void)cls; + return value + 21; +} + +static jlong relro_impl(JNIEnv *env, jclass cls, jbyteArray values) { + (void)env; (void)cls; (void)values; + return 22; +} + +static jint unicode_impl(JNIEnv *env, jclass cls, jstring value) { + (void)env; (void)cls; (void)value; + return 23; +} + +static jint ambiguous_impl(JNIEnv *env, jclass cls, jint value) { + (void)env; (void)cls; + return value + 24; +} + +/* Explicit Modified UTF-8: U+10400 is encoded as UTF-16 D801 DC00, not UTF-8's + * four-byte scalar encoding. The bytes are independently checked by oracle.py. */ +static JNINativeMethod data_methods[] __attribute__((used, section(".data"))) = { + {"registeredData", "(I)I", (void *)data_impl}, + {"ambiguous", "(I)I", (void *)ambiguous_impl}, + {"registered_\xe8\xa1\xa8\xed\xa0\x81\xed\xb0\x80", "(Ljava/lang/String;)I", (void *)unicode_impl}, +}; + +static const JNINativeMethod relro_methods[] __attribute__((used, section(".data.rel.ro"))) = { + {"registeredRelro", "([B)J", (void *)relro_impl}, +}; + +/* This is genuine JNI registration code, never invoked by verification. The + * FindClass string is not a proof of runtime attribution for a table scanner. */ +JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) { + (void)reserved; + JNIEnv *env = 0; + if ((*vm)->GetEnv(vm, (void **)&env, JNI_VERSION_1_6) != JNI_OK) return JNI_ERR; + jclass cls = (*env)->FindClass(env, "fixture/jni/Bindings"); + if (cls == 0) return JNI_ERR; + if ((*env)->RegisterNatives(env, cls, data_methods, 3) != JNI_OK) return JNI_ERR; + if ((*env)->RegisterNatives(env, cls, relro_methods, 1) != JNI_OK) return JNI_ERR; + return JNI_VERSION_1_6; +} diff --git a/tests/fixtures/android/jni-bindings/native/duplicate.c b/tests/fixtures/android/jni-bindings/native/duplicate.c new file mode 100644 index 0000000000..0e7fcd3d74 --- /dev/null +++ b/tests/fixtures/android/jni-bindings/native/duplicate.c @@ -0,0 +1,8 @@ +#include +#include "fixture_jni_Bindings.h" + +/* A distinct library exports the same JNI short name within each ABI. */ +JNIEXPORT jint JNICALL Java_fixture_jni_Bindings_sharedExport(JNIEnv *env, jclass cls) { + (void)env; (void)cls; + return 31; +} diff --git a/tests/fixtures/android/jni-bindings/oracle.py b/tests/fixtures/android/jni-bindings/oracle.py new file mode 100644 index 0000000000..c428ecea8c --- /dev/null +++ b/tests/fixtures/android/jni-bindings/oracle.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Small fixture oracle: known source symbols/slots, never a REA table scanner.""" +import argparse +import hashlib +import json +import struct +from pathlib import Path + + +def limited(path): + if path.stat().st_size > 16 * 1024 * 1024: + raise ValueError("Oracle admits only these small, explicit fixture artifacts") + return path.read_bytes() + + +def mutf8(value): + units = value.encode("utf-16-be", "surrogatepass") + result = bytearray() + for index in range(0, len(units), 2): + unit = int.from_bytes(units[index:index + 2], "big") + if 0 < unit < 128: + result.append(unit) + elif unit < 2048: + result.extend((0xC0 | (unit >> 6), 0x80 | (unit & 63))) + else: + result.extend((0xE0 | (unit >> 12), 0x80 | ((unit >> 6) & 63), 0x80 | (unit & 63))) + return bytes(result) + + +class Elf: + def __init__(self, data): + self.data = data + assert data[:4] == b"\x7fELF" and data[5] == 1 + self.width = {1: 4, 2: 8}[data[4]] + self.machine = self.unpack("H", 18)[0] + self.word = "Q" if self.width == 8 else "I" + if self.width == 8: + phoff, shoff = self.unpack("QQ", 32) + phsize, phnum, shsize, shnum, names = self.unpack("HHHHH", 54) + else: + phoff, shoff = self.unpack("II", 28) + phsize, phnum, shsize, shnum, names = self.unpack("HHHHH", 42) + self.sections = [] + for index in range(shnum): + row = self.unpack("IIQQQQIIQQ" if self.width == 8 else "IIIIIIIIII", shoff + index * shsize) + self.sections.append(dict(zip(("name_index", "type", "flags", "address", "offset", "size", "link", "info", "align", "entry_size"), row))) + names = self.section_data(self.sections[names]) + for section in self.sections: + section["name"] = self.zero(names, section["name_index"]).decode("ascii") + self.loads = [] + for index in range(phnum): + row = self.unpack("IIQQQQQQ" if self.width == 8 else "IIIIIIII", phoff + index * phsize) + if self.width == 8: + kind, flags, offset, address, _, filesz, memsz, _ = row + else: + kind, offset, address, _, filesz, memsz, flags, _ = row + if kind in (1, 0x6474E552): + self.loads.append({"type": kind, "flags": flags, "offset": offset, "address": address, "file_bytes": filesz, "memory_bytes": memsz}) + self.symbols = {} + self.exports = {} + for section in self.sections: + if section["type"] not in (2, 11): + continue + strings = self.section_data(self.sections[section["link"]]) + for offset in range(section["offset"], section["offset"] + section["size"], section["entry_size"]): + row = self.unpack("IBBHQQ" if self.width == 8 else "IIIBBH", offset) + if self.width == 8: + name, info, other, shndx, address, size = row + else: + name, address, size, info, other, shndx = row + name = self.zero(strings, name).decode("utf8") + symbol = {"address": address, "size": size, "section_index": shndx} + if section["type"] == 2: + self.symbols[name] = symbol + if section["type"] == 11 and shndx != 0 and info & 15 == 2 and info >> 4 in (1, 2) and other & 3 == 0: + self.exports[name] = symbol + self.overlay = {} + self.encodings = set() + relative = {183: 1027, 62: 8, 3: 8, 40: 23}[self.machine] + for section in self.sections: + if section["type"] in (4, 9): + rela = section["type"] == 4 + encoding = "RELA" if rela else "REL" + self.encodings.add(encoding) + fmt = ("QQq" if rela else "QQ") if self.width == 8 else ("IIi" if rela else "II") + for offset in range(section["offset"], section["offset"] + section["size"], section["entry_size"]): + row = self.unpack(fmt, offset) + kind = row[1] & (0xFFFFFFFF if self.width == 8 else 255) + if kind == relative: + self.overlay[row[0]] = (row[2] if rela else self.pointer(row[0]), encoding) + elif section["type"] == 19: + self.encodings.add("RELR") + cursor = 0 + for offset in range(section["offset"], section["offset"] + section["size"], self.width): + encoded = self.unpack(self.word, offset)[0] + if encoded & 1 == 0: + self.overlay[encoded] = (self.pointer(encoded), "RELR") + cursor = encoded + self.width + else: + for bit in range(self.width * 8 - 1): + if encoded & (1 << (bit + 1)): + slot = cursor + bit * self.width + self.overlay[slot] = (self.pointer(slot), "RELR") + cursor += (self.width * 8 - 1) * self.width + elif section["type"] in (0x60000001, 0x60000002): + assert self.section_data(section).startswith(b"APS2") + self.encodings.add("APS2") + + def unpack(self, fmt, offset): + return struct.unpack_from("<" + fmt, self.data, offset) + + @staticmethod + def zero(data, offset): + end = data.index(0, offset, min(len(data), offset + 4096)) + return data[offset:end] + + def section_data(self, section): + return self.data[section["offset"]:section["offset"] + section["size"]] + + def file_offset(self, address): + for load in self.loads: + delta = address - load["address"] + if load["type"] == 1 and 0 <= delta < load["file_bytes"]: + return load["offset"] + delta + raise ValueError(f"No file-backed PT_LOAD at {address:#x}") + + def pointer(self, address): + return self.unpack(self.word, self.file_offset(address))[0] + + def entry(self, address, index, name, descriptor, function=None): + slots = [] + for slot in range(3): + location = address + (index * 3 + slot) * self.width + resolved, encoding = self.overlay[location] + slots.append({"address": hex(location), "file_offset": hex(self.file_offset(location)), "raw_word": hex(self.pointer(location)), "resolved_word": hex(resolved), "encoding": encoding}) + for slot, expected in zip(slots, (name, descriptor)): + actual = self.zero(self.data, self.file_offset(int(slot["resolved_word"], 16))) + assert actual == mutf8(expected), (actual, expected) + slot["string_bytes_hex"] = actual.hex() + target = int(slots[2]["resolved_word"], 16) + assert any(p["type"] == 1 and p["flags"] & 1 and p["address"] <= target < p["address"] + p["memory_bytes"] for p in self.loads) + if function is not None: + assert target == self.symbols[function]["address"] + return {"table_address": hex(address), "entry_index": index, "name": name, "descriptor": descriptor, "function_symbol": function, "function_address": hex(target), "slots": slots} + + +def elf_oracle(path, expected, conscrypt): + data = limited(path) + elf = Elf(data) + tables = [] + if "APS2" not in elf.encodings: + if conscrypt: + assert hashlib.sha256(data).hexdigest() == "a3b78516ca2398d4ab233c71cf5c41ebb14c36ce157ebbaf1e5fd903e8220181" + row = elf.entry(0x2ADA68, 0, "EVP_PKEY_new_RSA", "([B[B[B[B[B[B[B[B)J") + assert row["function_address"] == "0xfa11c" + tables.append(row) + else: + for table in expected["tables"]: + symbol = elf.symbols[table["symbol"]] + section = elf.sections[symbol["section_index"]] + assert section["name"] == table["section"] and section["flags"] & 1 + assert symbol["size"] == table["entries"] * 3 * elf.width + rows = [r for r in expected["native_declarations"] if r.get("table_symbol") == table["symbol"]] + by_index = {r["entry_index"]: r for r in rows} + for index, row in sorted(by_index.items()): + tables.append(elf.entry(symbol["address"], index, row["name"], row["descriptor"], row["function_symbol"])) + return {"path": str(path), "sha256": hashlib.sha256(data).hexdigest(), "machine": elf.machine, "pointer_bytes": elf.width, "relocation_encodings": sorted(elf.encodings), "table_scope": "known source symbols/entries only; APS2 deliberately not independently decoded here", "tables": tables, "exports": {name: {**value, "address": hex(value["address"])} for name, value in sorted(elf.exports.items())}, "loads": elf.loads} + + +def dex_oracle(path, expected): + data = limited(path) + assert data[:4] == b"dex\n" + u32 = lambda offset: struct.unpack_from(" { + const u16 = (at: number, value: number) => + little ? bytes.writeUInt16LE(value, at) : bytes.writeUInt16BE(value, at); + const u32 = (at: number, value: number) => + little ? bytes.writeUInt32LE(value, at) : bytes.writeUInt32BE(value, at); + const u64 = (at: number, value: bigint) => + little + ? bytes.writeBigUInt64LE(value, at) + : bytes.writeBigUInt64BE(value, at); + const ptr = (at: number, value: bigint) => + bits === 64 ? u64(at, value) : u32(at, Number(value)); + return { u16, u32, u64, ptr }; +}; +type FixtureWriter = ReturnType; + +/** Header declarations have one owner for both pointer widths. */ +const writeHeaders = ( + input: { + bytes: Buffer; + bits: 32 | 64; + little: boolean; + machine: number; + base: bigint; + dynamicBytes: number; + }, + writer: FixtureWriter, +) => { + const { bytes, bits, little, machine, base, dynamicBytes } = input; + const { u16, u32, u64, ptr } = writer; + bytes.set([0x7f, 0x45, 0x4c, 0x46, bits === 64 ? 2 : 1, little ? 1 : 2, 1]); + u16(16, 3); + u16(18, machine); + u32(20, 1); + const ph = bits === 64 ? 64 : 52, + phWidth = bits === 64 ? 56 : 32; + if (bits === 64) { + u64(32, BigInt(ph)); + u64(40, 0x1400n); + } else { + u32(28, ph); + u32(32, 0x1400); + } + u16(bits === 64 ? 52 : 40, ph); + u16(bits === 64 ? 54 : 42, phWidth); + u16(bits === 64 ? 56 : 44, 3); + u16(bits === 64 ? 58 : 46, bits === 64 ? 64 : 40); + u16(bits === 64 ? 60 : 48, 10); + u16(bits === 64 ? 62 : 50, 9); + const program = ( + row: number, + type: number, + offset: number, + size: number, + flags: number, + ) => { + const at = ph + row * phWidth; + u32(at, type); + if (bits === 64) { + u32(at + 4, flags); + ptr(at + 8, BigInt(offset)); + ptr(at + 16, base + BigInt(offset)); + ptr(at + 32, BigInt(size)); + ptr(at + 40, BigInt(size)); + ptr(at + 48, 0x1000n); + } else { + u32(at + 4, offset); + u32(at + 8, Number(base) + offset); + u32(at + 16, size); + u32(at + 20, size); + u32(at + 24, flags); + u32(at + 28, 0x1000); + } + }; + program(0, 1, 0, 0x400, 5); + program(1, 1, 0x400, bytes.length - 0x400, 6); + program(2, 2, 0xc00, dynamicBytes, 6); +}; + +/** Section and dynamic-symbol declarations refer to the fixture's fixed byte ranges. */ +const writeDirectory = ( + input: { + bytes: Buffer; + bits: 32 | 64; + word: number; + base: bigint; + encoding: FixtureEncoding; + rowWidth: number; + relocationSize: number; + dynamicBytes: number; + functions: readonly bigint[]; + unnamedData: boolean; + }, + writer: FixtureWriter, +) => { + const { + bytes, + bits, + word, + base, + encoding, + rowWidth, + relocationSize, + dynamicBytes, + functions, + unnamedData, + } = input; + const { u16, u32, ptr } = writer; + const dynstr = Buffer.from("\0JNI_OnLoad\0Java_fixture_Owner_exported\0"); + bytes.set(dynstr, 0x1100); + const symbolWidth = bits === 64 ? 24 : 16; + for (const [i, name] of [1, 12].entries()) { + const at = 0x1000 + (i + 1) * symbolWidth; + u32(at, name); + if (bits === 64) { + bytes[at + 4] = 0x12; + u16(at + 6, 1); + ptr(at + 8, functions[0] ?? 0n); + ptr(at + 16, 1n); + } else { + ptr(at + 4, functions[0] ?? 0n); + ptr(at + 8, 1n); + bytes[at + 12] = 0x12; + u16(at + 14, 1); + } + } + const names = [ + "", + ".text", + ".rodata", + ".data", + ".data.rel.ro", + `.${encoding}.dyn`, + ".dynamic", + ".dynsym", + ".dynstr", + ".shstrtab", + ]; + const shstrings = Buffer.from(names.join("\0") + "\0"); + bytes.set(shstrings, 0x1200); + const rows = [ + [0, 0, 0, 0, 0, 0, 0], + [1, 6, 0x100, 0x30, 0, 0, 0], + [1, 2, 0x400, 0x100, 0, 0, 0], + [1, 3, 0x500, word * 6, 0, 0, 0], + [1, 3, 0x600, word * 3, 0, 0, 0], + [ + encoding === "rela" ? 4 : encoding === "rel" ? 9 : 19, + 2, + 0x800, + relocationSize, + 7, + 0, + encoding === "relr" ? word : rowWidth, + ], + [6, 3, 0xc00, dynamicBytes, 8, 0, word * 2], + [11, 2, 0x1000, symbolWidth * 3, 8, 1, symbolWidth], + [3, 2, 0x1100, dynstr.length, 0, 0, 0], + [3, 0, 0x1200, shstrings.length, 0, 0, 0], + ]; + let nameOffset = 0; + rows.forEach( + ( + [ + type = 0, + flags = 0, + offset = 0, + size = 0, + link = 0, + info = 0, + entry = 0, + ], + i, + ) => { + if (i === 0) { + nameOffset++; + return; + } + const at = 0x1400 + i * (bits === 64 ? 64 : 40); + u32(at, unnamedData === true && i === 3 ? 0 : nameOffset); + u32(at + 4, type); + if (bits === 64) { + ptr(at + 8, BigInt(flags)); + ptr(at + 16, base + BigInt(offset)); + ptr(at + 24, BigInt(offset)); + ptr(at + 32, BigInt(size)); + u32(at + 40, link); + u32(at + 44, info); + ptr(at + 48, BigInt(word)); + ptr(at + 56, BigInt(entry)); + } else { + u32(at + 8, flags); + u32(at + 12, Number(base) + offset); + u32(at + 16, offset); + u32(at + 20, size); + u32(at + 24, link); + u32(at + 28, info); + u32(at + 32, word); + u32(at + 36, entry); + } + nameOffset += Buffer.byteLength(names[i] ?? "") + 1; + }, + ); +}; + +/** Hand-produced loader/section fixture, independent of the JNI parser. Never executed. */ +export const jniElfFixture = ( + options: { + readonly bits?: 32 | 64; + readonly machine?: number; + readonly encoding?: "rel" | "rela" | "relr"; + readonly little?: boolean; + readonly symbolicFirstFunction?: boolean; + readonly packed?: boolean; + readonly unnamedData?: boolean; + } = {}, +) => { + const bits = options.bits ?? 64, + word = bits / 8; + const little = options.little ?? true; + const machine = options.machine ?? (bits === 64 ? 183 : 40); + const encoding = options.encoding ?? (bits === 64 ? "rela" : "rel"); + const bytes = Buffer.alloc(0x1800); + const base = 0x10000n; + const writer = fixtureWriter(bytes, bits, little); + const { ptr } = writer; + const functions = [base + 0x100n, base + 0x110n, base + 0x120n].map((x) => + machine === 40 ? x | 1n : x, + ); + bytes[0x100] = 0xc3; + bytes[0x110] = 0xc3; + bytes[0x120] = 0xc3; + const mutf = (text: string) => { + const encoded: number[] = []; + for (let at = 0; at < text.length; at++) { + const c = text.charCodeAt(at); + if (c > 0 && c < 128) encoded.push(c); + else if (c < 2048) encoded.push(0xc0 | (c >> 6), 0x80 | (c & 63)); + else + encoded.push(0xe0 | (c >> 12), 0x80 | ((c >> 6) & 63), 0x80 | (c & 63)); + } + return Buffer.from([...encoded, 0]); + }; + const stringOffsets = [0x400, 0x420, 0x440, 0x460, 0x480, 0x4a0]; + ["call_็Œซ", "()V", "overload", "(I)I", "๐€", "([Ljava/lang/String;)J"].forEach( + (s, i) => bytes.set(mutf(s), stringOffsets[i] ?? 0), + ); + const slots = [ + [0x500, base + 0x400n], + [0x500 + word, base + 0x420n], + [0x500 + word * 2, functions[0] ?? 0n], + [0x500 + word * 3, base + 0x440n], + [0x500 + word * 4, base + 0x460n], + [0x500 + word * 5, functions[1] ?? 0n], + [0x600, base + 0x480n], + [0x600 + word, base + 0x4a0n], + [0x600 + word * 2, functions[2] ?? 0n], + ] as const; + const relative = machine === 183 ? 1027 : machine === 40 ? 23 : 8; + const rowWidth = word * (encoding === "rela" ? 3 : 2); + if (encoding === "relr") { + slots.forEach(([at, value]) => ptr(at, value)); + [base + 0x500n, 63n, base + 0x600n, 7n].forEach((value, i) => + ptr(0x800 + i * word, value), + ); + } else + slots.forEach(([at, value], i) => { + const row = 0x800 + i * rowWidth; + const symbolic = options.symbolicFirstFunction === true && i === 2; + const kind = symbolic + ? machine === 183 + ? 1025 + : machine === 40 + ? 21 + : 6 + : relative; + ptr(row, base + BigInt(at)); + ptr( + row + word, + bits === 64 + ? BigInt(kind) | (symbolic ? 1n << 32n : 0n) + : BigInt(kind | (symbolic ? 1 << 8 : 0)), + ); + if (encoding === "rela") ptr(row + word * 2, value); + else ptr(at, value); + }); + const relocationSize = + encoding === "relr" ? word * 4 : rowWidth * slots.length; + const tags: [bigint, bigint][] = + encoding === "relr" + ? [ + [36n, base + 0x800n], + [35n, BigInt(relocationSize)], + [37n, BigInt(word)], + ] + : encoding === "rela" + ? [ + [7n, base + 0x800n], + [8n, BigInt(relocationSize)], + [9n, BigInt(rowWidth)], + ] + : [ + [17n, base + 0x800n], + [18n, BigInt(relocationSize)], + [19n, BigInt(rowWidth)], + ]; + if (options.packed === true) tags.push([0x60000011n, base + 0x800n]); + tags.push([0n, 0n]); + tags.forEach(([tag, value], i) => { + ptr(0xc00 + i * word * 2, tag); + ptr(0xc00 + i * word * 2 + word, value); + }); + const dynamicBytes = tags.length * word * 2; + writeHeaders({ bytes, bits, little, machine, base, dynamicBytes }, writer); + writeDirectory( + { + bytes, + bits, + word, + base, + encoding, + rowWidth, + relocationSize, + dynamicBytes, + functions, + unnamedData: options.unnamedData === true, + }, + writer, + ); + return { bytes, base, word, functions, slots, relocationSize }; +}; diff --git a/tests/fixtures/android/static-initializers/NativeDeclarations.smali b/tests/fixtures/android/static-initializers/NativeDeclarations.smali new file mode 100644 index 0000000000..563e7ce89f --- /dev/null +++ b/tests/fixtures/android/static-initializers/NativeDeclarations.smali @@ -0,0 +1,19 @@ +.class public Lfixture/Native_ๅŽŸ; +.super Ljava/lang/Object; + +.method public static native read_ๅŽŸ([B)J +.end method + +.method public static native overloaded(I)I +.end method + +.method public static native overloaded(Ljava/lang/String;)I +.end method + +.method public static native do()V +.end method + +.method public static overloaded(Z)Z + .registers 1 + return p0 +.end method diff --git a/tests/fixtures/binaryDiagnostics/nativeExports.ts b/tests/fixtures/binaryDiagnostics/nativeExports.ts index 12d02547bc..ea6a1a644b 100644 --- a/tests/fixtures/binaryDiagnostics/nativeExports.ts +++ b/tests/fixtures/binaryDiagnostics/nativeExports.ts @@ -413,6 +413,62 @@ export const elfExportImage = (options: { return bytes; }; +/** Small source table whose headers replay one shared maximum-length auxiliary chain. */ +export const elfAliasedVersionImage = ( + kind: "requirements" | "definitions", + nameLength = 1, +): Buffer => { + const name = "V".repeat(nameLength); + const original = elfExportImage({ + symbols: [{ name: "versioned_export", version: 2 }], + versionDefinitions: ["BASE", name], + }); + const sectionAt = Number(original.readBigUInt64LE(40)); + const namesAt = Number(original.readBigUInt64LE(sectionAt + 2 * 64 + 24)); + const nameOffset = + original.indexOf(Buffer.from(name + "\0"), namesAt) - namesAt; + if (nameOffset < 0) throw new Error("Version fixture name not found"); + const headers = 10_000, + auxiliaries = 65_535; + const headerBytes = kind === "requirements" ? 16 : 20; + const auxBytes = kind === "requirements" ? 16 : 8; + const start = Math.ceil(original.length / 16) * 16; + const tableBytes = headers * headerBytes + auxiliaries * auxBytes; + const bytes = Buffer.alloc(start + tableBytes); + original.copy(bytes); + const section = sectionAt + (kind === "requirements" ? 5 : 4) * 64; + bytes.writeBigUInt64LE(BigInt(start), section + 24); + bytes.writeBigUInt64LE(BigInt(tableBytes), section + 32); + bytes.writeUInt32LE(headers, section + 44); + // The other version table is absent, so a refusal cannot recover through it. + const other = sectionAt + (kind === "requirements" ? 4 : 5) * 64; + bytes.writeUInt32LE(0, other + 4); + for (let index = 0; index < headers; index++) { + const at = start + index * headerBytes; + bytes.writeUInt16LE(1, at); + bytes.writeUInt16LE(auxiliaries, at + (kind === "requirements" ? 2 : 6)); + if (kind === "definitions") bytes.writeUInt16LE(2, at + 4); + bytes.writeUInt32LE( + headers * headerBytes - index * headerBytes, + at + (kind === "requirements" ? 8 : 12), + ); + bytes.writeUInt32LE( + index + 1 < headers ? headerBytes : 0, + at + (kind === "requirements" ? 12 : 16), + ); + } + for (let index = 0; index < auxiliaries; index++) { + const at = start + headers * headerBytes + index * auxBytes; + if (kind === "requirements") bytes.writeUInt16LE(2, at + 6); + bytes.writeUInt32LE(nameOffset, at + (kind === "requirements" ? 8 : 0)); + bytes.writeUInt32LE( + index + 1 < auxiliaries ? auxBytes : 0, + at + (kind === "requirements" ? 12 : 4), + ); + } + return bytes; +}; + type ElfSectionRow = readonly [ name: number, type: number, diff --git a/tests/fixtures/il2cpp/binary.ts b/tests/fixtures/il2cpp/binary.ts index db9eecb78b..3cdb0ad376 100644 --- a/tests/fixtures/il2cpp/binary.ts +++ b/tests/fixtures/il2cpp/binary.ts @@ -509,6 +509,52 @@ export interface ElfFixtureOptions { readonly zeroFilledTable?: boolean; } +/** A valid large RELA image; extra file-backed pointers do not add IL2CPP registrations. */ +export const createLargeElfRelativeFixture = ( + image: Buffer, + extraRecords: number, + machine = 183, +): Buffer => { + const original = createElfFixture(image, { machine }); + const originalDynamic = Number(original.readBigUInt64LE(128)); + const originalRelaSize = Number( + original.readBigUInt64LE(originalDynamic + 24), + ); + const pointersAt = Math.ceil(original.length / 8) * 8; + const relaAt = pointersAt + extraRecords * 8; + const relaSize = originalRelaSize + extraRecords * 24; + const dynamicAt = relaAt + relaSize; + const bytes = Buffer.alloc(dynamicAt + 64); + original.copy(bytes); + original.copy(bytes, relaAt, 0x1000, 0x1000 + originalRelaSize); + for (let index = 0; index < extraRecords; index++) { + const row = relaAt + originalRelaSize + index * 24; + bytes.writeBigUInt64LE( + FIXTURE_BINARY_BASE + BigInt(pointersAt + index * 8), + row, + ); + bytes.writeBigUInt64LE(machine === 62 ? 8n : 1027n, row + 8); + bytes.writeBigUInt64LE(FIXTURE_BINARY_BASE + 0x100n, row + 16); + } + for (const [index, [tag, value]] of [ + [7n, FIXTURE_BINARY_BASE + BigInt(relaAt)], + [8n, BigInt(relaSize)], + [9n, 24n], + [0n, 0n], + ].entries()) { + bytes.writeBigUInt64LE(tag!, dynamicAt + index * 16); + bytes.writeBigUInt64LE(value!, dynamicAt + index * 16 + 8); + } + bytes.writeBigUInt64LE(BigInt(bytes.length), 96); + bytes.writeBigUInt64LE(BigInt(bytes.length), 104); + bytes.writeBigUInt64LE(BigInt(dynamicAt), 128); + bytes.writeBigUInt64LE(FIXTURE_BINARY_BASE + BigInt(dynamicAt), 136); + bytes.writeBigUInt64LE(FIXTURE_BINARY_BASE + BigInt(dynamicAt), 144); + bytes.writeBigUInt64LE(64n, 152); + bytes.writeBigUInt64LE(64n, 160); + return bytes; +}; + /** * Rewrap a fixture image as a 64-bit ELF shared object loaded at the same * base: every pointer slot into the image is zeroed on disk and restored by diff --git a/tests/support/analysis-efficiency/android-jni-costs.py b/tests/support/analysis-efficiency/android-jni-costs.py new file mode 100644 index 0000000000..2b51d4edcf --- /dev/null +++ b/tests/support/analysis-efficiency/android-jni-costs.py @@ -0,0 +1,88 @@ +"""Estimate complete JNI task context; these counts are not model billing.""" +import argparse +import hashlib +import json +from pathlib import Path + +import tiktoken + + +def compact(value): + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + + +def inspect(path, encoding): + summary = json.loads((path / "summary.json").read_text()) + catalog = json.loads((path / "catalog.json").read_text()) + calls = json.loads((path / "calls.json").read_text()) + qualifications = summary.get("qualifications", []) + task_calls = sum(run["calls"] for run in summary["runs"]) + assert len(calls) == summary["calls"] == task_calls + sum(row["calls"] for row in qualifications) + assert summary.get("task_calls", task_calls) == task_calls + assert all(not call["result"].get("isError") for call in calls) + count = lambda value: len(encoding.encode(value, disallowed_special=())) + inputs = [{key: tool[key] for key in ("name", "description", "inputSchema")} for tool in catalog["tools"]] + question = count(summary["question"]) + discovery = count(compact(inputs)) + offset = 0 + runs = [] + def costs(selected, prompt, catalog_tokens): + args = sum(count(compact({"name": call["name"], "arguments": call["arguments"]})) for call in selected) + replies = sum(count(block["text"]) for call in selected for block in call["result"]["content"] if block["type"] == "text") + return {"question_tokens": count(prompt), "input_catalog_tokens": catalog_tokens, + "argument_tokens": args, "all_text_reply_tokens": replies, + "text_tokens": count(prompt) + catalog_tokens + args + replies} + for run in summary["runs"]: + selected = calls[offset:offset + run["calls"]] + offset += run["calls"] + assert all(call.get("phase", "task") == "task" for call in selected) + measured = costs(selected, summary["question"], discovery) + wire = [{key: call[key] for key in ("name", "arguments", "result")} for call in selected] + runs.append({**run, "question_tokens": question, "input_catalog_once_tokens": discovery, + "argument_tokens": measured["argument_tokens"], "all_text_reply_tokens": measured["all_text_reply_tokens"], + "whole_task_text_tokens": measured["text_tokens"], + "sdk_catalog_and_calls_tokens": count(compact({"catalog": catalog, "calls": wire}))}) + qualified = [] + for row in qualifications: + assert row["first_call"] == offset + selected = calls[offset:offset + row["calls"]] + offset += row["calls"] + assert all(call["phase"] == "qualification" for call in selected) + qualified.append({**row, **costs(selected, row["question"], 0), + "catalog_scope": "Reuses already discovered catalog; incremental qualification costs, separate from investigation tasks."}) + assert offset == len(calls) + receipt_paths = ["summary.json", "catalog.json", "calls.json"] + parity = path / "cli-parity.json" + cli = None + if parity.exists(): + data = json.loads(parity.read_text()) + receipt_paths.append(parity.name) + for name in ["cli-stdout.json", "cli-stderr.log"]: + if (path / name).exists(): + receipt_paths.append(name) + cli = {"elapsed_ms": data["elapsed_ms"], "fact_digest": data["fact_digest"], + "response_tokens": count((path / "cli-stdout.json").read_text()) if (path / "cli-stdout.json").exists() else count(compact(data["response"])), + "scope": "Separate single cold CLI parity check; not added to the MCP task/token totals."} + wire = [{key: call[key] for key in ("name", "arguments", "result")} for call in calls] + return {**summary, "case": str(path), "runs": runs, "qualifications": qualified, "cli_parity": cli, + "all_recorded_sdk_catalog_and_calls_tokens": count(compact({"catalog": catalog, "calls": wire})), + "receipts_sha256": {name: hashlib.sha256((path / name).read_bytes()).hexdigest() for name in receipt_paths}} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("cases", nargs="+", type=Path) + args = parser.parse_args() + encoding = tiktoken.get_encoding("o200k_base") + records = [inspect(path, encoding) for path in args.cases] + args.output.write_text(json.dumps({ + "tokenizer": f"tiktoken {tiktoken.__version__}", "encoding": encoding.name, + "scope": "Each completed run counts its fixed analyst question, discovered input catalog once, every argument and text reply, including pages and proof-export receipts. SDK totals additionally count output schemas and structuredContent duplicates. Full exported files are verified locally, not charged as agent context unless returned inline. Startup/discovery and final cleanup wall time are separate; outer resource receipts include CPU and process-family memory. Actual billing, hidden prompts, host rendering and pruning are unmeasured.", + "baseline": "Earlier main lacks this JNI batch/table capability; manual export lookup has unequal coverage and is not an equivalent before workflow.", + "actual_model_usage": None, "records": records, + }, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/tests/support/analysis-efficiency/android-jni-library.mjs b/tests/support/analysis-efficiency/android-jni-library.mjs new file mode 100644 index 0000000000..24fc24bbab --- /dev/null +++ b/tests/support/analysis-efficiency/android-jni-library.mjs @@ -0,0 +1,156 @@ +// Static parser qualification against the separately produced NDK/readelf/byte oracle. +// The caller supplies built parser and fixture paths; this script never executes an ELF. +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { dirname, extname, join, resolve } from "node:path"; +import { performance } from "node:perf_hooks"; +import { pathToFileURL } from "node:url"; +import { parseArgs } from "node:util"; + +const { values } = parseArgs({ + options: { + parser: { type: "string" }, + "fixture-root": { type: "string" }, + output: { type: "string" }, + }, +}); +for (const name of ["parser", "fixture-root", "output"]) + assert.ok(values[name]?.startsWith("/"), `select absolute --${name}`); +const parserBytes = await readFile(values.parser); +const { inspectAndroidJniLibrary } = await import( + pathToFileURL(values.parser).href +); +const oracleBytes = await readFile( + join(values["fixture-root"], "full-expectations.json"), +); +const oracle = JSON.parse(oracleBytes); +const receipts = []; +const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex"); + +async function verify(expected, variant) { + const bytes = await readFile(expected.path); + assert.equal(sha256(bytes), expected.sha256); + const before = performance.now(); + const parsed = await inspectAndroidJniLibrary(bytes); + const elapsed = performance.now() - before; + assert.ok(parsed.ok, JSON.stringify(parsed)); + const facts = parsed.value; + assert.equal(facts.bits, expected.pointer_bytes * 8); + assert.equal(facts.byte_order, "little"); + const expectedNames = Object.keys(expected.exports) + .filter((name) => name === "JNI_OnLoad" || name.startsWith("Java_")) + .sort(); + assert.deepEqual(facts.exports.map((row) => row.name).sort(), expectedNames); + for (const row of facts.exports) + assert.equal(row.address, expected.exports[row.name].address); + const rows = facts.tables.flatMap((table) => table.entries); + if (variant === "aps2") { + assert.equal(facts.coverage.tables_status, "unknown"); + assert.equal(rows.length, 0); + assert.ok(facts.unknowns.some((unknown) => unknown.includes("APS2"))); + } else + for (const row of expected.tables) { + const actual = rows.find( + (candidate) => candidate.slots[0]?.address === row.slots[0]?.address, + ); + assert.ok( + actual, + `missing ${expected.path} ${row.name} at ${row.slots[0]?.address}`, + ); + assert.equal(actual.name, row.name); + assert.equal(actual.descriptor, row.descriptor); + assert.equal(actual.function_address, row.function_address); + const code = + expected.machine === 40 + ? BigInt(row.function_address) & ~1n + : BigInt(row.function_address); + assert.equal(actual.code_address, `0x${code.toString(16)}`); + const load = expected.loads.find( + (load) => + load.type === 1 && + (load.flags & 1) !== 0 && + code >= BigInt(load.address) && + code < BigInt(load.address + load.file_bytes), + ); + assert.ok(load, "independent executable segment must cover the function"); + assert.equal( + actual.function_file_offset, + Number(code - BigInt(load.address) + BigInt(load.offset)), + ); + for (const [index, slot] of row.slots.entries()) { + assert.equal(actual.slots[index]?.address, slot.address); + assert.equal( + actual.slots[index]?.file_offset, + Number(BigInt(slot.file_offset)), + ); + assert.equal(actual.slots[index]?.value, slot.resolved_word); + assert.equal( + actual.slots[index]?.relocation?.encoding, + slot.encoding.toLowerCase(), + ); + } + } + receipts.push({ + path: expected.path, + sha256: expected.sha256, + variant, + elapsed_ms: elapsed, + exports: facts.exports.length, + entries: rows.length, + verified_entries: variant === "aps2" ? 0 : expected.tables.length, + coverage: facts.coverage, + unknowns: facts.unknowns, + }); +} +for (const [variant, value] of Object.entries(oracle.variants)) + for (const expected of Object.values(value.libraries)) + await verify(expected, variant); +if (Object.hasOwn(oracle, "conscrypt")) { + const conscrypt = JSON.parse( + await readFile( + join(values["fixture-root"], "oracle/conscrypt-arm64.json"), + "utf8", + ), + ); + await verify(conscrypt, "conscrypt"); +} +const fingerprints = {}; +for (const relative of [ + ".", + "../metadata/ElfPointerRelocations", + "../exports/NativeExportParser", + "../metadata/ElfImage", + "../metadata/NativeSectionDirectory", + "../../domain/elfRelocation", + "../../domain/elfDynamicEntry", +]) { + const path = + relative === "." + ? values.parser + : resolve(dirname(values.parser), relative + extname(values.parser)); + fingerprints[relative] = { path, sha256: sha256(await readFile(path)) }; +} +const output = { + parser: values.parser, + parser_sha256: sha256(parserBytes), + oracle_sha256: sha256(oracleBytes), + node: process.version, + static_only: true, + module_fingerprints: fingerprints, + receipts, +}; +await writeFile(values.output, JSON.stringify(output, null, 2) + "\n", { + flag: "wx", + mode: 0o600, +}); +console.log( + JSON.stringify({ + libraries: receipts.length, + verified_entries: receipts.reduce( + (sum, row) => sum + row.verified_entries, + 0, + ), + output: values.output, + }), +); diff --git a/tests/support/analysis-efficiency/android-jni.mjs b/tests/support/analysis-efficiency/android-jni.mjs new file mode 100644 index 0000000000..9bb11d06f8 --- /dev/null +++ b/tests/support/analysis-efficiency/android-jni.mjs @@ -0,0 +1,1104 @@ +/** Public SDK/stdio JNI investigation, checked against independent NDK/DEX facts. */ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { + readFile, + writeFile, + mkdir, + copyFile, + unlink, + stat, +} from "node:fs/promises"; +import { constants } from "node:fs"; +import { createRequire } from "node:module"; +import { join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { parseArgs } from "node:util"; +import { performance } from "node:perf_hooks"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const { values } = parseArgs({ + options: { + repository: { type: "string" }, + target: { type: "string" }, + output: { type: "string" }, + "java-home": { type: "string" }, + jar: { type: "string" }, + expectations: { type: "string" }, + variant: { type: "string", default: "ordinary" }, + repeats: { type: "string", default: "2" }, + proof: { type: "boolean", default: false }, + detail: { type: "string", default: "complete" }, + "omit-detail": { type: "boolean", default: false }, + pages: { type: "boolean", default: false }, + cli: { type: "boolean", default: false }, + "retained-removal": { type: "boolean", default: false }, + }, +}); +for (const name of [ + "repository", + "target", + "output", + "java-home", + "jar", + "expectations", +]) + assert.ok(values[name], `Missing --${name}`); +assert.ok(["complete", "summary"].includes(values.detail)); +assert.ok( + !values["omit-detail"] || values.detail === "summary", + "--omit-detail verifies the summary default", +); +assert.ok( + values.detail !== "summary" || values.proof, + "Summary verification requires its full proof export", +); +assert.ok(["ordinary", "relr", "aps2", "conscrypt"].includes(values.variant)); +const repeats = Number(values.repeats); +assert.ok(Number.isSafeInteger(repeats) && repeats >= 1 && repeats <= 20); +const repo = resolve(values.repository), + target = resolve(values.target), + output = resolve(values.output); +await mkdir(output, { mode: 0o700 }); +const expectations = JSON.parse( + await readFile(resolve(values.expectations), "utf8"), +); +const targetSha = createHash("sha256") + .update(await readFile(target)) + .digest("hex"); +const expected = + values.variant === "conscrypt" + ? expectations.conscrypt + : expectations.variants[values.variant]; +assert.equal( + targetSha, + expected.sha256, + "Selected target must be the independent oracle's exact APK", +); +const canonical = (value) => + Array.isArray(value) + ? value.map(canonical) + : value !== null && typeof value === "object" + ? Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, canonical(value[key])]), + ) + : value; +const factDigest = (report) => { + const { engine: _engine, ...facts } = report; + return createHash("sha256") + .update(JSON.stringify(canonical(facts))) + .digest("hex"); +}; +const declarationKey = (row) => + JSON.stringify([ + row.class_name ?? row.class, + row.method_name ?? row.name, + row.descriptor, + ]); +const goldenMethods = new Map( + expectations.dex.native_methods.map((row) => [declarationKey(row), row]), +); +const expectedSlots = (candidate, row) => { + assert.equal(candidate.address, row.function_address); + if (row.table_address !== undefined) + assert.equal(candidate.table.table_address, row.table_address); + if (row.entry_index !== undefined) + assert.equal(candidate.table.entry.entry_index, row.entry_index); + assert.equal(candidate.table.entry.function_address, row.function_address); + assert.equal(candidate.table.entry.name, row.name); + assert.equal(candidate.table.entry.descriptor, row.descriptor); + assert.equal(candidate.table.entry.slots.length, 3); + for (let index = 0; index < 3; index++) { + const slot = candidate.table.entry.slots[index], + gold = row.slots[index]; + assert.equal(slot.role, ["name", "signature", "function"][index]); + assert.equal(slot.address, gold.address); + assert.equal(slot.file_offset, Number(BigInt(gold.file_offset))); + assert.equal(slot.value, gold.resolved_word); + assert.equal(slot.relocation?.encoding, gold.encoding.toLowerCase()); + } +}; + +function checkFixture(report, query) { + assert.equal(report.native_inventory.coverage.status, "complete"); + assert.equal(report.library_inventory.status, "complete"); + assert.equal( + report.declarations.length, + goldenMethods.size, + "Selection must not erase global pair ambiguity", + ); + assert.deepEqual( + new Set(report.declarations.map(declarationKey)), + new Set(goldenMethods.keys()), + ); + const declarations = new Map( + report.declarations.map((row) => [row.declaration_id, row]), + ); + const libraries = new Map( + report.libraries.map((row) => [row.library_id, row]), + ); + const candidates = new Map( + report.candidates.map((row) => [row.candidate_id, row]), + ); + const expectedLibraries = Object.entries(expected.libraries).filter( + ([path]) => + (query.abis === undefined || query.abis.includes(path.split("/")[1])) && + (query.library_paths === undefined || query.library_paths.includes(path)), + ); + assert.deepEqual( + report.libraries.map((row) => row.path).sort(), + expectedLibraries.map(([path]) => path).sort(), + ); + for (const library of report.libraries) { + const gold = expected.libraries[library.path]; + assert.equal(library.sha256, gold.sha256); + assert.equal(library.abi, library.path.split("/")[1]); + assert.equal(library.failure, null); + if (values.variant === "aps2") { + assert.notEqual(library.coverage.tables_status, "complete"); + assert.ok(library.unknowns.some((text) => /APS2|packed/i.test(text))); + } else { + assert.ok(["complete", "partial"].includes(library.coverage.status)); + if (library.coverage.status === "partial") + assert.ok(library.unknowns.length > 0); + } + } + const selectedMethods = report.declarations.filter( + (row) => + (query.class_name === undefined || row.class_name === query.class_name) && + (query.method_names === undefined || + query.method_names.includes(row.method_name)) && + (query.descriptors === undefined || + query.descriptors.includes(row.descriptor)), + ); + assert.deepEqual( + report.selected_declaration_ids, + selectedMethods.map((row) => row.declaration_id), + ); + const abis = query.abis ?? [ + ...new Set(report.libraries.map((row) => row.abi)), + ]; + assert.equal(report.bindings.length, selectedMethods.length * abis.length); + for (const binding of report.bindings) { + const declaration = declarations.get(binding.declaration_id); + assert.ok(selectedMethods.includes(declaration)); + const gold = expected.expected_methods.find( + (row) => + declarationKey(row) === declarationKey(declaration) && + row.abi === binding.abi, + ); + assert.ok(gold); + const wanted = gold.implementations.filter((row) => + expectedLibraries.some(([path]) => path === row.library), + ); + const observed = binding.candidate_ids.map((id) => candidates.get(id)); + assert.equal(observed.length, wanted.length); + for (const impl of wanted) { + const candidate = observed.find( + (row) => + libraries.get(row.library_id)?.path === impl.library && + row.address === (impl.address ?? impl.function_address), + ); + assert.ok( + candidate, + `Missing source-known candidate for ${declarationKey(declaration)}`, + ); + assert.equal( + libraries.get(candidate.library_id).sha256, + impl.library_sha256, + ); + assert.ok(candidate.declaration_ids.includes(binding.declaration_id)); + if (gold.basis === "export") { + assert.equal(candidate.basis, "jni-export"); + assert.equal(candidate.name, gold.symbol); + } else { + assert.equal(candidate.basis, "static-registration-table"); + expectedSlots(candidate, impl); + if (gold.candidate_classes) { + assert.equal(candidate.class_attribution, "ambiguous-native-pair"); + assert.deepEqual( + candidate.declaration_ids + .map((id) => declarations.get(id).class_name) + .sort(), + [...gold.candidate_classes].sort(), + ); + } else assert.equal(candidate.class_attribution, "unique-native-pair"); + } + } + const ambiguousPair = gold.candidate_classes !== undefined; + const distinct = new Set( + wanted.map((row) => + JSON.stringify([row.library, row.address ?? row.function_address]), + ), + ).size; + if ((ambiguousPair && wanted.length > 0) || distinct > 1) + assert.equal(binding.status, "ambiguous"); + else if (distinct === 1) + assert.equal( + binding.status, + "matched", + "Known candidate presence survives partial coverage", + ); + else if (binding.coverage !== "complete") { + assert.equal( + binding.status, + "unknown", + "Incomplete coverage must not establish absence", + ); + assert.ok(binding.unknowns.length > 0); + } else assert.equal(binding.status, "unmatched"); + } +} + +function checkConscrypt(report, query) { + const descriptor = "([B[B[B[B[B[B[B[B)J"; + assert.equal(report.native_inventory.coverage.status, "complete"); + const declaration = report.declarations.filter( + (row) => + row.class_name === query.class_name && + row.method_name === "EVP_PKEY_new_RSA" && + row.descriptor === descriptor, + ); + assert.equal(declaration.length, 1); + assert.deepEqual(report.selected_declaration_ids, [ + declaration[0].declaration_id, + ]); + assert.equal(report.bindings.length, 1); + const binding = report.bindings[0]; + assert.equal(binding.declaration_id, declaration[0].declaration_id); + assert.equal(binding.abi, "arm64-v8a"); + assert.equal(binding.status, "matched"); + const library = report.libraries.find( + (row) => row.path === "lib/arm64-v8a/libconscrypt_jni.so", + ); + assert.ok(library); + assert.equal( + library.sha256, + expected.libraries.find((row) => row.member === library.path).sha256, + ); + assert.equal(library.jni_on_load, true); + const candidate = report.candidates.find( + (row) => + binding.candidate_ids.includes(row.candidate_id) && + row.address === "0xfa11c", + ); + assert.ok(candidate); + assert.equal(candidate.library_id, library.library_id); + assert.equal(candidate.basis, "static-registration-table"); + assert.equal(candidate.class_attribution, "unique-native-pair"); + expectedSlots(candidate, { + name: "EVP_PKEY_new_RSA", + descriptor, + function_address: "0xfa11c", + slots: [ + { + address: "0x2ada68", + file_offset: "0x2a5a68", + resolved_word: "0x749a8", + encoding: "RELA", + }, + { + address: "0x2ada70", + file_offset: "0x2a5a70", + resolved_word: "0x7affa", + encoding: "RELA", + }, + { + address: "0x2ada78", + file_offset: "0x2a5a78", + resolved_word: "0xfa11c", + encoding: "RELA", + }, + ], + }); + assert.equal( + BigInt(candidate.table.table_address) + + BigInt(candidate.table.entry.entry_index) * 24n, + 0x2ada68n, + ); +} + +function checkSummary(view, report) { + assert.equal(view.kind, "summary"); + assert.equal(view.parent_operation, "inspect_android_jni_bindings"); + assert.deepEqual(view.artifact, report.apk); + const summary = view.summary; + for (const collection of [ + "bindings", + "declarations", + "candidates", + "libraries", + ]) + assert.equal(summary.counts[collection], report[collection].length); + assert.equal(summary.bindings.length, Math.min(64, report.bindings.length)); + assert.deepEqual(summary.bindings, report.bindings.slice(0, 64)); + assert.equal(view.coverage.total, report.bindings.length); + assert.deepEqual(summary.native_inventory, report.native_inventory); + assert.deepEqual(summary.library_inventory, report.library_inventory); + assert.deepEqual( + summary.libraries, + report.libraries, + "Summary retains diagnostics even without positive candidate edges", + ); + assert.equal( + summary.counts.selected_declarations, + report.selected_declaration_ids.length, + ); + assert.deepEqual( + summary.selected_declaration_ids, + report.selected_declaration_ids.slice(0, 64), + ); + assert.equal( + summary.declaration_coverage.total, + report.selected_declaration_ids.length, + ); + assert.equal( + summary.declaration_coverage.examined, + Math.min(64, report.selected_declaration_ids.length), + ); + assert.equal( + summary.declaration_coverage.exhausted, + report.selected_declaration_ids.length <= 64, + ); + assert.equal( + summary.declaration_coverage.next_offset, + report.selected_declaration_ids.length <= 64 ? null : 64, + ); + for (const [collection, identity] of [ + ["declarations", "declaration_id"], + ["candidates", "candidate_id"], + ["libraries", "library_id"], + ]) { + const originals = new Map( + report[collection].map((row) => [row[identity], row]), + ); + for (const row of summary[collection]) + assert.deepEqual(row, originals.get(row[identity])); + } + const declarations = new Set( + summary.declarations.map((row) => row.declaration_id), + ); + const libraries = new Set(summary.libraries.map((row) => row.library_id)); + const candidates = new Map( + summary.candidates.map((row) => [row.candidate_id, row]), + ); + for (const id of summary.selected_declaration_ids) + assert.ok(declarations.has(id)); + for (const binding of summary.bindings) { + assert.ok(declarations.has(binding.declaration_id)); + for (const id of binding.candidate_ids) { + const candidate = candidates.get(id); + assert.ok(candidate && libraries.has(candidate.library_id)); + for (const declaration of candidate.declaration_ids) + assert.ok(declarations.has(declaration)); + } + } +} + +function checkPage(view, report) { + assert.equal(view.kind, "android-jni"); + assert.equal(view.parent_operation, "inspect_android_jni_bindings"); + assert.deepEqual(view.artifact, report.apk); + const request = view.view; + assert.deepEqual(view.source, { + native_inventory: report.native_inventory, + library_inventory: report.library_inventory, + selection: report.selection, + }); + const accepts = (values, value) => + values === undefined || values.includes(value); + const rows = report[request.collection] + .map((row, row_index) => ({ ...row, row_index })) + .filter((row) => { + if (!accepts(request.indices, row.row_index)) return false; + switch (request.collection) { + case "declarations": + return ( + accepts(request.declaration_ids, row.declaration_id) && + accepts(request.descriptors, row.descriptor) && + accepts(request.class_names, row.class_name) && + accepts(request.method_names, row.method_name) + ); + case "libraries": + return ( + accepts(request.library_ids, row.library_id) && + accepts(request.abis, row.abi) + ); + case "candidates": + return ( + accepts(request.candidate_ids, row.candidate_id) && + accepts(request.library_ids, row.library_id) && + (request.declaration_ids === undefined || + row.declaration_ids.some((id) => + request.declaration_ids.includes(id), + )) + ); + case "bindings": + return ( + accepts(request.declaration_ids, row.declaration_id) && + accepts(request.abis, row.abi) && + accepts(request.statuses, row.status) + ); + default: + throw new Error(`Unknown retained collection ${request.collection}`); + } + }); + const selected = rows + .slice(request.offset, request.offset + request.limit) + .map((row) => + Object.fromEntries( + Object.entries(row).filter( + ([key]) => + key === "row_index" || + request.fields === undefined || + request.fields.includes(key), + ), + ), + ); + assert.deepEqual(view.items, selected); + assert.equal(view.coverage.total, rows.length); + assert.equal(view.coverage.examined, selected.length); + const exhausted = + request.offset >= rows.length || + request.offset + selected.length >= rows.length; + assert.equal(view.coverage.exhausted, exhausted); + assert.equal( + view.coverage.next_offset, + exhausted ? null : request.offset + selected.length, + ); +} + +function collectionPages(preview) { + const declarationIds = preview.selected_declaration_ids.slice(0, 2); + const candidateIds = preview.candidates + .slice(0, 2) + .map((row) => row.candidate_id); + assert.ok(declarationIds.length > 0 && candidateIds.length > 0); + const page = (collection, extra) => ({ + kind: "android-jni", + collection, + offset: 0, + limit: 2, + ...extra, + }); + return [ + page("bindings", { + limit: 7, + fields: ["declaration_id", "abi", "status", "coverage", "candidate_ids"], + }), + page("bindings", { + offset: 1, + statuses: ["ambiguous"], + fields: ["declaration_id", "abi", "status"], + }), + page("bindings", { statuses: [], fields: ["declaration_id", "coverage"] }), + page("declarations", { + offset: 1, + fields: [ + "declaration_id", + "class_name", + "method_name", + "descriptor", + "source_index", + "dex_entry", + "dex_entry_index", + "class_index", + "method_index", + ], + }), + page("declarations", { + class_names: [], + fields: ["declaration_id", "source_index"], + }), + page("libraries", { + offset: 1, + fields: [ + "library_id", + "path", + "source_path", + "abi", + "sha256", + "source_key", + "source_index", + "coverage", + "failure", + ], + }), + page("libraries", { library_ids: [], fields: ["library_id", "coverage"] }), + page("candidates", { + indices: [0, 2], + fields: [ + "candidate_id", + "library_id", + "basis", + "name", + "descriptor", + "address", + "declaration_ids", + "table", + ], + }), + page("candidates", { + declaration_ids: [], + fields: ["candidate_id", "table"], + }), + page("declarations", { limit: 1, fields: [] }), + page("declarations", { + declaration_ids: declarationIds, + fields: [ + "declaration_id", + "class_name", + "method_name", + "descriptor", + "dex_entry_index", + "class_index", + "method_index", + "source_index", + ], + }), + page("candidates", { + candidate_ids: candidateIds, + fields: [ + "candidate_id", + "library_id", + "declaration_ids", + "address", + "table", + ], + }), + ]; +} + +async function qualifyRetainedRemoval(invoke) { + const ownedCopy = join(output, "owned-retained-source.apk"); + assert.notEqual(ownedCopy, target); + let copyOwned = false; + const start = performance.now(), + firstCall = calls.length; + try { + await copyFile(target, ownedCopy, constants.COPYFILE_EXCL); + copyOwned = true; + const query = + values.variant === "conscrypt" + ? { ...queries[0], path: ownedCopy } + : { + path: ownedCopy, + class_name: "fixture.jni.Bindings", + method_names: ["registeredData"], + descriptors: ["(I)I"], + abis: ["arm64-v8a"], + library_paths: ["lib/arm64-v8a/libfixture.so"], + }; + const parent = await invoke("inspect_android_jni_bindings", { + ...query, + detail: "complete", + }); + assert.equal(parent.operation, "inspect_android_jni_bindings"); + assert.equal(parent.subject.digest.sha256, targetSha); + assert.equal(parent.normalized_result.apk.path, ownedCopy); + const report = parent.normalized_result; + (values.variant === "conscrypt" ? checkConscrypt : checkFixture)( + report, + query, + ); + const before = factDigest(report); + await unlink(ownedCopy); + copyOwned = false; + await assert.rejects(stat(ownedCopy), { code: "ENOENT" }); + // After deletion only retained views and a bundle export are requested. + // No target-analysis call or caller's original path is used for recovery. + const source = { + kind: "retained-evidence", + evidence_id: parent.evidence_id, + }; + const summary = await invoke("inspect_analysis_view", { + source, + view: { kind: "summary" }, + }); + assert.equal( + summary.normalized_result.parent_evidence_id, + parent.evidence_id, + ); + checkSummary(summary.normalized_result, report); + const declarations = report.bindings.map( + (binding) => binding.declaration_id, + ); + const pages = [ + { + kind: "android-jni", + collection: "candidates", + offset: 0, + limit: 64, + declaration_ids: declarations, + fields: [ + "candidate_id", + "library_id", + "basis", + "name", + "descriptor", + "address", + "declaration_ids", + "class_attribution", + "table", + ], + }, + { + kind: "android-jni", + collection: "bindings", + offset: 0, + limit: 1, + fields: [ + "declaration_id", + "abi", + "status", + "coverage", + "candidate_ids", + ], + }, + ]; + let tableEntries = 0; + for (const view of pages) { + const evidence = await invoke("inspect_analysis_view", { source, view }); + assert.equal( + evidence.normalized_result.parent_evidence_id, + parent.evidence_id, + ); + checkPage(evidence.normalized_result, report); + tableEntries += evidence.normalized_result.items.filter( + (row) => row.table != null, + ).length; + } + if (values.variant !== "aps2") + assert.ok( + tableEntries > 0, + "Relocated slot evidence must remain available after APK removal", + ); + const proof = join(output, "retained-removal-evidence.json"); + await invoke("export_evidence_bundle", { path: proof }); + const bundle = JSON.parse(await readFile(proof, "utf8")); + const retained = bundle.records.find( + (row) => row.evidence_id === parent.evidence_id, + ); + assert.ok(retained); + assert.deepEqual(retained, parent); + assert.equal(factDigest(retained.normalized_result), before); + assert.equal( + createHash("sha256") + .update(await readFile(target)) + .digest("hex"), + targetSha, + ); + return { + name: "retained-removal", + elapsed_ms: performance.now() - start, + calls: calls.length - firstCall, + first_call: firstCall, + question: + "After deleting only the harness-owned APK copy, retrieve retained summary, exact candidate slots and selected binding fields, then export unchanged complete parent Evidence.", + owned_copy: ownedCopy, + original_path: target, + original_sha256: targetSha, + parent_evidence_id: parent.evidence_id, + fact_digest: before, + retained_table_rows: tableEntries, + scope: + "Observed public SDK success after owned-copy removal; only retained-view/export calls follow deletion. No target code executes; internal engine calls are not directly instrumented by this public harness.", + }; + } finally { + if (copyOwned) await unlink(ownedCopy); + } +} + +const require = createRequire(join(repo, "package.json")); +const { Client } = await import( + pathToFileURL(require.resolve("@modelcontextprotocol/client")) +); +const { StdioClientTransport } = await import( + pathToFileURL(require.resolve("@modelcontextprotocol/client/stdio")) +); +const { Ajv2020 } = await import( + pathToFileURL(require.resolve("ajv/dist/2020.js")) +); +const environment = { + PATH: process.env.PATH, + JAVA_HOME: resolve(values["java-home"]), + REA_JADX_MCP_JAR: resolve(values.jar), + REA_JADX_HEAP_MIB: "1024", + REA_JADX_ACTIVE_PROCESSOR_COUNT: "2", + NODE_OPTIONS: "--max-old-space-size=3072", + UV_THREADPOOL_SIZE: "2", + REA_LOG_LEVEL: "silent", + REA_MCP_TOOL_FAMILIES: + values.pages || values["retained-removal"] + ? "android,application,session" + : values.proof + ? "android,session" + : "android", + REA_MCP_INPUT_SCHEMA_PROFILE: "compact", +}; +const transport = new StdioClientTransport({ + command: process.execPath, + args: [join(repo, "scripts/rea.mjs"), "mcp"], + cwd: repo, + env: environment, + stderr: "pipe", + maxBufferSize: 16 * 1024 * 1024, +}); +let stderr = ""; +transport.stderr?.on("data", (chunk) => { + if (stderr.length < 65536) + stderr += chunk.toString().slice(0, 65536 - stderr.length); +}); +const client = new Client({ name: "rea-real-android-jni", version: "1" }); +const calls = [], + runs = [], + parents = []; +let firstReport; +let receipt; +let callPhase = "task"; +let validateJniOutput; +const queries = + values.variant === "conscrypt" + ? [ + { + path: target, + class_name: "org.conscrypt.NativeCrypto", + method_names: ["EVP_PKEY_new_RSA"], + descriptors: ["([B[B[B[B[B[B[B[B)J"], + abis: ["arm64-v8a"], + library_paths: ["lib/arm64-v8a/libconscrypt_jni.so"], + }, + ] + : [ + { path: target }, + { + path: target, + class_name: "fixture.jni.Bindings", + method_names: ["ambiguous"], + descriptors: ["(I)I"], + abis: ["arm64-v8a"], + }, + { path: target, method_names: ["registered_่กจ๐€"], abis: ["arm64-v8a"] }, + { + path: target, + method_names: ["sharedExport"], + abis: ["arm64-v8a"], + library_paths: ["lib/arm64-v8a/libduplicate.so"], + }, + ]; +try { + const discoveryStart = performance.now(); + await client.connect(transport); + const catalog = await client.listTools(); + const startup_catalog_elapsed_ms = performance.now() - discoveryStart; + await writeFile(join(output, "catalog.json"), JSON.stringify(catalog)); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const contracts = new Map(); + for (const name of [ + "inspect_android_jni_bindings", + ...(values.proof || values["retained-removal"] + ? ["export_evidence_bundle"] + : []), + ...(values.pages || values["retained-removal"] + ? ["inspect_analysis_view"] + : []), + ]) { + const selected = catalog.tools.filter((tool) => tool.name === name); + assert.equal(selected.length, 1); + assert.ok(selected[0].outputSchema); + contracts.set(name, { + definition: selected[0], + input: ajv.compile(selected[0].inputSchema), + output: ajv.compile(selected[0].outputSchema), + }); + } + validateJniOutput = contracts.get("inspect_android_jni_bindings").output; + const invoke = async (name, args) => { + const contract = contracts.get(name); + const sent = { ...args }; + if ( + values["omit-detail"] && + name === "inspect_android_jni_bindings" && + sent.detail === "summary" + ) + delete sent.detail; + assert.ok(contract.input(sent), JSON.stringify(contract.input.errors)); + const start = performance.now(); + const result = await client.callTool( + { name, arguments: sent }, + { toolDefinition: contract.definition, timeout: 120000 }, + ); + calls.push({ + phase: callPhase, + name, + arguments: sent, + result, + elapsed_ms: performance.now() - start, + }); + assert.notEqual(result.isError, true, JSON.stringify(result.content)); + assert.ok( + contract.output(result.structuredContent), + JSON.stringify(contract.output.errors), + ); + return result.structuredContent; + }; + let coldDigests; + for (let pass = 0; pass < repeats; pass++) { + const start = performance.now(), + firstCall = calls.length, + firstParent = parents.length; + const digests = []; + for (const query of queries) { + const selectedView = + values.detail === "summary" && values.pages && query === queries[1] + ? { + kind: "android-jni", + collection: "bindings", + offset: 0, + limit: 1, + fields: [ + "declaration_id", + "abi", + "status", + "coverage", + "candidate_ids", + ], + } + : undefined; + const evidence = await invoke("inspect_android_jni_bindings", { + ...query, + detail: values.detail, + ...(selectedView === undefined ? {} : { view: selectedView }), + }); + assert.equal( + evidence.operation, + values.detail === "summary" + ? "inspect_analysis_view" + : "inspect_android_jni_bindings", + ); + assert.equal(evidence.subject.digest.sha256, targetSha); + const parent = { + id: + values.detail === "summary" + ? evidence.normalized_result.parent_evidence_id + : evidence.evidence_id, + query, + normalized: + values.detail === "complete" ? evidence.normalized_result : undefined, + view: + values.detail === "summary" ? evidence.normalized_result : undefined, + pages: [], + }; + if (values.detail === "summary") assert.equal(evidence.raw_result, null); + parents.push(parent); + if (values.pages && query === queries[0]) { + const preview = + values.detail === "summary" + ? evidence.normalized_result.summary + : evidence.normalized_result; + for (const view of collectionPages(preview)) { + const result = await invoke("inspect_analysis_view", { + source: { kind: "retained-evidence", evidence_id: parent.id }, + view, + }); + parent.pages.push(result.normalized_result); + assert.equal(result.normalized_result.parent_evidence_id, parent.id); + } + } + } + let bundle; + if (values.proof) { + const path = join(output, `pass-${pass}-evidence.json`); + await invoke("export_evidence_bundle", { path }); + bundle = JSON.parse(await readFile(path, "utf8")); + } + for (const delivered of parents.slice(firstParent)) { + let report = delivered.normalized; + if (values.proof) { + const parent = bundle.records.find( + (record) => record.evidence_id === delivered.id, + ); + assert.ok(parent, "Complete source Evidence must remain exportable"); + assert.equal(parent.operation, "inspect_android_jni_bindings"); + assert.equal(parent.subject.digest.sha256, targetSha); + if (report !== undefined) + assert.deepEqual(parent.normalized_result, report); + report = parent.normalized_result; + for (const [key, value] of Object.entries(delivered.query)) + assert.deepEqual(parent.parameters[key], value); + } + assert.ok(report); + const { path: _path, ...selection } = delivered.query; + assert.deepEqual(report.selection, selection); + assert.equal(report.apk.path, target); + assert.equal(report.apk.sha256, targetSha); + (values.variant === "conscrypt" ? checkConscrypt : checkFixture)( + report, + delivered.query, + ); + if (delivered.view?.kind === "android-jni") + checkPage(delivered.view, report); + else if (delivered.view) checkSummary(delivered.view, report); + for (const page of delivered.pages) checkPage(page, report); + digests.push(factDigest(report)); + firstReport ??= report; + } + if (pass === 0) coldDigests = digests; + else + assert.deepEqual( + digests, + coldDigests, + "Warm reuse must preserve exact normalized facts", + ); + runs.push({ + temperature: pass === 0 ? "cold-provider" : "warm-session", + elapsed_ms: performance.now() - start, + calls: calls.length - firstCall, + fact_digests: digests, + }); + } + const summary = { + targetSha, + variant: values.variant, + detail: values.detail, + omit_detail: values["omit-detail"], + proof: values.proof, + pages: values.pages, + question: + values.variant === "conscrypt" + ? "Resolve NativeCrypto.EVP_PKEY_new_RSA to its static arm64 Conscrypt implementation and retain exact library identity and relocated slot evidence." + : "Resolve every fixture native declaration by ABI and library; preserve overloads, Unicode, global class ambiguity, duplicate libraries, unmatched methods and relocation uncertainty.", + baseline: + "Earlier main has no batch JNI operation. Manual inventory/export lookup cannot recover RegisterNatives with equivalent coverage; no equivalent before-speedup claim.", + startup_catalog_elapsed_ms, + elapsed_scope: + "Per-run task includes investigation calls, selected pages, proof export and local oracle checks. Startup/catalog and final transport cleanup are separate one-time costs. Resource supervisor measures the whole process family, including these phases.", + runs, + task_calls: calls.length, + calls: calls.length, + qualifications: [], + }; + receipt = summary; + await writeFile( + join(output, "summary.json"), + JSON.stringify(summary, null, 2), + ); + await writeFile(join(output, "calls.json"), JSON.stringify(calls)); + if (values["retained-removal"]) { + callPhase = "qualification"; + summary.qualifications.push(await qualifyRetainedRemoval(invoke)); + summary.calls = calls.length; + await writeFile( + join(output, "summary.json"), + JSON.stringify(summary, null, 2), + ); + } + console.log(JSON.stringify(summary)); +} finally { + const closeStart = performance.now(); + try { + await client.close(); + } finally { + if (receipt) { + receipt.transport_close_elapsed_ms = performance.now() - closeStart; + await writeFile( + join(output, "summary.json"), + JSON.stringify(receipt, null, 2), + ); + } + await writeFile(join(output, "calls.json"), JSON.stringify(calls)); + await writeFile(join(output, "stderr.log"), stderr); + } +} + +// CLI parity runs once after the MCP transport has joined its provider cleanup, +// keeping a second JVM out of the live warm-session measurement and memory set. +if (values.cli) { + const { path: _path, ...query } = queries[0]; + const proof = join(output, "cli-evidence.json"); + const args = [ + join(repo, "scripts/rea.mjs"), + "inspect-android-jni-bindings", + target, + "--query", + JSON.stringify(query), + "--detail", + values.detail, + "--json", + ]; + if (values.detail === "summary") args.push("--evidence_output", proof); + const page = + values.pages && values.detail === "summary" + ? { + kind: "android-jni", + collection: "bindings", + offset: 0, + limit: 7, + fields: [ + "declaration_id", + "abi", + "status", + "coverage", + "candidate_ids", + ], + } + : undefined; + if (page) args.push("--view", JSON.stringify(page)); + const start = performance.now(); + let command; + try { + command = await promisify(execFile)(process.execPath, args, { + cwd: repo, + env: environment, + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + timeout: 120000, + killSignal: "SIGINT", + }); + } catch (cause) { + await writeFile(join(output, "cli-stdout.json"), cause.stdout ?? ""); + await writeFile(join(output, "cli-stderr.log"), cause.stderr ?? ""); + throw cause; + } + const { stdout, stderr } = command; + await writeFile(join(output, "cli-stdout.json"), stdout); + await writeFile(join(output, "cli-stderr.log"), stderr); + const response = JSON.parse(stdout); + const evidence = values.detail === "summary" ? response.summary : response; + assert.ok( + validateJniOutput(evidence), + JSON.stringify(validateJniOutput.errors), + ); + let report = evidence.normalized_result; + if (values.detail === "summary") { + const bundle = JSON.parse(await readFile(proof, "utf8")); + const parent = bundle.records.find( + (row) => + row.evidence_id === evidence.normalized_result.parent_evidence_id, + ); + assert.ok(parent); + report = parent.normalized_result; + if (page) checkPage(evidence.normalized_result, report); + else checkSummary(evidence.normalized_result, report); + } + assert.equal(factDigest(report), factDigest(firstReport)); + await writeFile( + join(output, "cli-parity.json"), + JSON.stringify( + { + arguments: args, + elapsed_ms: performance.now() - start, + fact_digest: factDigest(report), + response, + stderr, + scope: + "One cold CLI parity check after MCP cleanup; separate from MCP task/token measurements.", + }, + null, + 2, + ), + ); +}