From 52ca222bd0da4835f2b88e49f13d153e1a832c8c Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:28:05 +0800 Subject: [PATCH 01/21] fix(il2cpp): preserve loader ownership and ABI field boundaries Resolve pointer and scalar facts through their shared loader-aware owners before registration search or type rendering. Preserve unresolved names and attributes, read count fields at their declared width, and admit declared zero-fill registration tails without accepting unmapped bytes. Cover relocation overlap and adjacency, loader-bound fields, ABI padding, nullable names, and CLI/MCP behavior; document format and verification limits. Fixes #6881 Fixes #6873 Fixes #6871 --- docs/il2cpp-metadata.md | 26 +++- src/domain/il2cpp/il2cppMetadata.ts | 2 +- src/il2cpp/Il2cppBinaryFile.ts | 16 +-- src/il2cpp/Il2cppBinaryImage.ts | 34 +++-- src/il2cpp/Il2cppBinaryRegistration.ts | 50 +++++-- src/il2cpp/Il2cppBinaryTypes.ts | 28 +++- src/il2cpp/Il2cppMetadataProvider.ts | 2 +- src/native/AppleMachoFixups.test.ts | 11 ++ src/native/AppleMachoFixups.ts | 66 ++++++++- .../AppleObjcClassImportBindings.test.ts | 1 + src/native/AppleSwiftVtables.test.ts | 2 + tests/acceptance/il2cppMetadataCli.test.ts | 131 ++++++++++++++++++ .../boundary/il2cpp/il2cppBinaryFile.test.ts | 70 ++++++++++ tests/fixtures/il2cpp/binary.ts | 40 +++++- 14 files changed, 420 insertions(+), 59 deletions(-) diff --git a/docs/il2cpp-metadata.md b/docs/il2cpp-metadata.md index 72d55efba6..6d2e22bed5 100644 --- a/docs/il2cpp-metadata.md +++ b/docs/il2cpp-metadata.md @@ -204,9 +204,13 @@ assembly image. Each module record starts with its module name, its method pointer count and its method pointer array. The reader finds, for every metadata image, the single record whose name pointer reaches that image's name string and whose count equals the image's method count. It then requires exactly one -`codeGenModulesCount`/`codeGenModules` pair that lists exactly those records. No -symbols are needed, so stripped builds work. A method's token row id selects its -slot in the module's method pointer array. Within one image, each method must +`codeGenModulesCount`/`codeGenModules` pair that lists exactly those records. +`codeGenModulesCount` and module method counts own four bytes, followed by +alignment padding before their eight-byte pointers. Unresolved loader writes +overlapping those four bytes prevent the registration from being established; +padding outside the field is neither required nor interpreted as part of its +count. No symbols are needed, so stripped builds work. A method's token row id +selects its slot in the module's method pointer array. Within one image, each method must claim a distinct MethodDef token row; duplicate rows are refused before any linked method table or seed output is published. Row ids may repeat across different images, and distinct rows may still resolve to the same address when @@ -265,12 +269,26 @@ The result adds: metadata name (`TKey`). Nested generic arguments stay with the component that declares them, for example `Game.Outer+Inner` or `Game.Outer+Leaf` when the nested type adds no parameters. A trailing - `&` marks a by-reference type. `name` is `null` only for type index `-1`. + `&` marks a by-reference type. `name` is `null` for type index `-1`, metadata + version 27 as described below, or an unresolved type-table slot. A dyld bind + or unresolved ELF relocation retains the type index without interpreting its + stored word as a resolved pointer. If an unresolved loader value occurs + inside a type graph, linking is refused with `unsupported-pointer-encoding`; + a concrete name is not synthesized from that placeholder. `binary` also records `types_field` and `types_count`: the address of `Il2CppMetadataRegistration.typesCount` and its value. That record is found as the single place whose `fieldOffsetsCount` and `typeDefinitionsSizesCount` both equal the metadata type-definition count and whose type table is file mapped. +The complete 128-byte registration must have unambiguous loader coverage. Its +used fields through byte 112 must be file-backed; a trailing declared zero-fill +range can cover the final unused fields. An unmapped or conflicting tail is +refused. These layouts follow the versioned [Unity 2022.3.8f1 producer header](https://github.com/MlgmXyysd/libil2cpp/blob/master/libil2cpp/Unity_2022.3/2022.3.8f1/il2cpp-class-internals.h#L596-L637) +(redistributed source) and the [AArch64 aggregate alignment rules](https://github.com/ARM-software/abi-aa/blob/2025Q1/aapcs64/aapcs64.rst#aggregates). +Dyld pointer ownership comes from the parsed bind stream, as in Apple's +[dyld analyzer](https://github.com/apple-oss-distributions/dyld/blob/dyld-1165.3/common/MachOAnalyzer.cpp#L1508-L1511), rather than the slot's placeholder word. +The padding, zero-fill-tail and unresolved-type cases have authored regression +fixtures; they do not add real Unity-build verification to the checks above. Without `binary_path`, `binary` is `null` and the four binary collections are empty in the complete producer report to indicate unrequested observations. Their summary counts, including state counts, are `null`, and native page requests report unavailable observations. This does not establish an empty native inventory. Missing native fields in historical Evidence remain unknown too. diff --git a/src/domain/il2cpp/il2cppMetadata.ts b/src/domain/il2cpp/il2cppMetadata.ts index 88ca5a70ce..4e1a02e6d2 100644 --- a/src/domain/il2cpp/il2cppMetadata.ts +++ b/src/domain/il2cpp/il2cppMetadata.ts @@ -207,7 +207,7 @@ export const il2cppMethodAddressSchema = z.strictObject({ /** Index into the binary Il2CppType table and its C#-like display name. */ export const il2cppTypeReferenceSchema = z.strictObject({ type: optionalIndex, - /** Null for type index -1, and for every row of a version 27 report. */ + /** Null for type index -1, unresolved native type-table slots, and version 27. */ name: z.string().nullable(), }); diff --git a/src/il2cpp/Il2cppBinaryFile.ts b/src/il2cpp/Il2cppBinaryFile.ts index f096e861db..e4b843c69b 100644 --- a/src/il2cpp/Il2cppBinaryFile.ts +++ b/src/il2cpp/Il2cppBinaryFile.ts @@ -1,11 +1,6 @@ import type { Il2cppMethodAddress } from "../domain/il2cpp/il2cppMetadata.js"; import { Il2cppBinaryFormatFailure } from "./Il2cppBinaryFormat.js"; -import { - openImageSteps, - readPointer, - readSlot, - isExecutable, -} from "./Il2cppBinaryImage.js"; +import { openImageSteps, readSlot, isExecutable } from "./Il2cppBinaryImage.js"; import { findModules, findCodeRegistration, @@ -180,8 +175,13 @@ export function* linkIl2cppBinarySteps( ); // 27.0/27.1 and 27.2 share header version 27 but place byref differently. if (version === 27) return { type, name: null }; - const address = - readPointer(image, registration.types + BigInt(type * 8)) ?? 0n; + const address = readSlot(image, registration.types + BigInt(type * 8)); + if (address === "unknown") return { type, name: null }; + if (address === null) + throw new Il2cppBinaryFormatFailure( + "invalid-type", + `type ${type} has no mapped type-table slot`, + ); if (address === 0n) throw new Il2cppBinaryFormatFailure( "invalid-type", diff --git a/src/il2cpp/Il2cppBinaryImage.ts b/src/il2cpp/Il2cppBinaryImage.ts index af254ba299..dc12b2ba1c 100644 --- a/src/il2cpp/Il2cppBinaryImage.ts +++ b/src/il2cpp/Il2cppBinaryImage.ts @@ -76,7 +76,7 @@ export interface NativeImage { readonly imageBase: bigint; readonly unresolvedPointerSlots: ReadonlyMap; readonly zeroFillPointers: ReadonlyMap; - /** Mach-O loader fixups, used when interpreting method pointer slots. */ + /** Mach-O loader fixups shared by every native pointer consumer. */ readonly machoPointerFixups?: PointerFixups; } @@ -191,13 +191,8 @@ export function* openImageSteps( }; } -export const readPointer = ( - image: NativeImage, - address: bigint, -): bigint | null => { - if (hasUnresolvedPointer(image, address)) return null; - const virtual = image.zeroFillPointers.get(address); - if (virtual !== undefined) return virtual; +/** Stored bytes only; loader interpretation belongs to readSlot. */ +const readStoredWord = (image: NativeImage, address: bigint): bigint | null => { try { const offset = mappedFileOffset( image.container.mappings, @@ -216,8 +211,12 @@ export const readPointer = ( export const readUInt32 = ( image: NativeImage, address: bigint, -): number | null => { - if (hasUnresolvedPointer(image, address, 4)) return null; +): number | "unknown" | null => { + if ( + hasUnresolvedElfBytes(image, address, 4) || + image.machoPointerFixups?.hasUnresolvedLegacyBytes(address, 4) === true + ) + return "unknown"; const virtual = image.zeroFillPointers.get(address); if (virtual !== undefined) return Number(virtual & 0xffffffffn); try { @@ -329,7 +328,7 @@ export const isExecutable = (image: NativeImage, address: bigint): boolean => { } }; -const hasUnresolvedPointer = ( +const hasUnresolvedElfBytes = ( image: NativeImage, address: bigint, size = 8, @@ -346,13 +345,13 @@ export const readSlot = ( image: NativeImage, address: bigint, ): bigint | "unknown" | null => { - if (hasUnresolvedPointer(image, address)) return "unknown"; + if (hasUnresolvedElfBytes(image, address)) return "unknown"; try { const virtual = image.zeroFillPointers.get(address); let raw: bigint | null; if (virtual !== undefined) raw = virtual; else if (isZeroFilled(image, address, 8)) raw = 0n; - else raw = readPointer(image, address); + else raw = readStoredWord(image, address); if (raw === null) return null; if (image.machoPointerFixups !== undefined) { const resolution = image.machoPointerFixups.decode(address, raw); @@ -372,3 +371,12 @@ export const readSlot = ( throw cause; } }; + +/** Require an observed resolved pointer; unknown and unmapped slots stay null. */ +export const readPointer = ( + image: NativeImage, + address: bigint, +): bigint | null => { + const value = readSlot(image, address); + return typeof value === "bigint" ? value : null; +}; diff --git a/src/il2cpp/Il2cppBinaryRegistration.ts b/src/il2cpp/Il2cppBinaryRegistration.ts index 0452477ec5..02a98caa68 100644 --- a/src/il2cpp/Il2cppBinaryRegistration.ts +++ b/src/il2cpp/Il2cppBinaryRegistration.ts @@ -7,6 +7,7 @@ import { isMapped, hasPointerArrayCoverage, readPointer, + readUInt32, type NativeImage, } from "./Il2cppBinaryImage.js"; import type { Il2cppMetadataDecoded } from "./Il2cppMetadataFile.js"; @@ -136,6 +137,8 @@ export function* findModules( const indices = names.get(target); if (indices === undefined) continue; const address = mapping.address + BigInt(at); + // Matching stored bytes nominate a candidate, not a resolved pointer. + if (readPointer(image, address) !== target) continue; let recordOffset: number; try { recordOffset = mappedFileOffset( @@ -160,9 +163,10 @@ export function* findModules( let work = 0; for (const index of indices) { if (++work % 256 === 0) yield; - const count = image.bytes.readUInt32LE(offset + 8); + const count = readUInt32(image, address + 8n); if (count !== counts[index]) continue; - const pointers = image.bytes.readBigUInt64LE(offset + 16); + const pointers = readPointer(image, address + 16n); + if (pointers === null) continue; if (!hasPointerArrayCoverage(image, pointers, count)) continue; const list = candidates.get(index) ?? []; if (list.some((candidate) => candidate.address === address)) continue; @@ -207,7 +211,6 @@ export function* findCodeRegistration( modules: readonly Il2cppCodeGenModule[], ): Generator { const expected = new Set(modules.map((module) => module.address)); - const total = BigInt(modules.length); const matches: { field: bigint; array: bigint }[] = []; const matchedFields = new Set(); for (const mapping of image.container.mappings) { @@ -215,9 +218,13 @@ export function* findCodeRegistration( for (let at = start; at + 16 <= mapping.size; at += 8) { if ((at & 0xffff) < 8) yield; const offset = mapping.offset + at; - if (image.bytes.readBigUInt64LE(offset) !== total) continue; - const array = image.bytes.readBigUInt64LE(offset + 8); - if (!isMapped(image, array, modules.length * 8)) continue; + // The count owns four bytes; the following four only align its pointer. + if (image.bytes.readUInt32LE(offset) !== modules.length) continue; + const field = mapping.address + BigInt(at); + if (readUInt32(image, field) !== modules.length) continue; + const array = readPointer(image, field + 8n); + if (array === null || !isMapped(image, array, modules.length * 8)) + continue; const seen = new Set(); for (let slot = 0; slot < modules.length; slot++) { if (slot % 256 === 0) yield; @@ -226,7 +233,6 @@ export function* findCodeRegistration( seen.add(pointer); } if (seen.size === modules.length) { - const field = mapping.address + BigInt(at); // Exact load aliases can visit one virtual registration field twice. if (matchedFields.has(field)) continue; matchedFields.add(field); @@ -272,8 +278,11 @@ function* findModuleArrayByCode( const lastPosition = new Map(); for (let at = start; at + 8 <= mapping.size; at += 8) { if ((at & 0xffff) < 8) yield; - const value = image.bytes.readBigUInt64LE(mapping.offset + at); - if (!expected.has(value)) { + const raw = image.bytes.readBigUInt64LE(mapping.offset + at); + const value = expected.has(raw) + ? readPointer(image, mapping.address + BigInt(at)) + : null; + if (value === null || !expected.has(value)) { runStart = at + 8; lastPosition.clear(); continue; @@ -333,21 +342,32 @@ export function* findMetadataRegistration( const matches: { field: bigint; count: number; types: bigint }[] = []; const matchedFields = new Set(); const low = (offset: number) => image.bytes.readUInt32LE(offset); - const word = (offset: number) => image.bytes.readBigUInt64LE(offset); for (const mapping of image.container.mappings) { const start = Number((8n - (mapping.address % 8n)) % 8n); - for (let at = start; at + 128 <= mapping.size; at += 8) { + // The fields used below end at byte 112. The full 128-byte registration + // must still have loader coverage, including any declared zero-fill tail. + for (let at = start; at + 112 <= mapping.size; at += 8) { if ((at & 0xffff) < 8) yield; const base = mapping.offset + at; if (low(base + 80) !== typeDefinitions) continue; if (low(base + 96) !== typeDefinitions) continue; - const count = low(base + 48); - const types = word(base + 56); + const address = mapping.address + BigInt(at); + if (!hasPointerArrayCoverage(image, address, 16)) continue; + if (readUInt32(image, address + 80n) !== typeDefinitions) continue; + if (readUInt32(image, address + 96n) !== typeDefinitions) continue; + const count = readUInt32(image, address + 48n); + const types = readPointer(image, address + 56n); + const fieldOffsets = readPointer(image, address + 88n); + const typeSizes = readPointer(image, address + 104n); if ( + typeof count !== "number" || count === 0 || count > 0x7fffffff || - !isMapped(image, word(base + 88), 8) || - !isMapped(image, word(base + 104), 8) || + types === null || + fieldOffsets === null || + typeSizes === null || + !isMapped(image, fieldOffsets, 8) || + !isMapped(image, typeSizes, 8) || !isMapped(image, types, count * 8) ) continue; diff --git a/src/il2cpp/Il2cppBinaryTypes.ts b/src/il2cpp/Il2cppBinaryTypes.ts index 4fa69ee93d..2fe7fcb3b2 100644 --- a/src/il2cpp/Il2cppBinaryTypes.ts +++ b/src/il2cpp/Il2cppBinaryTypes.ts @@ -1,6 +1,6 @@ import { Il2cppBinaryFormatFailure } from "./Il2cppBinaryFormat.js"; import { - readPointer, + readSlot, readUInt32, isMapped, type NativeImage, @@ -46,12 +46,26 @@ export const createTypeNamer = ( const fail = (message: string): never => { throw new Il2cppBinaryFormatFailure("invalid-type", message); }; - const pointer = (address: bigint, label: string) => - readPointer(image, address) ?? - fail(`${label} at 0x${address.toString(16)} is not file mapped`); - const u32 = (address: bigint, label: string) => - readUInt32(image, address) ?? - fail(`${label} at 0x${address.toString(16)} is not file mapped`); + const unresolved = (address: bigint, label: string): never => { + throw new Il2cppBinaryFormatFailure( + "unsupported-pointer-encoding", + `${label} at 0x${address.toString(16)} is loader-unresolved; its stored bytes are not a resolved type value`, + ); + }; + const pointer = (address: bigint, label: string): bigint => { + const value = readSlot(image, address); + if (value === "unknown") return unresolved(address, label); + return ( + value ?? fail(`${label} at 0x${address.toString(16)} is not file mapped`) + ); + }; + const u32 = (address: bigint, label: string): number => { + const value = readUInt32(image, address); + if (value === "unknown") return unresolved(address, label); + return ( + value ?? fail(`${label} at 0x${address.toString(16)} is not file mapped`) + ); + }; let cachedBytes = 0; let work = 0; const guard = (bytes: number) => { diff --git a/src/il2cpp/Il2cppMetadataProvider.ts b/src/il2cpp/Il2cppMetadataProvider.ts index 2de3d83c06..dce515bcd0 100644 --- a/src/il2cpp/Il2cppMetadataProvider.ts +++ b/src/il2cpp/Il2cppMetadataProvider.ts @@ -531,7 +531,7 @@ export class Il2cppMetadataProvider implements Il2cppMetadataPort { ? "Field, parameter and return types are not recovered without binary_path; type indices refer to the native binary." : decoded.metadata_version === 27 ? "Field, parameter and return type names are null for metadata version 27: versions 27.0 to 27.2 share that header number but place the Il2CppType byref flag differently, so a name could gain or lose & without warning; type indices are still reported." - : "Field, parameter and return type names are rendered from the binary's Il2CppType table in a C#-like form (keywords for primitives, Namespace.Outer+Inner, generic parameters by metadata name); nested generic arguments stay with the type component that declares them. Custom modifiers, array bounds and attributes are not reported.", + : "Field, parameter and return type names are rendered from the binary's Il2CppType table in a C#-like form (keywords for primitives, Namespace.Outer+Inner, generic parameters by metadata name); nested generic arguments stay with the type component that declares them. An unresolved type-table slot retains its type index with a null name; a loader-unresolved value inside the type graph is refused rather than rendered from stored placeholder bytes. Custom modifiers, array bounds and attributes are not reported.", ...(linked.binary !== null && linked.binary.code_gen_modules_field === null ? [ diff --git a/src/native/AppleMachoFixups.test.ts b/src/native/AppleMachoFixups.test.ts index 6e0ca4ac91..43cda5621b 100644 --- a/src/native/AppleMachoFixups.test.ts +++ b/src/native/AppleMachoFixups.test.ts @@ -187,6 +187,7 @@ it("marks only opcode-selected pointer slots rebased, including target zero", () kind: "rebase", target: BASE + 0x200n, }); + expect(fixups.hasUnresolvedLegacyBytes(BASE + 0x100n, 4)).toBe(false); expect(fixups.decode(BASE + 0x118n, 0n)).toEqual({ kind: "rebase", target: 0n, @@ -252,6 +253,7 @@ it("retains known prefix rebases and leaves unmatched locations unknown on an in expect(fixups.issues).toMatchObject([ { code: "rebase_stream_decode_failed" }, ]); + expect(fixups.hasUnresolvedLegacyBytes(BASE + 0x108n, 4)).toBe(true); }); it("does not treat 32-bit text rebases as 64-bit metadata pointers", () => { @@ -263,6 +265,10 @@ it("does not treat 32-bit text rebases as 64-bit metadata pointers", () => { kind: "unsupported", reason: "Legacy non-pointer rebase is not a 64-bit metadata pointer", }); + expect(fixups.hasUnresolvedLegacyBytes(BASE + 0x100n, 4)).toBe(true); + expect(fixups.hasUnresolvedLegacyBytes(BASE + 0x103n, 4)).toBe(true); + expect(fixups.hasUnresolvedLegacyBytes(BASE + 0x104n, 4)).toBe(false); + expect(fixups.hasUnresolvedLegacyBytes(BASE + 0x108n, 4)).toBe(false); expect(fixups.decode(BASE + 0x108n, BASE + 0x300n)).toMatchObject({ kind: "raw", target: BASE + 0x300n, @@ -438,6 +444,7 @@ describe("incomplete legacy bind applicability", () => { expect(fixups.issues).toMatchObject([ { code: "bind_stream_decode_failed" }, ]); + expect(fixups.hasUnresolvedLegacyBytes(BASE + 0x1008n, 4)).toBe(true); }); it("retains a positive bind prefix while leaving unmatched slots unknown", () => { @@ -1285,6 +1292,10 @@ describe("legacy pointer fixup streams", () => { kind: "raw", target: 0x99n, }); + expect(fixups.hasUnresolvedLegacyBytes(BASE + 0x4014n, 4)).toBe(true); + expect(fixups.hasUnresolvedLegacyBytes(BASE + 0x4018n, 4)).toBe(false); + expect(fixups.hasUnresolvedLegacyBytes(BASE + 0x401cn, 4)).toBe(false); + expect(fixups.hasUnresolvedLegacyBytes(BASE + 0x401cn, 8)).toBe(true); }); it("wraps a negative DO_BIND_ADD_ADDR_ULEB step and keeps decoding", () => { diff --git a/src/native/AppleMachoFixups.ts b/src/native/AppleMachoFixups.ts index 78de93e90c..de2c6939cc 100644 --- a/src/native/AppleMachoFixups.ts +++ b/src/native/AppleMachoFixups.ts @@ -65,6 +65,10 @@ export interface PointerFixups { readonly legacyBindStreams: number; /** Bytes in rebase streams that could not be completely decoded. */ readonly unreadRebaseBytes: number; + /** Unresolved legacy writes intersecting a scalar span, including incomplete + * legacy applicability. Chained pointer values still require decode(). + */ + hasUnresolvedLegacyBytes(address: bigint, size: number): boolean; /** Decode the pointer stored at `address` whose raw 64-bit value is `raw`. */ decode(address: bigint, raw: bigint): PointerResolution; } @@ -508,6 +512,8 @@ const chainedFixups = ( legacyRebaseStreams: rebases.examined, legacyBindStreams: binds.examined, unreadRebaseBytes: rebases.unreadBytes, + hasUnresolvedLegacyBytes: (address, size) => + hasUnresolvedLegacyBytes(segments, rebases, binds, address, size), decode(address, raw) { const rawPointer = (): PointerResolution => applicabilityComplete @@ -805,13 +811,13 @@ const readRebaseStream = ( } }; -const resolveLegacyRebase = ( +/** Read loader-owned membership without interpreting a stored pointer word. */ +const legacyRebaseState = ( segments: readonly Segment[], membership: Uint8Array | null, address: bigint, - raw: bigint, -): PointerResolution | undefined => { - if (membership === null) return undefined; +): number | Extract => { + if (membership === null) return 0; let fileOffset: number | undefined; for (const segment of segments) { const relative = address - segment.address; @@ -830,10 +836,20 @@ const resolveLegacyRebase = ( }; fileOffset = Number(mapped); } - if (fileOffset === undefined) return undefined; + if (fileOffset === undefined) return 0; const at = Math.floor(fileOffset / 4); const shift = (fileOffset % 4) * 2; - const state = ((membership[at] ?? 0) >> shift) & 3; + return ((membership[at] ?? 0) >> shift) & 3; +}; + +const resolveLegacyRebase = ( + segments: readonly Segment[], + membership: Uint8Array | null, + address: bigint, + raw: bigint, +): PointerResolution | undefined => { + const state = legacyRebaseState(segments, membership, address); + if (typeof state !== "number") return state; if (state === 0) return undefined; if (state === REBASE_POINTER) return { kind: "rebase", target: raw }; return { @@ -1153,6 +1169,41 @@ const parseLegacyBinds = ( }; }; +/** Query retained legacy facts; never expand bind runs or read scalar padding. */ +const hasUnresolvedLegacyBytes = ( + segments: readonly Segment[], + rebases: RebaseFacts, + binds: BindFacts, + address: bigint, + size: number, +): boolean => { + if (size <= 0) return false; + if (!rebases.complete || !binds.complete) return true; + const first = address - 7n; + const end = address + BigInt(size); + for (const run of binds.runs) { + // An eight-byte bind overlaps [address,end) exactly when its start is + // in [address-7,end). Find the first such member without expanding it. + const index = + first <= run.start + ? 0n + : (first - run.start + run.stride - 1n) / run.stride; + if (index < run.count && run.start + index * run.stride < end) return true; + } + if (rebases.membership !== null) + for (let slot = first; slot < end; slot++) { + const state = legacyRebaseState(segments, rebases.membership, slot); + // Text rebases own four bytes; conflicting kinds may include a pointer. + if ( + typeof state !== "number" || + state === REBASE_CONFLICT || + (state === REBASE_NON_POINTER && slot + 4n > address) + ) + return true; + } + return false; +}; + /** Build pointer decoding for one slice from its fixup load commands. */ export const parsePointerFixups = ( bytes: Buffer, @@ -1187,6 +1238,7 @@ export const parsePointerFixups = ( legacyRebaseStreams: 0, legacyBindStreams: 0, unreadRebaseBytes: 0, + hasUnresolvedLegacyBytes: () => false, decode: (_address, raw) => rawPointer(raw), }; const failures = [...rebases.failures, ...binds.failures]; @@ -1201,6 +1253,8 @@ export const parsePointerFixups = ( legacyRebaseStreams: rebases.examined, legacyBindStreams: binds.examined, unreadRebaseBytes: rebases.unreadBytes, + hasUnresolvedLegacyBytes: (address, size) => + hasUnresolvedLegacyBytes(segments, rebases, binds, address, size), decode(address, raw) { const match = resolveLegacyBind(binds.runs, address); const rebase = resolveLegacyRebase( diff --git a/src/native/AppleObjcClassImportBindings.test.ts b/src/native/AppleObjcClassImportBindings.test.ts index bf7244f71a..bddc53eafc 100644 --- a/src/native/AppleObjcClassImportBindings.test.ts +++ b/src/native/AppleObjcClassImportBindings.test.ts @@ -19,6 +19,7 @@ const createInput = ( legacyRebaseStreams: 0, legacyBindStreams: 0, unreadRebaseBytes: 0, + hasUnresolvedLegacyBytes: () => false, decode: (address) => address === 0x1000n ? { diff --git a/src/native/AppleSwiftVtables.test.ts b/src/native/AppleSwiftVtables.test.ts index 24f490d6ab..bd24420095 100644 --- a/src/native/AppleSwiftVtables.test.ts +++ b/src/native/AppleSwiftVtables.test.ts @@ -176,6 +176,7 @@ it("counts examined slots and reports an unresolved async slot", () => { legacyRebaseStreams: 0, legacyBindStreams: 0, unreadRebaseBytes: 0, + hasUnresolvedLegacyBytes: () => false, }, declarations: { facet: "swift_declarations", @@ -314,6 +315,7 @@ describe("Swift vtable coverage facts: unsupported pointer fixup coverage", () = legacyRebaseStreams: 0, legacyBindStreams: 0, unreadRebaseBytes: 0, + hasUnresolvedLegacyBytes: () => false, }, declarations: { facet: "swift_declarations", diff --git a/tests/acceptance/il2cppMetadataCli.test.ts b/tests/acceptance/il2cppMetadataCli.test.ts index 7a0f816201..9431232fe4 100644 --- a/tests/acceptance/il2cppMetadataCli.test.ts +++ b/tests/acceptance/il2cppMetadataCli.test.ts @@ -34,6 +34,7 @@ import { createElfFixture, createIl2cppBinaryFixture, createIl2cppDyldBindFixture, + createIl2cppRegistrationTailFixture, createUniversalFixture, FIXTURE_BINARY_BASE, FIXTURE_METHOD_ADDRESSES, @@ -899,6 +900,136 @@ cliTest( 30000, ); +cliTest( + "retains unknown dyld-bound type names and source identity through CLI and MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-il2cpp-type-bind-"); + const path = join(root, "global-metadata.dat"); + const binaryPath = join(root, "GameAssembly.dylib"); + await writeFile(path, createLinkedIl2cppMetadataFixture().bytes); + const inspect = async (bytes: Buffer) => { + await writeFile(binaryPath, bytes); + const output = await runFileCli(cli, { + arguments: [ + "inspect-il2cpp-metadata", + path, + "--binary", + binaryPath, + "--json", + ], + }); + expect(output.exitCode, output.stderr).toBe(0); + const report = parseCliMetadata(output.json); + expect(report.binary?.artifact).toEqual({ + path: binaryPath, + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.length, + }); + expect(await readFile(binaryPath)).toEqual(bytes); + return report; + }; + const control = await inspect(createIl2cppBinaryFixture()); + expect(control.return_types[0]?.name).toBe(FIXTURE_RETURN_TYPE_NAME); + const { client, call } = await connectLocalToolsMcp(); + for (const raw of [ + 0n, + FIXTURE_BINARY_BASE + 0x980n, + FIXTURE_BINARY_BASE + 0x9b0n, + ]) { + const report = await inspect(createIl2cppDyldBindFixture(raw, 0x900)); + for (const collection of [ + "return_types", + "field_types", + "parameter_types", + ] as const) + expect(report[collection]).toEqual( + control[collection].map((row) => + row.type === 0 ? { ...row, name: null } : row, + ), + ); + expect(report.methods).toEqual(control.methods); + expect(report.method_addresses).toEqual(control.method_addresses); + expect(report.limitations).toContainEqual( + expect.stringContaining("unresolved type-table slot"), + ); + // One transport round trip covers the nullable name contract; all stored + // placeholder variants above cross the real CLI workflow. + if (raw === 0n) { + const response = await call("inspect_il2cpp_metadata", { + path, + binary_path: binaryPath, + detail: "complete", + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(parseCliMetadata(response.structuredContent)).toEqual(report); + } + } + await client.ping(); + }, + 30000, +); + +cliTest( + "links ABI count padding and declared zero-fill registration tails through CLI and MCP", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-il2cpp-registration-layout-", + ); + const path = join(root, "global-metadata.dat"); + await writeFile(path, createLinkedIl2cppMetadataFixture().bytes); + const { client, call } = await connectLocalToolsMcp(); + for (const tailBytes of [0, 8, 16] as const) { + const bytes = createIl2cppRegistrationTailFixture(tailBytes); + bytes.fill(0xff, 0x604, 0x608); + const binaryPath = join(root, `registration-${tailBytes}.dylib`); + await writeFile(binaryPath, bytes); + const output = await runFileCli(cli, { + arguments: [ + "inspect-il2cpp-metadata", + path, + "--binary", + binaryPath, + "--json", + ], + }); + expect(output.exitCode, output.stderr).toBe(0); + const report = parseCliMetadata(output.json); + expect(report.binary).toMatchObject({ + code_gen_modules_field: "0x100000600", + types_field: "0x100000fb0", + types_count: 4, + artifact: { + path: binaryPath, + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.length, + }, + }); + expect(report.return_types.map((row) => row.name)).toEqual( + Array.from({ length: 5 }, () => FIXTURE_RETURN_TYPE_NAME), + ); + expect(report.method_addresses.map((row) => row.state)).toEqual([ + "code", + "code", + "code", + "code", + "no-body", + ]); + if (tailBytes === 16) { + const response = await call("inspect_il2cpp_metadata", { + path, + binary_path: binaryPath, + detail: "complete", + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(parseCliMetadata(response.structuredContent)).toEqual(report); + } + expect(await readFile(binaryPath)).toEqual(bytes); + } + await client.ping(); + }, + 30000, +); + cliTest( "keeps nested IL2CPP generic arguments on their declaring type components", async ({ cli }) => { diff --git a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts index c8c656191b..910451636b 100644 --- a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts +++ b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts @@ -1,4 +1,8 @@ import { Il2cppBinaryFormatFailure } from "../../../src/il2cpp/Il2cppBinaryFormat.js"; +import { + openImageSteps, + readUInt32, +} from "../../../src/il2cpp/Il2cppBinaryImage.js"; import { expect, it } from "vitest"; import { linkIl2cppBinary, @@ -9,6 +13,7 @@ import { createElfFixture, createIl2cppBinaryFixture, createIl2cppDyldBindFixture, + createIl2cppRegistrationTailFixture, createUniversalFixture, createSplitPointerPeFixture, FIXTURE_BINARY_BASE, @@ -593,6 +598,71 @@ it("keeps unbound Mach-O method words classified from their values", () => { expect(linked.method_addresses[1]).toEqual(expectedAddresses[1]); expect(linked.method_addresses[4]).toEqual(expectedAddress("no-body")); }); +it("refuses an unresolved inner type value instead of rendering its stored placeholder", () => { + expect( + failure(createIl2cppDyldBindFixture(FIXTURE_BINARY_BASE + 0x990n, 0x980)), + ).toEqual({ + reason: "unsupported-pointer-encoding", + kind: "unsupported", + }); +}); +it.each([ + [0x600, 2n, "code-registration-not-found"], + [0x408, 3n, "code-gen-modules-not-found"], + [0x850, 4n, "metadata-registration-not-found"], + [0x860, 4n, "metadata-registration-not-found"], + [0x988, 0x001d0000n, "unsupported-pointer-encoding"], +] as const)("does not trust a dyld-owned scalar at %s", (slot, raw, reason) => { + expect(failure(createIl2cppDyldBindFixture(raw, slot)).reason).toBe(reason); +}); +it.each([ + [0x5f8, 0n, 2], + [0x5fc, 2n << 32n, "unknown"], + [0x600, 2n, "unknown"], + [0x604, 0n, 2], +] as const)( + "uses the four-byte count span for a dyld bind at %s", + (slot, raw, expected) => { + const steps = openImageSteps( + createIl2cppDyldBindFixture(raw, slot), + undefined, + ); + let step = steps.next(); + while (!step.done) step = steps.next(); + expect(readUInt32(step.value, FIXTURE_BINARY_BASE + 0x600n)).toBe(expected); + }, +); +it.each([ + [0x400, "code-gen-modules-not-found"], + [0x410, "code-gen-modules-not-found"], + [0x500, "code-registration-not-found"], + [0x608, "code-registration-not-found"], + [0x838, "metadata-registration-not-found"], +] as const)( + "does not nominate a registration from dyld-owned word at %s", + (slot, reason) => { + const raw = createIl2cppBinaryFixture().readBigUInt64LE(slot); + expect(failure(createIl2cppDyldBindFixture(raw, slot)).reason).toBe(reason); + }, +); +it.each([1, 0xff])( + "ignores ABI count padding byte %d without accepting a changed count", + (padding) => { + const bytes = createIl2cppBinaryFixture(); + bytes.fill(padding, 0x604, 0x608); + expect(link(bytes)).toEqual(link(createIl2cppBinaryFixture())); + bytes.writeUInt32LE(3, 0x600); + expect(failure(bytes).reason).toBe("code-registration-not-found"); + }, +); +it.each([8, 16] as const)( + "rejects %d absent registration bytes without declared zero-fill", + (tailBytes) => { + expect( + failure(createIl2cppRegistrationTailFixture(tailBytes, false)).reason, + ).toBe("metadata-registration-not-found"); + }, +); it.each([ ["PC32", 2], diff --git a/tests/fixtures/il2cpp/binary.ts b/tests/fixtures/il2cpp/binary.ts index 27795c0795..47e772044c 100644 --- a/tests/fixtures/il2cpp/binary.ts +++ b/tests/fixtures/il2cpp/binary.ts @@ -187,15 +187,25 @@ export const createIl2cppBinaryFixture = ( return bytes; }; -/** Add one classic dyld bind for a method slot, retaining its chosen raw word. */ -export const createIl2cppDyldBindFixture = (raw: bigint): Buffer => { +/** Add one classic dyld bind at the selected slot, retaining its stored word. */ +export const createIl2cppDyldBindFixture = ( + raw: bigint, + slotOffset = POINTERS, +): Buffer => { const bytes = Buffer.from(createIl2cppBinaryFixture()); const command = 104; const streamOffset = 0xf00; + const offsetBytes: number[] = []; + let remaining = slotOffset; + do { + const byte = remaining & 0x7f; + remaining >>>= 7; + offsetBytes.push(byte | (remaining === 0 ? 0 : 0x80)); + } while (remaining !== 0); const stream = Buffer.concat([ Buffer.from([0x3e, 0x40]), Buffer.from("_FixtureMethod\0", "utf8"), - Buffer.from([0x51, 0x70, 0x80, 0x06, 0x90, 0x00]), + Buffer.from([0x51, 0x70, ...offsetBytes, 0x90, 0x00]), ]); bytes.writeUInt32LE(2, 16); bytes.writeUInt32LE(120, 20); @@ -204,7 +214,29 @@ export const createIl2cppDyldBindFixture = (raw: bigint): Buffer => { bytes.writeUInt32LE(streamOffset, command + 16); bytes.writeUInt32LE(stream.length, command + 20); stream.copy(bytes, streamOffset); - bytes.writeBigUInt64LE(raw, 0x300); + // A classic pointer bind names writable loader storage. + bytes.writeUInt32LE(7, 88); + bytes.writeUInt32LE(7, 92); + bytes.writeBigUInt64LE(raw, slotOffset); + return bytes; +}; + +/** Complete registration whose final zero words may occupy declared zero-fill. */ +export const createIl2cppRegistrationTailFixture = ( + tailBytes: 0 | 8 | 16, + zeroFilled = true, +): Buffer => { + const source = createIl2cppBinaryFixture(); + source.copy( + source, + SIZE - 128, + METADATA_REGISTRATION, + METADATA_REGISTRATION + 128, + ); + source.fill(0, METADATA_REGISTRATION, METADATA_REGISTRATION + 128); + const bytes = source.subarray(0, SIZE - tailBytes); + bytes.writeBigUInt64LE(BigInt(bytes.length), 80); + if (!zeroFilled) bytes.writeBigUInt64LE(BigInt(bytes.length), 64); return bytes; }; From 3b94f2c2ef3fea23efb370af98b1f4929358a23e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:28:06 +0800 Subject: [PATCH 02/21] fix(signals): honor selected files and input format options Keep staged recording names as relative file operands, including a literal dash. Recognize exact selected input-option refusal captions even when sigrok exits successfully or a caller key contains a newline; retain producer diagnostics and typed request failures across CLI/MCP. Fixes #6875 Fixes #6839 --- docs/signal-recordings.md | 18 +- src/signals/SigrokRecordingProvider.ts | 8 +- src/signals/sigrokSelectorRejection.ts | 54 ++- .../signals/recordingProvider.test.ts | 323 +++++++++++++----- 4 files changed, 298 insertions(+), 105 deletions(-) diff --git a/docs/signal-recordings.md b/docs/signal-recordings.md index e89745b7fd..04d51708b2 100644 --- a/docs/signal-recordings.md +++ b/docs/signal-recordings.md @@ -40,12 +40,22 @@ unrelated producer warnings do not change request status. Omitting `input_format` requests sigrok's autodetection. REA uses an immutable private copy whose filename retains the selected recording's final suffix, including a dotfile basename such as `.csv`. Filename-based format detection -receives that hint without exposing the -caller's directory to sigrok. Omitting `annotations` -retains every annotation class. Both omitted selections are recorded as null. +receives that hint without exposing the caller's directory to sigrok. The +input operand is an explicit relative path within the private workspace, +so a recording named `-` is opened as a file rather than +[sigrok's stdin operand](https://github.com/sigrokproject/sigrok-cli/blob/sigrok-cli-0.7.2/input.c#L49-L75). +Omitting `annotations` retains every annotation class. Both omitted selections +are recorded as null. If sigrok reports an explicitly selected input format as unknown, REA returns `invalid_request` for `input_format`, retains the producer diagnostic, and -advises correcting the format or omitting it for autodetection. See the +advises correcting the format or omitting it for autodetection. The +[tagged option parser](https://github.com/sigrokproject/sigrok-cli/blob/sigrok-cli-0.7.2/parsers.c#L387-L404) +warns when an option key is unknown. When that diagnostic's key exactly matches +a selected `input_format` option, REA returns `invalid_request`, even on exit zero. +Fallback output and the diagnostic remain in `captured_output`; unrelated +option warnings stay visible without rejecting the request. Matching keeps +the complete diagnostic caption, including an LF inside a selected key; +embedded warning substrings do not establish a refusal. See the [upstream manual](https://sigrok.org/wiki/Sigrok-cli) for formats, decoder stacking and annotation selectors. No arbitrary command flags are accepted. diff --git a/src/signals/SigrokRecordingProvider.ts b/src/signals/SigrokRecordingProvider.ts index f37c8cdcbc..50a9c5056e 100644 --- a/src/signals/SigrokRecordingProvider.ts +++ b/src/signals/SigrokRecordingProvider.ts @@ -240,8 +240,8 @@ export class SigrokRecordingProvider implements SignalRecordingPort { }, }, ); - // Some selected-channel refusals are diagnostics, not process failures: - // sigrok can emit fallback annotations and still exit successfully. + // Selected-channel and input-option refusals can be diagnostics rather + // than process failures: fallback annotations may still exit successfully. const rejection = selection === undefined ? undefined @@ -331,7 +331,9 @@ export class SigrokRecordingProvider implements SignalRecordingPort { phase = "producer"; const args = [ "--dont-scan", - `--input-file=${snapshotName}`, + // An explicit relative path preserves filename hints while keeping + // a recording named "-" distinct from sigrok's stdin operand. + `--input-file=./${snapshotName}`, `--protocol-decoders=${input.decoders}`, "--protocol-decoder-samplenum", ...(input.input_format === null diff --git a/src/signals/sigrokSelectorRejection.ts b/src/signals/sigrokSelectorRejection.ts index 3b54b9f50c..e1f0b1b084 100644 --- a/src/signals/sigrokSelectorRejection.ts +++ b/src/signals/sigrokSelectorRejection.ts @@ -14,6 +14,8 @@ export type SigrokSelection = > | Pick; +type SigrokDiagnosticMatcher = (message: string) => boolean; + const rejectedSelector = ( field: "decoder" | "decoders" | "annotations" | "input_format", producerMessage: string, @@ -32,6 +34,8 @@ const CHANNEL_RECOVERY = "Set each decoder channel target to a channel present in the recording; inspect_signal_decoders shows the selected decoder's channel IDs."; const INPUT_FORMAT_RECOVERY = "Use a supported sigrok input format name, or omit input_format to let sigrok-cli autodetect the recording."; +const INPUT_OPTION_RECOVERY = + "Use options supported by the selected sigrok input format, or remove the rejected option."; /** * sigrok-cli loads selected decoders before decoding or showing help, so an @@ -40,13 +44,13 @@ const INPUT_FORMAT_RECOVERY = * whose own imports or code fail to load. */ const unknownDecoderRefusal = ( - lines: ReadonlySet, + hasDiagnostic: SigrokDiagnosticMatcher, id: string, ): string | undefined => { const producerMessage = `Failed to load protocol decoder ${id}.`; return id !== "" && - lines.has(producerMessage) && - lines.has( + hasDiagnostic(producerMessage) && + hasDiagnostic( `srd: ModuleNotFoundError: Failed to load decoder ${id}: import by name failed: No module named '${id}'`, ) ? producerMessage @@ -55,12 +59,12 @@ const unknownDecoderRefusal = ( /** * The sigrok-cli 0.7.2 channel mapper reports missing and unknown targets as - * plain stderr lines, even though it continues with the other mappings. Match + * plain stderr diagnostics, even though it continues with the other mappings. Match * the exact selected channel key or target value rather than searching the * selector text for a substring. */ const missingSelectedChannelRefusal = ( - lines: ReadonlySet, + hasDiagnostic: SigrokDiagnosticMatcher, decoders: string, ): string | undefined => { const selectedValues = new Set(); @@ -69,14 +73,14 @@ const missingSelectedChannelRefusal = ( const separator = assignment.indexOf("="); if (separator < 0 && assignment !== "") { const producerMessage = `Channel name for "${assignment}" missing.`; - if (lines.has(`cli: ${producerMessage}`)) return producerMessage; + if (hasDiagnostic(`cli: ${producerMessage}`)) return producerMessage; } if (separator > 0) selectedValues.add(assignment.slice(separator + 1)); } } for (const channel of selectedValues) { const producerMessage = `No channel with name "${channel}" found.`; - if (lines.has(`cli: ${producerMessage}`)) return producerMessage; + if (hasDiagnostic(`cli: ${producerMessage}`)) return producerMessage; } return undefined; }; @@ -89,32 +93,54 @@ export const parseSigrokSelectorRejection = ( stderr: string, selection: SigrokSelection, ): SigrokSelectorRejection | undefined => { - const lines = new Set(stderr.split("\n")); + // Selected option keys may contain LF, which the tagged producer prints verbatim. + // Match the complete caption at diagnostic boundaries without splitting it. + const hasDiagnostic: SigrokDiagnosticMatcher = (message) => + stderr === message || + stderr.startsWith(`${message}\n`) || + stderr.endsWith(`\n${message}`) || + stderr.includes(`\n${message}\n`); if ("decoder" in selection) { const refusal = selection.decoder === null ? undefined - : unknownDecoderRefusal(lines, selection.decoder); + : unknownDecoderRefusal(hasDiagnostic, selection.decoder); return refusal === undefined ? undefined : rejectedSelector("decoder", refusal, HELP_RECOVERY); } - const inputFormat = selection.input_format?.split(":")[0] ?? ""; + const [inputFormat = "", ...inputOptions] = + selection.input_format?.split(":") ?? []; const inputFormatRefusal = `Error: unknown input module '${inputFormat}'.`; - if (inputFormat !== "" && lines.has(`cli: ${inputFormatRefusal}`)) + if (inputFormat !== "" && hasDiagnostic(`cli: ${inputFormatRefusal}`)) return rejectedSelector( "input_format", inputFormatRefusal, INPUT_FORMAT_RECOVERY, ); + // sigrok-cli 0.7.2 parses format:option=value with colon separators and + // warns on unknown option keys, then continues with the supported options. + for (const option of inputOptions) { + const key = option.split("=", 1)[0] ?? ""; + const producerMessage = `Unknown keyword: ${key}.`; + if (key !== "" && hasDiagnostic(producerMessage)) + return rejectedSelector( + "input_format", + producerMessage, + INPUT_OPTION_RECOVERY, + ); + } // sigrok-cli stacks decoders with "," and appends options with ":". for (const entry of selection.decoders.split(",")) { - const refusal = unknownDecoderRefusal(lines, entry.split(":")[0] ?? ""); + const refusal = unknownDecoderRefusal( + hasDiagnostic, + entry.split(":")[0] ?? "", + ); if (refusal !== undefined) return rejectedSelector("decoders", refusal, DECODE_RECOVERY); } const missingChannel = missingSelectedChannelRefusal( - lines, + hasDiagnostic, selection.decoders, ); if (missingChannel !== undefined) @@ -125,7 +151,7 @@ export const parseSigrokSelectorRejection = ( if (decoder === "" || classes === undefined || rest.length > 0) continue; for (const annotation of classes.split(":")) { const producerMessage = `Annotation '${annotation}' not found for protocol decoder '${decoder}'.`; - if (annotation !== "" && lines.has(producerMessage)) + if (annotation !== "" && hasDiagnostic(producerMessage)) return rejectedSelector( "annotations", producerMessage, diff --git a/tests/boundary/signals/recordingProvider.test.ts b/tests/boundary/signals/recordingProvider.test.ts index 94b4c61aaf..1e483ed76c 100644 --- a/tests/boundary/signals/recordingProvider.test.ts +++ b/tests/boundary/signals/recordingProvider.test.ts @@ -6,7 +6,8 @@ import { signalDecodersSchema } from "../../../src/domain/signals/signalDecoders import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { OwnedCommandFailure } from "../../../src/process/OwnedCommand.js"; import { it, expect, onTestFinished } from "vitest"; -import { writeFile, readdir, chmod, symlink } from "node:fs/promises"; +import { writeFile, readFile, readdir, chmod, symlink } from "node:fs/promises"; +import { createHash } from "node:crypto"; import { basename, join } from "node:path"; import { readStableArtifact } from "../../../src/artifacts/readStableArtifact.js"; import { SigrokRecordingProvider } from "../../../src/signals/SigrokRecordingProvider.js"; @@ -30,7 +31,7 @@ const producerSource = (mode: string, snapshotName: string): string => `import {readFileSync, writeFileSync, chmodSync, truncateSync} from 'node:fs'; const inputFileArgument = process.argv.find((argument) => argument.startsWith('--input-file=')); const snapshotName = inputFileArgument?.slice('--input-file='.length); -if (inputFileArgument !== undefined && inputFileArgument !== ${JSON.stringify(`--input-file=${snapshotName}`)}) throw Error('wrong snapshot name'); +if (${JSON.stringify(mode)} !== 'input-operand' && inputFileArgument !== undefined && inputFileArgument !== ${JSON.stringify(`--input-file=./${snapshotName}`)}) throw Error('wrong snapshot name'); if (process.argv.includes('--version')) { if (${JSON.stringify(mode)} === 'invalid-version-stdout') process.stdout.write(Buffer.concat([Buffer.from('sigrok-cli 0.7.2\\nlibsigrokdecode 0.5.3\\n'), Buffer.from([0xff])])); else console.log(${JSON.stringify(mode)} === 'bad-version' ? 'sigrok-cli 0.7.1' : 'sigrok-cli 0.7.2\\nlibsigrokdecode 0.5.3'); @@ -40,6 +41,10 @@ if (process.argv.includes('--version')) { else if (${JSON.stringify(mode)} === 'bad-version') console.log('unexpected version fallback'); else if (${JSON.stringify(mode)} === 'nonzero') { console.error('invalid decoder'); process.exitCode = 1; } else if (${JSON.stringify(mode)} === 'unknown-input-format') { if (!process.argv.includes('--input-format=missingformat')) console.error('fixture: selected input format was not forwarded'); console.error("cli: Error: unknown input module 'missingformat'."); process.exitCode = 1; } +else if (${JSON.stringify(mode)} === 'unknown-input-option') { if (!process.argv.includes('--input-format=vcd:down_sample=2')) throw Error('selected input option was not forwarded'); process.stdout.write('1104-1938 uart-1: 52\\n'); console.error('Unknown keyword: down_sample.'); } +else if (${JSON.stringify(mode)} === 'unknown-multiline-input-option') { if (!process.argv.includes(${JSON.stringify("--input-format=vcd:down\nsample=2")})) throw Error('selected input option was not forwarded'); process.stdout.write('1104-1938 uart-1: 52\\n'); console.error(${JSON.stringify("Unknown keyword: down\nsample.")}); } +else if (${JSON.stringify(mode)} === 'other-input-option-warning') { process.stdout.write('10-20 uart-1: 42\\n'); console.error('Unknown keyword: down_sample.'); } +else if (${JSON.stringify(mode)} === 'embedded-input-option-warning') { process.stdout.write('10-20 uart-1: 42\\n'); console.error('context: Unknown keyword: down_sample. suffix'); } else if (${JSON.stringify(mode)} === 'unknown-decoder') { console.error("srd: ModuleNotFoundError: Failed to load decoder nosuchdecoder: import by name failed: No module named 'nosuchdecoder'\\nFailed to load protocol decoder nosuchdecoder."); process.exitCode = 1; } else if (${JSON.stringify(mode)} === 'broken-decoder') { console.error("srd: ModuleNotFoundError: Failed to load decoder uart: import by name failed: No module named 'serial'\\nFailed to load protocol decoder uart."); process.exitCode = 1; } else if (${JSON.stringify(mode)} === 'unknown-annotation') { console.error("Annotation 'nosuchclass' not found for protocol decoder 'uart'."); process.exitCode = 1; } @@ -60,7 +65,8 @@ else if (${JSON.stringify(mode)} === 'wait') { console.log('ready'); setInterval else if (${JSON.stringify(mode)} === 'many') process.stdout.write(Array.from({length:90},(_,i)=>'line '+(i+1)+'\\r\\n').join('')); else if (process.argv.includes('--list-supported')) { console.log(${JSON.stringify(mode)} === 'unicode-search' ? 'Supported protocol decoders:\\n uart Universal Asynchronous Receiver/Transmitter\\n sigma ΟΣ\\n' : 'Supported hardware drivers:\\n ignored Device\\nSupported protocol decoders:\\n uart Universal Asynchronous Receiver/Transmitter\\n i2c Inter-Integrated Circuit\\n'); console.error('catalog warning'); } else if (process.argv.includes('--show')) console.log(${JSON.stringify(mode)} === 'unicode-search' ? 'ID: uart\\nOptions:\\n- label: ΟΣ' : 'ID: uart\\nOptions:\\n- baudrate: Baud rate (default 115200)\\n- parity: Parity'); -else if (${JSON.stringify(mode)} === 'csv-autodetect') { if (!process.argv.includes(${JSON.stringify(`--input-file=${snapshotName}`)}) || readFileSync(snapshotName, 'utf8') !== 'CLK,MOSI\\n0,0\\n1,0\\n') throw Error('CSV suffix or copy bytes lost'); console.log('10-20 spi-1: 1'); } +else if (${JSON.stringify(mode)} === 'csv-autodetect') { if (!process.argv.includes(${JSON.stringify(`--input-file=./${snapshotName}`)}) || readFileSync(snapshotName, 'utf8') !== 'CLK,MOSI\\n0,0\\n1,0\\n') throw Error('CSV suffix or copy bytes lost'); console.log('10-20 spi-1: 1'); } +else if (${JSON.stringify(mode)} === 'input-operand') { console.log(readFileSync(snapshotName === '-' ? 0 : snapshotName).toString('hex')); } else if (${JSON.stringify(mode)} === 'long-basename') { if (!process.argv.includes('--input-format=vcd') || readFileSync(snapshotName, 'utf8') !== 'recording bytes') throw Error('long source basename or explicit format was lost'); console.log('10-20 uart-1: 42'); } else { if(readFileSync(snapshotName, 'utf8') !== 'recording bytes') throw Error('wrong copy'); console.log('10-20 uart-1: 42'); console.error('producer warning'); } `; @@ -215,7 +221,7 @@ it("pages literal output without changing decoder arguments or normalizing text" expect(report.output_window.returned_lines).toBe(count); expect(report.tool.arguments).toEqual([ "--dont-scan", - `--input-file=${basename(f.path)}`, + `--input-file=./${basename(f.path)}`, "--protocol-decoders=uart", "--protocol-decoder-samplenum", ]); @@ -258,7 +264,7 @@ it("decodes an immutable copy, preserves options and warnings, and releases its expect(report.selection.input_format).toBeNull(); expect(report.tool.arguments).toEqual([ "--dont-scan", - "--input-file=recorded bytes.vcd", + "--input-file=./recorded bytes.vcd", "--protocol-decoders=uart:rx=RX:baudrate=9600", "--protocol-decoder-samplenum", "--protocol-decoder-annotations=uart=rx-data", @@ -285,7 +291,7 @@ it.each(["capture.csv", ".capture.csv", ".csv"])( }); expect(report.tool.arguments).toEqual([ "--dont-scan", - `--input-file=${recordingName}`, + `--input-file=./${recordingName}`, "--protocol-decoders=spi:clk=CLK:mosi=MOSI", "--protocol-decoder-samplenum", ]); @@ -307,13 +313,44 @@ it("stages a long source basename without expanding it before decoding", async ( expect(report.artifact.path).toBe(f.path); expect(report.selection.input_format).toBe("vcd"); expect(report.annotations_text).toBe("10-20 uart-1: 42\n"); - expect(report.tool.arguments).toContain(`--input-file=${recordingName}`); + expect(report.tool.arguments).toContain(`--input-file=./${recordingName}`); expect(f.calls()).toBe(2); expect( (await readdir(f.directory)).filter((name) => name.startsWith("decoder-")), ).toEqual([]); }); +it.each(["control.vcd", "-"])( + "opens the complete private recording for %s rather than stdin", + async (recordingName) => { + // Model only the tagged sigrok input operand rule, not protocol decoding. + const f = await fixture("input-operand", false, recordingName); + const source = await readFile(f.path); + const result = await f.service.decode({ + path: f.path, + decoders: "uart", + input_format: "vcd", + }); + if (!result.ok) throw result.error; + const report = signalRecordingSchema.parse(result.value.normalized_result); + expect(report.artifact).toEqual({ + path: f.path, + bytes: source.length, + sha256: createHash("sha256").update(source).digest("hex"), + }); + expect(report.selection).toMatchObject({ + path: f.path, + input_format: "vcd", + }); + expect(report.annotations_text).toBe(`${source.toString("hex")}\n`); + expect(report.tool.arguments).not.toContain("--input-file=-"); + expect(await readFile(f.path)).toEqual(source); + expect((await readdir(f.directory)).sort()).toEqual( + [recordingName, "producer.mjs"].sort(), + ); + }, +); + it.each([ ["nonzero", "ProviderAdapterError"], ["malformed-copy", "AnalysisArtifactChangedError"], @@ -395,93 +432,211 @@ it.each([ }, ); -it("reports an unknown selected input format through CLI and MCP", async () => { - const f = await fixture("unknown-input-format"); - const refusal = "Error: unknown input module 'missingformat'."; - const issueMessage = - `sigrok-cli rejected the input_format selector: ${refusal} ` + - "Use a supported sigrok input format name, or omit input_format to let sigrok-cli autodetect the recording."; - const harness = await createApplicationMcpHarness({ - serverOptions: { environment: {}, signalRecording: f.service }, - }); - onTestFinished(harness.close); - const { error } = parseMcpToolError( - await harness.client.callTool({ - name: "decode_signal_recording", - arguments: { - path: f.path, - decoders: "uart", - input_format: "missingformat", - }, - }), - ); - expect(error).toMatchObject({ - code: "invalid_request", - category: "invalid_input", - message: expect.stringContaining(issueMessage), - details: { - issues: [ - { - path: ["input_format"], - reason: "invalid_value", - message: expect.stringContaining(issueMessage), +it.each([ + { + mode: "unknown-input-format", + inputFormat: "missingformat", + refusal: "Error: unknown input module 'missingformat'.", + recovery: + "Use a supported sigrok input format name, or omit input_format to let sigrok-cli autodetect the recording.", + producerExit: 1, + producerStdout: "", + }, + { + mode: "unknown-input-option", + inputFormat: "vcd:down_sample=2", + refusal: "Unknown keyword: down_sample.", + recovery: + "Use options supported by the selected sigrok input format, or remove the rejected option.", + producerExit: 0, + producerStdout: "1104-1938 uart-1: 52\n", + }, + { + mode: "unknown-multiline-input-option", + inputFormat: "vcd:down\nsample=2", + refusal: "Unknown keyword: down\nsample.", + recovery: + "Use options supported by the selected sigrok input format, or remove the rejected option.", + producerExit: 0, + producerStdout: "1104-1938 uart-1: 52\n", + }, +])( + "reports $mode through CLI and MCP without successful fallback Evidence", + async ({ + mode, + inputFormat, + refusal, + recovery, + producerExit, + producerStdout, + }) => { + const f = await fixture(mode); + const issueMessage = + `sigrok-cli rejected the input_format selector: ${refusal} ` + recovery; + const harness = await createApplicationMcpHarness({ + serverOptions: { environment: {}, signalRecording: f.service }, + }); + onTestFinished(harness.close); + const { error } = parseMcpToolError( + await harness.client.callTool({ + name: "decode_signal_recording", + arguments: { + path: f.path, + decoders: "uart", + input_format: inputFormat, + }, + }), + ); + expect(error).toMatchObject({ + code: "invalid_request", + category: "invalid_input", + message: expect.stringContaining(issueMessage), + details: { + issues: [ + { + path: ["input_format"], + reason: "invalid_value", + message: expect.stringContaining(issueMessage), + }, + ], + captured_output: { + exit_code: producerExit, + stdout: producerStdout, + stderr: expect.stringContaining(refusal), }, - ], - captured_output: { - exit_code: 1, - stderr: expect.stringContaining(refusal), }, - }, - }); - expect(JSON.stringify(error)).not.toContain( - "fixture: selected input format was not forwarded", - ); + }); + expect(JSON.stringify(error)).not.toContain( + "fixture: selected input format was not forwarded", + ); - const cli = Cli.create("signal-input-format-refusal-test", { sync: false }); - registerSignalCommands(cli, silentLogger, {}, f.service); - let stdout = ""; - await cli.serve( - [ - "decode-signal-recording", - f.path, - "--decoders", - "uart", - "--input_format", - "missingformat", - "--format", - "json", - ], - { - stdout: (text) => { - stdout += text; - }, - exit: () => {}, - }, - ); - expect(JSON.parse(stdout)).toMatchObject({ - code: "invalid_request", - message: expect.stringContaining(issueMessage), - details: { - issues: [ + const cli = Cli.create("signal-input-format-refusal-test", { sync: false }); + registerSignalCommands(cli, silentLogger, {}, f.service); + let stdout = ""; + let commandExitCode: typeof process.exitCode; + const previousExitCode = process.exitCode; + try { + process.exitCode = 0; + await cli.serve( + [ + "decode-signal-recording", + f.path, + "--decoders", + "uart", + "--input_format", + inputFormat, + "--format", + "json", + ], { - path: ["input_format"], - reason: "invalid_value", - message: expect.stringContaining(issueMessage), + stdout: (text) => { + stdout += text; + }, + exit: () => {}, }, - ], + ); + commandExitCode = process.exitCode; + } finally { + process.exitCode = previousExitCode; + } + expect(commandExitCode).toBe(1); + const envelope: unknown = JSON.parse(stdout); + expect(envelope).not.toHaveProperty("evidence_id"); + expect(envelope).toMatchObject({ + code: "invalid_request", + message: expect.stringContaining(issueMessage), + details: { + issues: [ + { + path: ["input_format"], + reason: "invalid_value", + message: expect.stringContaining(issueMessage), + }, + ], + captured_output: { + exit_code: producerExit, + stdout: producerStdout, + stderr: expect.stringContaining(refusal), + }, + }, + }); + expect(stdout).not.toContain( + "fixture: selected input format was not forwarded", + ); + expect(f.calls()).toBe(4); + expect( + (await readdir(f.directory)).filter((name) => + name.startsWith("decoder-"), + ), + ).toEqual([]); + }, +); + +it.each([ + "vcd:downsample=2", + "vcd:down_sample_extra=2", + "vcd:label=down_sample", + null, +])( + "retains an unrelated input-option warning with selection %s", + async (inputFormat) => { + const f = await fixture("other-input-option-warning"); + const result = await f.service.decode({ + path: f.path, + decoders: "uart", + input_format: inputFormat, + }); + if (!result.ok) throw result.error; + const report = signalRecordingSchema.parse(result.value.normalized_result); + expect(report.selection.input_format).toBe(inputFormat); + expect(report.annotations_text).toBe("10-20 uart-1: 42\n"); + expect(report.diagnostics).toBe("Unknown keyword: down_sample.\n"); + if (inputFormat !== null) + expect(report.tool.arguments).toContain(`--input-format=${inputFormat}`); + else + expect( + report.tool.arguments.some((argument) => + argument.startsWith("--input-format="), + ), + ).toBe(false); + }, +); + +it("recognizes a rejected bare input-format option key", async () => { + const f = await fixture("other-input-option-warning"); + const result = await f.service.decode({ + path: f.path, + decoders: "uart", + input_format: "vcd:down_sample", + }); + if (result.ok) throw new Error("Expected selected bare option refusal"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "invalid_request", + details: { + issues: [{ path: ["input_format"], reason: "invalid_value" }], captured_output: { - exit_code: 1, - stderr: expect.stringContaining(refusal), + exit_code: 0, + stdout: "10-20 uart-1: 42\n", + stderr: "Unknown keyword: down_sample.\n", }, }, }); - expect(stdout).not.toContain( - "fixture: selected input format was not forwarded", +}); + +it("does not treat an embedded warning substring as a selected option refusal", async () => { + const f = await fixture("embedded-input-option-warning"); + const result = await f.service.decode({ + path: f.path, + decoders: "uart", + input_format: "vcd:down_sample=2", + }); + if (!result.ok) throw result.error; + const report = signalRecordingSchema.parse(result.value.normalized_result); + expect(report.selection.input_format).toBe("vcd:down_sample=2"); + expect(report.annotations_text).toBe("10-20 uart-1: 42\n"); + expect(report.diagnostics).toBe( + "context: Unknown keyword: down_sample. suffix\n", ); - expect(f.calls()).toBe(4); - expect( - (await readdir(f.directory)).filter((name) => name.startsWith("decoder-")), - ).toEqual([]); }); it("projects sigrok's zero-exit missing-channel refusal through CLI and MCP with partial output", async () => { From 3e3c9eadb5371794a5df1046f1c0d3bb31f8e9cb Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:33:04 +0800 Subject: [PATCH 03/21] fix(protobuf): reject ambiguous descriptor identities Validate message-local field numbers and fully qualified registry identities during existing descriptor admission before constructing the schema registry. Preserve repeated wire occurrences, enum aliases and independent scopes, with bounded name allocation and retained capture provenance through CLI/MCP refusals. Fixes #6834 --- docs/web-network-captures.md | 10 +- .../history/protobuf/ProtobufSchema.ts | 54 ++-- .../tools/protobufDescriptorIdentity.test.ts | 257 ++++++++++++++++++ .../tools/protobufRegistryIdentity.test.ts | 218 +++++++++++++++ 4 files changed, 520 insertions(+), 19 deletions(-) create mode 100644 tests/acceptance/tools/protobufDescriptorIdentity.test.ts create mode 100644 tests/acceptance/tools/protobufRegistryIdentity.test.ts diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index 459bd42b45..6a1621c864 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -166,7 +166,15 @@ fully qualified message type. All imports must be included. REA never fetches schemas or executes the selected application. Descriptor identity and parser version remain in the body facts. Scalars, repeated numeric packing and nested messages are interpreted alongside the unchanged wire facts; disagreements -remain explicit. Map semantics, extensions and required-field validation are +remain explicit. Descriptor admission rejects duplicate field numbers within +each message, including nested messages and oneof fields, before schema +annotation. This does not reject repeated occurrences in captured wire bytes +or reuse of a number in a different message. Fully qualified message, enum, +service and extension names must also identify one declaration across the +descriptor set, including nested and synthetic map-entry messages. Different +packages or enclosing messages may reuse a local type name. Schema-admission +failures retain the partial capture Evidence and its original artifact identity. +Map semantics, extensions and required-field validation are unsupported. A supplied schema does not establish the sender's actual schema. An invalid nested interpretation records `schema_status: "invalid-message"` and `schema_error` locally; later outer fields remain available. Valid string diff --git a/src/browser/history/protobuf/ProtobufSchema.ts b/src/browser/history/protobuf/ProtobufSchema.ts index 8135b6c58d..0c4ec98733 100644 --- a/src/browser/history/protobuf/ProtobufSchema.ts +++ b/src/browser/history/protobuf/ProtobufSchema.ts @@ -39,42 +39,60 @@ const scalarWire = (scalar: ScalarType): ProtobufField["wire_type"] => { const floatingValue = (value: number): string => Object.is(value, -0) ? "-0" : String(value); -/** Qualified names can expand a small descriptor into a much larger registry. */ -const chargeRegistryExpansion = async ( +/** Admit descriptor identities and bound qualified names before registry construction. */ +const admitRegistryDescriptor = async ( descriptor: FileDescriptorSet, budget: ProtobufBudget, ): Promise => { + const registryNames = new Set(); + const chargeQualifiedName = (name: string, prefix: string) => + budget.charge((prefix.length + 1 + name.length) * 8 + 128); + const admitTypeName = (name: string, prefix: string): string => { + chargeQualifiedName(name, prefix); + budget.charge(32); + const qualified = prefix === "" ? name : `${prefix}.${name}`; + if (registryNames.has(qualified)) + throw new Error( + "Duplicate fully qualified type name in FileDescriptorSet", + ); + registryNames.add(qualified); + return qualified; + }; for (const file of descriptor.file) { budget.charge(file.name.length * 8 + 128); const stack = file.messageType.map((message) => ({ message, - prefix: file.package.length, + prefix: file.package, })); - const chargeQualifiedName = (name: string, prefix: number) => - budget.charge((prefix + 1 + name.length) * 8 + 128); for (const enumeration of file.enumType) - chargeQualifiedName(enumeration.name, file.package.length); + admitTypeName(enumeration.name, file.package); for (const extension of file.extension) - chargeQualifiedName(extension.name, file.package.length); + admitTypeName(extension.name, file.package); for (const service of file.service) { - const length = file.package.length + 1 + service.name.length; - budget.charge(length * 8 + 128); + const qualified = admitTypeName(service.name, file.package); for (const method of service.method) - chargeQualifiedName(method.name, length); + chargeQualifiedName(method.name, qualified); } while (stack.length > 0) { await budget.checkpoint(); const { message, prefix } = stack.pop()!; - const length = prefix + 1 + message.name.length; - budget.charge(length * 8 + 128); - for (const field of message.field) - chargeQualifiedName(field.name, length); + const qualified = admitTypeName(message.name, prefix); + budget.charge(message.field.length * 32); + const fieldNumbers = new Set(); + for (const field of message.field) { + chargeQualifiedName(field.name, qualified); + if (fieldNumbers.has(field.number)) + throw new Error( + "Duplicate field number in FileDescriptorSet message", + ); + fieldNumbers.add(field.number); + } for (const extension of message.extension) - chargeQualifiedName(extension.name, length); + admitTypeName(extension.name, qualified); for (const enumeration of message.enumType) - chargeQualifiedName(enumeration.name, length); + admitTypeName(enumeration.name, qualified); for (const nested of message.nestedType) - stack.push({ message: nested, prefix: length }); + stack.push({ message: nested, prefix: qualified }); } } }; @@ -151,7 +169,7 @@ export class ProtobufSchema implements WireSchema { readUnknownFields: false, recursionLimit: WEB_NETWORK_CAPTURE_LIMITS.depth, }); - await chargeRegistryExpansion(descriptor, budget); + await admitRegistryDescriptor(descriptor, budget); const message = createFileRegistry(descriptor).getMessage( typeName.replace(/^\./, ""), ); diff --git a/tests/acceptance/tools/protobufDescriptorIdentity.test.ts b/tests/acceptance/tools/protobufDescriptorIdentity.test.ts new file mode 100644 index 0000000000..dc40a3847d --- /dev/null +++ b/tests/acceptance/tools/protobufDescriptorIdentity.test.ts @@ -0,0 +1,257 @@ +import { create, toBinary } from "@bufbuild/protobuf"; +import { + FieldDescriptorProtoSchema, + FileDescriptorSetSchema, + OneofDescriptorProtoSchema, +} from "@bufbuild/protobuf/wkt"; +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; +import { z } from "zod"; + +import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { webNetworkCaptureSchema } from "../../../src/domain/webNetworkCapture.js"; +import { protobufHar } from "../../fixtures/protobufCapture.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const sha256 = (bytes: string | Uint8Array) => + createHash("sha256").update(bytes).digest("hex"); +// Two occurrences of first=1, second=3, then child { value=4 }. +const body = Buffer.from("0801080210031a020804", "hex"); +const descriptorDefinition = () => + create(FileDescriptorSetSchema, { + file: [ + { + name: "identity.proto", + package: "capture", + syntax: "proto3", + messageType: [ + { + name: "Message", + field: [ + { name: "first", number: 1, type: 5, label: 3 }, + { name: "second", number: 2, type: 5, label: 1 }, + { + name: "child", + number: 3, + type: 11, + typeName: ".capture.Message.Child", + label: 1, + }, + ], + nestedType: [ + { + name: "Child", + field: [{ name: "value", number: 1, type: 5, label: 1 }], + }, + ], + }, + ], + }, + ], + }); + +const retainedDetailsSchema = z.object({ + partial_observation: z.strictObject({ + kind: z.literal("retained-evidence"), + evidence_id: z.string(), + }), +}); + +cliTest( + "refuses ambiguous descriptor field identities through CLI/MCP while retaining source Evidence", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-protobuf-field-identity-"); + const capture = join(root, "capture.har"); + const text = protobufHar(body, "application/x-protobuf"); + await writeFile(capture, text); + const call = await cli.sharedMcpCall!(); + for (const variant of ["forward", "reverse", "nested", "oneof"]) { + const definition = descriptorDefinition(); + const message = definition.file[0]!.messageType[0]!; + if (variant === "nested") { + message.nestedType[0]!.field.push( + create(FieldDescriptorProtoSchema, { + name: "alias", + number: 1, + type: 5, + label: 1, + }), + ); + } else if (variant === "oneof") { + message.oneofDecl.push( + create(OneofDescriptorProtoSchema, { name: "choice" }), + ); + message.field.push( + create(FieldDescriptorProtoSchema, { + name: "alias", + number: 1, + type: 5, + label: 1, + oneofIndex: 0, + }), + ); + } else { + message.field[1]!.number = 1; + if (variant === "reverse") message.field.reverse(); + } + const descriptor = join(root, `${variant}.pb`); + const bytes = toBinary(FileDescriptorSetSchema, definition); + await writeFile(descriptor, bytes); + const output = await cli.sharedCliRun!({ + arguments: [ + "inspect-web-network-capture", + capture, + "har", + "--record", + "0", + "--protobuf-schema", + descriptor, + "--protobuf-message", + "capture.Message", + "--json", + ], + }); + const expected = { + code: "invalid_request", + details: { + issues: [{ path: ["protobuf", "schema"], reason: "invalid_format" }], + }, + }; + expect(output.exitCode, variant).toBe(1); + expect(output.json, variant).toMatchObject(expected); + const response = await call("inspect_web_network_capture", { + capture_path: capture, + format: "har", + record_ordinals: [0], + protobuf: { + schema: { + descriptor_set_path: descriptor, + message_type: "capture.Message", + }, + }, + }); + const error = parseMcpToolError(response).error; + expect(error, variant).toMatchObject(expected); + const source = retainedDetailsSchema.parse( + error.details, + ).partial_observation; + const retained = await call("inspect_analysis_view", { + source, + view: { kind: "summary" }, + }); + expect(retained.isError, JSON.stringify(retained)).not.toBe(true); + expect( + analysisViewResultSchema.parse( + parseEvidence(retained.structuredContent).normalized_result, + ), + ).toMatchObject({ + summary: { + artifact: { + path: capture, + bytes: Buffer.byteLength(text), + sha256: sha256(text), + }, + retained_records: 1, + protobuf_coverage: { status: "partial", inspected_records: 0 }, + }, + }); + expect(await readFile(descriptor)).toEqual(Buffer.from(bytes)); + } + expect(await readFile(capture, "utf8")).toBe(text); + }, +); + +cliTest( + "keeps repeated wire occurrences and message-local number reuse through CLI/MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-protobuf-field-control-"); + const capture = join(root, "capture.har"); + const descriptor = join(root, "valid.pb"); + const definition = toBinary( + FileDescriptorSetSchema, + descriptorDefinition(), + ); + await writeFile(capture, protobufHar(body, "application/x-protobuf")); + await writeFile(descriptor, definition); + const output = await cli.sharedCliRun!({ + arguments: [ + "inspect-web-network-capture", + capture, + "har", + "--record", + "0", + "--protobuf-schema", + descriptor, + "--protobuf-message", + "capture.Message", + "--json", + ], + }); + expect(output.exitCode, output.stdout + output.stderr).toBe(0); + const evidence = parseEvidence(output.json); + const report = webNetworkCaptureSchema.parse(evidence.normalized_result); + expect(report.protobuf_coverage?.status).toBe("complete"); + const decoded = report.records[0]?.protobuf?.[0]; + expect(decoded?.schema).toMatchObject({ + descriptor_set_path: descriptor, + descriptor_set_sha256: sha256(definition), + descriptor_set_bytes: definition.length, + message_type: "capture.Message", + }); + expect(decoded?.fields).toMatchObject([ + { + field_id: "offset:0", + number: 1, + schema_name: "first", + schema_value: "1", + }, + { + field_id: "offset:2", + number: 1, + schema_name: "first", + schema_value: "2", + }, + { + field_id: "offset:4", + number: 2, + schema_name: "second", + schema_value: "3", + }, + { + field_id: "offset:6", + number: 3, + schema_name: "child", + schema_type: "capture.Message.Child", + }, + { + field_id: "offset:8", + number: 1, + schema_name: "value", + schema_value: "4", + }, + ]); + expect(decoded?.fields).toHaveLength(5); + const call = await cli.sharedMcpCall!(); + const response = await call("inspect_web_network_capture", { + capture_path: capture, + format: "har", + record_ordinals: [0], + protobuf: { + schema: { + descriptor_set_path: descriptor, + message_type: "capture.Message", + }, + }, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(parseEvidence(response.structuredContent).evidence_id).toBe( + evidence.evidence_id, + ); + expect(await readFile(descriptor)).toEqual(Buffer.from(definition)); + }, +); diff --git a/tests/acceptance/tools/protobufRegistryIdentity.test.ts b/tests/acceptance/tools/protobufRegistryIdentity.test.ts new file mode 100644 index 0000000000..45e82db552 --- /dev/null +++ b/tests/acceptance/tools/protobufRegistryIdentity.test.ts @@ -0,0 +1,218 @@ +import { create, toBinary } from "@bufbuild/protobuf"; +import { + DescriptorProtoSchema, + EnumDescriptorProtoSchema, + FileDescriptorSetSchema, + ServiceDescriptorProtoSchema, +} from "@bufbuild/protobuf/wkt"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; + +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { webNetworkCaptureSchema } from "../../../src/domain/webNetworkCapture.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { protobufHar } from "../../fixtures/protobufCapture.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const scalarMessage = (name: string, fieldName: string) => + create(DescriptorProtoSchema, { + name, + field: [{ name: fieldName, number: 1, type: 5, label: 1 }], + }); +const descriptorDefinition = () => + create(FileDescriptorSetSchema, { + file: [ + { + name: "identity.proto", + package: "capture", + syntax: "proto3", + messageType: [scalarMessage("Message", "alpha")], + }, + ], + }); + +const collidingDescriptor = (scope: string, reverse: boolean) => { + const definition = descriptorDefinition(); + const file = definition.file[0]!; + let messageType = "capture.Message"; + if (scope === "same-file") { + file.messageType.push(scalarMessage("Message", "beta")); + if (reverse) file.messageType.reverse(); + } else if (["across-files", "enum", "service"].includes(scope)) { + const other = structuredClone(file); + other.name = "other.proto"; + if (scope === "enum") { + other.messageType = []; + other.enumType.push( + create(EnumDescriptorProtoSchema, { + name: "Message", + value: [{ name: "ZERO", number: 0 }], + }), + ); + } else if (scope === "service") { + other.messageType = []; + other.service.push( + create(ServiceDescriptorProtoSchema, { name: "Message" }), + ); + } else other.messageType[0]!.field[0]!.name = "beta"; + definition.file.push(other); + if (reverse) definition.file.reverse(); + } else { + const alternative = + scope === "map-entry" + ? create(DescriptorProtoSchema, { + name: "Message", + options: { mapEntry: true }, + field: [ + { name: "key", number: 1, type: 9, label: 1 }, + { name: "value", number: 2, type: 5, label: 1 }, + ], + }) + : scalarMessage("Message", "beta"); + const children = [file.messageType[0]!, alternative]; + if (reverse) children.reverse(); + file.messageType = [ + create(DescriptorProtoSchema, { name: "Outer", nestedType: children }), + ]; + messageType = "capture.Outer.Message"; + } + return { bytes: toBinary(FileDescriptorSetSchema, definition), messageType }; +}; + +cliTest( + "refuses ambiguous registry identities independently of declaration order through CLI/MCP", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-protobuf-registry-identity-", + ); + const capture = join(root, "capture.har"); + const text = protobufHar(Buffer.from([8, 1]), "application/x-protobuf"); + await writeFile(capture, text); + const call = await cli.sharedMcpCall!(); + for (const scope of [ + "same-file", + "across-files", + "nested", + "enum", + "service", + "map-entry", + ]) + for (const reverse of [false, true]) { + const { bytes, messageType } = collidingDescriptor(scope, reverse); + const descriptor = join(root, `${scope}-${reverse}.pb`); + await writeFile(descriptor, bytes); + const expected = { + code: "invalid_request", + details: { + issues: [ + { path: ["protobuf", "schema"], reason: "invalid_format" }, + ], + }, + }; + // Both orders cross the actual CLI adapter; other geometries reuse + // the production MCP session and the same application workflow. + if (scope === "same-file") { + const output = await cli.sharedCliRun!({ + arguments: [ + "inspect-web-network-capture", + capture, + "har", + "--record", + "0", + "--protobuf-schema", + descriptor, + "--protobuf-message", + messageType, + "--json", + ], + }); + expect(output.exitCode).toBe(1); + expect(output.json).toMatchObject(expected); + } + const response = await call("inspect_web_network_capture", { + capture_path: capture, + format: "har", + record_ordinals: [0], + protobuf: { + schema: { + descriptor_set_path: descriptor, + message_type: messageType, + }, + }, + }); + expect( + parseMcpToolError(response).error, + `${scope}-${reverse}`, + ).toMatchObject(expected); + expect(await readFile(descriptor)).toEqual(Buffer.from(bytes)); + } + expect(await readFile(capture, "utf8")).toBe(text); + }, +); + +cliTest( + "keeps empty-package, package, parent and case-sensitive registry identities distinct", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-protobuf-registry-scopes-"); + const capture = join(root, "capture.har"); + const descriptor = join(root, "scopes.pb"); + const definition = create(FileDescriptorSetSchema, { + file: [ + { + name: "root.proto", + syntax: "proto3", + messageType: [ + scalarMessage("Message", "root_value"), + scalarMessage("message", "case_value"), + create(DescriptorProtoSchema, { + name: "Outer", + nestedType: [scalarMessage("Message", "nested_value")], + }), + ], + }, + { + name: "package.proto", + package: "capture", + syntax: "proto3", + messageType: [scalarMessage("Message", "package_value")], + }, + ], + }); + const bytes = toBinary(FileDescriptorSetSchema, definition); + await writeFile( + capture, + protobufHar(Buffer.from([8, 1]), "application/x-protobuf"), + ); + await writeFile(descriptor, bytes); + const call = await cli.sharedMcpCall!(); + for (const [messageType, fieldName] of [ + ["Message", "root_value"], + ["message", "case_value"], + ["Outer.Message", "nested_value"], + ["capture.Message", "package_value"], + ] as const) { + const response = await call("inspect_web_network_capture", { + capture_path: capture, + format: "har", + record_ordinals: [0], + protobuf: { + schema: { + descriptor_set_path: descriptor, + message_type: messageType, + }, + }, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const report = webNetworkCaptureSchema.parse( + parseEvidence(response.structuredContent).normalized_result, + ); + expect(report.protobuf_coverage?.status).toBe("complete"); + expect(report.records[0]?.protobuf?.[0]?.fields).toMatchObject([ + { number: 1, schema_name: fieldName, schema_value: "1" }, + ]); + } + expect(await readFile(descriptor)).toEqual(Buffer.from(bytes)); + }, +); From 87a5be3000b7cf4c3038277a7e809319c13cb2cd Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:37:47 +0800 Subject: [PATCH 04/21] fix(environment): normalize values from owned response facts Require a real clock-field context for numeric time normalization and actual rendered write rules for generated strings. Preserve literal read, replay and missing-response values, and retain generated aliases and distinctness within each response. Keep clock context separate from entity identity while traversing arrays. Add domain and public CLI controls for measurements, business strings, confirmed writes, missing responses and generated-value relationships, with documented inference limits. Fixes #6878 Fixes #6837 --- docs/agent-environments.md | 24 +- .../environment/environmentCompare.ts | 22 +- .../environmentCompareDiff.test.ts | 258 +++++++++++++++++- .../environment/environmentCompareDiff.ts | 185 ++++++++++--- .../cli/environmentCompareCli.test.ts | 242 +++++++++++++++- 5 files changed, 685 insertions(+), 46 deletions(-) diff --git a/docs/agent-environments.md b/docs/agent-environments.md index 59106bb20a..cbd496594f 100644 --- a/docs/agent-environments.md +++ b/docs/agent-environments.md @@ -771,8 +771,28 @@ recording replayed in its place and never from a recording whose body the probe two collections can reuse a live id, and an equal pair stays in that collection. A live id in a later request's path, query or body is translated before the environment answers, a path parameter by the collection the live path names (a nested collection keeps its parent values, each decoded once, and a literal route segment stays), except a query value that pages a list, such as `limit` or `page`. Each call is checked against the served tool's input schema first, as the MCP server checks it. After mapping, two clock values compare by format only (ISO-8601 strings, and epoch numbers under -`at`, `time`, `date`, `timestamp`, `created`, `updated` or `modified` field names, including snake-case and camel-case forms), and two different values each side generated on its own compare equal: both UUIDs, or one length of -eight or more with the same character classes and a digit. The report counts each kind of normalised value. +`at`, `time`, `date`, `timestamp`, `created`, `updated` or `modified` field names, including snake-case and camel-case forms). +Numeric arrays retain that containing field's clock context through nested +arrays; an object's own fields establish their own context. Numeric values at +the body root, in root arrays, or under measurement fields remain literal. + +Generated strings normalize only where this confirmed, served write's entity +response rules identify generation. A `unique` rule or UUID identity rule must +still compare two UUIDs; a `renewed` rule applies only at its recorded pointers, +with `*` matching actual array items. Token/sequence strings retain the existing +format check: equal length of at least eight, the same character classes, and +a digit. Numeric sequences continue to compare literally unless an ID mapping +applies. String shape and names such as `token` or `sku` alone do not establish +generation. Read and replay responses without these facts stay literal except +for supported clocks and learned IDs. A missing write target replays an absence +response and does not inherit the successful entity response's generation rules. +Admitted generated values must also preserve aliases and distinct identities +within that response. UUID identity uses the generator's case-insensitive +matching; token and sequence strings retain their literal identity. This +correspondence ends with the answer and is not inferred across calls. +The report counts each kind of normalised +value; the generation rules remain model-derived facts, rather than new +observations of the application. ISO clock offsets are accepted only within RFC 3339's `00`–`23` hour and `00`–`59` minute ranges. Probe ordering retains every recorded fractional-second digit in ISO timestamps with a four-digit year, `T`, and a `Z` or `±HH:MM` suffix. Equal instants use ordinal order. Other native-Date diff --git a/src/application/environment/environmentCompare.ts b/src/application/environment/environmentCompare.ts index 2c0e470599..b75b82a021 100644 --- a/src/application/environment/environmentCompare.ts +++ b/src/application/environment/environmentCompare.ts @@ -1,4 +1,5 @@ import { decodedSegment } from "../../domain/analysisView/environmentStateRules.js"; +import type { FieldRule } from "../../domain/analysisView/environmentStateWriteRules.js"; import { fromJsonSchema } from "@modelcontextprotocol/server"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import type { JsonValue } from "../../domain/jsonValue.js"; @@ -139,22 +140,37 @@ const entityOf = ( run: ProbeRun, tool: EnvironmentManifestTool, segments: readonly string[], + answer: ReplayResult, ): { readonly scope: string; readonly at: string; readonly identity: boolean; + readonly rules: readonly FieldRule[]; } => { const view = run.store?.state.views.get(tool.name); const plan = run.store?.state.writes.get(tool.name); const response = plan?.response; const scope = collectionScope(tool, segments); if (view?.shape === "item" || view?.shape === "list") - return { scope, at: view.pointer, identity: false }; + return { scope, at: view.pointer, identity: false, rules: [] }; const identity = response?.shape === "identity"; const at = identity ? (response.pointer ?? "") : (response?.pointer ?? plan?.observation?.pointer ?? ""); - return { scope, at, identity }; + const provenance = (answer as Partial).state; + const renderedWrite = + provenance?.source === "state" && + (provenance.view === "create" || + provenance.view === "update" || + provenance.view === "document-write" || + provenance.view === "delete"); + // A missing target replays a recorded absence with state provenance too; + // only an applied write can render the plan's entity response. + const rules = + plan?.served === true && response?.shape === "entity" && renderedWrite + ? plan.rules + : []; + return { scope, at, identity, rules }; }; const fromState = (answer: ReplayResult): boolean => @@ -350,7 +366,7 @@ const compareStep = ( status: answered.value.status, body: envBody(run, answered.value), }; - const entity = entityOf(run, call.tool, liveSegments); + const entity = entityOf(run, call.tool, liveSegments, answered.value); // Another call's recording holds other entities, whose ids would map wrongly and, the first pairing winning, for good. if (answersCall(run, call, args, answered.value)) learnIds(identityValue(live.body), identityValue(env.body), run.map, { diff --git a/src/application/environment/environmentCompareDiff.test.ts b/src/application/environment/environmentCompareDiff.test.ts index 0d5b160adc..e039e8dcb4 100644 --- a/src/application/environment/environmentCompareDiff.test.ts +++ b/src/application/environment/environmentCompareDiff.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import type { JsonValue } from "../../domain/jsonValue.js"; +import type { FieldRule } from "../../domain/analysisView/environmentStateWriteRules.js"; import { compareAnswers } from "./environmentCompareDiff.js"; import { IdMap, learnIds } from "./environmentCompareIds.js"; @@ -9,11 +10,13 @@ const compare = ( env: unknown, map = new IdMap(), status: [number, number] = [200, 200], + rules: readonly FieldRule[] = [], ) => compareAnswers( { status: status[0], body: { kind: "json", value: live as JsonValue } }, { status: status[1], body: { kind: "json", value: env as JsonValue } }, map, + { rules }, ); describe("early-year clock formats", () => { @@ -32,7 +35,7 @@ describe("early-year clock formats", () => { }); describe("compareAnswers", () => { - it("agrees on mapped ids, clocks of one format and generated values, and counts each", () => { + it("agrees on mapped ids, clocks and confirmed generated values, and counts each", () => { const map = new IdMap(); map.learn(101, 1); const result = compare( @@ -49,6 +52,11 @@ describe("compareAnswers", () => { uuid: "0a1b2c3d-4e5f-4a6b-9c7d-8e9f0a1b2c3d", }, map, + [200, 200], + [ + { field: "token", rule: "identity", generator: "token" }, + { field: "uuid", rule: "unique" }, + ], ); expect(result.differences).toEqual([]); expect(result.normalised).toEqual({ @@ -105,7 +113,9 @@ describe("compareAnswers", () => { ), ).toEqual(["value"]); }); +}); +describe("mapped fields and literal collections", () => { it("compares an object keyed by mapped ids under the environment's keys", () => { const map = new IdMap(); map.learn("liveTable", "envTable"); @@ -167,7 +177,7 @@ describe("compareAnswers", () => { ).toEqual([]); }); - it("reports a reordered set of scalars as order alone and reads epoch array items as clocks", () => { + it("reports a reordered set of scalars as order alone and keeps nested numeric measurements literal", () => { expect( compare( { access: ["a", "b", "c"] }, @@ -177,7 +187,223 @@ describe("compareAnswers", () => { expect( compare({ stats: [[1792450800, 3]] }, { stats: [[1792285200, 3]] }) .differences, - ).toEqual([]); + ).toEqual([ + { + category: "value", + pointer: "/stats/0/0", + live: "1792450800", + env: "1792285200", + }, + ]); + }); +}); + +describe("normalization context", () => { + it.each([ + { live: 1800000000, env: 1700000000, pointer: "" }, + { live: [1800000000], env: [1700000000], pointer: "/0" }, + { + live: { samples: [1800000000] }, + env: { samples: [1700000000] }, + pointer: "/samples/0", + }, + { + live: { updated_at: [{ samples: 1800000000 }] }, + env: { updated_at: [{ samples: 1700000000 }] }, + pointer: "/updated_at/0/samples", + }, + ])( + "keeps epoch-shaped numbers literal at $pointer without a time field", + ({ live, env, pointer }) => { + const result = compare(live, env); + expect(result.differences).toEqual([ + { category: "value", pointer, live: "1800000000", env: "1700000000" }, + ]); + expect(result.normalised.clocks).toBe(0); + }, + ); + + it("preserves the time field through nested numeric clock arrays", () => { + const result = compare( + { updated_at: [[1700000000]] }, + { updated_at: [[1800000000]] }, + ); + expect(result.differences).toEqual([]); + expect(result.normalised.clocks).toBe(1); + }); + + it.each([ + ["product1", "product2"], + ["product1", "productX"], + ["3f2a9c8d7e", "9b8e7d6c5a"], + [ + "6f1c2b9e-2a3d-4f5e-8a9b-0c1d2e3f4a5b", + "0a1b2c3d-4e5f-4a6b-9c7d-8e9f0a1b2c3d", + ], + ])( + "keeps literal business strings %s versus %s without generation facts", + (live, env) => { + const result = compare( + { sku: live, items: [live] }, + { sku: env, items: [env] }, + ); + expect( + result.differences.map(({ category, pointer }) => [category, pointer]), + ).toEqual([ + ["value", "/sku"], + ["value", "/items/0"], + ]); + expect(result.normalised.generated).toBe(0); + }, + ); + + it("scopes confirmed renewal rules to their response entity and exact array pointers", () => { + const first = "6f1c2b9e-2a3d-4f5e-8a9b-0c1d2e3f4a5b"; + const second = "0a1b2c3d-4e5f-4a6b-9c7d-8e9f0a1b2c3d"; + const answer = (token: string) => ({ + status: 200, + body: { + kind: "json" as const, + value: { + data: { values: [{ token, sku: token }] }, + values: [{ token }], + }, + }, + }); + const result = compareAnswers(answer(first), answer(second), new IdMap(), { + at: "/data", + rules: [{ field: "values", rule: "renewed", pointers: ["/*/token"] }], + }); + expect(result.normalised.generated).toBe(1); + expect( + result.differences.map(({ category, pointer }) => [category, pointer]), + ).toEqual([ + ["value", "/data/values/0/sku"], + ["value", "/values/0/token"], + ]); + }); + + it("does not extend generated-string normalization to numeric sequences", () => { + const result = compare( + { code: 17 }, + { code: 18 }, + new IdMap(), + [200, 200], + [{ field: "code", rule: "sequence", scope: "type" }], + ); + expect(result.differences).toEqual([ + { category: "value", pointer: "/code", live: "17", env: "18" }, + ]); + expect(result.normalised.generated).toBe(0); + }); +}); + +describe("generated response identities", () => { + const first = "6f1c2b9e-2a3d-4f5e-8a9b-0c1d2e3f4a5b"; + const second = "0a1b2c3d-4e5f-4a6b-9c7d-8e9f0a1b2c3d"; + const fresh = "123e4567-e89b-42d3-a456-426614174000"; + const other = "223e4567-e89b-42d3-a456-426614174000"; + const rules: readonly FieldRule[] = [ + { field: "left", rule: "renewed", pointers: ["/*"] }, + { field: "right", rule: "renewed", pointers: ["/*"] }, + ]; + const answer = (left: string, right: string) => ({ + status: 200, + body: { kind: "json" as const, value: { left: [left], right: [right] } }, + }); + + it.each([ + { + name: "consistent fresh renaming", + liveLeft: fresh, + liveRight: fresh, + envLeft: first, + envRight: first, + differs: false, + generated: 2, + }, + { + name: "case-equivalent UUID aliases", + liveLeft: fresh.toUpperCase(), + liveRight: fresh, + envLeft: first, + envRight: first.toUpperCase(), + differs: false, + generated: 2, + }, + { + name: "split alias", + liveLeft: fresh, + liveRight: other, + envLeft: first, + envRight: first, + differs: true, + generated: 1, + }, + { + name: "merged identities", + liveLeft: fresh, + liveRight: fresh, + envLeft: first, + envRight: second, + differs: true, + generated: 1, + }, + { + name: "split after an unchanged array", + liveLeft: first, + liveRight: fresh, + envLeft: first, + envRight: first, + differs: true, + generated: 0, + }, + { + name: "merge after an unchanged array", + liveLeft: first, + liveRight: first, + envLeft: first, + envRight: second, + differs: true, + generated: 0, + }, + ])( + "preserves $name within one confirmed renewal response", + ({ liveLeft, liveRight, envLeft, envRight, differs, generated }) => { + const result = compareAnswers( + answer(liveLeft, liveRight), + answer(envLeft, envRight), + new IdMap(), + { rules }, + ); + expect(result.normalised.generated).toBe(generated); + expect(result.differences).toEqual( + differs + ? [ + { + category: "value", + pointer: "/right/0", + live: JSON.stringify(liveRight), + env: JSON.stringify(envRight), + }, + ] + : [], + ); + }, + ); + + it("retains order-only comparison for reordered generated scalar sets", () => { + const result = compare( + { left: [first, second] }, + { left: [second, first] }, + new IdMap(), + [200, 200], + rules, + ); + expect(result.differences).toEqual([ + { category: "order", pointer: "/left" }, + ]); + expect(result.normalised.generated).toBe(0); }); }); @@ -316,6 +542,32 @@ describe("identity responses two collections reuse", () => { }); describe("wrapped answers", () => { + it("keeps collection identity context separate from a wrapped array's clock field context", () => { + const map = new IdMap(); + map.learn(101, 1, "/api/books"); + map.learn(101, 2, "/api/authors"); + const result = compareAnswers( + { + status: 200, + body: { + kind: "json", + value: { data: [{ id: 101, updated_at: [1700000000] }] }, + }, + }, + { + status: 200, + body: { + kind: "json", + value: { data: [{ id: 2, updated_at: [1800000000] }] }, + }, + }, + map, + { scope: "/api/authors", at: "/data" }, + ); + expect(result.differences).toEqual([]); + expect(result.normalised).toMatchObject({ ids: 1, clocks: 1 }); + }); + it("compares the identity of an entity inside an envelope within the call's collection", () => { const map = new IdMap(); map.learn(101, 1, "/api/books"); diff --git a/src/application/environment/environmentCompareDiff.ts b/src/application/environment/environmentCompareDiff.ts index 350363d842..b7ac1dc5f9 100644 --- a/src/application/environment/environmentCompareDiff.ts +++ b/src/application/environment/environmentCompareDiff.ts @@ -1,4 +1,5 @@ import type { JsonObject } from "../../domain/analysisView/environmentStateRules.js"; +import type { FieldRule } from "../../domain/analysisView/environmentStateWriteRules.js"; import { isUuid, readClock, @@ -74,16 +75,15 @@ const formatKey = (format: ClockFormat): string => ? `epoch-${format.unit}` : `iso${format.separator}${String(format.fraction)}${format.zone === "" ? "" : format.zone === "Z" ? "Z" : "offset"}`; -/** The format of a clock value: an ISO-8601 string, or an epoch number under a time-named field or as an array item. */ +/** The format of a clock value: an ISO-8601 string, or an epoch number under a time-named field. */ const clockFormat = ( value: JsonValue, key: string | undefined, ): string | undefined => { if ( typeof value === "number" && - key !== undefined && - !TIME_FIELD.test(key) && - !TIME_FIELD_CAMEL.test(key) + (key === undefined || + (!TIME_FIELD.test(key) && !TIME_FIELD_CAMEL.test(key))) ) return undefined; const clock = readClock(value); @@ -102,16 +102,42 @@ const characterClasses = (text: string): string => .sort() .join(""); -/** Two values each side generated on its own: both UUIDs, or one length of eight or more with the same character classes and a digit. */ -const generatedPair = (live: string, env: string): boolean => - (isUuid(live) && isUuid(env)) || - (live.length === env.length && +/** A confirmed response field's rule, and the pointer within that field (`*` marks actual array traversal). */ +interface Generation { + readonly rule: FieldRule; + readonly pointer: string; +} + +/** Whether a confirmed generation rule applies here and both strings retain its generated format. */ +const generatedPair = ( + live: string, + env: string, + generation: Generation | undefined, +): boolean => { + if (generation === undefined) return false; + const { rule, pointer } = generation; + if (rule.rule === "renewed") + return rule.pointers.includes(pointer) && isUuid(live) && isUuid(env); + if (pointer !== "") return false; + if ( + rule.rule === "unique" || + (rule.rule === "identity" && rule.generator === "uuid") + ) + return isUuid(live) && isUuid(env); + if (rule.rule !== "sequence" && rule.rule !== "identity") return false; + return ( + live.length === env.length && live.length >= 8 && /\d/u.test(live) && /\d/u.test(env) && - characterClasses(live) === characterClasses(env)); + characterClasses(live) === characterClasses(env) + ); +}; class Differ { + readonly #rules: ReadonlyMap; + readonly #generatedToEnv = new Map(); + readonly #generatedToLive = new Map(); readonly differences: Difference[] = []; readonly normalised: Normalised = { ids: 0, @@ -128,13 +154,39 @@ class Differ { readonly scope: string | undefined; readonly at: string; readonly identity: boolean; + /** Parsed rules of the confirmed write whose entity response is being compared. */ + readonly rules: readonly FieldRule[]; }, - ) {} + ) { + this.#rules = new Map(entity.rules.map((rule) => [rule.field, rule])); + } add(difference: Difference): void { this.differences.push(difference); } + /** One answer's generated identities must rename consistently in both directions. */ + #pairGenerated(live: string, env: string, rule: FieldRule): boolean { + const uuid = + rule.rule === "unique" || + rule.rule === "renewed" || + (rule.rule === "identity" && rule.generator === "uuid"); + const identity = (value: string) => + uuid ? `uuid:${value.toLowerCase()}` : `string:${value}`; + const liveKey = identity(live); + const envKey = identity(env); + const knownEnv = this.#generatedToEnv.get(liveKey); + const knownLive = this.#generatedToLive.get(envKey); + if ( + (knownEnv !== undefined && knownEnv !== envKey) || + (knownLive !== undefined && knownLive !== liveKey) + ) + return false; + this.#generatedToEnv.set(liveKey, envKey); + this.#generatedToLive.set(envKey, liveKey); + return true; + } + /** Whether a value compares within the call's collection, as an entity's own identity or a create's bare id does; other ids compare by the probe's mapping. */ #inCollection( pointer: string, @@ -162,10 +214,29 @@ class Differ { live: JsonValue, env: JsonValue, pointer: string, - at: { readonly key?: string; readonly own?: boolean } = {}, + at: { + readonly key?: string; + readonly own?: boolean; + readonly clockKey?: string | undefined; + } = {}, + generation?: Generation, ): void { + const key = at.clockKey ?? at.key; + const liveClock = clockFormat(live, key); + const envClock = clockFormat(env, key); + const generatedMatch = + generation !== undefined && + typeof live === "string" && + typeof env === "string" && + liveClock === undefined && + envClock === undefined && + !this.map.hasLive(live) && + !this.map.hasEnv(env) && + generatedPair(live, env, generation) + ? this.#pairGenerated(live, env, generation.rule) + : undefined; if (live === env) { - if (this.#reassigned(live, pointer, at)) + if (generatedMatch === false || this.#reassigned(live, pointer, at)) this.add({ category: "value", pointer, @@ -174,17 +245,17 @@ class Differ { }); return; } - const { key } = at; + const field = at.key; if ( (typeof live === "string" || typeof live === "number") && (typeof env === "string" || typeof env === "number") ) { const mapsAsId = - (key !== undefined && ID_FIELD.test(key)) || + (field !== undefined && ID_FIELD.test(field)) || this.#inCollection(pointer, at) || (typeof live === "string" && - key !== undefined && - this.map.isStateReference(live, key)); + field !== undefined && + this.map.isStateReference(live, field)); const mapped = mapsAsId ? this.map.env( live, @@ -195,8 +266,6 @@ class Differ { this.normalised.ids += 1; return; } - const liveClock = clockFormat(live, key); - const envClock = clockFormat(env, key); if (liveClock !== undefined && envClock !== undefined) { if (liveClock === envClock) this.normalised.clocks += 1; else @@ -208,13 +277,7 @@ class Differ { }); return; } - if ( - typeof live === "string" && - typeof env === "string" && - !this.map.hasLive(live) && - !this.map.hasEnv(env) && - generatedPair(live, env) - ) { + if (generatedMatch === true) { this.normalised.generated += 1; return; } @@ -227,9 +290,28 @@ class Differ { }); } - array(live: JsonValue[], env: JsonValue[], pointer: string): void { + array( + live: JsonValue[], + env: JsonValue[], + pointer: string, + clockKey: string | undefined, + generation: Generation | undefined, + ): void { const own = pointer === this.entity.at; - if (this.sameItems(live, env, pointer)) return; + if (this.sameItems(live, env, pointer)) { + // Equal/reordered scalar sets still establish unchanged generated pairs; + // otherwise another field could split an alias hidden by this fast path. + if (generation !== undefined) + for (const [index, item] of live.entries()) + this.scalar( + item, + item, + `${pointer}/${String(index)}`, + { own, clockKey }, + { ...generation, pointer: `${generation.pointer}/*` }, + ); + return; + } const pairs = pairElements(live, env); if (live.length !== env.length || pairs.length !== live.length) this.add({ @@ -244,9 +326,18 @@ class Differ { ) this.add({ category: "order", pointer }); for (const [left, right] of pairs) - this.value(live[left]!, env[right]!, `${pointer}/${String(left)}`, { - own, - }); + this.value( + live[left]!, + env[right]!, + `${pointer}/${String(left)}`, + { + own, + clockKey, + }, + generation === undefined + ? undefined + : { ...generation, pointer: `${generation.pointer}/*` }, + ); } /** Arrays of scalars holding the same items in another order differ only in order, as sets an application returns unordered do. */ @@ -282,6 +373,7 @@ class Differ { env: JsonObject, pointer: string, own = false, + generation?: Generation, ): void { const self = own || pointer === this.entity.at; // An object keyed by ids compares each member under the environment's key; any other compares field by field. @@ -306,11 +398,25 @@ class Differ { }); else { matched.add(other); - this.value(value, env[other]!, child, { - key, - own: self, - keyed: self && keyed, - }); + const rule = self ? this.#rules.get(key) : undefined; + this.value( + value, + env[other]!, + child, + { + key, + own: self, + keyed: self && keyed, + }, + rule !== undefined + ? { rule, pointer: "" } + : generation === undefined + ? undefined + : { + ...generation, + pointer: `${generation.pointer}/${escapePointer(key)}`, + }, + ); } } for (const [key, value] of Object.entries(env)) @@ -331,16 +437,19 @@ class Differ { readonly key?: string; readonly own?: boolean; readonly keyed?: boolean; + readonly clockKey?: string | undefined; } = {}, + generation?: Generation, ): void { if (Array.isArray(live) && Array.isArray(env)) - this.array(live, env, pointer); + this.array(live, env, pointer, at.clockKey ?? at.key, generation); else if (isObject(live) && isObject(env)) this.object( live, env, pointer, at.own === true && (at.key === undefined || at.keyed === true), + generation, ); else if ( Array.isArray(live) || @@ -354,7 +463,7 @@ class Differ { live: shown(live), env: shown(env), }); - else this.scalar(live, env, pointer, at); + else this.scalar(live, env, pointer, at, generation); } } @@ -375,12 +484,14 @@ export const compareAnswers = ( readonly scope?: string; readonly at?: string; readonly identity?: boolean; + readonly rules?: readonly FieldRule[]; } = {}, ): { readonly differences: Difference[]; readonly normalised: Normalised } => { const differ = new Differ(map, { scope: entity.scope, at: entity.at ?? "", identity: entity.identity === true, + rules: entity.rules ?? [], }); if (live.status !== env.status) differ.add({ diff --git a/tests/boundary/cli/environmentCompareCli.test.ts b/tests/boundary/cli/environmentCompareCli.test.ts index 49cfe168bd..e8d2308939 100644 --- a/tests/boundary/cli/environmentCompareCli.test.ts +++ b/tests/boundary/cli/environmentCompareCli.test.ts @@ -7,13 +7,18 @@ import { analysisViewCaptureEvidence, environmentManifestCaptureFixture, timedCaptureEvidence, + type TimedCaptureRow, } from "../../fixtures/analysisView.js"; import { environmentStateServeFixture } from "../../fixtures/environmentServe.js"; import { BOOK_NOT_FOUND, BOOK_ROWS, FIRST_BOOK, + SECOND_BOOK, + THIRD_BOOK, book, + numberedUuid, + renamedBook, } from "../../fixtures/environmentStateWrites.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; @@ -98,7 +103,242 @@ const writeExactClockFiles = async ( return originals; }; -describe("rea environment-compare", () => { +/** Capture-confirmed generated writes and literal/clock probes, retained as unchanged Evidence files. */ +const writeValueContextFiles = async ( + root: string, +): Promise> => { + const recorded = { + samples: [1700000000], + nested_samples: [[1700000000]], + scalar_samples: 1700000000, + sku: "product1", + catalog: ["product1"], + serial: numberedUuid(1), + updated_at: [1700000000], + nested: { updated_at: 1700000000 }, + fixed_samples: [1700000000], + }; + const clocks = { + ...recorded, + updated_at: [1800000000], + nested: { updated_at: 1800000000 }, + }; + const changed = { + ...clocks, + samples: [1800000000], + nested_samples: [[1800000000]], + scalar_samples: 1800000000, + sku: "product2", + catalog: ["product2"], + serial: numberedUuid(2), + }; + const first = { ...FIRST_BOOK, ref: numberedUuid(1) }; + const second = { ...SECOND_BOOK, ref: numberedUuid(2) }; + const third = { ...THIRD_BOOK, ref: numberedUuid(3) }; + const renamed = { + ...renamedBook(first, "Renamed Book", 6), + ref: numberedUuid(4), + }; + const missing = { error: "missing", ref: numberedUuid(99) }; + const rows: TimedCaptureRow[] = [ + ["GET", "/api/config", recorded], + ["GET", "/api/books", [first, second]], + ["GET", "/api/books/1", first], + ["GET", "/api/books/2", second], + ["POST", "/api/books", third, 201, { name: "Third Book" }], + ["GET", "/api/books/3", third], + ["PATCH", "/api/books/1", renamed, 200, { name: "Renamed Book" }], + ["GET", "/api/books/1", renamed], + ["GET", "/api/books/99", missing, 404], + ]; + const capture = timedCaptureEvidence(rows); + const fixture = environmentStateServeFixture(capture); + expect(fixture.state.normalized_result).toMatchObject({ + writes: expect.arrayContaining([ + expect.objectContaining({ + tool: "post_books", + served: true, + response: { ordinal: 4, shape: "entity", pointer: "" }, + rules: expect.arrayContaining([{ field: "ref", rule: "unique" }]), + }), + expect.objectContaining({ + tool: "patch_books_by_id", + served: true, + response: { ordinal: 6, shape: "entity", pointer: "" }, + rules: expect.arrayContaining([{ field: "ref", rule: "unique" }]), + }), + ]), + }); + const generated = { + ...book(101, "Third Book", "2026-11-01T12:00:00Z"), + ref: numberedUuid(101), + }; + const originals = new Map(); + for (const [name, value] of [ + ["manifest.json", fixture.manifest], + ["capture.json", capture], + ["changed.json", timedCaptureEvidence([["GET", "/api/config", changed]])], + ["clocks.json", timedCaptureEvidence([["GET", "/api/config", clocks]])], + [ + "generated.json", + timedCaptureEvidence([ + ["POST", "/api/books", generated, 201, { name: "Third Book" }], + [ + "PATCH", + "/api/books/1", + { ...renamed, ref: numberedUuid(102) }, + 200, + { name: "Renamed Book" }, + ], + ]), + ], + [ + "missing.json", + timedCaptureEvidence([ + [ + "PATCH", + "/api/books/99", + { ...missing, ref: numberedUuid(100) }, + 404, + { name: "Ghost" }, + ], + ]), + ], + ] as const) { + const path = join(root, name); + const text = JSON.stringify(value); + originals.set(path, text); + await writeFile(path, text); + } + return originals; +}; + +describe("rea environment-compare value context", () => { + cliTest( + "keeps business values and numeric measurements literal while normalizing confirmed generation and clocks", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-environment-value-context-", + ); + const originals = await writeValueContextFiles(root); + const result = await cli.run({ + arguments: [ + "environment-compare", + join(root, "manifest.json"), + "--evidence", + join(root, "capture.json"), + "--probe", + join(root, "changed.json"), + "--probe", + join(root, "clocks.json"), + "--probe", + join(root, "generated.json"), + "--probe", + join(root, "missing.json"), + "--json", + ], + }); + expect(result.exitCode, result.stderr).toBe(0); + expect(result.json).toMatchObject({ + status: "complete", + summary: { + probes: 4, + steps: 5, + equal: 3, + agreement: 0.6, + normalised: { generated: 2 }, + }, + probes: [ + { + label: "changed.json", + normalised: { generated: 0, clocks: 2, ids: 0 }, + results: [ + { + equal: false, + categories: ["value"], + differences: [ + { + category: "value", + pointer: "/samples/0", + live: "1800000000", + env: "1700000000", + }, + { + category: "value", + pointer: "/nested_samples/0/0", + live: "1800000000", + env: "1700000000", + }, + { + category: "value", + pointer: "/scalar_samples", + live: "1800000000", + env: "1700000000", + }, + { + category: "value", + pointer: "/sku", + live: '"product2"', + env: '"product1"', + }, + { + category: "value", + pointer: "/catalog/0", + live: '"product2"', + env: '"product1"', + }, + { + category: "value", + pointer: "/serial", + live: JSON.stringify(numberedUuid(2)), + env: JSON.stringify(numberedUuid(1)), + }, + ], + }, + ], + }, + { + label: "clocks.json", + normalised: { clocks: 2, generated: 0 }, + results: [{ equal: true, differences: [] }], + }, + { + label: "generated.json", + normalised: { generated: 2, ids: 1 }, + results: [ + { equal: true, differences: [] }, + { equal: true, differences: [] }, + ], + }, + { + label: "missing.json", + normalised: { generated: 0 }, + results: [ + { + tool: "patch_books_by_id", + live_status: 404, + env_status: 404, + equal: false, + differences: [ + { + category: "value", + pointer: "/ref", + live: JSON.stringify(numberedUuid(100)), + env: JSON.stringify(numberedUuid(99)), + }, + ], + }, + ], + }, + ], + }); + for (const [path, text] of originals) + expect(await readFile(path, "utf8")).toBe(text); + }, + ); +}); + +describe("rea environment-compare replay and chronology", () => { cliTest( "preserves exact probe chronology and first-century clock-format differences", async ({ cli }) => { From 6d4cc6e6a4877a03bf9407022a32406651232779 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:48:45 +0800 Subject: [PATCH 05/21] refactor(managed): share compressed integer format decoding --- src/dotnet/ManagedArtifactInspector.test.ts | 89 +++++++++++++++++++++ src/dotnet/ManagedMemberInspector.test.ts | 66 +++++++++++++++ src/dotnet/ManagedMemberInspectorCore.ts | 34 ++------ src/dotnet/ManagedMetadataHeaps.ts | 79 +++++++++++------- src/dotnet/ManagedMetadataInventoryRows.ts | 43 ++-------- src/dotnet/ManagedPe.fixture.ts | 45 ++++++----- tests/acceptance/managedCliOutput.test.ts | 28 +++++++ 7 files changed, 272 insertions(+), 112 deletions(-) diff --git a/src/dotnet/ManagedArtifactInspector.test.ts b/src/dotnet/ManagedArtifactInspector.test.ts index 814e15e991..1c10c87718 100644 --- a/src/dotnet/ManagedArtifactInspector.test.ts +++ b/src/dotnet/ManagedArtifactInspector.test.ts @@ -318,6 +318,95 @@ describe("managed artifact inventory", () => { ); }); +describe("managed attribute SerString encoding", () => { + it.each([ + [0, [0x00]], + [0x7f, [0x7f]], + [0x80, [0x80, 0x80]], + [0x3fff, [0xbf, 0xff]], + [0x4000, [0xc0, 0x00, 0x40, 0x00]], + ] as const)( + "reads a target-framework SerString of %i UTF-8 bytes", + (length, prefix) => { + const targetFramework = + "é".repeat(Math.floor(length / 2)) + (length % 2 === 0 ? "" : "A"); + const value = Buffer.concat([ + Buffer.from([1, 0, ...prefix]), + Buffer.from(targetFramework), + Buffer.from([0, 0]), + ]); + const bytes = buildManagedPeFixture({ targetFramework }); + const result = inspectManagedArtifactBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + + expect(result.target_frameworks).toEqual([targetFramework]); + expect(result.attributes).toMatchObject([ + { + decoded_fixed_string: targetFramework, + value_length: value.length, + value_sha256: createHash("sha256").update(value).digest("hex"), + }, + ]); + expect(result.coverage).toEqual({ state: "complete", issues: [] }); + }, + ); + + it.each([ + { name: "missing length", value: [1, 0], decoded: null }, + { name: "null sentinel", value: [1, 0, 0xff], decoded: null }, + { name: "truncated two-byte prefix", value: [1, 0, 0x80], decoded: null }, + { + name: "truncated four-byte prefix", + value: [1, 0, 0xc0, 0, 0], + decoded: null, + }, + { name: "reserved prefix", value: [1, 0, 0xe0], decoded: null }, + { name: "content outside value", value: [1, 0, 5, 0x41], decoded: null }, + { + name: "maximum length without content", + value: [1, 0, 0xdf, 0xff, 0xff, 0xff], + decoded: null, + }, + { name: "invalid UTF-8", value: [1, 0, 1, 0xff, 0, 0], decoded: null }, + { name: "invalid prolog", value: [0, 0, 1, 0x41, 0, 0], decoded: null }, + { + name: "overlong two-byte length", + value: [1, 0, 0x80, 1, 0x41, 0, 0], + decoded: "A", + }, + { + name: "overlong four-byte length", + value: [1, 0, 0xc0, 0, 0, 1, 0x41, 0, 0], + decoded: "A", + }, + ])("preserves attribute evidence for $name", ({ value, decoded }) => { + const customAttributeValue = Buffer.from(value); + const bytes = buildManagedPeFixture({ customAttributeValue }); + const result = inspectManagedArtifactBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + + expect(result.attributes).toMatchObject([ + { + parent_token: "0x20000001", + constructor_token: "0x0a000001", + type_name: "System.Runtime.Versioning.TargetFrameworkAttribute", + decoded_fixed_string: decoded, + value_length: customAttributeValue.length, + value_sha256: createHash("sha256") + .update(customAttributeValue) + .digest("hex"), + }, + ]); + expect(result.target_frameworks).toEqual(decoded === null ? [] : [decoded]); + expect(result.metadata.status).toBe("complete"); + expect(result.coverage).toEqual({ state: "complete", issues: [] }); + }); +}); + describe("managed artifact coded indexes", () => { it("reports an invalid ManifestResource implementation while retaining the resource", () => { const bytes = buildManagedPeFixture({ resourceImplementationRaw: 9 }); diff --git a/src/dotnet/ManagedMemberInspector.test.ts b/src/dotnet/ManagedMemberInspector.test.ts index 737e66d2b1..9829618f90 100644 --- a/src/dotnet/ManagedMemberInspector.test.ts +++ b/src/dotnet/ManagedMemberInspector.test.ts @@ -1993,6 +1993,72 @@ it.each([ }, ); +it.each([ + [[], "truncated compressed integer"], + [[0x80], "reserved compressed integer"], + [[0xc0, 0, 0], "reserved compressed integer"], + [[0xe0], "reserved compressed integer"], +] as const)( + "retains TypeSpec prefix failure %j at its owning row", + (prefix, issue) => { + const bytes = buildManagedPeFixture({ + typeSpecSignatures: [Buffer.from([0x12, ...prefix])], + memberRefParentRaw: (1 << 3) | 4, + }); + const pe = readManagedPeLayout(bytes); + if (pe.cli === null) throw new Error("fixture has no CLI header"); + const { rva, size } = pe.cli.metadata; + const layout = readManagedMetadataLayout( + bytes, + pe.rvaToOffset(rva, size, "metadata"), + size, + ); + const result = inspectManagedMembersBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + + expect(result.member_refs[0]?.parent_name).toBeNull(); + expect(result.coverage.issues).toContainEqual({ + code: "invalid-blob", + scope: "metadata.TypeSpec:0x1b000001", + offset: metadataRowOffset(layout, 27, 1), + detail: issue, + }); + }, +); + +it.each([ + [[0x01], -64], + [[0x80, 0x01], -8192], + [[0xc0, 0x00, 0x00, 0x01], -268435456], +] as const)( + "preserves compressed signed width for array bound %j", + (bound, lower) => { + const bytes = buildManagedPeFixture({ + fieldSignature: Buffer.from([ + 0x06, + 0x14, + 0x08, + 0x01, + 0x00, + 0x01, + ...bound, + ]), + }); + const result = inspectManagedMembersBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + + expect(result.fields[0]?.signature).toMatchObject({ + parse_status: "decoded", + field_type: `i4[${lower}...]`, + }); + expect(result.coverage).toEqual({ state: "complete", issues: [] }); + }, +); + it("decodes generic instantiation member signatures as complete coverage", () => { // System.Runtime.Caching 0x06000001: instance void (Dictionary) // shape, with TypeRef/TypeSpec coded tokens kept as encoded. diff --git a/src/dotnet/ManagedMemberInspectorCore.ts b/src/dotnet/ManagedMemberInspectorCore.ts index 8039619ceb..7ffe2baddc 100644 --- a/src/dotnet/ManagedMemberInspectorCore.ts +++ b/src/dotnet/ManagedMemberInspectorCore.ts @@ -23,6 +23,7 @@ import type { } from "./ManagedMetadataLayout.js"; import { metadataRowCursor, + readManagedCompressedUnsigned, decodeMetadataCodedIndex, metadataCodedIndexCoordinate, type MetadataCodedIndexCoordinate, @@ -285,38 +286,17 @@ export const readCompressed = ( bytes: Buffer, offset: number, ): { readonly value: number; readonly next: number } => { - const first = bytes[offset]; - if (first === undefined) + const decoded = readManagedCompressedUnsigned(bytes, offset); + if (!decoded.ok) throw managedFailure( "invalid-blob", "signature", - "truncated compressed integer", + decoded.error.kind === "truncated" && decoded.error.width === 1 + ? "truncated compressed integer" + : "reserved compressed integer", offset, ); - if ((first & 0x80) === 0) return { value: first, next: offset + 1 }; - const second = bytes[offset + 1]; - if ((first & 0xc0) === 0x80 && second !== undefined) - return { value: ((first & 0x3f) << 8) | second, next: offset + 2 }; - const b1 = bytes[offset + 1]; - const b2 = bytes[offset + 2]; - const b3 = bytes[offset + 3]; - if ( - (first & 0xe0) === 0xc0 && - b1 !== undefined && - b2 !== undefined && - b3 !== undefined - ) - return { - value: - (first & 0x1f) * 0x01_00_00_00 + b1 * 0x01_00_00 + b2 * 0x0100 + b3, - next: offset + 4, - }; - throw managedFailure( - "invalid-blob", - "signature", - "reserved compressed integer", - offset, - ); + return { value: decoded.value.value, next: offset + decoded.value.width }; }; const ELEMENT_TYPES = new Map([ diff --git a/src/dotnet/ManagedMetadataHeaps.ts b/src/dotnet/ManagedMetadataHeaps.ts index ce1b2f8960..a212f6aacf 100644 --- a/src/dotnet/ManagedMetadataHeaps.ts +++ b/src/dotnet/ManagedMetadataHeaps.ts @@ -1,5 +1,6 @@ import type { ManagedParseIssue } from "../domain/managed/managedArtifact.js"; import { createHash } from "node:crypto"; +import { err, ok, type Result } from "../domain/result.js"; import { managedDecodeBudget } from "./ManagedDecodeBudget.js"; @@ -55,6 +56,39 @@ export class MetadataRowCursor { } } +interface ManagedCompressedUnsigned { + readonly value: number; + readonly width: 1 | 2 | 4; + readonly minimal: boolean; +} + +type ManagedCompressedUnsignedFailure = + | { readonly kind: "truncated"; readonly width: 1 | 2 | 4 } + | { readonly kind: "reserved" }; + +/** ECMA-335 unsigned encoding facts, without heap or signature admission policy. */ +export const readManagedCompressedUnsigned = ( + bytes: Buffer, + offset: number, +): Result => { + const first = bytes[offset]; + if (first === undefined) return err({ kind: "truncated", width: 1 }); + if ((first & 0x80) === 0) + return ok({ value: first, width: 1, minimal: true }); + const width = + (first & 0xc0) === 0x80 ? 2 : (first & 0xe0) === 0xc0 ? 4 : null; + if (width === null) return err({ kind: "reserved" }); + if (offset > bytes.length - width) return err({ kind: "truncated", width }); + const value = + width === 2 + ? ((first & 0x3f) << 8) | (bytes[offset + 1] ?? 0) + : (first & 0x1f) * 0x01_00_00_00 + + (bytes[offset + 1] ?? 0) * 0x01_00_00 + + (bytes[offset + 2] ?? 0) * 0x0100 + + (bytes[offset + 3] ?? 0); + return ok({ value, width, minimal: value >= (width === 2 ? 0x80 : 0x4000) }); +}; + // Issue code and label for the heap item a compressed length prefixes. const PREFIXED_ITEMS = { blob: { code: "invalid-blob", label: "Metadata blob" }, @@ -73,38 +107,21 @@ const compressedLength = ( readonly minimal: boolean; } => { const { code, label } = PREFIXED_ITEMS[item]; - const requirePrefix = (size: number): void => { - if (offset < 0 || offset > bytes.length - size) - throw managedFailure( - code, - scope, - `${label} length prefix leaves the artifact byte range`, - offset >= 0 ? offset : null, - ); + const decoded = readManagedCompressedUnsigned(bytes, offset); + if (!decoded.ok) + throw managedFailure( + code, + scope, + decoded.error.kind === "truncated" + ? `${label} length prefix leaves the artifact byte range` + : `${label} uses a reserved compressed-length prefix`, + offset >= 0 ? offset : null, + ); + return { + length: decoded.value.value, + prefix: decoded.value.width, + minimal: decoded.value.minimal, }; - requirePrefix(1); - const first = bytes[offset] ?? 0; - if ((first & 0x80) === 0) return { length: first, prefix: 1, minimal: true }; - if ((first & 0xc0) === 0x80) { - requirePrefix(2); - const length = ((first & 0x3f) << 8) | (bytes[offset + 1] ?? 0); - return { length, prefix: 2, minimal: length >= 0x80 }; - } - if ((first & 0xe0) === 0xc0) { - requirePrefix(4); - const length = - (first & 0x1f) * 0x01_00_00_00 + - (bytes[offset + 1] ?? 0) * 0x01_00_00 + - (bytes[offset + 2] ?? 0) * 0x0100 + - (bytes[offset + 3] ?? 0); - return { length, prefix: 4, minimal: length >= 0x4000 }; - } - throw managedFailure( - code, - scope, - `${label} uses a reserved compressed-length prefix`, - offset, - ); }; /** Read one UTF-8 value from #Strings within its PE/CLI stream extent. */ diff --git a/src/dotnet/ManagedMetadataInventoryRows.ts b/src/dotnet/ManagedMetadataInventoryRows.ts index 6bf3ccbb0b..0fba855438 100644 --- a/src/dotnet/ManagedMetadataInventoryRows.ts +++ b/src/dotnet/ManagedMetadataInventoryRows.ts @@ -16,6 +16,7 @@ import { } from "./ManagedMetadataLayout.js"; import { metadataRowCursor, + readManagedCompressedUnsigned, decodeMetadataCodedIndex, type ManagedCodedIndex, metadataToken, @@ -432,52 +433,24 @@ const attributeTypeName = ( return null; }; -const decodeCompressedLength = ( - blob: Buffer, - offset: number, -): { readonly length: number; readonly prefix: number } | undefined => { - const first = blob[offset]; - if (first === undefined) return undefined; - if ((first & 0x80) === 0) return { length: first, prefix: 1 }; - if ((first & 0xc0) === 0x80 && blob[offset + 1] !== undefined) - return { - length: ((first & 0x3f) << 8) | (blob[offset + 1] ?? 0), - prefix: 2, - }; - if ( - (first & 0xe0) === 0xc0 && - blob[offset + 1] !== undefined && - blob[offset + 2] !== undefined && - blob[offset + 3] !== undefined - ) - return { - length: - (first & 0x1f) * 0x01_00_00_00 + - (blob[offset + 1] ?? 0) * 0x01_00_00 + - (blob[offset + 2] ?? 0) * 0x0100 + - (blob[offset + 3] ?? 0), - prefix: 4, - }; - return undefined; -}; - const decodeFixedString = ( blob: Buffer, layout: ManagedMetadataLayout, ): string | null => { if (blob.length < 3 || blob.readUInt16LE(0) !== 1 || blob[2] === 0xff) return null; - const length = decodeCompressedLength(blob, 2); - if (length === undefined) return null; - const start = 2 + length.prefix; - if (start > blob.length - length.length) return null; + const decoded = readManagedCompressedUnsigned(blob, 2); + if (!decoded.ok) return null; + const { value: length, width } = decoded.value; + const start = 2 + width; + if (start > blob.length - length) return null; managedDecodeBudget(layout).reserveString( - length.length, + length, layout.blob?.offset ?? layout.rootOffset, ); try { return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - blob.subarray(start, start + length.length), + blob.subarray(start, start + length), ); } catch (cause: unknown) { // Invalid UTF-8 blobs decode as null. diff --git a/src/dotnet/ManagedPe.fixture.ts b/src/dotnet/ManagedPe.fixture.ts index f296ed7162..564a506f69 100644 --- a/src/dotnet/ManagedPe.fixture.ts +++ b/src/dotnet/ManagedPe.fixture.ts @@ -8,6 +8,7 @@ interface ManagedPeFixtureOptions { readonly cliFlags?: number; readonly corruptMetadataSignature?: boolean; readonly customAttributeTypeRaw?: number; + readonly customAttributeValue?: Buffer; readonly extendsRaw?: number; readonly exportedTypes?: readonly { readonly namespace: string; @@ -100,6 +101,18 @@ class StringHeap { } } +const compressedUnsigned = (value: number): Buffer => + value <= 0x7f + ? Buffer.from([value]) + : value <= 0x3fff + ? Buffer.from([0x80 | (value >> 8), value & 0xff]) + : Buffer.from([ + 0xc0 | (value >> 24), + (value >> 16) & 0xff, + (value >> 8) & 0xff, + value & 0xff, + ]); + class BlobHeap { readonly #chunks: Buffer[] = [Buffer.from([0])]; #size = 1; @@ -111,17 +124,7 @@ class BlobHeap { add(value: Buffer): number { if (value.length === 0) return 0; const index = this.#size; - const prefix = - value.length <= 0x7f - ? Buffer.from([value.length]) - : value.length <= 0x3fff - ? Buffer.from([0x80 | (value.length >> 8), value.length & 0xff]) - : Buffer.from([ - 0xc0 | (value.length >> 24), - (value.length >> 16) & 0xff, - (value.length >> 8) & 0xff, - value.length & 0xff, - ]); + const prefix = compressedUnsigned(value.length); this.#chunks.push(prefix, value); this.#size += value.length + prefix.length; return index; @@ -428,9 +431,12 @@ const exportedTypeRow = ( const fixedStringAttributeBlob = (value: string): Buffer => { const bytes = Buffer.from(textEncoder.encode(value)); - if (bytes.length > 0x7f) - throw new RangeError("Managed PE fixture target framework is too long"); - return Buffer.concat([Buffer.from([1, 0, bytes.length]), bytes, u16(0)]); + return Buffer.concat([ + Buffer.from([1, 0]), + compressedUnsigned(bytes.length), + bytes, + u16(0), + ]); }; /** @@ -1061,11 +1067,12 @@ const buildManagedFixtureMetadata = ( ); const referenceBlobs = referenceBlobIndexes(blobs, options); const attributeBlob = blobs.add( - fixedStringAttributeBlob( - options.targetFramework === null - ? ".NETCoreApp,Version=v8.0" - : (options.targetFramework ?? ".NETCoreApp,Version=v8.0"), - ), + options.customAttributeValue ?? + fixedStringAttributeBlob( + options.targetFramework === null + ? ".NETCoreApp,Version=v8.0" + : (options.targetFramework ?? ".NETCoreApp,Version=v8.0"), + ), ); const stringSize = indexSize(strings.size); const blobSize = indexSize(blobs.size); diff --git a/tests/acceptance/managedCliOutput.test.ts b/tests/acceptance/managedCliOutput.test.ts index 44d217bbce..0eb91e9853 100644 --- a/tests/acceptance/managedCliOutput.test.ts +++ b/tests/acceptance/managedCliOutput.test.ts @@ -526,3 +526,31 @@ cliTest( } }, ); + +cliTest( + "managed CLI retains target frameworks with multi-byte SerString lengths", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-managed-serstring-"); + for (const length of [0x80, 0x4000]) { + const targetFramework = "A".repeat(length); + const path = join(root, `framework-${length}.dll`); + await writeFile(path, buildManagedPeFixture({ targetFramework })); + + const result = await cli.run({ + arguments: ["inspect-managed-artifact", path, "--json"], + }); + + expect(result.exitCode).toBe(0); + expect(result.json).toMatchObject({ + operation: "inspect_managed_artifact", + normalized_result: { + target_frameworks: [targetFramework], + attributes: [ + expect.objectContaining({ decoded_fixed_string: targetFramework }), + ], + coverage: { state: "complete", issues: [] }, + }, + }); + } + }, +); From 171f56a5ed78085003c0076b2df72af30d26f9f2 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:49:05 +0800 Subject: [PATCH 06/21] refactor(setup): own parsed registration command facts --- src/application/ClientRegistrationIdentity.ts | 68 +++++++++++++------ src/application/ClientRegistrationStatus.ts | 41 +++++------ .../clientRegistrationStatus.test.ts | 67 +++++++++++------- 3 files changed, 109 insertions(+), 67 deletions(-) diff --git a/src/application/ClientRegistrationIdentity.ts b/src/application/ClientRegistrationIdentity.ts index 7b4d457c3a..ade744f5b5 100644 --- a/src/application/ClientRegistrationIdentity.ts +++ b/src/application/ClientRegistrationIdentity.ts @@ -80,7 +80,7 @@ const unwrapCmdCommand = (command: readonly string[]): readonly string[] => { }; /** Preserve the package argument from a supported package-runner command. */ -export const clientRegistrationPackageReference = ( +const clientRegistrationPackageReference = ( command: readonly string[], ): string | undefined => { const arguments_ = unwrapCmdCommand(command); @@ -100,20 +100,6 @@ export const clientRegistrationPackageReference = ( return undefined; }; -const isOwnedPackageReference = (packageReference: string): boolean => { - if ( - packageReference === PRODUCT_IDENTITY.packageName || - packageReference === PRODUCT_IDENTITY.packageSpecifier || - packageReference === PRODUCT_IDENTITY.registrationPackageSpecifier - ) - return true; - const versionPrefix = `${PRODUCT_IDENTITY.packageName}@`; - return ( - packageReference.startsWith(versionPrefix) && - valid(packageReference.slice(versionPrefix.length)) !== null - ); -}; - /** Check that one parsed command points to REA's MCP entry point. */ export type ClientRegistrationCommandIdentity = "owned" | "foreign" | "unknown"; @@ -149,14 +135,58 @@ const directEntrypointIdentity = ( : "unknown"; }; -/** Classify the command once; uncertainty never grants positive ownership. */ +export interface ClientRegistrationCommandFacts { + readonly identity: ClientRegistrationCommandIdentity; + readonly package: + | { + readonly reference: string; + readonly exactVersion: string | undefined; + } + | undefined; +} + +/** Parse package identity once, retaining exact source spelling and version facts. */ +export const inspectClientRegistrationCommand = ( + command: readonly string[], + currentCommandPath: string | undefined, +): ClientRegistrationCommandFacts => { + const reference = clientRegistrationPackageReference(command); + if (reference !== undefined) { + const prefix = `${PRODUCT_IDENTITY.packageName}@`; + const version = reference.startsWith(prefix) + ? reference.slice(prefix.length) + : undefined; + const normalized = version === undefined ? null : valid(version); + const owned = + reference === PRODUCT_IDENTITY.packageName || + reference === PRODUCT_IDENTITY.packageSpecifier || + reference === PRODUCT_IDENTITY.registrationPackageSpecifier || + normalized !== null; + return { + identity: owned ? "owned" : "foreign", + package: { + reference, + exactVersion: normalized === version ? version : undefined, + }, + }; + } + return { + identity: classifyDirectRegistrationCommand(command, currentCommandPath), + package: undefined, + }; +}; + +/** Ownership-only consumers share the same parsed command facts. */ export const classifyClientRegistrationCommand = ( command: readonly string[], currentCommandPath: string | undefined, +): ClientRegistrationCommandIdentity => + inspectClientRegistrationCommand(command, currentCommandPath).identity; + +const classifyDirectRegistrationCommand = ( + command: readonly string[], + currentCommandPath: string | undefined, ): ClientRegistrationCommandIdentity => { - const packageReference = clientRegistrationPackageReference(command); - if (packageReference !== undefined) - return isOwnedPackageReference(packageReference) ? "owned" : "foreign"; const currentPathAvailable = currentCommandPath !== undefined && currentCommandPath.length > 0; if (command.length === 3 && command[2] === "mcp") { diff --git a/src/application/ClientRegistrationStatus.ts b/src/application/ClientRegistrationStatus.ts index 649bb27992..5b1d7c6058 100644 --- a/src/application/ClientRegistrationStatus.ts +++ b/src/application/ClientRegistrationStatus.ts @@ -11,8 +11,6 @@ import { lstat } from "node:fs/promises"; import { err, ok, type Result } from "../domain/result.js"; import { isAbsolute } from "node:path"; -import { valid } from "semver"; - import { PRODUCT_IDENTITY } from "../identity.js"; import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import { CURRENT_ENTRYPOINT_PATH } from "../process/CurrentEntrypointPath.js"; @@ -24,8 +22,8 @@ import { type McpToolTimeoutInputs, } from "../mcpStartupPolicy.js"; import { - clientRegistrationPackageReference, - classifyClientRegistrationCommand, + inspectClientRegistrationCommand, + type ClientRegistrationCommandFacts, packageRunnerRegistrationCommand, } from "./ClientRegistrationIdentity.js"; import { @@ -279,16 +277,21 @@ export const readClientRegistrationInspections = async ( registration.command, ...registration.args, ]; - const packageReference = clientRegistrationPackageReference(command); + const commandFacts = inspectClientRegistrationCommand( + command, + currentCommandPath, + ); + const packageReference = commandFacts.package?.reference; const activation = registrationActivation(registration, client, parsed); - const packageVersion = exactRegisteredPackageVersion(packageReference); + const packageVersion = commandFacts.package?.exactVersion; const packageVersionDrift = packageVersion !== undefined && packageVersion !== PRODUCT_IDENTITY.packageVersion; const alignment = registrationAligned({ registration, client: effectiveClient, - currentCommandPath, + command, + commandFacts, platform: options.platform ?? process.platform, kimiInputSchemaProfileSelected: registrationEnvironment?.ok === true && @@ -373,21 +376,19 @@ const registrationActivation = ( const registrationAligned = ({ registration, client, - currentCommandPath, + command, + commandFacts, platform, kimiInputSchemaProfileSelected, }: { readonly registration: ClientRegistration; readonly client: SetupClient; - readonly currentCommandPath: string | undefined; + readonly command: readonly string[]; + readonly commandFacts: ClientRegistrationCommandFacts; readonly platform: NodeJS.Platform; readonly kimiInputSchemaProfileSelected: boolean; }): "aligned" | "stale" | "unknown" => { - const command = [registration.command, ...registration.args]; - const identity = classifyClientRegistrationCommand( - command, - currentCommandPath, - ); + const { identity } = commandFacts; if (identity === "unknown") return "unknown"; if (identity === "foreign") return "stale"; if ( @@ -427,7 +428,7 @@ const registrationAligned = ({ ) return "aligned"; // Other owned package references remain stale until pinned to this version. - if (clientRegistrationPackageReference(command) !== undefined) return "stale"; + if (commandFacts.package !== undefined) return "stale"; // The identity owner already established an exact direct launcher match. // A bare command still depends on the client's PATH, so cannot prove alignment. return isAbsolute(command[0] ?? "") ? "aligned" : "stale"; @@ -505,16 +506,6 @@ const timeoutAdvisory = ( return `The configured ${setting} (${currentValue}) is shorter than REA's known provider allowance (${String(providerAllowance)} ms), so provider startup or an upstream request may exceed this client deadline. REA recommends ${setting} = ${recommendedValue} to cover that allowance plus ${headroom} of operation headroom; this additional headroom is a recommendation, not a guarantee for every analysis.`; }; -const exactRegisteredPackageVersion = ( - packageReference: string | undefined, -): string | undefined => { - if (packageReference === undefined) return undefined; - const prefix = `${PRODUCT_IDENTITY.packageName}@`; - if (!packageReference.startsWith(prefix)) return undefined; - const version = packageReference.slice(prefix.length); - return valid(version) === version ? version : undefined; -}; - const packageVersionRemediation = ( packageReference: string, client: SetupClient, diff --git a/tests/boundary/filesystem/clientRegistrationStatus.test.ts b/tests/boundary/filesystem/clientRegistrationStatus.test.ts index cd4c9d0001..37b1766823 100644 --- a/tests/boundary/filesystem/clientRegistrationStatus.test.ts +++ b/tests/boundary/filesystem/clientRegistrationStatus.test.ts @@ -7,6 +7,7 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { clearClientLocationEnvironment } from "../../fixtures/clientEnvironment.js"; import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; +import { classifyClientRegistrationCommand } from "../../../src/application/ClientRegistrationIdentity.js"; import { existingMaintenanceScope } from "../../../src/application/UpdateMaintenance.js"; import { PRODUCT_IDENTITY } from "../../../src/identity.js"; @@ -229,31 +230,51 @@ describe("client registration launcher identity", () => { }); }); - it("reports the exact requested package version when it differs from this CLI", async () => { - const home = await createTestTempDirectory( - "rea-registration-version-drift-", - ); - await mkdir(join(home, ".codex")); - const command = ["npx", "-y", "rea-agents@4.1.0", "mcp"]; - await writeFile( - join(home, ".codex/config.toml"), - `[mcp_servers.rea]\ncommand = "${command[0]}"\nargs = ["${command[1]}", "${command[2]}", "${command[3]}"]\n`, - ); + it.each([ + ["4.1.0", true], + ["v4.1.0", false], + ] as const)( + "preserves package version %s ownership with canonical-only drift reporting", + async (version, reportsDrift) => { + const home = await createTestTempDirectory( + "rea-registration-version-drift-", + ); + await mkdir(join(home, ".codex")); + const reference = `rea-agents@${version}`; + const command = ["npx", "-y", reference, "mcp"]; + await writeFile( + join(home, ".codex/config.toml"), + `[mcp_servers.rea]\ncommand = "${command[0]}"\nargs = ["${command[1]}", "${command[2]}", "${command[3]}"]\n`, + ); - const [status] = await readClientRegistrationStatuses(home); + const [status] = await readClientRegistrationStatuses(home); - expect(status).toMatchObject({ - client: "codex", - command, - package_reference: "rea-agents@4.1.0", - state: "stale", - remediation: expect.stringContaining( - `current REA CLI package is ${PRODUCT_IDENTITY.packageVersion}`, - ), - }); - expect(status?.remediation).toContain("does not resolve package-runner"); - expect(status?.remediation).toContain("rea setup --client codex"); - }); + expect(classifyClientRegistrationCommand(command, undefined)).toBe( + "owned", + ); + expect(status).toMatchObject({ + client: "codex", + command, + package_reference: reference, + state: "stale", + }); + if (reportsDrift) { + expect(status?.remediation).toContain( + `current REA CLI package is ${PRODUCT_IDENTITY.packageVersion}`, + ); + expect(status?.remediation).toContain( + "does not resolve package-runner", + ); + expect(status?.remediation).toContain("rea setup --client codex"); + } else { + expect(status?.remediation).not.toContain("current REA CLI package"); + expect(status?.remediation).not.toContain( + "does not resolve package-runner", + ); + } + expect(status?.remediation).toContain("rea setup"); + }, + ); }); describe("commandcode registration status", () => { From 5f4bd6ba341e3ce0efd89a70b088b54a0b8feeee Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:49:06 +0800 Subject: [PATCH 07/21] refactor(cli): execute validated model and GPU requests --- src/cli/gpuCommands.ts | 2 +- src/cli/modelCommands.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli/gpuCommands.ts b/src/cli/gpuCommands.ts index e42d2066fd..039bdfe99c 100644 --- a/src/cli/gpuCommands.ts +++ b/src/cli/gpuCommands.ts @@ -101,7 +101,7 @@ export const registerGpuCommands = (cli: CliInstance, logger: Logger): void => { run: () => service.inspect( { - path: absoluteCallerPath(args.path), + path: selected.data.path, analysis: selected.data.analysis, ...(selected.data.expected_sha256 === undefined ? {} diff --git a/src/cli/modelCommands.ts b/src/cli/modelCommands.ts index e6b3803bf6..0df6e94408 100644 --- a/src/cli/modelCommands.ts +++ b/src/cli/modelCommands.ts @@ -84,8 +84,8 @@ export const registerModelCommands = ( run: () => service.inspect( { - path: absoluteCallerPath(args.path), - hash: options.hash, + path: selected.data.path, + hash: selected.data.hash, detail: "complete", }, { signal }, From 2b8e388bbe5129a956ba1527dced9f9999347613 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:49:20 +0800 Subject: [PATCH 08/21] fix(native): unify legacy Mach-O platform interpretation --- .../apple/MachoLoadCommandReader.test.ts | 1 + src/artifacts/apple/MachoLoadCommandReader.ts | 59 ++++----------- src/domain/apple/machoPlatform.ts | 19 +++++ src/native/parsers/otool.ts | 41 ++++++----- .../native/nativeMachoHeader.golden.test.ts | 73 ++++++++++++++----- 5 files changed, 109 insertions(+), 84 deletions(-) create mode 100644 src/domain/apple/machoPlatform.ts diff --git a/src/artifacts/apple/MachoLoadCommandReader.test.ts b/src/artifacts/apple/MachoLoadCommandReader.test.ts index 87942cf51e..04d5b27c1b 100644 --- a/src/artifacts/apple/MachoLoadCommandReader.test.ts +++ b/src/artifacts/apple/MachoLoadCommandReader.test.ts @@ -108,6 +108,7 @@ it.each([ { command: LC.VERSION_MIN_IPHONEOS, cpu: CPU.i386, platform: 7 }, { command: LC.VERSION_MIN_IPHONEOS, cpu: CPU.x86_64, platform: 7 }, { command: LC.VERSION_MIN_IPHONEOS, cpu: CPU.arm64, platform: 2 }, + { command: LC.VERSION_MIN_TVOS, cpu: CPU.i386, platform: 3 }, { command: LC.VERSION_MIN_TVOS, cpu: CPU.x86_64, platform: 8 }, { command: LC.VERSION_MIN_TVOS, cpu: CPU.arm64, platform: 3 }, { command: LC.VERSION_MIN_WATCHOS, cpu: CPU.i386, platform: 9 }, diff --git a/src/artifacts/apple/MachoLoadCommandReader.ts b/src/artifacts/apple/MachoLoadCommandReader.ts index f08b1bd302..7b2141a6e0 100644 --- a/src/artifacts/apple/MachoLoadCommandReader.ts +++ b/src/artifacts/apple/MachoLoadCommandReader.ts @@ -4,6 +4,10 @@ import type { MachoSlice, } from "../../domain/apple/dylibResolution.js"; import { machoCpuSubtypeBase } from "../../domain/apple/machoCpuIdentity.js"; +import { + legacyMachoPlatform, + type MachoVersionMinPlatform, +} from "../../domain/apple/machoPlatform.js"; import { machoLoadCommandAlignment, classifyFatContainerIdentity, @@ -53,53 +57,18 @@ const LC_LOAD_UPWARD_DYLIB = 0x80000023; const LC_RPATH = 0x8000001c; const LC_DYLD_ENVIRONMENT = 0x27; const LC_CODE_SIGNATURE = 0x1d; -const LC_VERSION_MIN_MACOSX = 0x24; -const LC_VERSION_MIN_IPHONEOS = 0x25; -const LC_VERSION_MIN_TVOS = 0x2f; -const LC_VERSION_MIN_WATCHOS = 0x30; const LC_BUILD_VERSION = 0x32; -// `mach-o/loader.h` `PLATFORM_*` numbers observed via Apple linker output. -const PLATFORM_MACOS = 1; -const PLATFORM_IOS = 2; -const PLATFORM_TVOS = 3; -const PLATFORM_WATCHOS = 4; -const PLATFORM_IOSSIMULATOR = 7; -const PLATFORM_TVOSSIMULATOR = 8; -const PLATFORM_WATCHOSSIMULATOR = 9; - -const CPU_TYPE_I386 = 7; -const CPU_TYPE_X86_64 = 0x01000007; const CPU_TYPE_ARM64 = 0x0100000c; -/** - * Legacy `LC_VERSION_MIN_*` commands predate explicit simulator platform - * IDs. Per Apple's dyld rules the image CPU type distinguishes simulator - * from device: Intel images are simulator, ARM images are device - * (tvOS simulator applies to x86_64 only). `LC_BUILD_VERSION` platform - * numbers are preserved verbatim; unknown future numbers stay unknown - * downstream, never mapped to nearest. - */ -const legacyVersionMinPlatform = ( - command: number, - cpuType: number, -): number | undefined => { - const isIntel = cpuType === CPU_TYPE_I386 || cpuType === CPU_TYPE_X86_64; - switch (command) { - case LC_VERSION_MIN_MACOSX: - return PLATFORM_MACOS; - case LC_VERSION_MIN_IPHONEOS: - return isIntel ? PLATFORM_IOSSIMULATOR : PLATFORM_IOS; - case LC_VERSION_MIN_TVOS: - return cpuType === CPU_TYPE_X86_64 - ? PLATFORM_TVOSSIMULATOR - : PLATFORM_TVOS; - case LC_VERSION_MIN_WATCHOS: - return isIntel ? PLATFORM_WATCHOSSIMULATOR : PLATFORM_WATCHOS; - default: - return undefined; - } -}; +/** loader.h command codes and their device platform identities. */ +const VERSION_MIN_PLATFORMS: ReadonlyMap = + new Map([ + [0x24, 1], // LC_VERSION_MIN_MACOSX + [0x25, 2], // LC_VERSION_MIN_IPHONEOS + [0x2f, 3], // LC_VERSION_MIN_TVOS + [0x30, 4], // LC_VERSION_MIN_WATCHOS + ]); /** `mach-o/loader.h` `dylib_use_command`: `nameoff == 28` and this marker. */ const DYLIB_USE_MARKER = 0x1a741800; @@ -691,14 +660,14 @@ const decodeCommand = ( slice.platforms.push(platform); return; } - const legacyPlatform = legacyVersionMinPlatform(command, slice.cpu_type); + const legacyPlatform = VERSION_MIN_PLATFORMS.get(command); if (legacyPlatform !== undefined) { if (body.byteLength < 16) throw new MachoFormatIssue( "malformed", `load command ${index} is too short for a version-min command`, ); - slice.platforms.push(legacyPlatform); + slice.platforms.push(legacyMachoPlatform(legacyPlatform, slice.cpu_type)); return; } if (command === LC_LAZY_LOAD_DYLIB_INFO) diff --git a/src/domain/apple/machoPlatform.ts b/src/domain/apple/machoPlatform.ts new file mode 100644 index 0000000000..997f3c6c63 --- /dev/null +++ b/src/domain/apple/machoPlatform.ts @@ -0,0 +1,19 @@ +/** Device platform encoded by a legacy LC_VERSION_MIN_* command. */ +export type MachoVersionMinPlatform = 1 | 2 | 3 | 4; + +/** + * dyld derives legacy simulator platforms from the CPU type. tvOS uses + * x86_64 only; iOS and watchOS also recognize i386. Explicit LC_BUILD_VERSION + * platforms do not use this rule. + */ +export const legacyMachoPlatform = ( + platform: MachoVersionMinPlatform, + cpuType: number | null, +): number => { + if (platform === 3) return cpuType === 0x01000007 ? 8 : 3; + if (cpuType === 7 || cpuType === 0x01000007) { + if (platform === 2) return 7; + if (platform === 4) return 9; + } + return platform; +}; diff --git a/src/native/parsers/otool.ts b/src/native/parsers/otool.ts index 37ade5ac33..f3fc0dfb56 100644 --- a/src/native/parsers/otool.ts +++ b/src/native/parsers/otool.ts @@ -1,3 +1,8 @@ +import { + legacyMachoPlatform, + type MachoVersionMinPlatform, +} from "../../domain/apple/machoPlatform.js"; + interface ParsedSection { readonly segment: string; readonly name: string; @@ -70,6 +75,14 @@ const DYLIB_FIELDS = [ const DYLIB_ENCODING_FIELDS = ["time stamp", "options"]; const EMPTY_VALUE_FIELDS = new Set(["options", "segname", "sectname"]); +const VERSION_MIN_PLATFORMS: ReadonlyMap = + new Map([ + ["LC_VERSION_MIN_MACOSX", 1], + ["LC_VERSION_MIN_IPHONEOS", 2], + ["LC_VERSION_MIN_TVOS", 3], + ["LC_VERSION_MIN_WATCHOS", 4], + ]); + /** * Non-verbose otool prints LC_BUILD_VERSION platform and tool codes as * numbers. Name the codes declared in the SDK's , without @@ -408,8 +421,16 @@ const collectBuild = ( } if (kind.startsWith("LC_VERSION_MIN_")) { const platform = kind.slice("LC_VERSION_MIN_".length) || null; + const devicePlatform = VERSION_MIN_PLATFORMS.get(kind); + const derivedPlatform = + devicePlatform === undefined + ? undefined + : legacyMachoPlatform(devicePlatform, state.cpuType); state.builds.push({ - platform: legacySimulatorPlatform(platform, state.cpuType), + platform: + derivedPlatform === undefined || derivedPlatform === devicePlatform + ? platform + : buildCodeName(BUILD_PLATFORM_NAMES, String(derivedPlatform)), minimum_os: stringField(fields, "version"), sdk: stringField(fields, "sdk"), tools: [], @@ -417,24 +438,6 @@ const collectBuild = ( } }; -// Apple dyld interprets legacy iOS/tvOS/watchOS minimum-version commands on -// Intel slices as simulator platforms. Newer commands encode this directly. -const legacySimulatorPlatform = ( - platform: string | null, - cpuType: number | null, -): string | null => { - if (![7, 0x01000007].includes(cpuType ?? -1)) return platform; - return ( - ( - { - IPHONEOS: "IOSSIMULATOR", - TVOS: "TVOSSIMULATOR", - WATCHOS: "WATCHOSSIMULATOR", - } as const - )[platform as "IPHONEOS" | "TVOS" | "WATCHOS"] ?? platform - ); -}; - const collectDependency = ( kind: string, fields: Readonly>, diff --git a/tests/boundary/providers/native/nativeMachoHeader.golden.test.ts b/tests/boundary/providers/native/nativeMachoHeader.golden.test.ts index 61722e921b..aeae2b523d 100644 --- a/tests/boundary/providers/native/nativeMachoHeader.golden.test.ts +++ b/tests/boundary/providers/native/nativeMachoHeader.golden.test.ts @@ -269,33 +269,66 @@ Section }); it.each([ - ["LC_VERSION_MIN_IPHONEOS", "IOSSIMULATOR"], - ["LC_VERSION_MIN_TVOS", "TVOSSIMULATOR"], - ["LC_VERSION_MIN_WATCHOS", "WATCHOSSIMULATOR"], -])("classifies legacy Intel simulator command %s", (command, platform) => { + ["LC_VERSION_MIN_IPHONEOS", 7, "IOSSIMULATOR"], + ["LC_VERSION_MIN_IPHONEOS", 16777223, "IOSSIMULATOR"], + ["LC_VERSION_MIN_IPHONEOS", 16777228, "IPHONEOS"], + ["LC_VERSION_MIN_TVOS", 7, "TVOS"], + ["LC_VERSION_MIN_TVOS", 16777223, "TVOSSIMULATOR"], + ["LC_VERSION_MIN_TVOS", 16777228, "TVOS"], + ["LC_VERSION_MIN_WATCHOS", 7, "WATCHOSSIMULATOR"], + ["LC_VERSION_MIN_WATCHOS", 16777223, "WATCHOSSIMULATOR"], + ["LC_VERSION_MIN_WATCHOS", 16777228, "WATCHOS"], + ["LC_VERSION_MIN_MACOSX", 7, "MACOSX"], + ["LC_VERSION_MIN_MACOSX", 16777223, "MACOSX"], + ["LC_VERSION_MIN_MACOSX", 16777228, "MACOSX"], +])( + "classifies legacy command %s on CPU %s as %s", + (command, cpuType, platform) => { + const text = [ + "Mach header", + " magic cputype cpusubtype caps filetype ncmds sizeofcmds flags", + ` ${cpuType === 7 ? "0xfeedface" : "0xfeedfacf"} ${cpuType} 3 0x00 2 1 16 0x0`, + "Load command 0", + ` cmd ${command}`, + " cmdsize 16", + " version 12.0", + " sdk 12.1", + ].join("\n"); + expect(parseOtoolLoadCommands(text).builds[0]?.platform).toBe(platform); + }, +); + +it.each([ + ["3", "TVOS"], + ["4294967295", "4294967295"], +])("preserves explicit build platform %s on Intel", (code, platform) => { + const text = [ + "Mach header", + " magic cputype cpusubtype caps filetype ncmds sizeofcmds flags", + " 0xfeedfacf 16777223 3 0x00 2 1 24 0x0", + "Load command 0", + " cmd LC_BUILD_VERSION", + " cmdsize 24", + ` platform ${code}`, + " minos 12.0", + " sdk 12.1", + " ntools 0", + ].join("\n"); + expect(parseOtoolLoadCommands(text).builds[0]?.platform).toBe(platform); +}); + +it("preserves an unknown legacy platform spelling on Intel", () => { const text = [ "Mach header", " magic cputype cpusubtype caps filetype ncmds sizeofcmds flags", " 0xfeedfacf 16777223 3 0x00 2 1 16 0x0", "Load command 0", - ` cmd ${command}`, + " cmd LC_VERSION_MIN_FUTURE", " cmdsize 16", " version 12.0", " sdk 12.1", ].join("\n"); - expect(parseOtoolLoadCommands(text).builds[0]?.platform).toBe(platform); -}); - -it("keeps legacy device platforms on arm64 slices", () => { - const text = [ - "Mach header", - " magic cputype cpusubtype caps filetype ncmds sizeofcmds flags", - " 0xfeedfacf 16777228 0 0x00 2 1 16 0x0", - "Load command 0", - " cmd LC_VERSION_MIN_IPHONEOS", - " cmdsize 16", - " version 12.0", - " sdk 12.1", - ].join("\n"); - expect(parseOtoolLoadCommands(text).builds[0]?.platform).toBe("IPHONEOS"); + const result = parseOtoolLoadCommands(text); + expect(result.builds[0]?.platform).toBe("FUTURE"); + expect(result.commands[0]?.kind).toBe("LC_VERSION_MIN_FUTURE"); }); From beb41f8145017346e588b13c193a66dce9769ace Mon Sep 17 00:00:00 2001 From: Cassie_Fox <249835913+CassieFox23@users.noreply.github.com> Date: Fri, 23 Oct 2026 16:53:00 -0400 Subject: [PATCH 09/21] docs(website): add dark-mode styles to the favicon and social card (#6909) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The favicon tile (#20252b) is almost invisible on a dark browser tab strip, and the social card opened on its own is light only. Add one prefers-color-scheme: dark block to each SVG. Each edit is a pure insertion, and the light render is 0 pixels different in magick and in CairoSVG. CairoSVG ignores the block, so the PNG that link previews fetch stays light. website/figures.md says so. 🤖 Generated with Claude Code (Sonnet 5.5) 🦊 Co-authored-by: Claude Sonnet 5.5 --- website/figures.md | 5 +++++ website/public/assets/favicon.svg | 2 +- website/public/assets/social-card.svg | 12 ++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/website/figures.md b/website/figures.md index 73fc70c42c..cef6c871e6 100644 --- a/website/figures.md +++ b/website/figures.md @@ -52,6 +52,11 @@ preparation. Check the rendered image for readable text and spacing before publishing. The build uses DejaVu Sans for consistent typography; only the SVG source is committed. Metadata and build instructions are in the website README. +The card and `public/assets/favicon.svg` each carry a dark +`prefers-color-scheme` block for people who open the SVG directly or use a dark +browser tab strip. CairoSVG ignores it, so the PNG that link previews fetch +stays light. + ## Beginner homepage and dinosaur reconstruction The manual/agent comparison and Calculator percentage controls are semantic diff --git a/website/public/assets/favicon.svg b/website/public/assets/favicon.svg index 78512ef981..10a2e315c7 100644 --- a/website/public/assets/favicon.svg +++ b/website/public/assets/favicon.svg @@ -1 +1 @@ - + diff --git a/website/public/assets/social-card.svg b/website/public/assets/social-card.svg index faab21bd25..2e30ac9eec 100644 --- a/website/public/assets/social-card.svg +++ b/website/public/assets/social-card.svg @@ -1,6 +1,18 @@ REA — Find out how software works. Reverse engineering with your coding agent. REA connects your agent to analysis tools and returns evidence. + REA From d29ae43aeb522559da6f25faae29b5d907e1fe40 Mon Sep 17 00:00:00 2001 From: PJY | ProviDER <128478616+uyg7x@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:23:25 +0530 Subject: [PATCH 10/21] docs: add missing src/ directories to architecture.mermaid (#6908) * docs: add 21 missing src/ dirs to architecture.mermaid (#6901) * docs: restore KUNA and AUX node declarations lost in merge * docs: remove final conflict marker --- docs/architecture.mermaid | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/architecture.mermaid b/docs/architecture.mermaid index ddb1d785b4..e223aba6ca 100644 --- a/docs/architecture.mermaid +++ b/docs/architecture.mermaid @@ -38,7 +38,11 @@ graph TD IDA["src/ida/
Upstream MCP contract adaptation
live GUI binding or owned headless database"] KUNA["src/kuna/
BYO Kuna deep analysis candidate
discovery reads metadata; version probe on target open"] IDA_MCP["External ida-pro-mcp
GUI proxy or database supervisor
upstream installation and engine lifecycle"] + + KUNA["src/kuna/
BYO Kuna decompiler, a deep analysis candidate
discovery reads metadata only; version probe when a target opens"] AUX["src/artifacts/ + src/native/
artifact inventory, bounded directory triage and extraction
format readers and native capabilities"] + + BROWSER["src/browser/
CDP / Playwright capture
captured scripts and native module resolution
execution/: coverage, listener and source attribution"] JS_CODEC["src/javascript/sourceMaps/
pinned upstream map codec
owned heap-limited Node process, complete point evidence"] INSPECTOR["src/inspector/
passive V8 Inspector discovery and capture
script and target locations"] @@ -50,12 +54,19 @@ graph TD FIRMWARE["src/firmware/
Binwalk / Unblob adapters
owned processes, report normalization, private extraction"] ANDROID["src/android/
Static APK adapter
owned SDK stdio, pinned headless JADX, normalization"] REFERENCE["src/reference/
Filesystem reference-source reader
bounded traversal, validation, errors"] + + ANDROID_TOOLS["src/adb/ + src/apktool/ + src/jeb/
BYO device, resource-decoding and JEB adapters
owned adb and Apktool processes; JEB engine client over MCP"] + CAPTURE_TOOLS["src/captures/ + src/pcap/ + src/signals/ + src/sqlite/
BYO tshark, pcap, sigrok and SQLite child inspection
selected captures, recordings and databases; owned processes"] + NATIVE_READERS["src/go/ + src/rust/ + src/il2cpp/ + src/flutter/ + src/gpu/
Bundled in-process binary metadata readers
Go build info, Rust dependencies, IL2CPP, Dart build ID, GPU code objects
src/compression/: shared codec-result helper for rust and gpu"] + FORMAT_READERS["src/model/ + src/python/ + src/erlang/ + src/electron/
Bundled in-process format readers
GGUF and safetensors headers, Python .pyc, Erlang BEAM, Electron fuses"] + RUNTIME["src/frida/ + src/cutter/
Frida instrumentation sessions and live Cutter bridge client"] end subgraph "Provider Process Foundation" PROCESS["src/process/
owned process groups, private roots
deadlines, bounded file reads and diagnostics, cleanup"] WINDOWS["src/windows/
Windows host boundary adapters
package loader, opaque handles, owned jobs and private runtimes"] WINDOWS_NATIVE["native/windows/src/
Node-API implementation
NTFS admission, protected DACLs and atomic Job Object assignment"] + FILESYSTEM["src/filesystem/
owned file and directory handles
stable reads, staged output, caller paths"] end subgraph "Contracts Layer" @@ -181,3 +192,23 @@ graph TD APP -->|EVM interface port| EVM EVM --> PROCESS EVM --> DOMAIN + COMPOSITION --> KUNA + KUNA --> PROCESS + KUNA --> FILESYSTEM + KUNA --> DOMAIN + COMPOSITION --> ANDROID_TOOLS + ANDROID_TOOLS --> PROCESS + ANDROID_TOOLS --> DOMAIN + COMPOSITION --> CAPTURE_TOOLS + CAPTURE_TOOLS --> PROCESS + CAPTURE_TOOLS --> DOMAIN + COMPOSITION --> NATIVE_READERS + NATIVE_READERS --> DOMAIN + COMPOSITION --> FORMAT_READERS + FORMAT_READERS --> DOMAIN + SERVER --> RUNTIME + CLI --> RUNTIME + RUNTIME --> DOMAIN + PROCESS --> FILESYSTEM + AUX --> FILESYSTEM + APP --> FILESYSTEM \ No newline at end of file From 57b1f8236717185815f21e0bf421fbeb380cc8e7 Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:00:05 +0800 Subject: [PATCH 11/21] perf(native): query retained export table facts (#6907) * perf(native): query retained export table facts * docs(perf): record native export task comparisons * fix(native): preserve omitted export projection fields * docs(perf): refresh final export view qualification * test(native): verify sparse export view digests --- .../references/evidence-workflows.md | 11 + docs/binary-diagnostics.md | 47 +- .../native-export-costs.py | 115 + .../native-export-oracle.py | 48 + .../native-export-views.json | 2404 +++++++++++++++++ .../native-export-views.md | 167 ++ .../native-export-views.mjs | 224 ++ .../binaryDiagnostics/NativeExportsService.ts | 3 +- src/cli/analysisViewCommands.ts | 2 +- src/cli/binaryDiagnosticsCommands.ts | 19 +- src/contracts/analysisViewToolContracts.ts | 2 +- .../native/binaryDiagnosticsToolContracts.ts | 17 +- src/domain/analysisView/analysisView.ts | 27 + src/domain/analysisView/nativeExportsView.ts | 251 ++ src/domain/native/nativeExports.ts | 77 +- src/native/exports/NativeExportsView.test.ts | 263 ++ src/server/createServer.ts | 1 + src/server/registerNativeExportsTool.ts | 31 +- .../applications/nativeExportsViews.test.ts | 168 ++ .../boundary/mcp/binaryDiagnosticsMcp.test.ts | 45 + 20 files changed, 3871 insertions(+), 51 deletions(-) create mode 100644 experiments/analysis-efficiency/native-export-costs.py create mode 100644 experiments/analysis-efficiency/native-export-oracle.py create mode 100644 experiments/analysis-efficiency/native-export-views.json create mode 100644 experiments/analysis-efficiency/native-export-views.md create mode 100644 experiments/analysis-efficiency/native-export-views.mjs create mode 100644 src/domain/analysisView/nativeExportsView.ts create mode 100644 src/native/exports/NativeExportsView.test.ts create mode 100644 tests/acceptance/applications/nativeExportsViews.test.ts diff --git a/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md b/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md index 90cf08eeb3..fe542fa90d 100644 --- a/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md +++ b/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md @@ -104,3 +104,14 @@ each plan states why in its `reason`. It replays the capture instead with `--replay-only` or when the state is unavailable (not derivable, or a catalog over the discovery budget), and its stopped document's `state` says which applied. + +For a PE/ELF export question, `inspect_native_exports` accepts `detail: "summary"` +and retains the complete table. Query `inspect_analysis_view` with +`view: {kind: "native-exports", collection: "exports", names: ["selected_name"], +fields: ["name"], offset: 0, limit: 32}` and the reported parent Evidence ID. +All same-name versions and aliases survive; use `source_index` with the parent ID +for stable identity. `collection: "issues"` pages interpretation failures. Page +coverage describes selected retained rows; `summary.coverage` or `exports.coverage` +still describes the source table, so an empty partial page does not prove absence. +CLI `inspect-native-exports --detail summary --evidence_output PATH` saves the +complete bundle for later `inspect-analysis-view --evidence-bundle PATH` requests. diff --git a/docs/binary-diagnostics.md b/docs/binary-diagnostics.md index b4b5bf5f12..4a1c62b8fd 100644 --- a/docs/binary-diagnostics.md +++ b/docs/binary-diagnostics.md @@ -326,7 +326,48 @@ rea inspect-native-exports ./runtimes/linux-x64/native/libexample.so --json ``` The complete Evidence identifies the stable artifact snapshot by path, byte -count and SHA-256. +count and SHA-256. Complete delivery remains the default. For a small initial +MCP reply, select `detail: "summary"`; this retains the complete Evidence and +returns artifact identity, table coverage and row counts. Use its +`normalized_result.parent_evidence_id` with `inspect_analysis_view`: + +```json +{ + "source": { + "kind": "retained-evidence", + "evidence_id": "ev_" + }, + "view": { + "kind": "native-exports", + "collection": "exports", + "names": ["selected_name"], + "fields": ["name"], + "offset": 0, + "limit": 32 + } +} +``` + +Names match exactly and preserve every same-name version or alias. Optional +`indices` selects original row positions, intersecting a name filter when both +are supplied; empty filters select nothing. `source_index` always remains and, +with the parent ID, identifies the original export or issue row. Fields default +to the complete row; an empty field list returns only its identity. ELF `name` +selection also returns raw-byte and encoding qualifications. Fields from the +other format are rejected. `collection: "issues"` pages the complete diagnostic +records, with optional original indices. Offset and limit default to 0 and 64 +and apply to matching rows in original order. + +View `coverage` describes the selected page; `summary.coverage` or +`exports.coverage` preserves source-table coverage. An exhausted or empty page +does not make a partial source complete. Retained views never reopen the file. +The CLI uses the same projector and requires a durable bundle for a summary: + +```sh +rea inspect-native-exports ./libexample.so --detail summary --evidence_output ./exports.bundle.json --json +# Pass the summary's parent ID and the same view as JSON to inspect-analysis-view, +# together with --evidence-bundle ./exports.bundle.json. +``` - PE rows reproduce the export directory: each used export-address-table slot with its biased `ordinal` (`OrdinalBase` plus the slot index), one row per @@ -368,8 +409,8 @@ Unresolved symbol versions keep their rows: the version name is null when no definition or requirement names the index, and `version` is `"unknown"` when the version table cannot be read; coverage is likewise `partial`. A dynamic symbol table without its null entry is malformed. An ELF -without section headers is `unsupported`: -enumerating dynamic symbols from the dynamic segment is not +without section headers, or with a dynamic segment but no dynamic-symbol +section, is `unsupported`: enumerating dynamic symbols from the dynamic segment is not implemented. Mach-O exports are reported by `inspect_macho`. `max_file_bytes` defaults to 256 MiB (maximum 512 MiB; CLI `--max-file-bytes`). diff --git a/experiments/analysis-efficiency/native-export-costs.py b/experiments/analysis-efficiency/native-export-costs.py new file mode 100644 index 0000000000..8b6111e0cd --- /dev/null +++ b/experiments/analysis-efficiency/native-export-costs.py @@ -0,0 +1,115 @@ +"""Check equivalent export questions and count explicitly projected tokenizer costs.""" +import argparse +import hashlib +import json +from pathlib import Path +import statistics + +import tiktoken + + +def compact(value): + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + + +def inspect(path, encoding): + report = json.loads((path / "report.json").read_text()) + catalog = json.loads((path / "catalog.json").read_text()) + transcript = json.loads((path / "transcript.json").read_text()) + scope = next(json.loads(line) for line in reversed(path.with_suffix(".log").read_text().splitlines()) + if line.startswith('{"resource_scope":')) + observed = scope.get("payload_resource_observation") or {} + events = observed.get("resource_events", {}) + assert report["status"] == "passed" and report.get("runtime_identity_sha256") + assert not report["cleanup_failures"] and not report["receipt_failures"] + assert scope["abort"] is None and scope.get("payload_resource_observation_verified") is True + assert observed.get("payload_exit_code") == 0 and observed.get("payload_cleanup_verified") is True + assert all(events.get("memory.events", {}).get(key) == 0 for key in ["high", "max", "oom", "oom_kill"]) + assert events.get("pids.events", {}).get("max") == 0 + assert len(catalog["tools"]) == 220 + count = lambda text: len(encoding.encode(text, disallowed_special=())) + directory = [{key: tool[key] for key in ["name", "description", "inputSchema"]} for tool in catalog["tools"]] + question = (f"For {report['artifact']['path']}, report the export count and every version/alias of " + f"{', '.join(dict.fromkeys(row['name'] for row in report['answer']['rows']))}, including addresses, " + "sizes, types, binding and symbol-version facts. Check that __rea_export_oracle_missing__ " + "has no recorded match, preserve source coverage and export the complete Evidence. " + "Report static declarations without claiming runtime execution.") + arguments = sum(count(compact({"name": event["name"], "arguments": event["input"]})) for event in transcript) + replies = sum(count(block["text"]) for event in transcript for block in event["reply"]["content"] if block["type"] == "text") + inputs = count(compact(directory)) + raw_catalog = count(compact(catalog)) + task_directory = [tool for tool in directory if tool["name"] in ["inspect_native_exports", "inspect_analysis_view", "export_evidence_bundle"]] + return {"case": path.name, "lane": report["lane"], "artifact": report["artifact"], + "answer": report["answer"], "parent_evidence_id": report["parent_evidence_id"], + "source_report_sha256": report["source_report_sha256"], + "runtime_identity_sha256": report["runtime_identity_sha256"], + "catalog_sha256": report["catalog_sha256"], "catalog_tools": len(catalog["tools"]), + "buffer_bytes": report["buffer_bytes"], "calls": report["calls"], "phases": report["phases"], + "wall_seconds": scope["elapsed_seconds"], "cpu_seconds": observed["cpu_stat"]["usage_usec"] / 1e6, + "kernel_family_peak_bytes": observed["memory_peak_bytes"], + "events_at_payload_exit": events, "payload_cleanup_verified": True, + "final_whole_scope_events": scope.get("resource_events_final"), + "token_proxy": {"question": count(question), "input_catalog_once": inputs, "arguments": arguments, + "text_replies": replies, "combined": count(question) + inputs + arguments + replies, + "only_three_task_tools": count(question) + count(compact(task_directory)) + arguments + replies, + "complete_transport_catalog": raw_catalog}, + "receipt_sha256": {file.name: hashlib.sha256(file.read_bytes()).hexdigest() for file in + [path / "report.json", path / "catalog.json", path / "transcript.json", path / "evidence.json", path.with_suffix(".log")]}} + + +def summarize(rows): + result = {} + for artifact in sorted({row["artifact"]["sha256"] for row in rows}): + selected = [row for row in rows if row["artifact"]["sha256"] == artifact] + assert len({row["parent_evidence_id"] for row in selected}) == 1 + assert len({row["source_report_sha256"] for row in selected}) == 1 + assert all(row["answer"] == selected[0]["answer"] for row in selected) + result[artifact] = {} + for lane in ["complete", "selected"]: + samples = [row for row in selected if row["lane"] == lane] + assert len(samples) == 3 + metrics = {key: [row[key] for row in samples] for key in ["wall_seconds", "cpu_seconds", "kernel_family_peak_bytes"]} + metrics["projected_tokens"] = [row["token_proxy"]["combined"] for row in samples] + metrics["mcp_call_count"] = [len(row["calls"]) for row in samples] + metrics["mcp_reply_bytes"] = [sum(call["reply_bytes"] for call in row["calls"]) for row in samples] + result[artifact][lane] = {metric: {"median": statistics.median(values), "minimum": min(values), "maximum": max(values)} for metric, values in metrics.items()} + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--cohort", type=Path, required=True) + parser.add_argument("cases", type=Path, nargs="+") + args = parser.parse_args() + manifest = json.loads((args.cohort / "manifest.json").read_text()) + index = json.loads((args.cohort / "index.json").read_text()) + assert len(index) == 12 and all(case["exit"] == 0 and case["status"] == "passed" for case in index) + assert {path.resolve() for path in args.cases} == {(args.cohort / case["name"]).resolve() for case in index} + assert len(args.cases) == len(index) + for file, digest in manifest["driver_files"].items(): + assert hashlib.sha256(Path(file).read_bytes()).hexdigest() == digest, file + runtimes = {} + for case in index: + path = Path(case["runtime"]) / "runtime-identity.json" + data = path.read_bytes() + identity = json.loads(data) + assert identity["source_clean"] is True + runtimes[hashlib.sha256(data).hexdigest()] = { + "source_revision": identity["source_revision"], + "compiled_files": len(identity["compiled_files"]), + "entrypoints": len(identity["entrypoints"]), + } + encoding = tiktoken.get_encoding("cl100k_base") + rows = [inspect(path, encoding) for path in args.cases] + assert all(row["runtime_identity_sha256"] in runtimes for row in rows) + receipt = {"tiktoken": tiktoken.__version__, "encoding": encoding.name, "actual_model_usage": None, + "cohort": manifest, "commands": index, "runtime_identities": runtimes, + "projection": "Scripted SDK tasks, not an actual model run. The estimate includes a fixed task description, every discovered tool name/description/input schema once, every tool argument and text reply, including Evidence export. Complete raw SDK directory token costs and a three-tool-only alternative are reported separately; output schemas are not included in the primary model-context projection. Host injection, history replay, inference quality and billable usage are unmeasured.", + "resources": "CPU quota 100%; launcher binds clients/servers to CPUs 0-1 (the outer observer starts on CPU15). The guard receipt affinity is initial placement, not child affinity; a live cohort observation records the override. 3 GiB early stop/4 GiB hard/no swap. Fresh client/server/HOME per task, Node compile cache disabled, OS caches unflushed. Later retained pages are warm queries; no separate warm-server cohort. Whole-family costs include client/oracle comparisons, source hashing, runtime binding, serialization and receipts. Tokenizer processing itself is outside these measured tasks. Verified counters are at payload exit while the observer remains alive; unavailable final whole-scope counters remain null.", + "summary": summarize(rows), "recordings": rows} + args.output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/experiments/analysis-efficiency/native-export-oracle.py b/experiments/analysis-efficiency/native-export-oracle.py new file mode 100644 index 0000000000..b0213e15d6 --- /dev/null +++ b/experiments/analysis-efficiency/native-export-oracle.py @@ -0,0 +1,48 @@ +"""Independent readelf symbol facts and directly mapped ELF64 version indices.""" +import hashlib +import json +from pathlib import Path +import re +import struct +import subprocess +import sys + +path = Path(sys.argv[1]).resolve() +selected_names = sys.argv[2:] +source = path.read_bytes() +assert source[:6] == b'\x7fELF\x02\x01', 'This experiment oracle only covers ELF64 little-endian' +section_offset = struct.unpack_from(' 0 +sections = [struct.unpack_from('