diff --git a/src/apktool/ApktoolProvider.test.ts b/src/apktool/ApktoolProvider.test.ts index 1fb114dc26..ed09cf6309 100644 --- a/src/apktool/ApktoolProvider.test.ts +++ b/src/apktool/ApktoolProvider.test.ts @@ -653,6 +653,63 @@ it("reuses normalized directory table for locale aliases without extra reads", a expect(invocations).toHaveLength(1); }); +it.each([2000, 2001])( + "retains observed alias counts after exhausting the aggregate entry budget (%i entries)", + async (entryCount) => { + const { command, root } = await writeApktoolStub((root) => + decodeStub({ withGerman: false, withStrings: false })(root).replace( + 'if [ "$1" = "d" ]; then', + `if [ "$1" = "d" ]; then + for suffix in pt-rBR de es it; do + mkdir -p "$5/res/values-$suffix" + cp '${join(root, "strings.xml")}' "$5/res/values-$suffix/strings.xml" + done`, + ), + ); + const entries = Array.from( + { length: entryCount }, + (_, i) => `value`, + ).join(""); + await writeFile( + join(root, "strings.xml"), + `${entries}unknown`, + ); + const apk = await apkFixture(root); + const provider = providerOver(command); + try { + const observation = await new ApktoolResourceAnalysisService( + provider, + ).execute("decode_android_resources", { + path: apk.path, + include_strings: true, + locale: "pt-BR", + locales: ["de", "es", "it", "pt-rBR"], + }); + if (!observation.ok) throw observation.error; + const result = apktoolResultSchemas.decode_android_resources.parse( + observation.value.normalized_result, + ); + expect(result.string_tables?.at(-1)).toEqual({ + locale: "pt-rBR", + status: "skipped", + strings: [], + total_count: entryCount, + unparsed_count: 1, + coverage: "partial", + }); + expect(result.strings).toHaveLength(2000); + expect( + observation.value.limitations.some((note) => + note.startsWith("Locale pt-rBR strings reports the first"), + ), + ).toBe(false); + expect(result.coverage).toBe("partial"); + } finally { + await provider.close(); + } + }, +); + it("skips all string tables when include_strings is false despite locales requested", async () => { const { command, root } = await writeApktoolStub((root) => decodeStub({ withGerman: false })(root).replace( diff --git a/src/apktool/ApktoolProvider.ts b/src/apktool/ApktoolProvider.ts index e86ea7efdf..c45aedc5e8 100644 --- a/src/apktool/ApktoolProvider.ts +++ b/src/apktool/ApktoolProvider.ts @@ -30,14 +30,8 @@ import { createAnalysisExecutionEvidence, type AnalysisExecution, } from "../application/AnalysisProvider.js"; -import { - androidLocaleDirectorySuffix, - AGGREGATE_STRINGS_XML_MAX_BYTES, - MAX_AGGREGATE_STRINGS, - MAX_STRINGS, - type ApktoolRequest, - type StringTable, -} from "../domain/apktool/apktoolResourceAnalysis.js"; +import type { ApktoolRequest } from "../domain/apktool/apktoolResourceAnalysis.js"; +import { projectApktoolStringTables } from "./ApktoolStringTables.js"; import { AnalysisCapabilityUnavailableError, AnalysisCancelledError, @@ -68,10 +62,7 @@ import { } from "./ApktoolConfiguration.js"; import { parseApktoolYml, - parseDecodedStrings, parseManifestPackage, - type ApktoolStringsDocumentError, - type DecodedStringEntry, } from "./ApktoolDecodeOutput.js"; /** Stable identity for the Apktool resource-decoding provider. */ @@ -84,7 +75,6 @@ export const APKTOOL_PROVIDER_IDENTITY = { const DECODE_TIMEOUT_MS = 600_000; const DECODE_MAX_OUTPUT_BYTES = 4 * 1024 * 1024; const MANIFEST_MAX_BYTES = 512 * 1024; -const STRINGS_XML_MAX_BYTES = 8 * 1024 * 1024; const MAX_DECODED_ENTRIES_VISITED = 100_000; const STDERR_EXCERPT_BYTES = 512; @@ -511,12 +501,10 @@ export class ApktoolProvider { apktool_version: string | null; }, provider: { id: string; name: string; version: string | null }, - input: { - readonly path: string; - readonly include_strings: boolean; - readonly locale?: string | undefined; - readonly locales?: readonly string[] | undefined; - }, + input: Extract< + ApktoolRequest, + { operation: "decode_android_resources" } + >["input"], clientLimitations: readonly string[], signal: AbortSignal | undefined, ): Promise> { @@ -591,19 +579,10 @@ export class ApktoolProvider { manifest === null || manifestRead.truncated ? null : parseManifestPackage(manifest); - let strings: Array<{ name: string; value: string }>; - let locale: string | null; - let stringTables: StringTable[] | undefined; const limitations: string[] = [ ...clientLimitations, "Decoding skips smali (--no-src); Java-source recovery belongs to the JADX family", ]; - let stringsTruncated = false; - let stringsUnparsedCount = 0; - let stringsTotalCount: number | null = null; - let stringsDocumentFailure: ApktoolStringsDocumentError | undefined; - let stringsMissingLimitation: string | undefined; - const files = await countDecodedFiles( resources, workspace, @@ -612,324 +591,15 @@ export class ApktoolProvider { ); throwIfCancelled(operation, signal); - let aggregateCapped = false; - let anyTablePartial = false; - - if (input.locales === undefined) { - locale = input.locale ?? null; - const stringsDirectory = - input.locale === undefined - ? "values" - : `values-${androidLocaleDirectorySuffix(input.locale)}`; - const stringsRead = input.include_strings - ? await resources.readText( - join(workspace, "res", stringsDirectory, "strings.xml"), - STRINGS_XML_MAX_BYTES, - signal, - ) - : { text: null, truncated: false }; - stringsTruncated = stringsRead.truncated; - const stringsXml = stringsRead.truncated ? null : stringsRead.text; - const stringsProjection = - stringsXml === null ? undefined : parseDecodedStrings(stringsXml); - const parsedStrings = - stringsProjection?.ok === true ? stringsProjection.value : undefined; - stringsDocumentFailure = - stringsProjection?.ok === false ? stringsProjection.error : undefined; - strings = parsedStrings?.entries.slice(0, MAX_STRINGS) ?? []; - stringsTotalCount = parsedStrings?.entries.length ?? null; - stringsUnparsedCount = parsedStrings?.unparsedCount ?? 0; - - if (input.include_strings && stringsRead.text === null) { - stringsMissingLimitation = - input.locale !== undefined - ? `No strings.xml was found for locale ${input.locale}` - : "No default res/values/strings.xml was found in the decoded workspace"; - } - if (stringsMissingLimitation !== undefined) { - limitations.push(stringsMissingLimitation); - } - if (stringsDocumentFailure !== undefined) { - limitations.push( - `String table ${JSON.stringify(`res/${stringsDirectory}/strings.xml`)} could not be projected: ${stringsDocumentFailure.message}; entry counts are unknown`, - ); - } - if (stringsUnparsedCount > 0) { - limitations.push( - `String entries omitted due to unsupported markup or missing names: ${String(stringsUnparsedCount)}`, - ); - } - if (stringsTotalCount !== null && stringsTotalCount > MAX_STRINGS) { - limitations.push( - `strings reports the first ${String(MAX_STRINGS)} of ${String(stringsTotalCount)} entries`, - ); - } - if (stringsTruncated) { - limitations.push( - `strings.xml exceeds the ${String(STRINGS_XML_MAX_BYTES)}-byte projection budget`, - ); - } - } else { - locale = input.locale ?? null; - stringTables = []; - - const requestedLocales: Array<{ - readonly locale: string | null; - readonly isPrimary: boolean; - }> = []; - - requestedLocales.push({ - locale, - isPrimary: true, - }); - - for (const loc of input.locales) { - if (!requestedLocales.some((r) => r.locale === loc)) { - requestedLocales.push({ locale: loc, isPrimary: false }); - } - } - - if (!input.include_strings) { - strings = []; - stringTables = []; - } else { - let aggregateXmlBytes = 0; - let aggregateStringsCount = 0; - const tableLimitations: string[] = []; - - type PhysicalCached = - | { - readonly status: "missing" | "oversized" | "malformed"; - readonly total_count: number | null; - readonly unparsed_count: number | null; - readonly coverage: "complete" | "partial"; - readonly limitation: string; - } - | { - readonly status: "present"; - readonly entries: readonly DecodedStringEntry[]; - readonly unparsed_count: number; - }; - - const physicalCache = new Map(); - - for (const item of requestedLocales) { - throwIfCancelled(operation, signal); - const loc = item.locale; - const dirName = - loc === null - ? "values" - : `values-${androidLocaleDirectorySuffix(loc)}`; - const filePath = join(workspace, "res", dirName, "strings.xml"); - const tableLabel = - loc === null - ? "default res/values/strings.xml" - : `res/${dirName}/strings.xml`; - - let cached = physicalCache.get(dirName); - if (cached === undefined) { - const remainingXmlBudget = Math.max( - 0, - AGGREGATE_STRINGS_XML_MAX_BYTES - aggregateXmlBytes, - ); - const remainingStringsBudget = Math.max( - 0, - MAX_AGGREGATE_STRINGS - aggregateStringsCount, - ); - if (remainingXmlBudget <= 0 || remainingStringsBudget <= 0) { - aggregateCapped = true; - stringTables.push({ - locale: loc, - status: "skipped", - strings: [], - total_count: null, - unparsed_count: null, - coverage: "partial", - }); - tableLimitations.push( - `Projection for ${tableLabel} skipped: aggregate resource limit reached`, - ); - continue; - } - - const tableBudget = Math.min( - STRINGS_XML_MAX_BYTES, - remainingXmlBudget, - ); - const stringsRead = await resources.readText( - filePath, - tableBudget, - signal, - ); - - if (stringsRead.text !== null) { - if (stringsRead.truncated) { - aggregateXmlBytes += tableBudget; - } else { - aggregateXmlBytes += Buffer.byteLength( - stringsRead.text, - "utf-8", - ); - } - } - - if (stringsRead.text === null) { - cached = { - status: "missing", - total_count: 0, - unparsed_count: 0, - coverage: "complete", - limitation: - loc !== null - ? `No strings.xml was found for locale ${loc}` - : "No default res/values/strings.xml was found in the decoded workspace", - }; - physicalCache.set(dirName, cached); - } else if (stringsRead.truncated) { - if (tableBudget < STRINGS_XML_MAX_BYTES) { - aggregateCapped = true; - } - cached = { - status: "oversized", - total_count: null, - unparsed_count: null, - coverage: "partial", - limitation: - loc === null - ? `strings.xml exceeds the ${String(tableBudget)}-byte projection budget` - : `strings.xml for locale ${loc} exceeds the ${String(tableBudget)}-byte projection budget`, - }; - physicalCache.set(dirName, cached); - } else { - const parsed = parseDecodedStrings(stringsRead.text); - if (!parsed.ok) { - cached = { - status: "malformed", - total_count: null, - unparsed_count: null, - coverage: "partial", - limitation: `String table ${JSON.stringify(`res/${dirName}/strings.xml`)} could not be projected: ${parsed.error.message}; entry counts are unknown`, - }; - physicalCache.set(dirName, cached); - } else { - cached = { - status: "present", - entries: parsed.value.entries, - unparsed_count: parsed.value.unparsedCount, - }; - physicalCache.set(dirName, cached); - } - } - } - - if (cached.status !== "present") { - stringTables.push({ - locale: loc, - status: cached.status, - strings: [], - total_count: cached.total_count, - unparsed_count: cached.unparsed_count, - coverage: cached.coverage, - }); - if (!tableLimitations.includes(cached.limitation)) { - tableLimitations.push(cached.limitation); - } - continue; - } - - const totalCount = cached.entries.length; - const remainingStringsBudget = Math.max( - 0, - MAX_AGGREGATE_STRINGS - aggregateStringsCount, - ); - const tableLimit = Math.min(MAX_STRINGS, remainingStringsBudget); - - if (remainingStringsBudget <= 0) { - aggregateCapped = true; - stringTables.push({ - locale: loc, - status: "skipped", - strings: [], - total_count: null, - unparsed_count: null, - coverage: "partial", - }); - tableLimitations.push( - `Projection for ${tableLabel} skipped: aggregate resource limit reached`, - ); - continue; - } - - const bounded = cached.entries.slice(0, tableLimit); - aggregateStringsCount += bounded.length; - - if (item.isPrimary) { - // Primary table copy in top-level strings also consumes output entry budget - aggregateStringsCount += bounded.length; - } - - if (bounded.length < Math.min(totalCount, MAX_STRINGS)) { - aggregateCapped = true; - } - - const tableCoverage = - totalCount > MAX_STRINGS || - bounded.length < totalCount || - cached.unparsed_count > 0 - ? "partial" - : "complete"; - - stringTables.push({ - locale: loc, - status: "present", - strings: bounded, - total_count: totalCount, - unparsed_count: cached.unparsed_count, - coverage: tableCoverage, - }); - - if (cached.unparsed_count > 0) { - tableLimitations.push( - loc === null - ? `String entries omitted due to unsupported markup or missing names: ${String(cached.unparsed_count)}` - : `String entries omitted in locale ${loc} due to unsupported markup or missing names: ${String(cached.unparsed_count)}`, - ); - } - if (totalCount > MAX_STRINGS) { - tableLimitations.push( - loc === null - ? `strings reports the first ${String(MAX_STRINGS)} of ${String(totalCount)} entries` - : `Locale ${loc} strings reports the first ${String(MAX_STRINGS)} of ${String(totalCount)} entries`, - ); - } else if (bounded.length < totalCount) { - tableLimitations.push( - `Locale ${loc ?? "default"} strings reports ${String(bounded.length)} entries due to aggregate strings limit`, - ); - } - } - - limitations.push(...tableLimitations); - - const primaryTable = stringTables.find((t) => t.locale === locale); - strings = primaryTable?.strings ?? []; - } - - anyTablePartial = stringTables.some( - (t) => - t.coverage === "partial" || - t.status === "oversized" || - t.status === "malformed" || - t.status === "skipped", - ); - } - + const projectedStrings = await projectApktoolStringTables( + resources, + workspace, + input, + signal, + ); + limitations.push(...projectedStrings.limitations); const coverage = - (input.locales === undefined - ? (stringsTotalCount !== null && stringsTotalCount > MAX_STRINGS) || - stringsTruncated || - stringsUnparsedCount > 0 || - stringsDocumentFailure !== undefined - : anyTablePartial || aggregateCapped) || + projectedStrings.coverage === "partial" || manifestRead.truncated || files.capped || files.unreadableStringsTables.length > 0 @@ -956,12 +626,6 @@ export class ApktoolProvider { `Decoded workspace enumeration stopped after ${String(MAX_DECODED_ENTRIES_VISITED)} entries; decoded_file_count and locales may be partial`, ); } - if (aggregateCapped) { - limitations.push( - "Aggregate string table projection stopped due to resource limits", - ); - } - result = ok( createAnalysisExecution( { @@ -975,12 +639,12 @@ export class ApktoolProvider { package_name: packageName, }, manifest, - strings, - locale, + strings: projectedStrings.strings, + locale: input.locale ?? null, locales: files.locales, - ...(stringTables !== undefined - ? { string_tables: stringTables } - : {}), + ...(input.locales === undefined + ? {} + : { string_tables: projectedStrings.tables }), decoded_file_count: files.count, coverage, }, diff --git a/src/apktool/ApktoolStringTables.test.ts b/src/apktool/ApktoolStringTables.test.ts new file mode 100644 index 0000000000..bd8c8cb18b --- /dev/null +++ b/src/apktool/ApktoolStringTables.test.ts @@ -0,0 +1,181 @@ +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + MAX_AGGREGATE_STRINGS, + MAX_STRINGS, +} from "../domain/apktool/apktoolResourceAnalysis.js"; +import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; +import { projectApktoolStringTables } from "./ApktoolStringTables.js"; + +const xml = (count: number) => + `${Array.from({ length: count }, (_, index) => `value`).join("")}`; + +const textRead = (text: string) => ({ + text, + truncated: false, + sourceBytes: Buffer.byteLength(text, "utf8"), +}); + +describe("physical Apktool string facts", () => { + it("reads one physical table for two caller-selected aliases", async () => { + const paths: string[] = []; + const result = await projectApktoolStringTables( + { + async readText(path) { + paths.push(path); + return textRead(xml(1)); + }, + }, + "decoded", + { include_strings: true, locale: "pt-BR", locales: ["pt-rBR"] }, + ); + expect(paths).toEqual([ + join("decoded", "res", "values-pt-rBR", "strings.xml"), + ]); + expect(result.tables.map((table) => table.locale)).toEqual([ + "pt-BR", + "pt-rBR", + ]); + expect(result.tables.map((table) => table.total_count)).toEqual([1, 1]); + }); + + it("counts complete source entries beyond the retained table prefix", async () => { + const result = await projectApktoolStringTables( + { + async readText() { + return textRead( + xml(MAX_STRINGS + 1).replace( + "", + 'omitted', + ), + ); + }, + }, + "decoded", + { include_strings: true }, + ); + expect(result.tables[0]).toMatchObject({ + total_count: MAX_STRINGS + 1, + unparsed_count: 1, + coverage: "partial", + }); + expect(result.strings).toHaveLength(MAX_STRINGS); + expect(result.limitations).toContain( + `strings reports the first ${String(MAX_STRINGS)} of ${String(MAX_STRINGS + 1)} entries`, + ); + }); + + it("does not read unexamined tables after output exhaustion", async () => { + const paths: string[] = []; + const result = await projectApktoolStringTables( + { + async readText(path) { + paths.push(path); + if (path.includes("values-fr")) + throw new Error("Unexamined table was read"); + return textRead(xml(MAX_STRINGS)); + }, + }, + "decoded", + { include_strings: true, locales: ["de", "es", "it", "fr"] }, + ); + expect(paths).toHaveLength(4); + expect(result.tables.at(-1)).toEqual({ + locale: "fr", + status: "skipped", + strings: [], + total_count: null, + unparsed_count: null, + coverage: "partial", + }); + expect( + result.strings.length + + result.tables.reduce((count, table) => count + table.strings.length, 0), + ).toBe(MAX_AGGREGATE_STRINGS); + }); + + it("keeps a known empty alias present after the output budget is spent", async () => { + const paths: string[] = []; + const result = await projectApktoolStringTables( + { + async readText(path) { + paths.push(path); + return textRead( + xml(path.includes("values-pt-rBR") ? 0 : MAX_STRINGS), + ); + }, + }, + "decoded", + { + include_strings: true, + locale: "pt-BR", + locales: ["de", "es", "it", "fr", "en", "pt-rBR"], + }, + ); + expect(paths.filter((path) => path.includes("values-pt-rBR"))).toHaveLength( + 1, + ); + expect(result.tables.at(-1)).toEqual({ + locale: "pt-rBR", + status: "present", + strings: [], + total_count: 0, + unparsed_count: 0, + coverage: "complete", + }); + expect(result.coverage).toBe("complete"); + }); +}); + +describe("Apktool string projection admission", () => { + it("honors omitted strings without reading any selected table", async () => { + const result = await projectApktoolStringTables( + { + async readText() { + throw new Error("Unexpected read"); + }, + }, + "decoded", + { include_strings: false, locale: "de", locales: ["fr"] }, + ); + expect(result).toEqual({ + strings: [], + tables: [], + limitations: [], + coverage: "complete", + }); + }); + + it("preserves the exact physical read failure", async () => { + const cause = new Error("Physical read and close failed"); + await expect( + projectApktoolStringTables( + { + async readText() { + throw cause; + }, + }, + "decoded", + { include_strings: true }, + ), + ).rejects.toBe(cause); + }); + + it("preserves opaque cancellation before any table read", async () => { + const reason = { selected: "caller" }; + const pending = projectApktoolStringTables( + { + async readText() { + throw new Error("Unexpected read"); + }, + }, + "decoded", + { include_strings: true }, + AbortSignal.abort(reason), + ); + await expect(pending).rejects.toBeInstanceOf(AnalysisCancelledError); + await expect(pending).rejects.toHaveProperty("cause", reason); + }); +}); diff --git a/src/apktool/ApktoolStringTables.ts b/src/apktool/ApktoolStringTables.ts new file mode 100644 index 0000000000..95966dfec7 --- /dev/null +++ b/src/apktool/ApktoolStringTables.ts @@ -0,0 +1,252 @@ +import { join } from "node:path"; + +import { + AGGREGATE_STRINGS_XML_MAX_BYTES, + MAX_AGGREGATE_STRINGS, + MAX_STRINGS, + androidLocaleDirectorySuffix, + type ApktoolRequest, + type StringTable, +} from "../domain/apktool/apktoolResourceAnalysis.js"; +import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; +import { + parseDecodedStrings, + type DecodedStringEntry, +} from "./ApktoolDecodeOutput.js"; +import type { ApktoolWorkspace } from "./ApktoolWorkspace.js"; + +const STRINGS_XML_MAX_BYTES = 8 * 1024 * 1024; +type Selection = Pick< + Extract["input"], + "include_strings" | "locale" | "locales" +>; +type PresentTable = { + readonly status: "present"; + readonly entries: readonly DecodedStringEntry[]; + readonly totalCount: number; + readonly unparsedCount: number; +}; +type UnavailableTable = + | { + readonly status: "missing"; + readonly totalCount: 0; + readonly unparsedCount: 0; + } + | { + readonly status: "oversized"; + readonly totalCount: null; + readonly unparsedCount: null; + readonly byteBudget: number; + } + | { + readonly status: "malformed"; + readonly totalCount: null; + readonly unparsedCount: null; + readonly message: string; + }; +type PhysicalTable = PresentTable | UnavailableTable; + +/** One physical read owns parsed counts independently of output allocation. */ +const readTable = async ( + resources: Pick, + path: string, + budget: number, + signal?: AbortSignal, +): Promise<{ readonly fact: PhysicalTable; readonly bytes: number }> => { + const source = await resources.readText(path, budget, signal); + if (source.text === null) + return { + bytes: 0, + fact: { status: "missing", totalCount: 0, unparsedCount: 0 }, + }; + if (source.truncated) + return { + bytes: source.sourceBytes, + fact: { + status: "oversized", + totalCount: null, + unparsedCount: null, + byteBudget: budget, + }, + }; + const bytes = source.sourceBytes; + const parsed = parseDecodedStrings(source.text); + if (!parsed.ok) + return { + bytes, + fact: { + status: "malformed", + totalCount: null, + unparsedCount: null, + message: parsed.error.message, + }, + }; + return { + bytes, + fact: { + status: "present", + // Future aliases need counts and at most one table's output, not every + // parsed entry. The complete source was examined before this retention cut. + entries: parsed.value.entries.slice(0, MAX_STRINGS), + totalCount: parsed.value.entries.length, + unparsedCount: parsed.value.unparsedCount, + }, + }; +}; + +const unavailableTableLimitation = ( + fact: UnavailableTable, + directory: string, + locale: string | null, + input: Selection, +): string => { + if (fact.status === "missing") + return locale === null + ? "No default res/values/strings.xml was found in the decoded workspace" + : `No strings.xml was found for locale ${locale}`; + if (fact.status === "oversized") + return input.locales === undefined || locale === null + ? `strings.xml exceeds the ${String(fact.byteBudget)}-byte projection budget` + : `strings.xml for locale ${locale} exceeds the ${String(fact.byteBudget)}-byte projection budget`; + return `String table ${JSON.stringify(`res/${directory}/strings.xml`)} could not be projected: ${fact.message}; entry counts are unknown`; +}; + +export interface ApktoolStringProjection { + readonly strings: readonly DecodedStringEntry[]; + readonly tables: readonly StringTable[]; + readonly limitations: readonly string[]; + readonly coverage: "complete" | "partial"; +} + +/** Project caller labels from shared physical facts for both request shapes. */ +export const projectApktoolStringTables = async ( + resources: Pick, + workspace: string, + input: Selection, + signal?: AbortSignal, +): Promise => { + if (!input.include_strings) + return { strings: [], tables: [], limitations: [], coverage: "complete" }; + const primary = input.locale ?? null; + const locales = [...new Set([primary, ...(input.locales ?? [])])]; + const physical = new Map(); + const tables: StringTable[] = []; + const limitations = new Set(); + let xmlBytes = 0; + let outputEntries = 0; + let aggregateCapped = false; + for (const [index, locale] of locales.entries()) { + if (signal?.aborted) + throw new AnalysisCancelledError("decode_android_resources", { + cause: signal.reason, + }); + const directory = + locale === null + ? "values" + : `values-${androidLocaleDirectorySuffix(locale)}`; + const label = + locale === null + ? "default res/values/strings.xml" + : `res/${directory}/strings.xml`; + let fact = physical.get(directory); + if (fact === undefined) { + const budget = Math.min( + STRINGS_XML_MAX_BYTES, + AGGREGATE_STRINGS_XML_MAX_BYTES - xmlBytes, + ); + if (budget <= 0 || outputEntries >= MAX_AGGREGATE_STRINGS) { + tables.push({ + locale, + status: "skipped", + strings: [], + total_count: null, + unparsed_count: null, + coverage: "partial", + }); + limitations.add( + `Projection for ${label} skipped: aggregate resource limit reached`, + ); + aggregateCapped = true; + continue; + } + const read = await readTable( + resources, + join(workspace, "res", directory, "strings.xml"), + budget, + signal, + ); + fact = read.fact; + xmlBytes += read.bytes; + if (fact.status === "oversized" && budget < STRINGS_XML_MAX_BYTES) + aggregateCapped = true; + physical.set(directory, fact); + } + if (fact.status !== "present") { + tables.push({ + locale, + status: fact.status, + strings: [], + total_count: fact.totalCount, + unparsed_count: fact.unparsedCount, + coverage: fact.status === "missing" ? "complete" : "partial", + }); + limitations.add( + unavailableTableLimitation(fact, directory, locale, input), + ); + continue; + } + // Batch output includes the primary strings twice, in the legacy field and + // in string_tables. Charge that representation before selecting entries. + const copies = index === 0 && input.locales !== undefined ? 2 : 1; + const remaining = Math.max(0, MAX_AGGREGATE_STRINGS - outputEntries); + const selected = fact.entries.slice(0, Math.floor(remaining / copies)); + outputEntries += selected.length * copies; + const skipped = selected.length === 0 && fact.totalCount > 0; + const limited = selected.length < Math.min(fact.totalCount, MAX_STRINGS); + if (limited) aggregateCapped = true; + tables.push({ + locale, + status: skipped ? "skipped" : "present", + strings: selected, + // Output exhaustion cannot erase counts from an already examined table. + total_count: fact.totalCount, + unparsed_count: fact.unparsedCount, + coverage: + selected.length < fact.totalCount || fact.unparsedCount > 0 + ? "partial" + : "complete", + }); + if (skipped) + limitations.add( + `Projection for ${label} skipped: aggregate resource limit reached`, + ); + if (fact.unparsedCount > 0) + limitations.add( + input.locales === undefined || locale === null + ? `String entries omitted due to unsupported markup or missing names: ${String(fact.unparsedCount)}` + : `String entries omitted in locale ${locale} due to unsupported markup or missing names: ${String(fact.unparsedCount)}`, + ); + if (limited && !skipped) + limitations.add( + `Locale ${locale ?? "default"} strings reports ${String(selected.length)} entries due to aggregate strings limit`, + ); + else if (fact.totalCount > MAX_STRINGS && !skipped) + limitations.add( + input.locales === undefined || locale === null + ? `strings reports the first ${String(selected.length)} of ${String(fact.totalCount)} entries` + : `Locale ${locale} strings reports the first ${String(selected.length)} of ${String(fact.totalCount)} entries`, + ); + } + if (aggregateCapped) + limitations.add( + "Aggregate string table projection stopped due to resource limits", + ); + return { + strings: tables[0]?.strings ?? [], + tables, + limitations: [...limitations], + coverage: tables.some((table) => table.coverage === "partial") + ? "partial" + : "complete", + }; +}; diff --git a/src/apktool/ApktoolWorkspace.lifecycle.test.ts b/src/apktool/ApktoolWorkspace.lifecycle.test.ts index 0ddb601628..f2ec967557 100644 --- a/src/apktool/ApktoolWorkspace.lifecycle.test.ts +++ b/src/apktool/ApktoolWorkspace.lifecycle.test.ts @@ -149,7 +149,7 @@ it("records cancellation observed during a successful output descriptor close", expect(failure.fileCancellation.reason).toBe(reason); expect(failure.fileCancellation.outcome).toEqual({ kind: "completed", - value: { text: "", truncated: false }, + value: { text: "", truncated: false, sourceBytes: 12 }, }); } finally { await workspace.close(); @@ -358,3 +358,32 @@ it("preserves a directory inspection failure instead of treating its locale tabl await commands.close(); } }); + +it("reports raw retained-prefix bytes when UTF-8 decoding removes a BOM", async () => { + const root = await PrivateRuntimeRoot.create({ + prefix: "rea-apktool-source-bytes-", + }); + const commands = new ApktoolCommandCleanup(); + const workspace = new ApktoolWorkspace(root, commands); + const output = join(root.path, "strings.xml"); + const bytes = Buffer.from("\uFEFF", "utf8"); + try { + await fs.writeFile(output, bytes); + await expect(workspace.readText(output, 100)).resolves.toEqual({ + text: "", + truncated: false, + sourceBytes: bytes.length, + }); + await expect(workspace.readText(output, 5)).resolves.toEqual({ + text: " { + ): Promise<{ + text: string | null; + truncated: boolean; + /** Raw bytes represented by the retained prefix, before UTF-8 decoding. */ + sourceBytes: number; + }> { let admitted = false; try { return await withArtifactRegularFile( @@ -175,13 +180,15 @@ export class ApktoolWorkspace { `Apktool output changed while being read: ${path}`, ); const truncated = length > budget; + const sourceBytes = Math.min(length, budget); try { return { text: new TextDecoder("utf-8", { fatal: true }).decode( - bytes.subarray(0, Math.min(length, budget)), + bytes.subarray(0, sourceBytes), { stream: truncated }, ), truncated, + sourceBytes, }; } catch (cause) { throw new AnalysisProtocolError( @@ -198,7 +205,7 @@ export class ApktoolWorkspace { "code" in cause && cause.code === "ENOENT" ) - return { text: null, truncated: false }; + return { text: null, truncated: false, sourceBytes: 0 }; throw cause; } } diff --git a/src/domain/apktool/apktoolResourceAnalysis.ts b/src/domain/apktool/apktoolResourceAnalysis.ts index 1da4aa92ea..33e0ebc681 100644 --- a/src/domain/apktool/apktoolResourceAnalysis.ts +++ b/src/domain/apktool/apktoolResourceAnalysis.ts @@ -138,7 +138,7 @@ export const stringTableSchema = z.strictObject({ .nonnegative() .nullable() .describe( - "Total valid string entries before per-table cap, or null if unmeasured/skipped/malformed", + "Total valid string entries before output limits, or null if the table was not examined or was malformed", ), unparsed_count: z .number() @@ -146,7 +146,7 @@ export const stringTableSchema = z.strictObject({ .nonnegative() .nullable() .describe( - "Entries skipped due to nested markup or missing attributes, or null if unmeasured/skipped/malformed", + "Entries skipped due to nested markup or missing attributes, or null if the table was not examined or was malformed", ), coverage: z.enum(["complete", "partial"]), });