From 1639189f269e016d37cfe18c2b617bd57c260dd6 Mon Sep 17 00:00:00 2001 From: Kaoru0822 Date: Thu, 22 Oct 2026 00:26:12 -0700 Subject: [PATCH] fix(javascript): keep encoded URL slashes inside one runtime path segment (#6259) A url-prefix runtime mapping decoded the whole pathname suffix at once, so `child%2Fpart.js` became the nested artifact path `child/part.js` and produced a medium-confidence location match and an inferred observed_as edge. Decode each URL path segment separately at the mapping boundary. A segment that decodes to contain a slash, like a malformed or non-UTF-8 escape, has no artifact path, so the runtime script stays unknown with no-authorized-location-mapping. Encoded spaces, Unicode, %25 and the literal %5C backslash name keep matching. File-root and URL-prefix mappings now share one tri-state translation (outside, untranslatable, translated) for matching and load-state scope. Static nodes beneath a prefix whose runtime script or worker could not be translated stay unknown instead of becoming not-observed-in-capture. Fixes #5878 Co-authored-by: Claude Opus 5.5 --- docs/javascript-runtime-reconciliation.md | 8 +- .../javascript/javascriptRuntimeLoadState.ts | 13 +- .../javascriptRuntimeStaticCandidates.ts | 163 +++++++--- .../cli/runtimeUrlPrefixMappingCli.test.ts | 279 ++++++++++++++++++ 4 files changed, 408 insertions(+), 55 deletions(-) create mode 100644 tests/boundary/cli/runtimeUrlPrefixMappingCli.test.ts diff --git a/docs/javascript-runtime-reconciliation.md b/docs/javascript-runtime-reconciliation.md index 068353661d..e3236e66bf 100644 --- a/docs/javascript-runtime-reconciliation.md +++ b/docs/javascript-runtime-reconciliation.md @@ -57,7 +57,11 @@ are complete Evidence objects, not only evidence IDs: `static_layers` requires exactly one `application` role and permits additional `cache` and `assets` roles. A `file-root` mapping translates a canonical runtime filesystem root to an artifact-relative prefix. A `url-prefix` mapping does the -same for one exact HTTP(S) origin and path prefix. Mappings cannot add CDP +same for one exact HTTP(S) origin and path prefix. Each URL path segment below +the prefix is percent-decoded separately: an encoded slash (`%2F`) stays inside +its segment and never names a nested artifact, so such a location, like one +with malformed or non-UTF-8 escapes, has no artifact path under that mapping. +An encoded backslash (`%5C`) remains a literal name byte. Mappings cannot add CDP origins, broaden Electron roots, read files, or authorize source-map access; they are caller-declared inference inputs applied only to already-authorized locations. @@ -133,6 +137,8 @@ classification: - `unknown` covers incomplete captures, layers outside runtime scope, and unresolved correspondence. Nodes outside a mapped artifact prefix stay unknown even when another prefix in the same static layer was observed. + Nodes beneath a mapped artifact prefix also stay unknown when a runtime + script or worker under that mapping has no artifact path. None of these states proves feature execution, initialization order, reachability, or causality. In particular, `not-observed-in-capture` never means diff --git a/src/domain/javascript/javascriptRuntimeLoadState.ts b/src/domain/javascript/javascriptRuntimeLoadState.ts index a1d064efcb..87ba847e8c 100644 --- a/src/domain/javascript/javascriptRuntimeLoadState.ts +++ b/src/domain/javascript/javascriptRuntimeLoadState.ts @@ -9,7 +9,7 @@ import type { RuntimeReconciliationEntity } from "./javascriptRuntimeReconciliat import { createStaticRuntimeScope, mappedRuntimePaths, - staticNodeWithinRuntimeScope, + staticNodeRuntimeScope, } from "./javascriptRuntimeStaticCandidates.js"; export interface StaticLoadStateProjection { @@ -93,8 +93,8 @@ const classifyLayer = ( ? "static-or-runtime-coverage-incomplete" : "ambiguous-runtime-candidate", }); - const nodeWithinScope = staticNodeWithinRuntimeScope(node, runtimeScope); - if (layerComplete && nodeWithinScope) + const membership = staticNodeRuntimeScope(node, runtimeScope); + if (layerComplete && membership === "within") return state(layer, node, { status: "not-observed-in-capture", runtimeNodeIds: [], @@ -103,9 +103,10 @@ const classifyLayer = ( return state(layer, node, { status: "unknown", runtimeNodeIds: [], - reason: !nodeWithinScope - ? "layer-outside-runtime-scope" - : "static-or-runtime-coverage-incomplete", + reason: + membership === "outside" + ? "layer-outside-runtime-scope" + : "static-or-runtime-coverage-incomplete", }); }); return { states }; diff --git a/src/domain/javascript/javascriptRuntimeStaticCandidates.ts b/src/domain/javascript/javascriptRuntimeStaticCandidates.ts index 5c3ef04f69..a870a8fe78 100644 --- a/src/domain/javascript/javascriptRuntimeStaticCandidates.ts +++ b/src/domain/javascript/javascriptRuntimeStaticCandidates.ts @@ -42,9 +42,14 @@ export interface MappedRuntimePath { /** Precomputed mapping scope and graph paths for static load-state decisions. */ export interface StaticRuntimeScope { readonly artifactPrefixes: readonly string[]; + /** Prefixes beneath which a runtime script or worker had no artifact path. */ + readonly untranslatedArtifactPrefixes: readonly string[]; readonly pathsByNodeId: ReadonlyMap; } +/** Where one static node falls relative to the runtime-exercised mappings. */ +export type StaticRuntimeScopeMembership = "outside" | "within" | "uncertain"; + /** Index statically reconstructed entities that can receive runtime matches. */ export const collectStaticRuntimeCandidates = ( layers: readonly ParsedStaticLayer[], @@ -79,13 +84,9 @@ export const mappedRuntimePaths = ( }); } for (const mapping of layer.runtimeMappings) { - const suffix = - mapping.kind === "file-root" && entity.location.kind === "file" - ? pathBelowRoot(mapping.root, entity.location.value) - : mapping.kind === "url-prefix" && entity.location.kind === "url" - ? pathBelowUrlPrefix(mapping.prefix, entity.location.value) - : null; - if (suffix === null) continue; + const translation = translateRuntimeLocation(mapping, entity.location); + if (translation.kind !== "translated") continue; + const suffix = translation.suffix; const path = safeArtifactPath(mapping.artifact_prefix, suffix); if (path === null) continue; paths.push({ @@ -107,18 +108,27 @@ export const createStaticRuntimeScope = ( layer: ParsedStaticLayer, entities: readonly RuntimeReconciliationEntity[], ): StaticRuntimeScope => ({ - artifactPrefixes: activeArtifactPrefixes(layer, entities), + ...exercisedArtifactPrefixes(layer, entities), pathsByNodeId: indexStaticPaths(layer.graph), }); -/** Whether one static node falls under a mapping exercised by this runtime set. */ -export const staticNodeWithinRuntimeScope = ( +/** + * Classify one static node against the mappings exercised by this runtime set. + * A node beneath a prefix whose runtime script or worker location could not be + * translated is uncertain: that location may name it, so absence is unproven. + */ +export const staticNodeRuntimeScope = ( node: ApplicationNode, scope: StaticRuntimeScope, -): boolean => - (scope.pathsByNodeId.get(node.node_id) ?? []).some((path) => - scope.artifactPrefixes.some((prefix) => pathWithinPrefix(path, prefix)), - ); +): StaticRuntimeScopeMembership => { + const paths = scope.pathsByNodeId.get(node.node_id) ?? []; + const beneath = (prefixes: readonly string[]): boolean => + paths.some((path) => + prefixes.some((prefix) => pathWithinPrefix(path, prefix)), + ); + if (beneath(scope.untranslatedArtifactPrefixes)) return "uncertain"; + return beneath(scope.artifactPrefixes) ? "within" : "outside"; +}; const candidatesForLayer = ( layer: ParsedStaticLayer, @@ -391,38 +401,87 @@ const absoluteFilePathStyle = (path: string): "posix" | "windows" | null => { return path.startsWith("/") ? "posix" : null; }; -const pathBelowUrlPrefix = (prefix: string, value: string): string | null => { - let base: URL; - let candidate: URL; - try { - base = new URL(prefix); - candidate = new URL(value); - } catch (cause: unknown) { - // Invalid input is represented by the null return. - void cause; - return null; - } +/** + * How a runtime location relates to one mapping: outside it, beneath it with + * an artifact-relative suffix, or beneath it without any artifact path. + */ +type RuntimeLocationTranslation = + | { readonly kind: "outside" } + | { readonly kind: "untranslatable" } + | { readonly kind: "translated"; readonly suffix: string }; + +const OUTSIDE_MAPPING: RuntimeLocationTranslation = { kind: "outside" }; + +const translateRuntimeLocation = ( + mapping: ParsedStaticLayer["runtimeMappings"][number], + location: RuntimeReconciliationEntity["location"], +): RuntimeLocationTranslation => { + if (mapping.kind === "url-prefix") + return location.kind === "url" + ? pathBelowUrlPrefix(mapping.prefix, location.value) + : OUTSIDE_MAPPING; + const suffix = + location.kind === "file" + ? pathBelowRoot(mapping.root, location.value) + : null; + return suffix === null ? OUTSIDE_MAPPING : { kind: "translated", suffix }; +}; + +const pathBelowUrlPrefix = ( + prefix: string, + value: string, +): RuntimeLocationTranslation => { + const base = URL.parse(prefix); + const candidate = URL.parse(value); if ( + base === null || + candidate === null || base.origin !== candidate.origin || !candidate.pathname.startsWith(base.pathname) ) - return null; - try { - return relativeLogicalPath( - decodeURIComponent(candidate.pathname.slice(base.pathname.length)), - ); - } catch (cause: unknown) { - // Undecodable path segments are represented by the null return. - void cause; - return null; - } + return OUTSIDE_MAPPING; + const encoded = candidate.pathname.slice(base.pathname.length); + // The prefix itself names the mapped directory or a page, not a file below it. + if (encoded === "") return OUTSIDE_MAPPING; + const decoded = decodeUrlPathSegments(encoded); + const suffix = decoded === null ? null : relativeLogicalPath(decoded); + return suffix === null + ? { kind: "untranslatable" } + : { kind: "translated", suffix }; }; -const activeArtifactPrefixes = ( +/** + * Decode each URL path segment separately. An encoded slash is a byte inside + * one segment, never an artifact directory boundary, so a segment containing + * one after decoding has no artifact path. An encoded backslash stays a literal + * name byte, and malformed or non-UTF-8 escapes have no artifact path. + */ +const decodeUrlPathSegments = (value: string): string | null => { + const segments: string[] = []; + for (const segment of value.split("/")) { + let decoded: string; + try { + decoded = decodeURIComponent(segment); + } catch (cause: unknown) { + // Malformed escapes are represented by the null return. + void cause; + return null; + } + if (decoded.includes("/")) return null; + segments.push(decoded); + } + return segments.join("/"); +}; + +const exercisedArtifactPrefixes = ( layer: ParsedStaticLayer, entities: readonly RuntimeReconciliationEntity[], -): string[] => { +): Pick< + StaticRuntimeScope, + "artifactPrefixes" | "untranslatedArtifactPrefixes" +> => { const prefixes = new Set(); + const untranslated = new Set(); if ( layer.result.format === "directory" && entities.some( @@ -432,18 +491,26 @@ const activeArtifactPrefixes = ( ) ) prefixes.add(""); - for (const mapping of layer.runtimeMappings) - if ( - entities.some(({ location }) => - mapping.kind === "file-root" && location.kind === "file" - ? pathBelowRoot(mapping.root, location.value) !== null - : mapping.kind === "url-prefix" && location.kind === "url" - ? pathBelowUrlPrefix(mapping.prefix, location.value) !== null - : false, + for (const mapping of layer.runtimeMappings) { + const prefix = normalizeArtifactPrefix(mapping.artifact_prefix); + for (const entity of entities) { + if (prefixes.has(prefix) && untranslated.has(prefix)) break; + const translation = translateRuntimeLocation(mapping, entity.location); + if (translation.kind === "translated") prefixes.add(prefix); + // Only scripts and workers can correspond to load-state nodes. + else if ( + translation.kind === "untranslatable" && + (entity.kind === "script" || entity.kind === "worker") ) - ) - prefixes.add(normalizeArtifactPrefix(mapping.artifact_prefix)); - return [...prefixes].sort(compareUnicodeCodePoints); + untranslated.add(prefix); + } + } + return { + artifactPrefixes: [...prefixes].sort(compareUnicodeCodePoints), + untranslatedArtifactPrefixes: [...untranslated].sort( + compareUnicodeCodePoints, + ), + }; }; const normalizeArtifactPrefix = (prefix: string): string => diff --git a/tests/boundary/cli/runtimeUrlPrefixMappingCli.test.ts b/tests/boundary/cli/runtimeUrlPrefixMappingCli.test.ts new file mode 100644 index 0000000000..c09e6ceda3 --- /dev/null +++ b/tests/boundary/cli/runtimeUrlPrefixMappingCli.test.ts @@ -0,0 +1,279 @@ +import { execFile } from "node:child_process"; +import { mkdir, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { afterEach, expect, it } from "vitest"; +import { z } from "zod"; + +import { createBrowserEvidence } from "../../../src/application/BrowserEvidence.js"; +import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js"; +import { analyzeFixture } from "../../fixtures/javascriptRuntimeReconciliation.js"; +import { passiveNetworkCapture } from "../../fixtures/passiveNetworkCapture.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const INTEGRATION_TEST_TIMEOUT_MS = 30_000; +const temporary: string[] = []; + +afterEach(async () => { + await Promise.all( + temporary + .splice(0) + .map(async (path) => rm(path, { recursive: true, force: true })), + ); +}); + +const reconciledSchema = z.object({ + normalized_result: z.object({ + reconciliations: z.array( + z.object({ + entity_kind: z.string(), + runtime_node_id: z.string(), + static_node_id: z.string().nullable(), + status: z.string(), + basis: z.string(), + confidence: z.string(), + reason: z.string(), + }), + ), + static_load_states: z.array( + z.object({ + static_node_id: z.string(), + kind: z.string(), + status: z.string(), + reason: z.string(), + }), + ), + graph: z.object({ + nodes: z.array( + z.object({ + node_id: z.string(), + observations: z.array( + z.object({ + properties: z.record(z.string(), z.unknown()), + }), + ), + }), + ), + edges: z.array( + z.object({ + relation: z.string(), + source_node_id: z.string(), + target_node_id: z.string(), + }), + ), + }), + }), +}); + +/** Reconcile one source-free browser capture under `${origin}/assets/`. */ +const reconcileThroughCli = async ( + root: string, + analysis: Awaited>, + suffixes: readonly string[], +) => { + const inspection = passiveNetworkCapture([]); + const origin = inspection.target.origin; + inspection.frames = [ + { frame_id: "frame-main", parent_frame_id: null, url: origin, origin }, + ]; + inspection.scripts = { + total: suffixes.length, + items: suffixes.map((suffix, index) => ({ + script_key: `scr_${String(index).padStart(64, "0")}`, + frame_id: "frame-main", + url: `${origin}/assets/${suffix}`, + origin, + cdp_hash: null, + length: null, + is_module: true, + language: "JavaScript", + source_map_url: null, + resource_reconciliation: { + status: "unmatched", + reason: "no_exact_sanitized_url", + }, + source: { included: false, reason: "not-requested" }, + })), + }; + const runtime = createBrowserEvidence( + "inspect_web_page", + inspectWebPageInputSchema.parse({ + cdp_endpoint: "http://127.0.0.1:9222", + target_id: "page", + allowed_origins: [origin], + observation_ms: 0, + include_script_sources: false, + }), + inspection, + { + id: "rea-cdp-browser", + name: "REA Chrome DevTools Protocol observation provider", + version: "2", + }, + ); + const inputPath = join(root, "reconciliation.json"); + await writeFile( + inputPath, + JSON.stringify({ + static_layers: [ + { + role: "application", + analysis, + runtime_mappings: [ + { + kind: "url-prefix", + prefix: `${origin}/assets/`, + artifact_prefix: "", + }, + ], + }, + ], + runtime_observations: [runtime], + }), + ); + const { stdout, stderr } = await execute( + process.execPath, + ["scripts/rea.mjs", "reconcile-javascript-runtime", inputPath, "--json"], + { cwd: process.cwd(), maxBuffer: 16 * 1_024 * 1_024 }, + ); + expect(stderr).toBe(""); + const result = reconciledSchema.parse(JSON.parse(stdout)).normalized_result; + const nodes = new Map(result.graph.nodes.map((node) => [node.node_id, node])); + const property = (nodeId: string | null, key: string) => + nodeId === null + ? null + : (nodes + .get(nodeId) + ?.observations.map(({ properties }) => properties[key]) + .find((value) => typeof value === "string") ?? null); + const pathOf = (nodeId: string | null) => property(nodeId, "path"); + const scripts = new Map( + result.reconciliations + .filter(({ entity_kind }) => entity_kind === "script") + .map((item) => [ + String(property(item.runtime_node_id, "location")).slice( + `${origin}/assets/`.length, + ), + { + status: item.status, + basis: item.basis, + reason: item.reason, + path: pathOf(item.static_node_id), + observed_as: result.graph.edges.filter( + ({ relation, source_node_id, target_node_id }) => + relation === "observed_as" && + [source_node_id, target_node_id].includes(item.runtime_node_id), + ).length, + }, + ]), + ); + const loadStates = new Map( + result.static_load_states + .filter(({ kind }) => kind === "javascript-asset") + .map((state) => [ + pathOf(state.static_node_id), + { status: state.status, reason: state.reason }, + ]), + ); + return { scripts, loadStates }; +}; + +const unmappedLocation = { + status: "unknown", + basis: "none", + reason: "no-authorized-location-mapping", + path: null, + observed_as: 0, +}; + +const mappedLocation = (path: string) => ({ + status: "matched", + basis: "operator-url-mapping", + reason: "unique-location-match", + path, + observed_as: 1, +}); + +it( + "keeps an encoded slash inside one URL segment rather than naming a nested artifact", + async () => { + const root = await createTestTempDirectory("rea-url-prefix-segments-"); + temporary.push(root); + await mkdir(join(root, "child")); + await Promise.all([ + writeFile(join(root, "child", "part.js"), "export const nested = 1;\n"), + writeFile(join(root, "child%2Fpart.js"), "export const literal = 2;\n"), + writeFile(join(root, "sp ace.js"), "export const space = 3;\n"), + writeFile(join(root, "é.js"), "export const accent = 4;\n"), + ]); + const analysis = await analyzeFixture(root); + + const { scripts } = await reconcileThroughCli(root, analysis, [ + "child/part.js", + "child/part.js?v=%2F#f%2F", + "child%2Fpart.js", + "child%2fpart.js", + "child%252Fpart.js", + "sp%20ace.js", + "%C3%A9.js", + "child/%E0.js", + ]); + + expect(Object.fromEntries(scripts)).toEqual({ + "child/part.js": mappedLocation("child/part.js"), + "child/part.js?v=%2F#f%2F": mappedLocation("child/part.js"), + "child%2Fpart.js": unmappedLocation, + "child%2fpart.js": unmappedLocation, + "child%252Fpart.js": mappedLocation("child%2Fpart.js"), + "sp%20ace.js": mappedLocation("sp ace.js"), + "%C3%A9.js": mappedLocation("é.js"), + "child/%E0.js": unmappedLocation, + }); + }, + INTEGRATION_TEST_TIMEOUT_MS, +); + +it( + "keeps static load state unknown beneath a prefix whose runtime script had no artifact path", + async () => { + const root = await createTestTempDirectory("rea-url-prefix-load-state-"); + temporary.push(root); + await mkdir(join(root, "child")); + await Promise.all([ + writeFile(join(root, "child", "part.js"), "export const nested = 1;\n"), + writeFile(join(root, "other.js"), "export const other = 2;\n"), + ]); + const analysis = await analyzeFixture(root); + + const control = await reconcileThroughCli(root, analysis, ["other.js"]); + expect(Object.fromEntries(control.loadStates)).toEqual({ + "child/part.js": { + status: "not-observed-in-capture", + reason: "not-observed-in-bounded-capture", + }, + "other.js": { + status: "loaded", + reason: "runtime-script-correspondence", + }, + }); + + const encoded = await reconcileThroughCli(root, analysis, [ + "other.js", + "child%2Fpart.js", + ]); + expect(encoded.scripts.get("child%2Fpart.js")).toEqual(unmappedLocation); + expect(Object.fromEntries(encoded.loadStates)).toEqual({ + "child/part.js": { + status: "unknown", + reason: "static-or-runtime-coverage-incomplete", + }, + "other.js": { + status: "loaded", + reason: "runtime-script-correspondence", + }, + }); + }, + INTEGRATION_TEST_TIMEOUT_MS, +);