diff --git a/docs/javascript-application-workflows.md b/docs/javascript-application-workflows.md index af0b6ba941..2d2d9e64aa 100644 --- a/docs/javascript-application-workflows.md +++ b/docs/javascript-application-workflows.md @@ -156,9 +156,12 @@ to the comparison Evidence in a live session. `compare_source_to_bundle` compares one cryptographically committed `HistoricalSourceGraph` with authenticated application-graph Evidence. The stable scoring model reports every admitted signal and weight: exact source -digest, source-map original path, exact current path, path suffix, basename, and -language extension. Exact digest wins over location inference; weak basename -or extension evidence never forces a mapping. +digest, source-map original path, exact current path, path suffix, Unicode- +equivalent path, basename, and language extension. Exact digest wins over +location inference; weak basename, extension, and Unicode-equivalence evidence +never forces a mapping. Unicode-equivalent spellings are candidate evidence +only because distinct filesystem paths can preserve those spellings as separate +files; multiple canonical-equivalent candidates remain unknown. Each historical source file is classified as `unchanged`, `modified`, `removed`, `split`, `merged`, `duplicated`, or `unknown`. Multiple equally diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.sourceMaps.test.ts b/src/application/javascript/JavaScriptArtifactAnalysis.sourceMaps.test.ts new file mode 100644 index 0000000000..cbd9c745cf --- /dev/null +++ b/src/application/javascript/JavaScriptArtifactAnalysis.sourceMaps.test.ts @@ -0,0 +1,71 @@ +import { expect, it } from "vitest"; +import { TraceMap } from "@jridgewell/trace-mapping"; + +import { analyzeJavaScriptArtifactFiles } from "./JavaScriptArtifactAnalysis.js"; + +const HASH = "a".repeat(64); + +it("retains batched TraceMap-resolved paths and null source positions", () => { + const sourceMapText = JSON.stringify({ + version: 3, + file: "main.js", + sourceRoot: "../src/", + sources: ["cafe\u0301.ts", null, "other.ts"], + sourcesContent: ["export const value = 1;", null, "export {};"], + names: [], + mappings: "", + }); + const analysis = analyzeJavaScriptArtifactFiles({ + files: [ + { + path: "dist/main.js.map", + container_sha256: HASH, + sha256: HASH, + bytes: Buffer.byteLength(sourceMapText), + inventory_artifact_id: "artifact_test", + kind: "source-map", + unpacked: false, + text: { included: true, value: sourceMapText }, + }, + ], + containers: [], + text_bytes_read: Buffer.byteLength(sourceMapText), + invalid_utf8_files: 0, + }); + const originals = analysis.source_maps[0]?.sources; + const original = originals?.[0]; + const producerPath = new TraceMap( + { + version: 3, + file: "main.js", + sourceRoot: "../src/", + sources: ["cafe\u0301.ts", "", "other.ts"], + names: [], + mappings: "", + }, + "file:///artifact/dist/main.js.map", + ).resolvedSources[0]; + + if (producerPath === undefined) + throw new Error("TraceMap should resolve its single source declaration"); + expect(originals?.[1]?.reference).toBeNull(); + expect(originals?.[2]?.reference?.resolution).toEqual({ + kind: "artifact-relative", + path: "src/other.ts", + }); + expect(original?.reference).toMatchObject({ + source_name: "cafe\u0301.ts", + source_root: "../src/", + map_path: "dist/main.js.map", + resolution: { + kind: "artifact-relative", + path: "src/cafe\u0301.ts", + }, + }); + expect(original?.reference?.resolution).toEqual({ + kind: "artifact-relative", + path: decodeURIComponent( + new URL(producerPath).pathname.slice("/artifact/".length), + ), + }); +}); diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts index e52ba04ddc..dd1a504883 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysis.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts @@ -3,7 +3,7 @@ import { createHash } from "node:crypto"; import { moduleKindForJavaScriptReference } from "./JavaScriptArtifactPathResolution.js"; import { setImmediate } from "node:timers/promises"; -import { resolveJavaScriptSourceMapReference } from "../../domain/javascript/javascriptSourceMapPaths.js"; +import { resolveJavaScriptSourceMapReferences } from "../../domain/javascript/javascriptSourceMapPaths.js"; import { analyzeParsedJavaScriptStaticSourceSteps } from "../../domain/javascript/javascriptStaticAnalysis.js"; import { analyzeParsedJavaScriptSemanticsSteps, @@ -561,21 +561,27 @@ const collectSourceMapOriginals = ( "Source map sourceRoot must be a string or null.", ); const root = typeof map.sourceRoot === "string" ? map.sourceRoot : null; - for (const [index, raw] of names.entries()) { + const sourceNames: (string | null)[] = []; + for (const raw of names) { if (raw !== null && typeof raw !== "string") return invalidSourceMap( file, "Source map contains a non-string source name.", ); + sourceNames.push(raw); + } + const references = resolveJavaScriptSourceMapReferences( + sourceNames, + root, + file.path, + ); + for (const [index, reference] of references.entries()) { const rawContent = Array.isArray(contents) ? contents[index] : undefined; const content = typeof rawContent === "string" ? rawContent : null; sources.push({ source_index: index, declaration_index: declarationIndex++, - reference: - typeof raw === "string" - ? resolveJavaScriptSourceMapReference(raw, root, file.path) - : null, + reference, content, content_sha256: content === null ? null : sha256(content), }); diff --git a/src/browser/WebSourceMapFetcher.ts b/src/browser/WebSourceMapFetcher.ts index 0b17df0cb5..b950926eeb 100644 --- a/src/browser/WebSourceMapFetcher.ts +++ b/src/browser/WebSourceMapFetcher.ts @@ -2,7 +2,6 @@ import * as t from "@babel/types"; import { decodeValidatedSourceMapLeaves, isBeforeSourceMapLeafStop, - resolveSourceMapSource, } from "../javascript/sourceMaps/DecodedSourceMap.js"; import { sanitizeBrowserUrl } from "../domain/browserObservation.js"; @@ -21,6 +20,7 @@ import { webSourceMapsSchema } from "../domain/webBundleAnalysis.js"; import { createWebTextArtifact } from "../domain/webContentArtifact.js"; import { safeParseJson } from "../domain/safeJson.js"; import { sourceMapJsonText } from "../domain/sourceMapEnvelope.js"; +import { resolveJavaScriptSourceMapUrl } from "../domain/javascript/javascriptSourceMapPaths.js"; import { jsonParts } from "../domain/jsonSerialization.js"; import { inspectSourceMapValue, @@ -867,7 +867,11 @@ const preflightResolvedSourceBytes = ( const sourceKey = source; let resolved = sourcesForRoot.get(sourceKey); if (resolved === undefined) - resolved = resolveSourceMapSource(source, map.sourceRoot, mapUrl); + resolved = resolveJavaScriptSourceMapUrl( + source, + map.sourceRoot, + mapUrl, + ); const bytes = Buffer.byteLength(resolved, "utf8"); if (bytes > SOURCE_MAP_RESPONSE_BYTES - resolvedBytes) throw resolvedSourceLimitFailure(); diff --git a/src/domain/analysisView/analysisView.test.ts b/src/domain/analysisView/analysisView.test.ts index 631f9d506f..f1aa3e8299 100644 --- a/src/domain/analysisView/analysisView.test.ts +++ b/src/domain/analysisView/analysisView.test.ts @@ -1,6 +1,5 @@ import { expect, it } from "vitest"; -import { resolveJavaScriptSourceMapReference } from "../javascript/javascriptSourceMapPaths.js"; import { functionDossierSchema } from "../hopperValues.js"; import { ghidraFunctionDossier } from "../ghidraValues.fixture.js"; import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "../../contracts/javascript/javascriptRuntimeReconciliationExample.js"; @@ -24,13 +23,14 @@ import { completeWithinViewCoverage, pageViewCoverage, } from "./analysisViewCoverage.js"; +import { sourceMapReference } from "../javascript/javascriptApplicationGraph.fixture.js"; import { javascriptModulePath } from "./javascriptApplicationView.js"; - import { inspectAnalysisViewInputSchema, projectAnalysisView, type AnalysisViewParent, } from "./analysisView.js"; + const layoutParent = (): AnalysisViewParent => { const evidence = analysisViewLayoutEvidence(); return { @@ -532,7 +532,7 @@ it("pages actual modules, including unknown and empty paths, without unrelated r observations: original.observations.map((observation) => ({ ...(kind === "source-module" ? { - source_map_reference: resolveJavaScriptSourceMapReference( + source_map_reference: sourceMapReference( "", null, "renderer.js", diff --git a/src/domain/javascript/javascriptApplicationGraph.fixture.ts b/src/domain/javascript/javascriptApplicationGraph.fixture.ts index 3002b822f3..0e78e5115b 100644 --- a/src/domain/javascript/javascriptApplicationGraph.fixture.ts +++ b/src/domain/javascript/javascriptApplicationGraph.fixture.ts @@ -6,6 +6,7 @@ import { } from "./javascriptApplicationGraph.js"; import type { ApplicationNode } from "./javascriptApplicationGraphSchemas.js"; import type { ApplicationGraphEvidence } from "./javascriptApplicationEvidenceSchemas.js"; +import { resolveJavaScriptSourceMapReferences } from "./javascriptSourceMapPaths.js"; export const APPLICATION_GRAPH_DIGESTS = { package: "1".repeat(64), @@ -22,6 +23,20 @@ export const completeCoverage = { limits: [], } satisfies ApplicationGraphEvidence["coverage"]; +export const sourceMapReference = ( + sourceName: string, + sourceRoot: string | null, + mapPath: string, +) => { + const reference = resolveJavaScriptSourceMapReferences( + [sourceName], + sourceRoot, + mapPath, + )[0]; + if (reference == null) throw new TypeError("Expected a source-map reference"); + return reference; +}; + const extractor = (operation: string) => ({ name: "rea-test-fixture", version: "1.0.0", diff --git a/src/domain/javascript/javascriptApplicationGraph.test.ts b/src/domain/javascript/javascriptApplicationGraph.test.ts index 5e396a5452..5bf9174809 100644 --- a/src/domain/javascript/javascriptApplicationGraph.test.ts +++ b/src/domain/javascript/javascriptApplicationGraph.test.ts @@ -20,10 +20,9 @@ import { inferredArtifactEvidence, nodeByLabel, runtimeEvidence, + sourceMapReference, unknownEvidence, } from "./javascriptApplicationGraph.fixture.js"; - -import { resolveJavaScriptSourceMapReference } from "./javascriptSourceMapPaths.js"; const createLargeNativeExportGraph = (): JavaScriptApplicationGraph => { const properties = Object.fromEntries( Array.from({ length: 1_001 }, (_, index) => [ @@ -438,7 +437,7 @@ describe("JavaScript Application Graph: source maps and observations", () => { observations: [ { label: "src/editor.ts", - source_map_reference: resolveJavaScriptSourceMapReference( + source_map_reference: sourceMapReference( "webpack:///src/editor.ts", null, "dist/app.js.map", @@ -535,7 +534,7 @@ describe("JavaScript Application Graph: source-map digest consistency", () => { observations: [ { label: null, - source_map_reference: resolveJavaScriptSourceMapReference( + source_map_reference: sourceMapReference( "src/editor.ts", null, "dist/app.js.map", diff --git a/src/domain/javascript/javascriptSourceMapPaths.ts b/src/domain/javascript/javascriptSourceMapPaths.ts index d897bdd26a..06b5f82656 100644 --- a/src/domain/javascript/javascriptSourceMapPaths.ts +++ b/src/domain/javascript/javascriptSourceMapPaths.ts @@ -1,5 +1,6 @@ import { posix } from "node:path"; import { domainToASCII, fileURLToPath, pathToFileURL } from "node:url"; +import { TraceMap } from "@jridgewell/trace-mapping"; import { z } from "zod"; @@ -25,22 +26,64 @@ export type JavaScriptSourceMapReference = z.infer< typeof javascriptSourceMapReferenceSchema >; -/** Interpret a declaration once at the source-map ingestion boundary. */ -export const resolveJavaScriptSourceMapReference = ( - sourceName: string, +/** Interpret declarations together once at their source-map ingestion boundary. */ +export const resolveJavaScriptSourceMapReferences = ( + sourceNames: readonly (string | null)[], sourceRoot: string | null, mapPath: string, -): JavaScriptSourceMapReference => { - // TraceMap's producer prefixes sourceRoot before resolving the URL reference. - const reference = `${sourceRoot ? `${sourceRoot}/` : ""}${sourceName}`; - return { - source_name: sourceName, - source_root: sourceRoot, - map_path: mapPath, - resolution: resolveJavaScriptSourceMapPath(reference, mapPath), - }; +): (JavaScriptSourceMapReference | null)[] => { + if (sourceNames.length === 0) return []; + const resolvedSources = resolveSourceMapUrls( + sourceNames.map((sourceName) => sourceName ?? ""), + sourceRoot, + "", + ); + return sourceNames.map((sourceName, index) => { + if (sourceName === null) return null; + const reference = resolvedSources[index]; + if (reference === undefined) + throw new TypeError( + `TraceMap did not resolve source declaration ${String(index)}`, + ); + return { + source_name: sourceName, + source_root: sourceRoot, + map_path: mapPath, + resolution: resolveJavaScriptSourceMapPath(reference, mapPath), + }; + }); }; +/** Resolve a source declaration using the same pinned format implementation as map consumers. */ +export const resolveJavaScriptSourceMapUrl = ( + source: string | null, + sourceRoot: string | null | undefined, + mapUrl: string, +): string => { + const resolved = resolveSourceMapUrls([source ?? ""], sourceRoot, mapUrl)[0]; + if (resolved === undefined) + throw new TypeError( + "TraceMap did not resolve its single source declaration", + ); + return resolved; +}; + +const resolveSourceMapUrls = ( + sources: string[], + sourceRoot: string | null | undefined, + mapUrl: string, +): readonly string[] => + new TraceMap( + { + version: 3, + names: [], + sources, + ...(sourceRoot == null ? {} : { sourceRoot }), + mappings: "", + }, + mapUrl, + ).resolvedSources; + /** Resolve a source-map URL reference without changing its recorded identity. */ const resolveJavaScriptSourceMapPath = ( source: string, diff --git a/src/domain/javascript/sourceToBundleComparison.test.ts b/src/domain/javascript/sourceToBundleComparison.test.ts index f488ee598d..d231cb5ec1 100644 --- a/src/domain/javascript/sourceToBundleComparison.test.ts +++ b/src/domain/javascript/sourceToBundleComparison.test.ts @@ -1,10 +1,11 @@ import { describe, expect, it } from "vitest"; +import { TraceMap } from "@jridgewell/trace-mapping"; -import { resolveJavaScriptSourceMapReference } from "./javascriptSourceMapPaths.js"; import { createJavaScriptApplicationGraph, createJavaScriptApplicationNode, } from "./javascriptApplicationGraph.js"; +import type { ApplicationNode } from "./javascriptApplicationGraphSchemas.js"; import { createHistoricalSourceGraph } from "../referenceSourceGraph.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; import { compareSourceToBundle } from "./sourceToBundleComparison.js"; @@ -12,7 +13,10 @@ import { SOURCE_TO_BUNDLE_SIGNAL_WEIGHTS, sourceToBundleComparisonResultSchema, } from "./sourceToBundleComparisonSchemas.js"; -import { artifactEvidence } from "./javascriptApplicationGraph.fixture.js"; +import { + artifactEvidence, + sourceMapReference, +} from "./javascriptApplicationGraph.fixture.js"; const HASH = { artifact: "1".repeat(64), @@ -42,9 +46,6 @@ describe("historical source to bundle comparison", () => { const second = compareSourceToBundle(input); expect(first).toEqual(second); - expect(() => - sourceToBundleComparisonResultSchema.parse(first), - ).not.toThrow(); expect(first.summary).toEqual({ unchanged: 1, modified: 0, @@ -153,7 +154,7 @@ describe("source to bundle digest changes", () => { observations: [ { label: "display label", - source_map_reference: resolveJavaScriptSourceMapReference( + source_map_reference: sourceMapReference( "webpack://fixture/./src/modified.ts", null, "dist/main.js.map", @@ -202,6 +203,90 @@ describe("source to bundle digest changes", () => { }); }); +describe("source to bundle Unicode path collisions", () => { + it("keeps colliding canonical-equivalent current paths ambiguous", () => { + const historicalPath = "src/a\u0301\u0323.ts"; + const currentPaths = ["src/a\u0323\u0301.ts", "src/\u1ea1\u0301.ts"]; + const nodes = currentPaths.map((path, index) => + moduleNode(`unicode-${index}`, { path }), + ); + const result = comparePathToNodes(historicalPath, nodes); + const mapping = item(result, historicalPath); + + expect(mapping).toMatchObject({ + status: "unknown", + confidence: "unknown", + current_node_ids: [], + candidates: [ + expect.objectContaining({ + signals: expect.arrayContaining([ + expect.objectContaining({ kind: "unicode-equivalent-path" }), + ]), + }), + expect.objectContaining({ + signals: expect.arrayContaining([ + expect.objectContaining({ kind: "unicode-equivalent-path" }), + ]), + }), + ], + limitations: [ + expect.stringContaining( + "filesystem paths with distinct Unicode spellings", + ), + ], + }); + }); + + it("prefers a raw source-map path over canonically equivalent alternatives", () => { + const historicalPath = "src/cafe\u0301.ts"; + const exact = sourceMapNode( + "exact", + "cafe\u0301.ts", + "../src/", + HASH.modified, + ); + const equivalent = sourceMapNode( + "equivalent", + "caf\u00e9.ts", + "../src/", + HASH.removed, + ); + const result = comparePathToNodes(historicalPath, [exact, equivalent]); + + expect(item(result, historicalPath)).toMatchObject({ + status: "modified", + current_node_ids: [exact.node_id], + }); + }); + + it("finds Unicode-equivalent suffixes of absolute source-map URLs", () => { + const historicalPath = "src/caf\u00e9.ts"; + const node = sourceMapNode( + "remote", + "cafe\u0301.ts", + "https://cdn.example/src/", + HASH.modified, + ); + const result = comparePathToNodes(historicalPath, [node]); + + expect(item(result, historicalPath)).toMatchObject({ + status: "unknown", + current_node_ids: [], + candidates: [ + expect.objectContaining({ + current_node_id: node.node_id, + signals: expect.arrayContaining([ + expect.objectContaining({ + kind: "unicode-equivalent-path", + current_values: ["cdn.example/src/cafe\u0301.ts"], + }), + ]), + }), + ], + }); + }); +}); + describe("source to bundle path syntax", () => { it.each([ "src/modified#part.ts", @@ -282,9 +367,60 @@ describe("source to bundle path syntax", () => { }), ], }); + expect( + item(result, "src/modified.ts").candidates.flatMap( + ({ signals }) => signals, + ), + ).not.toContainEqual( + expect.objectContaining({ kind: "unicode-equivalent-path" }), + ); }); }); +describe("source-map resolver equivalence", () => { + it.each([ + { + sourceRoot: "https://cdn.example/src/", + expectedSource: "https://cdn.example/src/a.js", + expectedResolution: { kind: "suffix", path: "cdn.example/src/a.js" }, + }, + { + sourceRoot: "https://cdn.example/src//", + expectedSource: "https://cdn.example/src/a.js", + expectedResolution: { kind: "suffix", path: "cdn.example/src/a.js" }, + }, + { + sourceRoot: "webpack://", + expectedSource: "webpack:///a.js", + expectedResolution: { kind: "suffix", path: "a.js" }, + }, + ])( + "matches TraceMap for sourceRoot $sourceRoot", + ({ sourceRoot, expectedSource, expectedResolution }) => { + const producerSource = new TraceMap( + { + version: 3, + file: "main.js", + sourceRoot, + sources: ["a.js"], + names: [], + mappings: "", + }, + "file:///artifact/dist/main.js.map", + ).resolvedSources[0]; + const reference = sourceMapReference( + "a.js", + sourceRoot, + "dist/main.js.map", + ); + + expect(producerSource).toBe(expectedSource); + expect(reference.source_root).toBe(sourceRoot); + expect(reference.resolution).toEqual(expectedResolution); + }, + ); +}); + describe("source to bundle complete metadata", () => { it("accepts long source metadata and all reported limitations", () => { const result = compareSourceToBundle({ @@ -328,6 +464,61 @@ describe("source to bundle complete metadata", () => { }); }); +const sourceMapNode = ( + key: string, + sourceName: string, + sourceRoot: string, + digest: string, +) => + createJavaScriptApplicationNode({ + kind: "source-module", + identity: { + strategy: "source-map-original", + source_root: sourceRoot, + stability: "source-map-exact", + source_map_sha256: HASH.artifact, + original_source: sourceName, + source_sha256: digest, + }, + observations: [ + { + label: `${key} source-map spelling`, + source_map_reference: sourceMapReference( + sourceName, + sourceRoot, + `dist/${key}.map`, + ), + properties: { content_available: true, content_sha256: digest }, + evidence: artifactEvidence(HASH.artifact, `dist/${key}.map`), + }, + ], + }); + +const comparePathToNodes = ( + sourcePath: string, + nodes: readonly ApplicationNode[], +) => + compareSourceToBundle({ + reference: historicalGraph("complete", [sourcePath]), + application: { + evidenceId: EVIDENCE_ID, + rootArtifactSha256: HASH.artifact, + graph: createJavaScriptApplicationGraph({ + schema: "JavaScriptApplicationGraph", + root_node_ids: nodes.map(({ node_id: nodeId }) => nodeId), + nodes, + edges: [], + coverage: { + status: "complete", + truncated: false, + omitted_count: 0, + limits: [], + }, + limitations: [], + }), + }, + }); + const historicalGraph = ( inventoryState: "complete" | "partial", paths = [ diff --git a/src/domain/javascript/sourceToBundleComparison.ts b/src/domain/javascript/sourceToBundleComparison.ts index ac0007cb46..5a70c57f37 100644 --- a/src/domain/javascript/sourceToBundleComparison.ts +++ b/src/domain/javascript/sourceToBundleComparison.ts @@ -258,7 +258,11 @@ const classifyMapping = ( confidence: "unknown", currentNodeIds: [], limitations: [ - "Only weak basename or extension signals were available; no mapping was forced.", + candidates.some((candidate) => + hasSignal(candidate, "unicode-equivalent-path"), + ) + ? "Unicode-equivalent paths are inferred candidates only; filesystem paths with distinct Unicode spellings may identify different files, so no mapping was forced." + : "Only weak basename or extension signals were available; no mapping was forced.", ], }; return absenceComplete diff --git a/src/domain/javascript/sourceToBundleComparisonSchemas.ts b/src/domain/javascript/sourceToBundleComparisonSchemas.ts index 2e27cadb36..9d6bd6b2f4 100644 --- a/src/domain/javascript/sourceToBundleComparisonSchemas.ts +++ b/src/domain/javascript/sourceToBundleComparisonSchemas.ts @@ -15,6 +15,7 @@ export const SOURCE_TO_BUNDLE_SIGNAL_WEIGHTS = [ ["source-map-original-path", 70], ["current-path-exact", 60], ["current-path-suffix", 40], + ["unicode-equivalent-path", 20], ["basename-match", 20], ["language-extension", 5], ] as const; @@ -24,6 +25,7 @@ const sourceToBundleSignalKindSchema = z.enum([ "source-map-original-path", "current-path-exact", "current-path-suffix", + "unicode-equivalent-path", "basename-match", "language-extension", ]); diff --git a/src/domain/javascript/sourceToBundleSignals.ts b/src/domain/javascript/sourceToBundleSignals.ts index 2190988fe1..b3546aecc6 100644 --- a/src/domain/javascript/sourceToBundleSignals.ts +++ b/src/domain/javascript/sourceToBundleSignals.ts @@ -102,8 +102,12 @@ export const buildSourceToBundleCandidateIndex = ( for (const path of projection.paths) { for (const suffix of path.allowSuffix ? pathSuffixes(path.value) - : [path.value]) + : [path.value]) { addIndexValue(byPathSuffix, suffix, projection.node.node_id); + const canonical = suffix.normalize("NFC"); + if (canonical !== suffix) + addIndexValue(byPathSuffix, canonical, projection.node.node_id); + } addIndexValue( byBasename, posix.basename(path.value), @@ -126,9 +130,13 @@ export const candidateIdsForSource = ( source: SourceFile, index: SourceToBundleCandidateIndex, ): string[] => { + const canonicalPath = source.path.normalize("NFC"); const groups = [ source.sha256 === null ? undefined : index.byDigest.get(source.sha256), index.byPathSuffix.get(source.path), + canonicalPath === source.path + ? undefined + : index.byPathSuffix.get(canonicalPath), index.byBasename.get(posix.basename(source.path)), ]; const seen = new Set(); @@ -220,6 +228,14 @@ const candidateSignals = ( signals.push( signal("source-map-original-path", source.path, sourceMapPaths), ); + const unicodeEquivalentPaths = matchingUnicodePaths( + source.path, + current.paths, + ); + if (unicodeEquivalentPaths.length > 0) + signals.push( + signal("unicode-equivalent-path", source.path, unicodeEquivalentPaths), + ); const otherPaths = current.paths.filter( ({ kind }) => kind !== "source-map-original", ); @@ -286,6 +302,27 @@ const matchingPaths = ( ) .map(({ value }) => value); +const matchingUnicodePaths = ( + sourcePath: string, + currentPaths: readonly CurrentPath[], +): string[] => { + const canonicalSource = sourcePath.normalize("NFC"); + return currentPaths + .filter(({ value, allowSuffix }) => { + if ( + value === sourcePath || + (allowSuffix && value.endsWith(`/${sourcePath}`)) + ) + return false; + const canonicalValue = value.normalize("NFC"); + return ( + canonicalValue === canonicalSource || + (allowSuffix && canonicalValue.endsWith(`/${canonicalSource}`)) + ); + }) + .map(({ value }) => value); +}; + const addJsonPath = (paths: CurrentPath[], value: unknown): void => { if (typeof value === "string") addPath(paths, "observation-path", value); }; diff --git a/src/javascript/sourceMaps/DecodedSourceMap.ts b/src/javascript/sourceMaps/DecodedSourceMap.ts index 60bfcfc0e5..e9ed304e1a 100644 --- a/src/javascript/sourceMaps/DecodedSourceMap.ts +++ b/src/javascript/sourceMaps/DecodedSourceMap.ts @@ -19,31 +19,6 @@ export const isBeforeSourceMapLeafStop = ( ); }; -/** Resolve one declaration with the same pinned upstream implementation used by full maps. */ -export const resolveSourceMapSource = ( - source: string | null, - sourceRoot: string | null | undefined, - mapUrl: string, -): string => { - const map = new TraceMap( - { - version: 3, - names: [], - sources: [source ?? ""], - ...(sourceRoot == null ? {} : { sourceRoot }), - mappings: "", - }, - mapUrl, - ); - const resolved = map.resolvedSources[0]; - if (resolved === undefined) - throw new SourceMapFormatFailure( - "format", - "Source-map resolver did not return the requested source identity.", - ); - return resolved; -}; - /** Decode leaf mappings and validate producer indices before any consumer publishes them. */ export const decodeValidatedSourceMapLeaves = ( leaves: readonly SourceMapLeaf[], diff --git a/tests/boundary/filesystem/sourceToBundlePaths.test.ts b/tests/boundary/filesystem/sourceToBundlePaths.test.ts index 19aa07a1a1..8bc9caa27b 100644 --- a/tests/boundary/filesystem/sourceToBundlePaths.test.ts +++ b/tests/boundary/filesystem/sourceToBundlePaths.test.ts @@ -6,7 +6,7 @@ import { analyzeJavaScriptApplication } from "../../support/javascriptApplicatio import { parseApplicationGraphEvidence } from "../../../src/application/javascript/JavaScriptApplicationEvidenceGraph.js"; import { findApplicationFeatureSeeds } from "../../../src/domain/javascript/javascriptFeatureSeed.js"; import { compareSourceToBundle } from "../../../src/domain/javascript/sourceToBundleComparison.js"; -import { resolveSourceMapSource } from "../../../src/javascript/sourceMaps/DecodedSourceMap.js"; +import { resolveJavaScriptSourceMapUrl } from "../../../src/domain/javascript/javascriptSourceMapPaths.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { importReferenceSource } from "../../support/referenceSourceResourceScope.js"; @@ -180,7 +180,7 @@ it.each([ decoy: decoyPath = "src/a.js", }) => { if (sourceRoot !== undefined) { - const resolved = resolveSourceMapSource( + const resolved = resolveJavaScriptSourceMapUrl( source, sourceRoot, `https://example.test/${mapPath}`, diff --git a/tests/boundary/mcp/applicationEvidenceMcp.test.ts b/tests/boundary/mcp/applicationEvidenceMcp.test.ts index 16dfc142c2..ba71b79213 100644 --- a/tests/boundary/mcp/applicationEvidenceMcp.test.ts +++ b/tests/boundary/mcp/applicationEvidenceMcp.test.ts @@ -2,6 +2,7 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import type { Client } from "@modelcontextprotocol/client"; import { describe, expect, it } from "vitest"; import { Ajv2020 } from "ajv/dist/2020.js"; +import type { AnySchema } from "ajv"; import type { BinarySession } from "../../../src/application/binary/BinarySession.js"; import { @@ -10,12 +11,143 @@ import { SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE, } from "../../../src/contracts/javascript/javascriptApplicationWorkflowExamples.js"; import { createEvidence } from "../../../src/domain/evidence.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { + createJavaScriptApplicationGraph, + createJavaScriptApplicationNode, +} from "../../../src/domain/javascript/javascriptApplicationGraph.js"; +import { createJavaScriptSemanticGraph } from "../../../src/domain/javascript/javascriptSemanticGraph.js"; +import { + artifactEvidence, + sourceMapReference, +} from "../../../src/domain/javascript/javascriptApplicationGraph.fixture.js"; import { createApplicationMcpHarness, type ApplicationMcpHarness, } from "../../fixtures/applicationMcpHarness.js"; import { createHistoricalSourceGraph } from "../../../src/domain/referenceSourceGraph.js"; +async function runUnicodeSourceMapComparison( + harness: ApplicationMcpHarness, +): Promise { + const baseApplication = SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE.application; + const subject = baseApplication.subject; + if (subject === null) throw new Error("Application Evidence has no subject"); + const baseResult = javascriptApplicationAnalysisResultSchema.parse( + baseApplication.normalized_result, + ); + const baseGraph = baseResult.graph; + const decomposedSource = "cafe\u0301.ts"; + const unicodeSourceNode = createJavaScriptApplicationNode({ + kind: "source-module", + identity: { + strategy: "source-map-original", + source_root: "../src/", + stability: "source-map-exact", + source_map_sha256: "1".repeat(64), + original_source: decomposedSource, + source_sha256: "3".repeat(64), + }, + observations: [ + { + label: "decomposed source-map spelling", + source_map_reference: sourceMapReference( + decomposedSource, + "../src/", + "dist/main.js.map", + ), + properties: { content_available: true, content_sha256: "3".repeat(64) }, + evidence: artifactEvidence("1".repeat(64), "dist/main.js.map"), + }, + ], + }); + const applicationGraph = createJavaScriptApplicationGraph({ + schema: "JavaScriptApplicationGraph", + root_node_ids: [...baseGraph.root_node_ids, unicodeSourceNode.node_id], + nodes: [...baseGraph.nodes, unicodeSourceNode], + edges: baseGraph.edges, + coverage: baseGraph.coverage, + limitations: baseGraph.limitations, + }); + const semantic = baseResult.semantic_graph; + const { graph_id: _semanticGraphId, ...semanticInput } = semantic; + const semanticGraph = createJavaScriptSemanticGraph({ + ...semanticInput, + application_graph_id: applicationGraph.graph_id, + }); + const unicodeApplication = createEvidence( + { + path: baseResult.input_path, + sha256: subject.digest.sha256, + format: subject.format, + }, + baseApplication.provider, + { + predicateType: baseApplication.predicate_type, + operation: baseApplication.operation, + parameters: baseApplication.parameters, + result: { + ...baseResult, + graph: applicationGraph, + semantic_graph: semanticGraph, + }, + confidence: baseApplication.confidence, + authority: baseApplication.authority, + limitations: baseApplication.limitations, + evidenceLinks: baseApplication.evidence_links, + }, + ); + const historicalUnicodePath = "src/caf\u00e9.ts"; + const unicodeReference = createHistoricalSourceGraph({ + ...SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE.reference, + root_sha256: undefined, + entries: SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE.reference.entries.map( + (entry) => + entry.path === "src/main.ts" + ? { ...entry, path: historicalUnicodePath } + : entry, + ), + }); + const reply = await harness.client.callTool({ + name: "compare_source_to_bundle", + arguments: { reference: unicodeReference, application: unicodeApplication }, + }); + expect(reply.isError).not.toBe(true); + expect(reply.structuredContent).toMatchObject({ + normalized_result: { + items: [ + { + source_path: historicalUnicodePath, + status: "unknown", + current_node_ids: [], + candidates: [ + expect.objectContaining({ + confidence: "low", + signals: expect.arrayContaining([ + expect.objectContaining({ + kind: "unicode-equivalent-path", + source_value: historicalUnicodePath, + current_values: ["src/cafe\u0301.ts"], + }), + ]), + }), + ], + }, + ], + }, + }); + const advertised = (await harness.client.listTools()).tools.find( + ({ name }) => name === "compare_source_to_bundle", + ); + if (advertised?.outputSchema === undefined) + throw new Error("Comparison tool output schema is missing"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const outputSchema = advertised.outputSchema as AnySchema; + expect(ajv.validateSchema(outputSchema)).toBe(true); + const validateOutput = ajv.compile(outputSchema); + expect(validateOutput(reply.structuredContent)).toBe(true); +} + async function runInlineWorkflowScenarios( harness: ApplicationMcpHarness, ): Promise { @@ -92,6 +224,8 @@ async function runInlineWorkflowScenarios( }, }); + await runUnicodeSourceMapComparison(harness); + const referenceWithExclusion = createHistoricalSourceGraph({ ...SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE.reference, root_sha256: undefined, diff --git a/tests/boundary/process/sourceMapUncHosts.test.ts b/tests/boundary/process/sourceMapUncHosts.test.ts index 293a3f71a3..99953509f8 100644 --- a/tests/boundary/process/sourceMapUncHosts.test.ts +++ b/tests/boundary/process/sourceMapUncHosts.test.ts @@ -73,9 +73,9 @@ processTest.for(cases)( [ "--input-type=module", "--eval", - `import { resolveJavaScriptSourceMapReference } from "./dist/domain/javascript/javascriptSourceMapPaths.js"; + `import { resolveJavaScriptSourceMapReferences } from "./dist/domain/javascript/javascriptSourceMapPaths.js"; const [source, root] = JSON.parse(process.argv[1]); - console.log(JSON.stringify(resolveJavaScriptSourceMapReference(source, root, "maps/main.js.map")));`, + console.log(JSON.stringify(resolveJavaScriptSourceMapReferences([source], root, "maps/main.js.map")[0]));`, JSON.stringify([scenario.source, sourceRoot]), ], { timeoutMs: 10_000 },