From f2e5fd148a85fa737302dd194a33172c4561093e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:25:03 +0800 Subject: [PATCH] fix(reference): preserve module syntax and loader request identity --- docs/cli.md | 18 +- .../ReferenceSourceImportEntries.ts | 182 ++++++++++++---- .../JavaScriptArtifactPathResolution.ts | 27 +-- .../javascript/javascriptModuleSpecifier.ts | 21 ++ src/domain/referenceSourceClassification.ts | 5 +- src/domain/referenceSourceGraph.ts | 3 +- src/domain/referenceSourceImportParsing.ts | 150 +++++-------- src/domain/referenceSourceRequireBindings.ts | 56 +++-- .../referenceSourceModuleSemantics.test.ts | 203 ++++++++++++++++++ .../referenceSourceComputedRequire.test.ts | 12 +- 10 files changed, 487 insertions(+), 190 deletions(-) create mode 100644 tests/boundary/cli/referenceSourceModuleSemantics.test.ts diff --git a/docs/cli.md b/docs/cli.md index 9d40e4a47b..fd917d6a72 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -456,11 +456,19 @@ does not infer module targets from replacement characters. Exact dynamic-import strings, expression-free templates and runtime-transparent TypeScript wrappers retain their targets; computed imports remain unknown. TypeScript import-equals dependencies remain `requires` relationships whether or not they are exported. +Historical dependencies also include TypeScript import types (`import("./dep").T` +and `typeof import("./dep")`) and declarations inside ambient module bodies; +these source relationships do not assert runtime loading. Language labels retain +the conventional case-sensitive `.C` (C++) and `.M` (Objective-C++) suffixes. REA derives historical CommonJS paths only when the selected loader and its base are proven: this includes the exact `createRequire(new URL("./file.cjs", import.meta.url))` form and -`require.resolve` with `{ paths: [__dirname] }`. Other custom bases and search +`require.resolve` with `{ paths: [__dirname] }`. Exact optional-call forms such +as `require.resolve?.("./dep.cjs")` use the same loader and mutation facts. The +nearest inventoried package manifest supplies `.js` module scope; an explicit +`type: "module"` supplies no CommonJS wrapper, while an unreadable, excluded or +malformed manifest leaves candidate loader resolution unknown. Other custom bases and search options remain partial with unknown resolution and the literal requested specifier. Overwritten resolver methods retain that uncertainty; they are not attributed to Node. `require.main` is a module object, so calling it does not @@ -470,6 +478,14 @@ performs lookup without loading the target. An excluded `package.json` also leaves package-directory selection unknown rather than treating the manifest as absent and applying an `index` fallback. +Import requests use URL pathname semantics: query and fragment remain request +identity, percent escapes decode exactly once for inventory lookup, and encoded +separators or malformed escapes retain an unresolved relationship and diagnostic. +URL-bearing relationships retain the authored `source_specifier` separately from +the inventoried `to` path; distinct query/fragment requests remain distinct even +when they name the same file. CommonJS requests retain literal filename semantics +for percent signs, query marks and fragments. + Historical-source import requires safe no-follow file opens on Linux or macOS. Native Windows returns `unsupported_host`; use Linux REA inside WSL or another supported host. See diff --git a/src/application/ReferenceSourceImportEntries.ts b/src/application/ReferenceSourceImportEntries.ts index d378071d79..2907fc75df 100644 --- a/src/application/ReferenceSourceImportEntries.ts +++ b/src/application/ReferenceSourceImportEntries.ts @@ -1,6 +1,8 @@ +import { err, ok, type Result } from "../domain/result.js"; import { createHash } from "node:crypto"; import { posix } from "node:path"; +import { decodeJavaScriptModuleUrlPath } from "../domain/javascript/javascriptModuleSpecifier.js"; import { logicalPathEscapesRoot } from "../domain/artifactIdentity.js"; import { createReferenceSourcePathLookup, @@ -100,59 +102,74 @@ type PackageMain = | { readonly state: "none" } | { readonly state: "unknown" }; -/** - * Read each inventoried `package.json` once. Node selects a non-empty string - * `main` before a directory index; an unreadable or invalid manifest leaves - * that selection unknown. - */ -const readPackageMains = ( +type PackageType = "module" | "commonjs" | "unknown" | undefined; +interface PackageMetadata { + readonly main: PackageMain; + readonly type: PackageType; +} +const UNKNOWN_PACKAGE: PackageMetadata = { + main: { state: "unknown" }, + type: "unknown", +}; + +/** Parse each manifest once; both entry selection and loader scope consume its facts. */ +const readPackageMetadata = ( read: ReferenceSourceRead, exclusions: HistoricalSourceGraphInput["exclusions"], ): { - readonly mains: ReadonlyMap; + readonly packages: ReadonlyMap; readonly pathLookup: ReferenceSourcePathLookup; } => { - const mains = new Map(); + const packages = new Map(); const manifestPaths: string[] = []; for (const exclusion of exclusions) { if (posix.basename(exclusion.path) !== "package.json") continue; manifestPaths.push(exclusion.path); - mains.set(posix.dirname(exclusion.path), { state: "unknown" }); + packages.set(posix.dirname(exclusion.path), UNKNOWN_PACKAGE); } for (const entry of read.entries) { if (posix.basename(entry.path) !== "package.json") continue; manifestPaths.push(entry.path); const directory = posix.dirname(entry.path); - if (mains.has(directory)) { - mains.set(directory, { state: "unknown" }); - continue; - } - if (entry.status !== "read" || entry.kind !== "file") { - mains.set(directory, { state: "unknown" }); + if ( + packages.has(directory) || + entry.status !== "read" || + entry.kind !== "file" + ) { + packages.set(directory, UNKNOWN_PACKAGE); continue; } try { const manifest: unknown = JSON.parse( new TextDecoder("utf-8", { fatal: true }).decode(entry.bytes), ); - const main = - typeof manifest === "object" && - manifest !== null && - Object.hasOwn(manifest, "main") - ? Reflect.get(manifest, "main") - : undefined; - mains.set( - directory, - typeof main === "string" && main !== "" - ? { state: "declared", main } - : { state: "none" }, - ); + if ( + typeof manifest !== "object" || + manifest === null || + Array.isArray(manifest) + ) { + packages.set(directory, UNKNOWN_PACKAGE); + continue; + } + const type: unknown = Object.hasOwn(manifest, "type") + ? Reflect.get(manifest, "type") + : undefined; + const main: unknown = Object.hasOwn(manifest, "main") + ? Reflect.get(manifest, "main") + : undefined; + packages.set(directory, { + type: type === "module" || type === "commonjs" ? type : undefined, + main: + typeof main === "string" && main !== "" + ? { state: "declared", main } + : { state: "none" }, + }); } catch { - mains.set(directory, { state: "unknown" }); + packages.set(directory, UNKNOWN_PACKAGE); } } return { - mains, + packages, pathLookup: createReferenceSourcePathLookup(manifestPaths), }; }; @@ -199,7 +216,7 @@ const nodeRequireIndexCandidates = (directory: string): string[] => /** Inventory facts an internal specifier resolves against. */ interface SpecifierContext { readonly lookupPath: ReferenceSourcePathLookup; - readonly packageMains: ReadonlyMap; + readonly packages: ReadonlyMap; readonly packageManifestPathLookup: ReferenceSourcePathLookup; } @@ -212,7 +229,7 @@ const packageMainForDirectory = ( ); if (manifestPath !== undefined) { const manifestDirectory = posix.dirname(manifestPath); - const main = context.packageMains.get(manifestDirectory); + const main = context.packages.get(manifestDirectory)?.main; return main === undefined ? undefined : { directory: manifestDirectory, main }; @@ -274,22 +291,71 @@ const selectPackageMain = ( : { to: selected, resolution: "internal" }; }; +/** Parse URL syntax before inventory lookup; require filenames remain literal. */ +const importLookupPath = ( + fromPath: string, + specifier: string, +): Result<{ readonly path: string; readonly directory: boolean }, string> => { + // A private origin is only a logical pathname base, never a filesystem access. + let source: URL; + try { + source = new URL( + fromPath.split("/").map(encodeURIComponent).join("/"), + "file:///rea-reference/", + ); + } catch (cause: unknown) { + if (!(cause instanceof URIError)) throw cause; + return err("The source path cannot be represented as a module URL."); + } + const target = URL.parse(specifier, source); + if (target === null || target.protocol !== "file:" || target.host !== "") + return err( + "The module URL cannot be resolved within the source inventory.", + ); + if (!target.pathname.startsWith("/rea-reference/")) + return err("The module URL escapes the selected source inventory."); + const decoded = decodeJavaScriptModuleUrlPath( + target.pathname.slice("/rea-reference/".length), + ); + return decoded.ok + ? ok({ path: decoded.value, directory: target.pathname.endsWith("/") }) + : decoded; +}; + const resolveInternalSpecifier = ( fromPath: string, specifier: string, context: SpecifierContext, mode: InternalSpecifierResolutionMode, -): { to: string; resolution: "internal" | "unresolved" } => { +): { + to: string; + resolution: "internal" | "unresolved"; + limitation?: string; +} => { if (posix.isAbsolute(specifier)) return { to: specifier, resolution: "unresolved" }; - const normalized = posix.normalize( - posix.join(posix.dirname(fromPath), specifier), - ); + const imported = + mode === "source-import" + ? importLookupPath(fromPath, specifier) + : undefined; + if (imported !== undefined && !imported.ok) + return { + to: specifier, + resolution: "unresolved", + limitation: imported.error, + }; + const normalized = + imported?.ok === true + ? imported.value.path + : posix.normalize(posix.join(posix.dirname(fromPath), specifier)); if (logicalPathEscapesRoot(normalized)) return { to: normalized, resolution: "unresolved" }; // Preserve directory intent before normalization removes terminal dot segments. const directoryReference = - specifier.endsWith("/") || [".", ".."].includes(posix.basename(specifier)); + imported?.ok === true + ? imported.value.directory + : specifier.endsWith("/") || + [".", ".."].includes(posix.basename(specifier)); // A directory resolves only to an index with an extension; neither Node nor // TypeScript selects an extensionless `index` file. const slots = resolutionSuffixes(mode); @@ -330,6 +396,29 @@ const resolveInternalSpecifier = ( type ReadFile = Extract; +/** The nearest inventoried manifest owns package type, even when it omits type. */ +const packageTypeForFile = ( + path: string, + context: SpecifierContext, +): "module" | "commonjs" | "unknown" | undefined => { + let directory = posix.dirname(path); + while (true) { + const manifest = packageMainForDirectory(directory, context); + if (manifest !== undefined) + return ( + context.packages.get(manifest.directory)?.type ?? + (manifest.main.state === "unknown" ? "unknown" : undefined) + ); + if ( + directory === "." || + directory === "/" || + posix.basename(directory) === "node_modules" + ) + return undefined; + directory = posix.dirname(directory); + } +}; + const appendFile = ( entry: ReadFile, context: SpecifierContext, @@ -339,7 +428,12 @@ const appendFile = ( const parsed = language === null || !PARSEABLE_REFERENCE_SOURCE_LANGUAGES.has(language) ? undefined - : parseReferenceSourceImports(entry.path, entry.bytes, language); + : parseReferenceSourceImports( + entry.path, + entry.bytes, + language, + packageTypeForFile(entry.path, context), + ); output.entries.push({ path: entry.path, kind: "file", @@ -371,8 +465,16 @@ const appendFile = ( context, mode, ); + if (resolved.limitation !== undefined) { + const limitation = `${entry.path}: ${JSON.stringify(graphRelationship.to)}: ${resolved.limitation}`; + output.limitations.push(limitation); + output.entries.at(-1)?.limitations.push(limitation); + } output.relationships.push({ ...graphRelationship, + ...(mode === "source-import" && /[?%#]/u.test(graphRelationship.to) + ? { source_specifier: graphRelationship.to } + : {}), to: resolved.to, resolution: resolved.resolution, }); @@ -434,11 +536,11 @@ export const parseReferenceSourceEntries = ( output.limitations.push( `Reference root enumeration is incomplete: ${read.rootFailure.message}`, ); - const packageMainInventory = readPackageMains(read, exclusions); + const packageInventory = readPackageMetadata(read, exclusions); const context: SpecifierContext = { lookupPath: createReferenceSourcePathLookup(filePaths), - packageMains: packageMainInventory.mains, - packageManifestPathLookup: packageMainInventory.pathLookup, + packages: packageInventory.packages, + packageManifestPathLookup: packageInventory.pathLookup, }; for (const entry of read.entries) { if (signal?.aborted === true) break; diff --git a/src/application/javascript/JavaScriptArtifactPathResolution.ts b/src/application/javascript/JavaScriptArtifactPathResolution.ts index b8d1ac4253..d46af98586 100644 --- a/src/application/javascript/JavaScriptArtifactPathResolution.ts +++ b/src/application/javascript/JavaScriptArtifactPathResolution.ts @@ -28,7 +28,10 @@ import { typescriptEmittedSourceCandidates, type TypeScriptEmittedSourceCandidate, } from "../TypeScriptEmittedSourceCandidates.js"; -import { isRelativeJavaScriptModuleSpecifier } from "../../domain/javascript/javascriptModuleSpecifier.js"; +import { + isRelativeJavaScriptModuleSpecifier, + decodeJavaScriptModuleUrlPath, +} from "../../domain/javascript/javascriptModuleSpecifier.js"; import { selectPackageSubpathPattern } from "../../domain/javascript/javascriptPackageSubpathPatterns.js"; import type { JavaScriptModuleResolutionMode } from "../../domain/javascript/javascriptModuleResolutionMode.js"; import type { JsonValue } from "../../domain/jsonValue.js"; @@ -752,24 +755,10 @@ const contextualCandidate = ( const decodeModuleUrlPath = ( declared: string, ): string | CandidateResolution => { - if (/%(?:2f|5c)/iu.test(declared)) - return unresolvedCandidate("rejected", [ - "ES module URL paths cannot contain percent-encoded slash or backslash bytes.", - ]); - let decoded: string; - try { - decoded = decodeURIComponent(declared); - } catch (cause: unknown) { - void cause; - return unresolvedCandidate("rejected", [ - "The module URL path contains malformed percent encoding.", - ]); - } - return decoded.includes("\0") - ? unresolvedCandidate("rejected", [ - "The decoded module URL path contains NUL.", - ]) - : decoded; + const decoded = decodeJavaScriptModuleUrlPath(declared); + return decoded.ok + ? decoded.value + : unresolvedCandidate("rejected", [decoded.error]); }; const relativeCandidate = (sourcePath: string, declared: string): string => diff --git a/src/domain/javascript/javascriptModuleSpecifier.ts b/src/domain/javascript/javascriptModuleSpecifier.ts index f52f081dcf..50d293ccd3 100644 --- a/src/domain/javascript/javascriptModuleSpecifier.ts +++ b/src/domain/javascript/javascriptModuleSpecifier.ts @@ -1,6 +1,27 @@ +import { err, ok, type Result } from "../result.js"; + /** Relative module syntax differs from an arbitrary dot-prefixed filename. */ export const isRelativeJavaScriptModuleSpecifier = (value: string): boolean => value === "." || value === ".." || value.startsWith("./") || value.startsWith("../"); + +/** Node file-module URL paths decode once and reject encoded separators. */ +export const decodeJavaScriptModuleUrlPath = ( + value: string, +): Result => { + if (/%(?:2f|5c)/iu.test(value)) + return err( + "ES module URL paths cannot contain percent-encoded slash or backslash bytes.", + ); + try { + const decoded = decodeURIComponent(value); + return decoded.includes("\0") + ? err("The decoded module URL path contains NUL.") + : ok(decoded); + } catch (cause: unknown) { + if (!(cause instanceof URIError)) throw cause; + return err("The module URL path contains malformed percent encoding."); + } +}; diff --git a/src/domain/referenceSourceClassification.ts b/src/domain/referenceSourceClassification.ts index fe8a6b28dd..fcfb8bd039 100644 --- a/src/domain/referenceSourceClassification.ts +++ b/src/domain/referenceSourceClassification.ts @@ -345,8 +345,9 @@ export const detectReferenceSourceLanguage = (path: string): string | null => { if (lower.endsWith(".d.cts")) return "TypeScript"; const parts = basenameParts(filename); - const extension = parts.ext.toLowerCase(); - return LANGUAGE_BY_EXTENSION.get(extension) ?? null; + if (parts.ext === "C") return "C++"; + if (parts.ext === "M") return "Objective-C++"; + return LANGUAGE_BY_EXTENSION.get(parts.ext.toLowerCase()) ?? null; }; /** Classify a reference source path deterministically. */ diff --git a/src/domain/referenceSourceGraph.ts b/src/domain/referenceSourceGraph.ts index deb1fb23fe..61aa2b4b66 100644 --- a/src/domain/referenceSourceGraph.ts +++ b/src/domain/referenceSourceGraph.ts @@ -119,6 +119,7 @@ const sourceEntrySchema = z.union([ const sourceRelationshipSchema = z.strictObject({ from_path: relativePathSchema, to: z.string().min(1), + source_specifier: z.string().min(1).exactOptional(), kind: z.enum(["imports", "requires", "references", "declares-module"]), resolution: z.enum(["internal", "external", "unresolved", "unknown"]), parse_state: z.enum(["parsed", "partial", "unknown"]), @@ -275,7 +276,7 @@ export const historicalSourceParseFailureKey = ( export const historicalSourceRelationshipKey = ( relationship: HistoricalSourceGraphInput["relationships"][number], ): string => - `${relationship.from_path}\u0000${relationship.to}\u0000${relationship.kind}\u0000${relationship.resolution}\u0000${relationship.parse_state}`; + `${relationship.from_path}\u0000${relationship.to}\u0000${relationship.kind}\u0000${relationship.resolution}\u0000${relationship.parse_state}${relationship.source_specifier === undefined ? "" : `\u0000${relationship.source_specifier}`}`; const checkSortedUnique = ( values: readonly string[], diff --git a/src/domain/referenceSourceImportParsing.ts b/src/domain/referenceSourceImportParsing.ts index ded8375331..5dbe082dee 100644 --- a/src/domain/referenceSourceImportParsing.ts +++ b/src/domain/referenceSourceImportParsing.ts @@ -1,7 +1,7 @@ import type { CallExpression, File, - ImportExpression, + OptionalCallExpression, Node, } from "@babel/types"; import { traverseJavaScriptAst } from "./javascript/javascriptSemanticTraversal.js"; @@ -21,6 +21,8 @@ import { isExportNamedDeclaration, isImportDeclaration, isImportExpression, + isOptionalCallExpression, + isTSImportType, isStringLiteral, isTSExternalModuleReference, isTSImportEqualsDeclaration, @@ -146,111 +148,57 @@ const appendImportEquals = ( }); }; -const extractImportDeclarations = ( - body: readonly Node[], +/** One AST owner retains historical declarations, types and runtime requests. */ +const extractImports = ( + ast: File, from_path: string, relationships: ReferenceSourceImportRelationship[], limitations: string[], + requireCalls: ReadonlyMap< + CallExpression | OptionalCallExpression, + ReferenceSourceRequireCall + >, ): void => { - for (const statement of body) { - if (isImportDeclaration(statement)) { - appendRelationship(relationships, limitations, { - fromPath: from_path, - to: safeModuleName(statement.source), - kind: "imports", - parseState: "parsed", - }); - continue; - } - - if (isExportNamedDeclaration(statement)) { - if (statement.source !== null && statement.source !== undefined) { + traverseJavaScriptAst(ast.program, { + enter: (node) => { + if ( + isImportDeclaration(node) || + isExportAllDeclaration(node) || + (isExportNamedDeclaration(node) && node.source != null) + ) { appendRelationship(relationships, limitations, { fromPath: from_path, - to: safeModuleName(statement.source), + to: safeModuleName(node.source), kind: "imports", parseState: "parsed", }); - } else if ( - statement.declaration !== null && - statement.declaration !== undefined - ) - appendImportEquals( - statement.declaration, - from_path, - relationships, - limitations, + } else if (isTSImportEqualsDeclaration(node)) { + appendImportEquals(node, from_path, relationships, limitations); + } else if (isTSModuleDeclaration(node)) { + if (node.declare === true && isStringLiteral(node.id)) + appendRelationship(relationships, limitations, { + fromPath: from_path, + to: node.id.value, + kind: "declares-module", + parseState: "parsed", + resolution: "unknown", + }); + } else if (isImportExpression(node) || isTSImportType(node)) { + const specifier = exactStringModuleName( + isTSImportType(node) ? node.argument : node.source, ); - continue; - } - - if (isExportAllDeclaration(statement)) { - appendRelationship(relationships, limitations, { - fromPath: from_path, - to: safeModuleName(statement.source), - kind: "imports", - parseState: "parsed", - }); - continue; - } - - if (isTSImportEqualsDeclaration(statement)) { - appendImportEquals(statement, from_path, relationships, limitations); - continue; - } - - if (isTSModuleDeclaration(statement)) { - if (statement.declare === true && isStringLiteral(statement.id)) { appendRelationship(relationships, limitations, { fromPath: from_path, - to: statement.id.value, - kind: "declares-module", - parseState: "parsed", - resolution: "unknown", + to: specifier ?? "", + kind: "imports", + parseState: specifier === undefined ? "partial" : "parsed", + ...(specifier === undefined ? { resolution: "unknown" } : {}), }); - } - } - } -}; - -const extractRequireAndDynamicImports = ( - body: readonly Node[], - from_path: string, - relationships: ReferenceSourceImportRelationship[], - limitations: string[], - requireCalls: ReadonlyMap, -): void => { - // Single traversal owner: iterative, VISITOR_KEYS-gated, no recursion over - // loc/comment objects. Survives deeply nested generated member chains that - // overflowed the previous hand-rolled Object.values walker. - const expressions: Array = []; - for (const statement of body) - traverseJavaScriptAst(statement, { - enter: (node) => { - if (isCallExpression(node) || isImportExpression(node)) - expressions.push(node); - }, - }); - - for (const call of expressions) { - if (isImportExpression(call)) { - const specifier = exactStringModuleName(call.source); - appendRelationship(relationships, limitations, { - fromPath: from_path, - to: specifier ?? "", - kind: "imports", - parseState: specifier === undefined ? "partial" : "parsed", - ...(specifier === undefined ? { resolution: "unknown" } : {}), - }); - continue; - } - // The shared exact-literal reader owns argument exactness: it unwraps - // runtime-transparent TS wrappers (`as const`, `!`, `satisfies`) and - // admits quoted and expression-free template spellings alike. - const requireCall = requireCalls.get(call); - if (requireCall !== undefined) { - const specifier = exactStringModuleName(requireCall.arguments?.[0]); - if (specifier !== undefined) { + } else if (isCallExpression(node) || isOptionalCallExpression(node)) { + const requireCall = requireCalls.get(node); + if (requireCall === undefined) return; + const specifier = exactStringModuleName(requireCall.arguments?.[0]); + if (specifier === undefined) return; const uncertainResolution = requireCall.resolutionBase === "unknown"; appendRelationship(relationships, limitations, { fromPath: from_path, @@ -262,14 +210,12 @@ const extractRequireAndDynamicImports = ( ? {} : { resolutionStrategy: "runtime-require" as const }), ...(typeof requireCall.resolutionBase === "object" - ? { - resolutionBasePath: requireCall.resolutionBase.path, - } + ? { resolutionBasePath: requireCall.resolutionBase.path } : {}), }); } - } - } + }, + }); }; const parseWithBabel = ( @@ -317,6 +263,7 @@ export const parseReferenceSourceImports = ( path: string, bytes: Uint8Array, language: string | null, + packageType?: "module" | "commonjs" | "unknown", ): ReferenceSourceImportParseResult => { if (language !== null && !PARSEABLE_REFERENCE_SOURCE_LANGUAGES.has(language)) return { relationships: [], parse_failures: [], limitations: [] }; @@ -364,13 +311,12 @@ export const parseReferenceSourceImports = ( } const relationships: ReferenceSourceImportRelationship[] = []; - extractImportDeclarations(ast.program.body, path, relationships, limitations); - extractRequireAndDynamicImports( - ast.program.body, + extractImports( + ast, path, relationships, limitations, - referenceSourceRequireCalls(ast, path), + referenceSourceRequireCalls(ast, path, packageType), ); return { diff --git a/src/domain/referenceSourceRequireBindings.ts b/src/domain/referenceSourceRequireBindings.ts index 4c9c15fd63..eb94ccf5d2 100644 --- a/src/domain/referenceSourceRequireBindings.ts +++ b/src/domain/referenceSourceRequireBindings.ts @@ -19,6 +19,7 @@ import { } from "./javascript/javascriptSemanticTraversal.js"; import { isExplicitModuleSourcePath, + javaScriptRuntimeContext, type ParsedJavaScriptSource, } from "./javascript/javascriptSourceParser.js"; @@ -36,10 +37,14 @@ export interface ReferenceSourceRequireCall { export const referenceSourceRequireCalls = ( file: Pick, path: string, -): ReadonlyMap => { + packageType?: "module" | "commonjs" | "unknown", +): ReadonlyMap< + t.CallExpression | t.OptionalCallExpression, + ReferenceSourceRequireCall +> => { // Unmarked paths depend on package scope or compiler options. Preserve their // candidate CommonJS edges; explicit module paths have no wrapper loader. - const wrapperMode = commonJsWrapperMode(path); + const wrapperMode = commonJsWrapperMode(path, file.program, packageType); const index = completeSemanticSteps( indexJavaScriptBindingsWithReceiversSteps( file.program, @@ -48,14 +53,15 @@ export const referenceSourceRequireCalls = ( ); const factoryBindings: ReadonlySet = new Set(); const candidates: { - readonly call: t.CallExpression; + readonly call: t.CallExpression | t.OptionalCallExpression; readonly identifier: t.Identifier; readonly arguments: JavaScriptRequireCallFact["arguments"]; readonly resolutionBase: ReferenceSourceRequireResolutionBase; }[] = []; traverseJavaScriptAst(file.program, { enter: (node) => { - if (!t.isCallExpression(node)) return; + if (!t.isCallExpression(node) && !t.isOptionalCallExpression(node)) + return; const identifier = callReceiverIdentifier(node.callee); if (identifier === undefined) return; let fact = semanticRequireLoaderAt(node, index.bindings, factoryBindings); @@ -97,7 +103,9 @@ export const referenceSourceRequireCalls = ( path: normalizedSourceRelativePath, } : fact === "commonjs-loader" && callFact.loader === "commonjs-loader" - ? "source-file" + ? path.endsWith(".js") && packageType === "unknown" + ? "unknown" + : "source-file" : "unknown"; const searchBase = requireResolveSearchBase( node, @@ -136,7 +144,7 @@ export const referenceSourceRequireCalls = ( /** `require.resolve` options replace the default search paths unless exact current-directory paths are supplied. */ const requireResolveSearchBase = ( - call: t.CallExpression, + call: t.CallExpression | t.OptionalCallExpression, callArguments: JavaScriptRequireCallFact["arguments"], bindings: JavaScriptSemanticAnalysisState, loaderBase: ReferenceSourceRequireResolutionBase, @@ -164,7 +172,9 @@ const requireResolveSearchBase = ( : "unknown"; }; -const isRequireResolveCall = (call: t.CallExpression): boolean => { +const isRequireResolveCall = ( + call: t.CallExpression | t.OptionalCallExpression, +): boolean => { const callee = call.callee; if (!t.isMemberExpression(callee) && !t.isOptionalMemberExpression(callee)) return false; @@ -175,24 +185,32 @@ const isRequireResolveCall = (call: t.CallExpression): boolean => { type CommonJsWrapperMode = "present" | "absent" | "unknown"; -const commonJsWrapperMode = (path: string): CommonJsWrapperMode => - path.endsWith(".cjs") - ? "present" - : path.endsWith(".mjs") - ? "absent" - : isExplicitModuleSourcePath(path) - ? "absent" - : "unknown"; +const commonJsWrapperMode = ( + path: string, + program: t.Program, + packageType: "module" | "commonjs" | "unknown" | undefined, +): CommonJsWrapperMode => { + if (path.endsWith(".cjs")) return "present"; + if (isExplicitModuleSourcePath(path)) return "absent"; + if (path.endsWith(".js")) { + if (packageType === "module") return "absent"; + if (packageType === "commonjs") return "present"; + if (packageType === "unknown") return "unknown"; + if (javaScriptRuntimeContext(program) === "module") return "absent"; + } + return "unknown"; +}; const hasPriorMemberMutation = ( index: JavaScriptReceiverIndex, - site: t.CallExpression, + site: t.CallExpression | t.OptionalCallExpression, identifier: t.Identifier, ): boolean => { const callee = site.callee; - const property = t.isMemberExpression(callee) - ? semanticStaticPropertyKey(callee.property, callee.computed) - : undefined; + const property = + t.isMemberExpression(callee) || t.isOptionalMemberExpression(callee) + ? semanticStaticPropertyKey(callee.property, callee.computed) + : undefined; if (property === undefined || property === null) return true; const facts = index.effectsForReferenceAt(site, identifier, [property]); return ( diff --git a/tests/boundary/cli/referenceSourceModuleSemantics.test.ts b/tests/boundary/cli/referenceSourceModuleSemantics.test.ts new file mode 100644 index 0000000000..c52bffbbf5 --- /dev/null +++ b/tests/boundary/cli/referenceSourceModuleSemantics.test.ts @@ -0,0 +1,203 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { expect } from "vitest"; + +import { historicalSourceGraphSchema } from "../../../src/domain/referenceSourceGraph.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "preserves historical module syntax, loader scope and URL request identity", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-reference-module-semantics-", + ); + const files: Record = { + "package.json": '{"type":"module"}\n', + "main.js": 'require("./dep.cjs"); export {};\n', + "dep.cjs": 'throw new Error("must never load");\n', + "optional.cjs": + 'require.resolve?.("./dep.cjs"); require?.resolve("./dep.cjs");\n', + "overwritten.cjs": + 'require.resolve = other; require.resolve?.("./dep.cjs");\n', + "shadowed.cjs": + 'function f(require) { require?.resolve("./dep.cjs"); }\n', + "types.d.ts": + 'declare module "consumer" { import type { Marker } from "dependency"; export type A = import("./dep").Marker; export type B = typeof import("./dep"); }\n', + "dep.ts": "export interface Marker { id: string }\n", + "nested/package.json": '{"type":"commonjs"}\n', + "nested/main.js": 'require("../dep.cjs");\n', + "unknown/package.json": "{invalid", + "unknown/main.js": 'require("../dep.cjs");\n', + "url.mjs": + 'import "./dep%20name.mjs?v=1#first"; import "./dep%20name.mjs?v=2#second"; import "./dep%2520name.mjs"; import "./bad%2Fname.mjs"; import "./bad%.mjs";\n', + "dep name.mjs": "export const actual = 1;\n", + "dep%20name.mjs": "export const literalPercent = 2;\n", + "literal.cjs": + 'require.resolve("./dep%20name.mjs"); require.resolve("./dep?literal.cjs");\n', + "dep?literal.cjs": 'throw new Error("must never load");\n', + "oracle.mjs": + 'import { fileURLToPath } from "node:url"; import { createRequire } from "node:module"; const require = createRequire(import.meta.url); console.log(JSON.stringify([fileURLToPath(import.meta.resolve("./dep%20name.mjs?v=1#first")), fileURLToPath(import.meta.resolve("./dep%2520name.mjs")), require.resolve("./dep%20name.mjs"), require.resolve("./dep?literal.cjs")]));\n', + }; + for (const directory of ["nested", "unknown"]) + await mkdir(join(root, directory)); + await Promise.all( + Object.entries(files).map(([path, source]) => + writeFile(join(root, path), source), + ), + ); + // This self-authored resolver control never loads a dependency. + const oracle = await promisify(execFile)( + process.execPath, + [join(root, "oracle.mjs")], + { timeout: 5_000 }, + ); + expect(JSON.parse(oracle.stdout)).toEqual( + [ + "dep name.mjs", + "dep%20name.mjs", + "dep%20name.mjs", + "dep?literal.cjs", + ].map((path) => join(root, path)), + ); + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const graph = historicalSourceGraphSchema.parse(result.json); + expect(graph.parse_failures).toEqual([]); + const edges = (from: string) => + graph.relationships.filter((edge) => edge.from_path === from); + expect(edges("main.js")).toEqual([]); + expect(edges("shadowed.cjs")).toEqual([]); + for (const from of ["optional.cjs", "nested/main.js"]) + expect(edges(from)).toEqual([ + { + from_path: from, + to: "dep.cjs", + kind: "requires", + resolution: "internal", + parse_state: "parsed", + }, + ]); + for (const [from, to] of [ + ["overwritten.cjs", "./dep.cjs"], + ["unknown/main.js", "../dep.cjs"], + ] as const) + expect(edges(from)).toEqual([ + { + from_path: from, + to, + kind: "requires", + resolution: "unknown", + parse_state: "partial", + }, + ]); + expect(edges("types.d.ts")).toEqual( + expect.arrayContaining([ + { + from_path: "types.d.ts", + to: "consumer", + kind: "declares-module", + resolution: "unknown", + parse_state: "parsed", + }, + { + from_path: "types.d.ts", + to: "dependency", + kind: "imports", + resolution: "external", + parse_state: "parsed", + }, + { + from_path: "types.d.ts", + to: "dep.ts", + kind: "imports", + resolution: "internal", + parse_state: "parsed", + }, + ]), + ); + for (const [source_specifier, to] of [ + ["./dep%20name.mjs?v=1#first", "dep name.mjs"], + ["./dep%20name.mjs?v=2#second", "dep name.mjs"], + ["./dep%2520name.mjs", "dep%20name.mjs"], + ]) + expect(edges("url.mjs")).toContainEqual({ + from_path: "url.mjs", + to, + source_specifier, + kind: "imports", + resolution: "internal", + parse_state: "parsed", + }); + expect( + edges("url.mjs").filter((edge) => edge.resolution === "unresolved"), + ).toHaveLength(2); + expect( + graph.entries.find((entry) => entry.path === "url.mjs")?.limitations, + ).toHaveLength(2); + expect( + edges("literal.cjs") + .map((edge) => edge.to) + .sort(), + ).toEqual(["dep%20name.mjs", "dep?literal.cjs"]); + for (const [path, source] of Object.entries(files)) + expect(graph.entries).toContainEqual( + expect.objectContaining({ + path, + sha256: createHash("sha256").update(source).digest("hex"), + }), + ); + }, +); + +cliTest( + "preserves case-sensitive compiler suffix languages in full graphs and summaries", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-reference-native-suffixes-", + ); + const languages = { + "upper.C": "C++", + "lower.c": "C", + "upper.M": "Objective-C++", + "lower.m": "Objective-C", + }; + await Promise.all( + Object.keys(languages).map((path) => + writeFile(join(root, path), "// inert source\n"), + ), + ); + const full = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + }); + expect(full.exitCode, full.stderr).toBe(0); + const graph = historicalSourceGraphSchema.parse(full.json); + expect( + Object.fromEntries( + graph.entries.map((entry) => [ + entry.path, + entry.kind === "file" ? entry.language : null, + ]), + ), + ).toEqual(languages); + const summary = await cli.run({ + arguments: [ + "import-reference-source", + root, + "--json", + "--detail", + "summary", + ], + }); + expect(summary.exitCode, summary.stderr).toBe(0); + expect(summary.json).toMatchObject({ + languages: ["C", "C++", "Objective-C", "Objective-C++"], + }); + }, +); diff --git a/tests/boundary/filesystem/referenceSourceComputedRequire.test.ts b/tests/boundary/filesystem/referenceSourceComputedRequire.test.ts index c8b19bcb65..3ad267d563 100644 --- a/tests/boundary/filesystem/referenceSourceComputedRequire.test.ts +++ b/tests/boundary/filesystem/referenceSourceComputedRequire.test.ts @@ -30,6 +30,12 @@ it("distinguishes computed require member values from literal members in source 'require["main"]', 'require[("resolve")]', 'require["res\\u006flve"]', + "require?.", + "require.resolve?.", + 'require["resolve"]?.', + "require?.resolve", + 'require?.["resolve"]', + "(require?.resolve)", ]; const ignoredSources = [ 'require[resolve]("./dep.js");', @@ -42,18 +48,12 @@ it("distinguishes computed require member values from literal members in source 'require[`resolve`]("./dep.js");', 'require["re" + "solve"]("./dep.js");', ...[ - "require?.", - "require.resolve?.", - 'require["resolve"]?.', "require.main?.", 'require["main"]?.', - "require?.resolve", - 'require?.["resolve"]', "require?.[resolve]", "require?.main", 'require?.["main"]', "require?.[main]", - "(require?.resolve)", "require.resolve.call", "require.main.require", "module.require",