diff --git a/docs/cli.md b/docs/cli.md index d052d8ba4a..a6957371ce 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -410,6 +410,13 @@ strings, expression-free templates and runtime-transparent TypeScript wrappers retain their targets; computed imports remain unknown. TypeScript import-equals dependencies remain `requires` relationships whether or not they are exported. +CommonJS `require` relationships retain the requested specifier. REA resolves +them against the source file only when Node's default source-relative loader is +proven; non-default `createRequire` bases and unsupported +`require.resolve` search options remain partial with unknown resolution. An +excluded `package.json` also leaves package-directory selection unknown rather +than treating the manifest as absent and applying an `index` fallback. + Historical-source import requires safe no-follow file opens on Linux or macOS. Native Windows returns `unsupported_host`; use Linux REA inside WSL or another supported host. See diff --git a/src/application/ReferenceSourceImport.ts b/src/application/ReferenceSourceImport.ts index f58043b0a5..4c27f6c457 100644 --- a/src/application/ReferenceSourceImport.ts +++ b/src/application/ReferenceSourceImport.ts @@ -210,7 +210,7 @@ export const importReferenceSource = async ( ); const { entries, relationships, parseFailures, limitations } = - parseReferenceSourceEntries(read, filePaths, options.signal); + parseReferenceSourceEntries(read, filePaths, exclusions, options.signal); if (isAborted(options.signal)) return err(cancelled(read)); diff --git a/src/application/ReferenceSourceImportEntries.test.ts b/src/application/ReferenceSourceImportEntries.test.ts index e895bcde86..9997c491c3 100644 --- a/src/application/ReferenceSourceImportEntries.test.ts +++ b/src/application/ReferenceSourceImportEntries.test.ts @@ -22,7 +22,7 @@ it("projects an unsupported non-regular entry as unknown kind instead of file", bytesRead: 0, limitations: [], }; - const parsed = parseReferenceSourceEntries(read, new Set()); + const parsed = parseReferenceSourceEntries(read, new Set(), []); expect(parsed.entries).toContainEqual({ path: "src/pipe", kind: "unknown", @@ -49,7 +49,7 @@ it("keeps a failed regular file as file kind", () => { bytesRead: 0, limitations: [], }; - const parsed = parseReferenceSourceEntries(read, new Set()); + const parsed = parseReferenceSourceEntries(read, new Set(), []); expect(parsed.entries).toContainEqual({ path: "src/broken.ts", kind: "file", @@ -88,6 +88,7 @@ it("keeps source entries when a module has an empty specifier", () => { const parsed = parseReferenceSourceEntries( read, new Set(files.map(([path]) => path)), + [], ); expect(parsed.entries.map(({ path }) => path)).toEqual([ @@ -136,6 +137,7 @@ it("retains Kotlin source identity and language across the inventory projection" const parsed = parseReferenceSourceEntries( read, new Set(files.map(([path]) => path)), + [], ); for (const [path, content, classifications, language] of files) expect(parsed.entries).toContainEqual({ @@ -193,6 +195,7 @@ it("resolves runtime require package mains without an index and before typed ind const parsed = parseReferenceSourceEntries( read, new Set(files.map(([path]) => path)), + [], ); expect(parsed.relationships).toContainEqual({ @@ -281,6 +284,7 @@ it("matches package manifests by exact or unique canonical directory identity", const parsed = parseReferenceSourceEntries( read, new Set(files.map(([path]) => path)), + [], ); expect(parsed.relationships.map(({ to }) => to)).toEqual([ @@ -319,6 +323,7 @@ it("resolves extensionless imports and requires to .mts and .cts files and direc const parsed = parseReferenceSourceEntries( read, new Set(files.map(([path]) => path)), + [], ); expect(parsed.relationships).toEqual( expect.arrayContaining([ diff --git a/src/application/ReferenceSourceImportEntries.ts b/src/application/ReferenceSourceImportEntries.ts index 268f9f8af0..2c22edb8a2 100644 --- a/src/application/ReferenceSourceImportEntries.ts +++ b/src/application/ReferenceSourceImportEntries.ts @@ -107,12 +107,18 @@ type PackageMain = */ const readPackageMains = ( read: ReferenceSourceRead, + exclusions: HistoricalSourceGraphInput["exclusions"], ): { readonly mains: ReadonlyMap; readonly pathLookup: ReferenceSourcePathLookup; } => { const mains = new Map(); const manifestPaths: string[] = []; + for (const exclusion of exclusions) { + if (posix.basename(exclusion.path) !== "package.json") continue; + manifestPaths.push(exclusion.path); + mains.set(posix.dirname(exclusion.path), { state: "unknown" }); + } for (const entry of read.entries) { if (posix.basename(entry.path) !== "package.json") continue; manifestPaths.push(entry.path); @@ -412,6 +418,7 @@ const appendReadEntry = ( export const parseReferenceSourceEntries = ( read: ReferenceSourceRead, filePaths: ReadonlySet, + exclusions: HistoricalSourceGraphInput["exclusions"], signal?: AbortSignal, ): ParsedReferenceSourceEntries => { const output: ParsedReferenceSourceEntries = { @@ -424,7 +431,7 @@ export const parseReferenceSourceEntries = ( output.limitations.push( `Reference root enumeration is incomplete: ${read.rootFailure.message}`, ); - const packageMainInventory = readPackageMains(read); + const packageMainInventory = readPackageMains(read, exclusions); const context: SpecifierContext = { lookupPath: createReferenceSourcePathLookup(filePaths), packageMains: packageMainInventory.mains, diff --git a/src/domain/referenceSourceImportParsing.ts b/src/domain/referenceSourceImportParsing.ts index 5b3259b7de..01306c6878 100644 --- a/src/domain/referenceSourceImportParsing.ts +++ b/src/domain/referenceSourceImportParsing.ts @@ -11,17 +11,15 @@ import { parseRecoveredJavaScript, parserPluginsForPath, } from "./javascript/javascriptSourceParser.js"; -import { unshadowedReferenceSourceRequireCalls } from "./referenceSourceRequireBindings.js"; +import { referenceSourceRequireCallBases } from "./referenceSourceRequireBindings.js"; import { isRelativeJavaScriptModuleSpecifier } from "./javascript/javascriptModuleSpecifier.js"; import { isCallExpression, isExportAllDeclaration, isExportNamedDeclaration, - isIdentifier, isImport, isImportDeclaration, isImportExpression, - isMemberExpression, isStringLiteral, isTSExternalModuleReference, isTSImportEqualsDeclaration, @@ -233,31 +231,12 @@ const extractImportDeclarations = ( } }; -const isRequireCallee = (callee: Node | null | undefined): boolean => { - if (isIdentifier(callee) && callee.name === "require") return true; - if ( - isMemberExpression(callee) && - isIdentifier(callee.object) && - callee.object.name === "require" && - ((!callee.computed && isIdentifier(callee.property)) || - isStringLiteral(callee.property)) - ) { - const propertyName = isIdentifier(callee.property) - ? callee.property.name - : isStringLiteral(callee.property) - ? callee.property.value - : undefined; - return propertyName === "resolve" || propertyName === "main"; - } - return false; -}; - const extractRequireAndDynamicImports = ( body: readonly Node[], from_path: string, relationships: ReferenceSourceImportRelationship[], limitations: string[], - unshadowedRequireCalls: ReadonlySet, + requireCallBases: ReadonlyMap, ): void => { // Single traversal owner: iterative, VISITOR_KEYS-gated, no recursion over // loc/comment objects. Survives deeply nested generated member chains that @@ -287,15 +266,20 @@ const extractRequireAndDynamicImports = ( // The shared exact-literal reader owns argument exactness: it unwraps // runtime-transparent TS wrappers (`as const`, `!`, `satisfies`) and // admits quoted and expression-free template spellings alike. - if (isRequireCallee(call.callee) && unshadowedRequireCalls.has(call)) { + const resolutionBase = requireCallBases.get(call); + if (resolutionBase !== undefined) { const result = moduleSpecifierFromExpression(first); if (result !== undefined) { + const uncertainResolution = resolutionBase === "unknown"; appendRelationship(relationships, limitations, { fromPath: from_path, to: result.specifier, kind: "requires", - parseState: result.parseState, - resolutionStrategy: "runtime-require", + parseState: uncertainResolution ? "partial" : result.parseState, + ...(uncertainResolution ? { resolution: "unknown" } : {}), + ...(uncertainResolution + ? {} + : { resolutionStrategy: "runtime-require" as const }), }); } continue; @@ -413,7 +397,7 @@ export const parseReferenceSourceImports = ( path, relationships, limitations, - unshadowedReferenceSourceRequireCalls(ast, path), + referenceSourceRequireCallBases(ast, path), ); return { diff --git a/src/domain/referenceSourceRequireBindings.ts b/src/domain/referenceSourceRequireBindings.ts index 773c052324..ec4c8563e1 100644 --- a/src/domain/referenceSourceRequireBindings.ts +++ b/src/domain/referenceSourceRequireBindings.ts @@ -1,11 +1,16 @@ import * as t from "@babel/types"; import { semanticStaticPropertyKey } from "./javascript/javascriptAstValues.js"; +import { isUnmodifiedGlobalReference } from "./javascript/javascriptSemanticAmbientWrites.js"; import { indexJavaScriptBindingsWithReceiversSteps, type JavaScriptReceiverIndex, } from "./javascript/javascriptSemanticAnalysis.js"; -import { semanticRequireLoaderAt } from "./javascript/javascriptSemanticRequireOrigin.js"; +import type { JavaScriptSemanticAnalysisState } from "./javascript/javascriptSemanticState.js"; +import { + semanticRequireCallAt, + semanticRequireLoaderAt, +} from "./javascript/javascriptSemanticRequireOrigin.js"; import { completeSemanticSteps, traverseJavaScriptAst, @@ -15,11 +20,13 @@ import { type ParsedJavaScriptSource, } from "./javascript/javascriptSourceParser.js"; -/** Recover source dependencies using the shared loader and mutation owners. */ -export const unshadowedReferenceSourceRequireCalls = ( +export type ReferenceSourceRequireResolutionBase = "source-file" | "unknown"; + +/** Identify exact calls through a source-owned Node loader and their resolver base. */ +export const referenceSourceRequireCallBases = ( file: Pick, path: string, -): ReadonlySet => { +): ReadonlyMap => { // Unmarked paths depend on package scope or compiler options. Preserve their // candidate CommonJS edges; explicit module paths have no wrapper loader. const wrapperMode = commonJsWrapperMode(path); @@ -33,6 +40,7 @@ export const unshadowedReferenceSourceRequireCalls = ( const candidates: { readonly call: t.CallExpression; readonly identifier: t.Identifier; + readonly resolutionBase: ReferenceSourceRequireResolutionBase; }[] = []; traverseJavaScriptAst(file.program, { enter: (node) => { @@ -47,22 +55,81 @@ export const unshadowedReferenceSourceRequireCalls = ( fact = semanticRequireLoaderAt(node, index.bindings, factoryBindings); } if ( - fact === "create-require" || - (fact === "commonjs-loader" && wrapperMode !== "absent") + fact !== "create-require" && + (fact !== "commonjs-loader" || wrapperMode === "absent") ) - candidates.push({ call: node, identifier }); + return; + const callFact = semanticRequireCallAt( + node, + index.bindings, + factoryBindings, + ); + const base = + fact === "create-require" + ? callFact?.sourceRelative === true + ? "source-file" + : "unknown" + : "source-file"; + candidates.push({ + call: node, + identifier, + resolutionBase: + base === "source-file" && + isDefaultRequireResolveSearch(node, index.bindings) + ? "source-file" + : "unknown", + }); }, }); if (candidates.some(({ call }) => !t.isIdentifier(call.callee))) completeSemanticSteps(index.indexMemberMutationFactsSteps()); - return new Set( + return new Map( candidates .filter( ({ call, identifier }) => t.isIdentifier(call.callee) || !hasPriorMemberMutation(index, call, identifier), ) - .map(({ call }) => call), + .map(({ call, resolutionBase }) => [call, resolutionBase] as const), + ); +}; + +/** `require.resolve` options replace the default search paths unless exact current-directory paths are supplied. */ +const isDefaultRequireResolveSearch = ( + call: t.CallExpression, + bindings: JavaScriptSemanticAnalysisState, +): boolean => { + if (!isRequireResolveCall(call) || call.arguments.length < 2) return true; + if (call.arguments.length !== 2) return false; + const options = t.isExpression(call.arguments[1]) + ? call.arguments[1] + : undefined; + if (options === undefined) return false; + if (!t.isObjectExpression(options)) return false; + if (options.properties.length === 0) return true; + if (options.properties.length !== 1) return false; + const property = options.properties[0]; + if (!t.isObjectProperty(property)) return false; + if (semanticStaticPropertyKey(property.key, property.computed) !== "paths") + return false; + const value = t.isExpression(property.value) ? property.value : undefined; + if (!t.isArrayExpression(value) || value.elements.length !== 1) return false; + const directory = value.elements[0]; + return ( + t.isIdentifier(directory, { name: "__dirname" }) && + isUnmodifiedGlobalReference(directory, bindings, "__dirname") + ); +}; + +const isRequireResolveCall = (call: t.CallExpression): boolean => { + const callee = call.callee; + if ( + !t.isMemberExpression(callee) || + !t.isIdentifier(callee.object, { name: "require" }) + ) + return false; + return ( + semanticStaticPropertyKey(callee.property, callee.computed) === "resolve" ); }; diff --git a/tests/boundary/cli/referenceSourceExcludedPackageManifest.test.ts b/tests/boundary/cli/referenceSourceExcludedPackageManifest.test.ts new file mode 100644 index 0000000000..0e30ef2772 --- /dev/null +++ b/tests/boundary/cli/referenceSourceExcludedPackageManifest.test.ts @@ -0,0 +1,64 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect } from "vitest"; + +import { historicalSourceGraphSchema } from "../../../src/domain/referenceSourceGraph.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "keeps package selection unknown when its manifest is excluded", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-reference-excluded-package-main-", + ); + const config = await createTestTempDirectory( + "rea-reference-excluded-package-main-config-", + ); + await mkdir(join(root, "pkg"), { recursive: true }); + await Promise.all([ + writeFile(join(root, ".gitignore"), "pkg/package.json\n"), + writeFile(join(root, "main.cjs"), 'require("./pkg");\n'), + writeFile(join(root, "pkg", "package.json"), '{"main":"actual.cjs"}\n'), + writeFile(join(root, "pkg", "actual.cjs"), "module.exports = 1;\n"), + writeFile(join(root, "pkg", "index.js"), "module.exports = 2;\n"), + ]); + + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + environment: { + HOME: config, + XDG_CONFIG_HOME: config, + XDG_CACHE_HOME: config, + }, + }); + expect(result.exitCode, result.stderr + JSON.stringify(result.json)).toBe( + 0, + ); + const graph = historicalSourceGraphSchema.parse(result.json); + expect(graph.exclusions).toContainEqual({ + path: "pkg/package.json", + entry_kind: "file", + reason: "project-ignored", + pattern: "pkg/package.json", + }); + expect(graph.entries.map(({ path }) => path)).toEqual( + expect.arrayContaining(["pkg/actual.cjs", "pkg/index.js"]), + ); + expect(graph.relationships).toContainEqual({ + from_path: "main.cjs", + to: "pkg", + kind: "requires", + resolution: "unresolved", + parse_state: "parsed", + }); + expect(graph.relationships).not.toContainEqual( + expect.objectContaining({ + from_path: "main.cjs", + to: "pkg/index.js", + resolution: "internal", + }), + ); + }, +); diff --git a/tests/boundary/cli/referenceSourceRequireResolutionBase.test.ts b/tests/boundary/cli/referenceSourceRequireResolutionBase.test.ts new file mode 100644 index 0000000000..5d1ab3cada --- /dev/null +++ b/tests/boundary/cli/referenceSourceRequireResolutionBase.test.ts @@ -0,0 +1,125 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect } from "vitest"; + +import { historicalSourceGraphSchema } from "../../../src/domain/referenceSourceGraph.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "preserves unknown CommonJS resolution when Node is given another base", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-reference-require-base-"); + const config = await createTestTempDirectory( + "rea-reference-require-base-config-", + ); + await mkdir(join(root, "sub"), { recursive: true }); + await Promise.all([ + writeFile( + join(root, "main.mjs"), + [ + 'import { createRequire } from "node:module";', + 'const custom = createRequire(new URL("./sub/entry.cjs", import.meta.url));', + 'custom("./dep.cjs");', + "const local = createRequire(import.meta.url);", + 'local("./dep.cjs");', + ].join("\n"), + ), + writeFile(join(root, "dep.cjs"), "module.exports = 'root';\n"), + writeFile(join(root, "sub", "dep.cjs"), "module.exports = 'sub';\n"), + ]); + + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + environment: { + HOME: config, + XDG_CONFIG_HOME: config, + XDG_CACHE_HOME: config, + }, + }); + expect(result.exitCode, result.stderr + JSON.stringify(result.json)).toBe( + 0, + ); + const graph = historicalSourceGraphSchema.parse(result.json); + expect(graph.relationships).toEqual( + expect.arrayContaining([ + { + from_path: "main.mjs", + to: "./dep.cjs", + kind: "requires", + resolution: "unknown", + parse_state: "partial", + }, + { + from_path: "main.mjs", + to: "dep.cjs", + kind: "requires", + resolution: "internal", + parse_state: "parsed", + }, + ]), + ); + }, +); + +cliTest( + "honors supported require.resolve search options and leaves alternate paths unknown", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-reference-require-resolve-paths-", + ); + const config = await createTestTempDirectory( + "rea-reference-require-resolve-paths-config-", + ); + await mkdir(join(root, "sub"), { recursive: true }); + await Promise.all([ + writeFile( + join(root, "main.cjs"), + [ + 'require.resolve("./dep.cjs");', + 'require.resolve("./dep.cjs", { paths: [__dirname] });', + 'require.resolve("./dep.cjs", { paths: [__dirname + "/sub"] });', + 'require.resolve("./dep.cjs", { paths: [] });', + ].join("\n"), + ), + writeFile(join(root, "dep.cjs"), "module.exports = 'root';\n"), + writeFile(join(root, "sub", "dep.cjs"), "module.exports = 'sub';\n"), + ]); + + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + environment: { + HOME: config, + XDG_CONFIG_HOME: config, + XDG_CACHE_HOME: config, + }, + }); + expect(result.exitCode, result.stderr + JSON.stringify(result.json)).toBe( + 0, + ); + const graph = historicalSourceGraphSchema.parse(result.json); + const relationships = graph.relationships.filter( + ({ from_path, kind }) => from_path === "main.cjs" && kind === "requires", + ); + expect(relationships).toEqual( + expect.arrayContaining([ + { + from_path: "main.cjs", + to: "dep.cjs", + kind: "requires", + resolution: "internal", + parse_state: "parsed", + }, + { + from_path: "main.cjs", + to: "./dep.cjs", + kind: "requires", + resolution: "unknown", + parse_state: "partial", + }, + ]), + ); + expect(relationships).toHaveLength(2); + }, +); diff --git a/tests/boundary/filesystem/referenceSourceImport.test.ts b/tests/boundary/filesystem/referenceSourceImport.test.ts index 9cb11b5053..e05c434d48 100644 --- a/tests/boundary/filesystem/referenceSourceImport.test.ts +++ b/tests/boundary/filesystem/referenceSourceImport.test.ts @@ -462,7 +462,7 @@ describe("reference source symlink import", () => { limitations: [], }; - const parsed = parseReferenceSourceEntries(read, new Set()); + const parsed = parseReferenceSourceEntries(read, new Set(), []); expect(parsed.entries).toContainEqual( expect.objectContaining({ kind: "symlink",