diff --git a/docs/testing.md b/docs/testing.md index fff59c9512..51c8424281 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -72,6 +72,16 @@ preparation use that entrypoint; a raw Turbo compiler task without matching identity fails instead of publishing an ambiguous successful cache entry. A missing binary on a supported host also fails without silently falling back. +Development verifiers load their shared command, terminal and file owners from +source using Node's in-process TypeScript transformation. A single process-lifetime +loader handles this checkout's `src/` and `scripts/lib/` graphs, resolving missing +relative `.js`/`.mjs` imports to their `.ts`/`.mts` source. Existing JavaScript and +modules outside those roots keep native resolution and failures. Node's module +cache shares source identities within the verifier process; transformed output +is not cached across Node versions. Restart the verifier after source edits. +This avoids a cold-cache compiler subprocess surviving command cleanup; the +live process-lineage checks remain required. + The Rust compiler defaults to at most two checker threads. For `--noEmit` checks, `--maxMemory 3G` also sets a soft memory target; emit builds do not use that target. Explicit compiler arguments can override the defaults. These controls do not diff --git a/scripts/lib/verifier-source-loader.mjs b/scripts/lib/verifier-source-loader.mjs index 5b3229faea..c4b5607297 100644 --- a/scripts/lib/verifier-source-loader.mjs +++ b/scripts/lib/verifier-source-loader.mjs @@ -1,17 +1,114 @@ -import { createRequire } from "node:module"; +import { statSync } from "node:fs"; +import { registerHooks, stripTypeScriptTypes } from "node:module"; +import { extname, isAbsolute, relative, sep } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; -const requireFromTsx = createRequire(import.meta.resolve("tsx/package.json")); -let imports = 0; +const sourceRoots = [ + fileURLToPath(new URL("../../src/", import.meta.url)), + fileURLToPath(new URL("./", import.meta.url)), +]; +let hooks; -/** Load a verifier's source graph, then release the transform service that loaded it. */ -export async function importVerifierSource(url, parent) { - const { tsImport } = await import("tsx/esm/api"); - imports++; +const ownedSourcePath = (url) => { + if (!url.startsWith("file:")) return undefined; + const path = fileURLToPath(url); + return sourceRoots.some((root) => { + const selected = relative(root, path); + return ( + selected !== ".." && + !selected.startsWith(`..${sep}`) && + !isAbsolute(selected) + ); + }) + ? path + : undefined; +}; + +const errorCode = (cause) => + typeof cause === "object" && cause !== null && "code" in cause + ? cause.code + : undefined; + +const javascriptFormat = (format) => + format === "module-typescript" + ? "module" + : format === "commonjs-typescript" + ? "commonjs" + : format; + +const sourceResolution = (resolution) => { + const path = ownedSourcePath(resolution.url); + return path !== undefined && [".ts", ".mts"].includes(extname(path)) + ? { ...resolution, format: javascriptFormat(resolution.format) } + : resolution; +}; + +const resolveSource = (specifier, context, nextResolve) => { try { - return await tsImport(url, parent); - } finally { - // tsx's esbuild service otherwise outlives every owned command and appears - // as a surviving verifier descendant on a cold transform cache. - if (--imports === 0) requireFromTsx("esbuild").stop(); + return sourceResolution(nextResolve(specifier, context)); + } catch (cause) { + if ( + errorCode(cause) !== "ERR_MODULE_NOT_FOUND" || + context.parentURL === undefined || + ownedSourcePath(context.parentURL) === undefined || + !(specifier.startsWith("./") || specifier.startsWith("../")) + ) + throw cause; + const selected = new URL(specifier, context.parentURL); + const path = ownedSourcePath(selected.href); + if (path === undefined) throw cause; + const extension = extname(path); + if (extension !== ".js" && extension !== ".mjs") throw cause; + const source = + path.slice(0, -extension.length) + (extension === ".js" ? ".ts" : ".mts"); + try { + if (!statSync(source).isFile()) throw cause; + } catch (failure) { + if (["ENOENT", "ENOTDIR"].includes(errorCode(failure))) throw cause; + throw failure; + } + const candidate = pathToFileURL(source); + candidate.search = selected.search; + candidate.hash = selected.hash; + return sourceResolution(nextResolve(candidate.href, context)); } -} +}; + +const loadSource = (url, context, nextLoad) => { + const path = ownedSourcePath(url); + if (path === undefined || ![".ts", ".mts"].includes(extname(path))) + return nextLoad(url, context); + const loaded = nextLoad(url, context); + const format = javascriptFormat(loaded.format); + if (format !== "module" && format !== "commonjs") return loaded; + let source; + if (typeof loaded.source === "string") source = loaded.source; + else if (ArrayBuffer.isView(loaded.source)) + source = Buffer.from( + loaded.source.buffer, + loaded.source.byteOffset, + loaded.source.byteLength, + ).toString("utf8"); + else if (loaded.source instanceof ArrayBuffer) + source = Buffer.from(loaded.source).toString("utf8"); + else throw new TypeError(`Verifier module source was unavailable: ${url}`); + return { + ...loaded, + format, + source: stripTypeScriptTypes(source, { + mode: "transform", + sourceMap: true, + sourceUrl: url, + }), + }; +}; + +/** + * Import this checkout's verifier source without starting a compiler process. + * One process-lifetime hook and Node's URL-keyed module cache share the source + * graph across owners. No transformed output is cached across Node versions. + */ +export const importVerifierSource = async (url, parent) => { + hooks ??= registerHooks({ resolve: resolveSource, load: loadSource }); + return import(new URL(url, parent).href); +}; diff --git a/tests/boundary/process/verifierSourceLoading.test.ts b/tests/boundary/process/verifierSourceLoading.test.ts new file mode 100644 index 0000000000..cd10725503 --- /dev/null +++ b/tests/boundary/process/verifierSourceLoading.test.ts @@ -0,0 +1,229 @@ +import { copyFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { expect, onTestFinished } from "vitest"; +import { z } from "zod"; + +import { processTest } from "../../support/process/processFixture.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; + +const scriptUrl = (path: string) => + JSON.stringify(pathToFileURL(resolve("scripts/lib", path)).href); + +// Windows' root lineage observer explicitly reports unavailable; its command +// owners and source loading are covered by the native Windows boundary lane. +processTest.skipIf(process.platform === "win32")( + "shares cold verifier commands and closes their compiler-free source graph", + async ({ processes }) => { + const source = ` +import { mkdtemp, rm, access } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createVerifierRun, completeVerifierRun } from ${scriptUrl("verifier-run.mjs")}; +import { createVerifierCommands } from ${scriptUrl("verify-commands.mjs")}; +import { createPackageVerifierCommands } from ${scriptUrl("verify-package-commands.mjs")}; +import { withVerifierCleanup } from ${scriptUrl("verifier-cleanup.mjs")}; +const run = createVerifierRun(); +const root = await mkdtemp(join(tmpdir(), "rea-verifier-source-")); +let generic; +let packaged; +let outputs; +let failures; +await withVerifierCleanup(async () => { + generic = await createVerifierCommands(); + packaged = await createPackageVerifierCommands({ workspace: { + path: root, close: () => rm(root, { recursive: true, force: true }), + } }); + const owners = [["generic", generic], ["packaged", packaged]]; + outputs = await Promise.all(owners.map(([name, commands]) => + commands.exec(process.execPath, ["-e", "process.stdout.write(" + JSON.stringify(name) + ")"]), + )); + failures = await Promise.all(owners.map(([, commands]) => + commands.exec(process.execPath, ["-e", "process.exitCode = 2"]).catch((cause) => cause), + )); +}, [ + () => generic?.close(), + () => packaged === undefined ? rm(root, { recursive: true, force: true }) : packaged.close(), +]); +let removed = false; +try { await access(root); } +catch (cause) { if (cause.code !== "ENOENT") throw cause; removed = true; } +const observation = await completeVerifierRun(run); +console.log(JSON.stringify({ + outputs: outputs.map(({ stdout }) => stdout), removed, + sameFailureClass: failures[0].constructor === failures[1].constructor, + exitCodes: failures.map(({ code }) => code), + process_lineage: observation.process_lineage, +})); +`; + const result = await processes.run( + process.execPath, + ["--input-type=module", "--eval", source], + { env: { ...process.env, TSX_DISABLE_CACHE: "1" }, timeoutMs: 20_000 }, + ); + expect(result.exitCode, result.stderr).toBe(0); + const observation = z + .object({ + outputs: z.array(z.string()), + removed: z.boolean(), + sameFailureClass: z.boolean(), + exitCodes: z.array(z.number()), + process_lineage: z.object({ + status: z.string(), + descendants: z.array(z.unknown()).optional(), + }), + }) + .parse(JSON.parse(result.stdout)); + expect(observation).toMatchObject({ + outputs: ["generic", "packaged"], + removed: true, + sameFailureClass: true, + exitCodes: [2, 2], + process_lineage: { status: "verified", descendants: [] }, + }); + }, +); + +processTest( + "preserves a caller-owned compiler context while loading verifier source", + async ({ processes }) => { + const source = ` +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createPackageVerifierCommands } from ${scriptUrl("verify-package-commands.mjs")}; +import { withVerifierCleanup } from ${scriptUrl("verifier-cleanup.mjs")}; +const compiler = createRequire(import.meta.resolve("tsx/package.json"))("esbuild"); +let context; +let root; +let commands; +await withVerifierCleanup(async () => { + context = await compiler.context({ + stdin: { contents: "module.exports = 7;" }, + format: "cjs", write: false, logLevel: "silent", + }); + await context.rebuild(); + root = await mkdtemp(join(tmpdir(), "rea-verifier-compiler-owner-")); + commands = await createPackageVerifierCommands({ workspace: { + path: root, close: () => rm(root, { recursive: true, force: true }), + } }); + await commands.close(); + const result = await context.rebuild(); + assert.equal(result.outputFiles.length, 1); + const module = { exports: undefined }; + new Function("module", result.outputFiles[0].text)(module); + assert.equal(module.exports, 7); + console.log(JSON.stringify({ retainedContext: true, value: module.exports })); +}, [ + () => commands?.close() ?? (root === undefined ? undefined : rm(root, { recursive: true, force: true })), + async () => { try { await context?.dispose(); } finally { compiler.stop(); } }, +]); +`; + const result = await processes.run( + process.execPath, + ["--input-type=module", "--eval", source], + { timeoutMs: 20_000, env: { ...process.env, TSX_DISABLE_CACHE: "1" } }, + ); + expect(result.exitCode, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + retainedContext: true, + value: 7, + }); + }, +); + +processTest( + "loads one scoped TypeScript graph and preserves native module failures", + async ({ processes }) => { + const workspace = await createTestWorkspace("rea-verifier-source-loader-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + await workspace.mkdir("scripts/lib"); + await copyFile( + resolve("scripts/lib/verifier-source-loader.mjs"), + workspace.path("scripts/lib/verifier-source-loader.mjs"), + ); + await Promise.all([ + workspace.write("package.json", '{"type":"module"}'), + workspace.write( + "src/value.ts", + "export enum Kind { Ready = 7 } export class Box { constructor(readonly value: number) {} }", + ), + workspace.write("src/peer.mts", "export const peer: number = 9;"), + workspace.write("src/preferred.mjs", 'export const preferred = "js";'), + workspace.write("src/attributes.ts", "export const value: number = 1;"), + workspace.write("src/common/package.json", '{"type":"commonjs"}'), + workspace.write( + "src/common/value.ts", + "class Box { constructor(readonly value: number) {} } module.exports = new Box(12);", + ), + workspace.write( + "src/preferred.mts", + 'throw new Error("Existing JavaScript must win");', + ), + workspace.write( + "src/selected space.mts", + "export const selected: string = import.meta.url;", + ), + workspace.write("src/broken.mjs", "export const broken = ;"), + workspace.write( + "src/broken.mts", + "globalThis.fallbackUsed = true; export const broken = 1;", + ), + workspace.write("outside/entry.mjs", 'import "./typed.js";'), + workspace.write("outside/typed.ts", "export const outside: number = 1;"), + workspace.write( + "scripts/lib/owner.mts", + [ + 'import { Kind, Box } from "../../src/value.js";', + 'import { peer } from "../../src/peer.mjs";', + 'import { preferred } from "../../src/preferred.mjs";', + 'import { selected } from "../../src/selected%20space.mjs?caller=one%20two#origin";', + "export const facts = { value: new Box(Kind.Ready).value, peer, preferred, selected };", + ].join("\n"), + ), + workspace.write( + "probe.mjs", + ` +import { importVerifierSource } from "./scripts/lib/verifier-source-loader.mjs"; +const url = new URL("./scripts/lib/owner.mts", import.meta.url); +const first = await importVerifierSource(url); +const second = await importVerifierSource(url); +const foreign = await import("./outside/entry.mjs").then(() => "accepted", (cause) => cause.code); +const broken = await importVerifierSource(new URL("./src/broken.mjs", import.meta.url)).then(() => "accepted", (cause) => cause.name); +const attributes = await import("./src/attributes.ts", { with: { type: "json" } }).then(() => "accepted", (cause) => cause.code); +const common = await importVerifierSource(new URL("./src/common/value.ts", import.meta.url)); +const selected = new URL(first.facts.selected); +console.log(JSON.stringify({ + ...first.facts, selected: { query: selected.search, fragment: selected.hash }, + sameModule: first === second, foreign, broken, attributes, common: common.default.value, + fallbackUsed: globalThis.fallbackUsed === true, +})); +`, + ), + ]); + const result = await processes.run( + process.execPath, + [workspace.path("probe.mjs")], + { cwd: workspace.root, timeoutMs: 20_000 }, + ); + expect(result.exitCode, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + value: 7, + peer: 9, + preferred: "js", + selected: { query: "?caller=one%20two", fragment: "#origin" }, + sameModule: true, + foreign: "ERR_MODULE_NOT_FOUND", + broken: "SyntaxError", + attributes: "ERR_IMPORT_ATTRIBUTE_TYPE_INCOMPATIBLE", + common: 12, + fallbackUsed: false, + }); + }, +);