mirror of
https://github.com/morluto/rea
synced 2026-10-25 22:08:11 -04:00
fix(verifiers): load shared source without compiler processes (#7012)
This commit is contained in:
gensokyo-dev
@@ -72,6 +72,16 @@ preparation use that entrypoint; a raw Turbo compiler task without matching
|
||||
identity fails instead of publishing an ambiguous successful cache entry.
|
||||
A missing binary on a supported host also fails without silently falling back.
|
||||
|
||||
Development verifiers load their shared command, terminal and file owners from
|
||||
source using Node's in-process TypeScript transformation. A single process-lifetime
|
||||
loader handles this checkout's `src/` and `scripts/lib/` graphs, resolving missing
|
||||
relative `.js`/`.mjs` imports to their `.ts`/`.mts` source. Existing JavaScript and
|
||||
modules outside those roots keep native resolution and failures. Node's module
|
||||
cache shares source identities within the verifier process; transformed output
|
||||
is not cached across Node versions. Restart the verifier after source edits.
|
||||
This avoids a cold-cache compiler subprocess surviving command cleanup; the
|
||||
live process-lineage checks remain required.
|
||||
|
||||
The Rust compiler defaults to at most two checker threads. For `--noEmit` checks,
|
||||
`--maxMemory 3G` also sets a soft memory target; emit builds do not use that target.
|
||||
Explicit compiler arguments can override the defaults. These controls do not
|
||||
|
||||
@@ -1,17 +1,114 @@
|
||||
import { createRequire } from "node:module";
|
||||
import { statSync } from "node:fs";
|
||||
import { registerHooks, stripTypeScriptTypes } from "node:module";
|
||||
import { extname, isAbsolute, relative, sep } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
const requireFromTsx = createRequire(import.meta.resolve("tsx/package.json"));
|
||||
let imports = 0;
|
||||
const sourceRoots = [
|
||||
fileURLToPath(new URL("../../src/", import.meta.url)),
|
||||
fileURLToPath(new URL("./", import.meta.url)),
|
||||
];
|
||||
let hooks;
|
||||
|
||||
/** Load a verifier's source graph, then release the transform service that loaded it. */
|
||||
export async function importVerifierSource(url, parent) {
|
||||
const { tsImport } = await import("tsx/esm/api");
|
||||
imports++;
|
||||
const ownedSourcePath = (url) => {
|
||||
if (!url.startsWith("file:")) return undefined;
|
||||
const path = fileURLToPath(url);
|
||||
return sourceRoots.some((root) => {
|
||||
const selected = relative(root, path);
|
||||
return (
|
||||
selected !== ".." &&
|
||||
!selected.startsWith(`..${sep}`) &&
|
||||
!isAbsolute(selected)
|
||||
);
|
||||
})
|
||||
? path
|
||||
: undefined;
|
||||
};
|
||||
|
||||
const errorCode = (cause) =>
|
||||
typeof cause === "object" && cause !== null && "code" in cause
|
||||
? cause.code
|
||||
: undefined;
|
||||
|
||||
const javascriptFormat = (format) =>
|
||||
format === "module-typescript"
|
||||
? "module"
|
||||
: format === "commonjs-typescript"
|
||||
? "commonjs"
|
||||
: format;
|
||||
|
||||
const sourceResolution = (resolution) => {
|
||||
const path = ownedSourcePath(resolution.url);
|
||||
return path !== undefined && [".ts", ".mts"].includes(extname(path))
|
||||
? { ...resolution, format: javascriptFormat(resolution.format) }
|
||||
: resolution;
|
||||
};
|
||||
|
||||
const resolveSource = (specifier, context, nextResolve) => {
|
||||
try {
|
||||
return await tsImport(url, parent);
|
||||
} finally {
|
||||
// tsx's esbuild service otherwise outlives every owned command and appears
|
||||
// as a surviving verifier descendant on a cold transform cache.
|
||||
if (--imports === 0) requireFromTsx("esbuild").stop();
|
||||
return sourceResolution(nextResolve(specifier, context));
|
||||
} catch (cause) {
|
||||
if (
|
||||
errorCode(cause) !== "ERR_MODULE_NOT_FOUND" ||
|
||||
context.parentURL === undefined ||
|
||||
ownedSourcePath(context.parentURL) === undefined ||
|
||||
!(specifier.startsWith("./") || specifier.startsWith("../"))
|
||||
)
|
||||
throw cause;
|
||||
const selected = new URL(specifier, context.parentURL);
|
||||
const path = ownedSourcePath(selected.href);
|
||||
if (path === undefined) throw cause;
|
||||
const extension = extname(path);
|
||||
if (extension !== ".js" && extension !== ".mjs") throw cause;
|
||||
const source =
|
||||
path.slice(0, -extension.length) + (extension === ".js" ? ".ts" : ".mts");
|
||||
try {
|
||||
if (!statSync(source).isFile()) throw cause;
|
||||
} catch (failure) {
|
||||
if (["ENOENT", "ENOTDIR"].includes(errorCode(failure))) throw cause;
|
||||
throw failure;
|
||||
}
|
||||
const candidate = pathToFileURL(source);
|
||||
candidate.search = selected.search;
|
||||
candidate.hash = selected.hash;
|
||||
return sourceResolution(nextResolve(candidate.href, context));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const loadSource = (url, context, nextLoad) => {
|
||||
const path = ownedSourcePath(url);
|
||||
if (path === undefined || ![".ts", ".mts"].includes(extname(path)))
|
||||
return nextLoad(url, context);
|
||||
const loaded = nextLoad(url, context);
|
||||
const format = javascriptFormat(loaded.format);
|
||||
if (format !== "module" && format !== "commonjs") return loaded;
|
||||
let source;
|
||||
if (typeof loaded.source === "string") source = loaded.source;
|
||||
else if (ArrayBuffer.isView(loaded.source))
|
||||
source = Buffer.from(
|
||||
loaded.source.buffer,
|
||||
loaded.source.byteOffset,
|
||||
loaded.source.byteLength,
|
||||
).toString("utf8");
|
||||
else if (loaded.source instanceof ArrayBuffer)
|
||||
source = Buffer.from(loaded.source).toString("utf8");
|
||||
else throw new TypeError(`Verifier module source was unavailable: ${url}`);
|
||||
return {
|
||||
...loaded,
|
||||
format,
|
||||
source: stripTypeScriptTypes(source, {
|
||||
mode: "transform",
|
||||
sourceMap: true,
|
||||
sourceUrl: url,
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Import this checkout's verifier source without starting a compiler process.
|
||||
* One process-lifetime hook and Node's URL-keyed module cache share the source
|
||||
* graph across owners. No transformed output is cached across Node versions.
|
||||
*/
|
||||
export const importVerifierSource = async (url, parent) => {
|
||||
hooks ??= registerHooks({ resolve: resolveSource, load: loadSource });
|
||||
return import(new URL(url, parent).href);
|
||||
};
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
import { copyFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
import { expect, onTestFinished } from "vitest";
|
||||
import { z } from "zod";
|
||||
|
||||
import { processTest } from "../../support/process/processFixture.js";
|
||||
import {
|
||||
createTestWorkspace,
|
||||
removeTestWorkspace,
|
||||
} from "../../support/workspace/workspaceFixture.js";
|
||||
|
||||
const scriptUrl = (path: string) =>
|
||||
JSON.stringify(pathToFileURL(resolve("scripts/lib", path)).href);
|
||||
|
||||
// Windows' root lineage observer explicitly reports unavailable; its command
|
||||
// owners and source loading are covered by the native Windows boundary lane.
|
||||
processTest.skipIf(process.platform === "win32")(
|
||||
"shares cold verifier commands and closes their compiler-free source graph",
|
||||
async ({ processes }) => {
|
||||
const source = `
|
||||
import { mkdtemp, rm, access } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { createVerifierRun, completeVerifierRun } from ${scriptUrl("verifier-run.mjs")};
|
||||
import { createVerifierCommands } from ${scriptUrl("verify-commands.mjs")};
|
||||
import { createPackageVerifierCommands } from ${scriptUrl("verify-package-commands.mjs")};
|
||||
import { withVerifierCleanup } from ${scriptUrl("verifier-cleanup.mjs")};
|
||||
const run = createVerifierRun();
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-verifier-source-"));
|
||||
let generic;
|
||||
let packaged;
|
||||
let outputs;
|
||||
let failures;
|
||||
await withVerifierCleanup(async () => {
|
||||
generic = await createVerifierCommands();
|
||||
packaged = await createPackageVerifierCommands({ workspace: {
|
||||
path: root, close: () => rm(root, { recursive: true, force: true }),
|
||||
} });
|
||||
const owners = [["generic", generic], ["packaged", packaged]];
|
||||
outputs = await Promise.all(owners.map(([name, commands]) =>
|
||||
commands.exec(process.execPath, ["-e", "process.stdout.write(" + JSON.stringify(name) + ")"]),
|
||||
));
|
||||
failures = await Promise.all(owners.map(([, commands]) =>
|
||||
commands.exec(process.execPath, ["-e", "process.exitCode = 2"]).catch((cause) => cause),
|
||||
));
|
||||
}, [
|
||||
() => generic?.close(),
|
||||
() => packaged === undefined ? rm(root, { recursive: true, force: true }) : packaged.close(),
|
||||
]);
|
||||
let removed = false;
|
||||
try { await access(root); }
|
||||
catch (cause) { if (cause.code !== "ENOENT") throw cause; removed = true; }
|
||||
const observation = await completeVerifierRun(run);
|
||||
console.log(JSON.stringify({
|
||||
outputs: outputs.map(({ stdout }) => stdout), removed,
|
||||
sameFailureClass: failures[0].constructor === failures[1].constructor,
|
||||
exitCodes: failures.map(({ code }) => code),
|
||||
process_lineage: observation.process_lineage,
|
||||
}));
|
||||
`;
|
||||
const result = await processes.run(
|
||||
process.execPath,
|
||||
["--input-type=module", "--eval", source],
|
||||
{ env: { ...process.env, TSX_DISABLE_CACHE: "1" }, timeoutMs: 20_000 },
|
||||
);
|
||||
expect(result.exitCode, result.stderr).toBe(0);
|
||||
const observation = z
|
||||
.object({
|
||||
outputs: z.array(z.string()),
|
||||
removed: z.boolean(),
|
||||
sameFailureClass: z.boolean(),
|
||||
exitCodes: z.array(z.number()),
|
||||
process_lineage: z.object({
|
||||
status: z.string(),
|
||||
descendants: z.array(z.unknown()).optional(),
|
||||
}),
|
||||
})
|
||||
.parse(JSON.parse(result.stdout));
|
||||
expect(observation).toMatchObject({
|
||||
outputs: ["generic", "packaged"],
|
||||
removed: true,
|
||||
sameFailureClass: true,
|
||||
exitCodes: [2, 2],
|
||||
process_lineage: { status: "verified", descendants: [] },
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
processTest(
|
||||
"preserves a caller-owned compiler context while loading verifier source",
|
||||
async ({ processes }) => {
|
||||
const source = `
|
||||
import assert from "node:assert/strict";
|
||||
import { createRequire } from "node:module";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { createPackageVerifierCommands } from ${scriptUrl("verify-package-commands.mjs")};
|
||||
import { withVerifierCleanup } from ${scriptUrl("verifier-cleanup.mjs")};
|
||||
const compiler = createRequire(import.meta.resolve("tsx/package.json"))("esbuild");
|
||||
let context;
|
||||
let root;
|
||||
let commands;
|
||||
await withVerifierCleanup(async () => {
|
||||
context = await compiler.context({
|
||||
stdin: { contents: "module.exports = 7;" },
|
||||
format: "cjs", write: false, logLevel: "silent",
|
||||
});
|
||||
await context.rebuild();
|
||||
root = await mkdtemp(join(tmpdir(), "rea-verifier-compiler-owner-"));
|
||||
commands = await createPackageVerifierCommands({ workspace: {
|
||||
path: root, close: () => rm(root, { recursive: true, force: true }),
|
||||
} });
|
||||
await commands.close();
|
||||
const result = await context.rebuild();
|
||||
assert.equal(result.outputFiles.length, 1);
|
||||
const module = { exports: undefined };
|
||||
new Function("module", result.outputFiles[0].text)(module);
|
||||
assert.equal(module.exports, 7);
|
||||
console.log(JSON.stringify({ retainedContext: true, value: module.exports }));
|
||||
}, [
|
||||
() => commands?.close() ?? (root === undefined ? undefined : rm(root, { recursive: true, force: true })),
|
||||
async () => { try { await context?.dispose(); } finally { compiler.stop(); } },
|
||||
]);
|
||||
`;
|
||||
const result = await processes.run(
|
||||
process.execPath,
|
||||
["--input-type=module", "--eval", source],
|
||||
{ timeoutMs: 20_000, env: { ...process.env, TSX_DISABLE_CACHE: "1" } },
|
||||
);
|
||||
expect(result.exitCode, result.stderr).toBe(0);
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
retainedContext: true,
|
||||
value: 7,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
processTest(
|
||||
"loads one scoped TypeScript graph and preserves native module failures",
|
||||
async ({ processes }) => {
|
||||
const workspace = await createTestWorkspace("rea-verifier-source-loader-");
|
||||
onTestFinished(() => removeTestWorkspace(workspace.root));
|
||||
await workspace.mkdir("scripts/lib");
|
||||
await copyFile(
|
||||
resolve("scripts/lib/verifier-source-loader.mjs"),
|
||||
workspace.path("scripts/lib/verifier-source-loader.mjs"),
|
||||
);
|
||||
await Promise.all([
|
||||
workspace.write("package.json", '{"type":"module"}'),
|
||||
workspace.write(
|
||||
"src/value.ts",
|
||||
"export enum Kind { Ready = 7 } export class Box { constructor(readonly value: number) {} }",
|
||||
),
|
||||
workspace.write("src/peer.mts", "export const peer: number = 9;"),
|
||||
workspace.write("src/preferred.mjs", 'export const preferred = "js";'),
|
||||
workspace.write("src/attributes.ts", "export const value: number = 1;"),
|
||||
workspace.write("src/common/package.json", '{"type":"commonjs"}'),
|
||||
workspace.write(
|
||||
"src/common/value.ts",
|
||||
"class Box { constructor(readonly value: number) {} } module.exports = new Box(12);",
|
||||
),
|
||||
workspace.write(
|
||||
"src/preferred.mts",
|
||||
'throw new Error("Existing JavaScript must win");',
|
||||
),
|
||||
workspace.write(
|
||||
"src/selected space.mts",
|
||||
"export const selected: string = import.meta.url;",
|
||||
),
|
||||
workspace.write("src/broken.mjs", "export const broken = ;"),
|
||||
workspace.write(
|
||||
"src/broken.mts",
|
||||
"globalThis.fallbackUsed = true; export const broken = 1;",
|
||||
),
|
||||
workspace.write("outside/entry.mjs", 'import "./typed.js";'),
|
||||
workspace.write("outside/typed.ts", "export const outside: number = 1;"),
|
||||
workspace.write(
|
||||
"scripts/lib/owner.mts",
|
||||
[
|
||||
'import { Kind, Box } from "../../src/value.js";',
|
||||
'import { peer } from "../../src/peer.mjs";',
|
||||
'import { preferred } from "../../src/preferred.mjs";',
|
||||
'import { selected } from "../../src/selected%20space.mjs?caller=one%20two#origin";',
|
||||
"export const facts = { value: new Box(Kind.Ready).value, peer, preferred, selected };",
|
||||
].join("\n"),
|
||||
),
|
||||
workspace.write(
|
||||
"probe.mjs",
|
||||
`
|
||||
import { importVerifierSource } from "./scripts/lib/verifier-source-loader.mjs";
|
||||
const url = new URL("./scripts/lib/owner.mts", import.meta.url);
|
||||
const first = await importVerifierSource(url);
|
||||
const second = await importVerifierSource(url);
|
||||
const foreign = await import("./outside/entry.mjs").then(() => "accepted", (cause) => cause.code);
|
||||
const broken = await importVerifierSource(new URL("./src/broken.mjs", import.meta.url)).then(() => "accepted", (cause) => cause.name);
|
||||
const attributes = await import("./src/attributes.ts", { with: { type: "json" } }).then(() => "accepted", (cause) => cause.code);
|
||||
const common = await importVerifierSource(new URL("./src/common/value.ts", import.meta.url));
|
||||
const selected = new URL(first.facts.selected);
|
||||
console.log(JSON.stringify({
|
||||
...first.facts, selected: { query: selected.search, fragment: selected.hash },
|
||||
sameModule: first === second, foreign, broken, attributes, common: common.default.value,
|
||||
fallbackUsed: globalThis.fallbackUsed === true,
|
||||
}));
|
||||
`,
|
||||
),
|
||||
]);
|
||||
const result = await processes.run(
|
||||
process.execPath,
|
||||
[workspace.path("probe.mjs")],
|
||||
{ cwd: workspace.root, timeoutMs: 20_000 },
|
||||
);
|
||||
expect(result.exitCode, result.stderr).toBe(0);
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
value: 7,
|
||||
peer: 9,
|
||||
preferred: "js",
|
||||
selected: { query: "?caller=one%20two", fragment: "#origin" },
|
||||
sameModule: true,
|
||||
foreign: "ERR_MODULE_NOT_FOUND",
|
||||
broken: "SyntaxError",
|
||||
attributes: "ERR_IMPORT_ATTRIBUTE_TYPE_INCOMPATIBLE",
|
||||
common: 12,
|
||||
fallbackUsed: false,
|
||||
});
|
||||
},
|
||||
);
|
||||
Reference in New Issue
Block a user