mirror of
https://github.com/morluto/rea
synced 2026-10-27 06:33:32 -04:00
fix(javascript): yield during per-file projection and graph construction (#1525)
analyze_javascript_application ran several long phases with no event-loop turn, so MCP ping and cancellation waited: each file's parse, analysis and semantic projection; building every application node and relationship before the application graph's first sealing step; and whole-graph schema parses inside both immutable graph factories. - Each file runs as steps (parse, static analysis, semantic analysis, then each projection pass), driven by the existing time-sliced checkpoint. - Application graph input construction yields between phases, per module link and per file of findings. - The pipeline transfers its privately built graph inputs to new sealing steps that validate the graph's own fields synchronously and elements, content checks and the digest in steps. The public copying factories keep their synchronous capture. Results are unchanged: graph IDs and the complete result digest match main on five inputs including Obsidian's 23 MB obsidian.asar. On the issue's 1,200-function fixture the longest event-loop block falls from about 290 ms to 90 ms, and ping after abort from up to 286 ms to at most 56 ms. On obsidian.asar it falls from 10.9 s to 5.4 s; the rest is one analyzer stage of a single very large file. Refs #1462 Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com>
This commit is contained in:
@@ -1,4 +1,9 @@
|
||||
import { z } from "zod";
|
||||
import {
|
||||
arrayContainerSchema,
|
||||
parseElementSteps,
|
||||
} from "../zodCollectionSteps.js";
|
||||
import { canonicalJsonDigestSteps } from "../canonicalJsonDigestSteps.js";
|
||||
import { canonicalJson } from "../comparisonSemantics.js";
|
||||
import { digestCanonicalValue } from "../canonicalDigest.js";
|
||||
import { uniqueSorted } from "../canonicalOrdering.js";
|
||||
@@ -93,11 +98,14 @@ type EdgeSemantic = Omit<ApplicationEdge, "edge_id">;
|
||||
const edgeId = (edge: EdgeSemantic): string =>
|
||||
`jag_edge_${digestCanonicalValue(edge, "JavaScript Application Graph")}`;
|
||||
|
||||
/** The issue sink shared by schema refinement and stepwise checking. */
|
||||
type GraphIssueContext = Pick<z.RefinementCtx, "addIssue">;
|
||||
|
||||
const sortedUniqueIssue = (
|
||||
values: readonly string[],
|
||||
path: PropertyKey[],
|
||||
label: string,
|
||||
context: z.RefinementCtx,
|
||||
context: GraphIssueContext,
|
||||
): void => {
|
||||
for (let index = 1; index < values.length; index += 1) {
|
||||
if (
|
||||
@@ -116,7 +124,7 @@ const sortedUniqueIssue = (
|
||||
const checkEvidenceNormalization = (
|
||||
evidence: ApplicationGraphEvidence,
|
||||
path: PropertyKey[],
|
||||
context: z.RefinementCtx,
|
||||
context: GraphIssueContext,
|
||||
): void => {
|
||||
sortedUniqueIssue(
|
||||
evidence.limitations,
|
||||
@@ -226,7 +234,7 @@ const observationMatchesIdentity = (
|
||||
const checkNode = (
|
||||
node: ApplicationNode,
|
||||
index: number,
|
||||
context: z.RefinementCtx,
|
||||
context: GraphIssueContext,
|
||||
): void => {
|
||||
if (node.node_id !== nodeId(node))
|
||||
context.addIssue({
|
||||
@@ -306,8 +314,16 @@ type GraphRecord = z.infer<typeof javascriptApplicationGraphRecordSchema>;
|
||||
|
||||
const checkGraphContent = (
|
||||
graph: JavaScriptApplicationGraphInput,
|
||||
context: z.RefinementCtx,
|
||||
context: GraphIssueContext,
|
||||
): void => {
|
||||
const steps = checkGraphContentSteps(graph, context);
|
||||
while (steps.next().done !== true);
|
||||
};
|
||||
|
||||
function* checkGraphContentSteps(
|
||||
graph: JavaScriptApplicationGraphInput,
|
||||
context: GraphIssueContext,
|
||||
): Generator<void, void> {
|
||||
sortedUniqueIssue(
|
||||
graph.root_node_ids,
|
||||
["root_node_ids"],
|
||||
@@ -353,9 +369,12 @@ const checkGraphContent = (
|
||||
message: "Root identifier must name a graph node",
|
||||
path: ["root_node_ids"],
|
||||
});
|
||||
for (const [index, node] of graph.nodes.entries())
|
||||
for (const [index, node] of graph.nodes.entries()) {
|
||||
if (index % 64 === 0) yield;
|
||||
checkNode(node, index, context);
|
||||
}
|
||||
for (const [index, edge] of graph.edges.entries()) {
|
||||
if (index % 64 === 0) yield;
|
||||
const { edge_id: identifier, ...semantic } = edge;
|
||||
if (identifier !== edgeId(semantic))
|
||||
context.addIssue({
|
||||
@@ -394,11 +413,11 @@ const checkGraphContent = (
|
||||
context,
|
||||
);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const checkGraphInvariants = (
|
||||
graph: GraphRecord,
|
||||
context: z.RefinementCtx,
|
||||
context: GraphIssueContext,
|
||||
): void => {
|
||||
checkGraphContent(graph, context);
|
||||
const { graph_id: identifier, ...semantic } = graph;
|
||||
@@ -475,22 +494,9 @@ export const createJavaScriptApplicationEdge = (
|
||||
export const createJavaScriptApplicationGraph = (
|
||||
input: unknown,
|
||||
): JavaScriptApplicationGraph => {
|
||||
const parsed = javascriptApplicationGraphInputSchema.parse(input);
|
||||
const semantic: JavaScriptApplicationGraphInput = {
|
||||
...parsed,
|
||||
root_node_ids: uniqueSorted(parsed.root_node_ids),
|
||||
nodes: [...parsed.nodes].sort((left, right) =>
|
||||
compareUnicodeCodePoints(left.node_id, right.node_id),
|
||||
),
|
||||
edges: [...parsed.edges].sort((left, right) =>
|
||||
compareUnicodeCodePoints(left.edge_id, right.edge_id),
|
||||
),
|
||||
coverage: {
|
||||
...parsed.coverage,
|
||||
limits: normalizeLimits(parsed.coverage.limits),
|
||||
},
|
||||
limitations: uniqueSorted(parsed.limitations),
|
||||
};
|
||||
const semantic = normalizeGraphInput(
|
||||
javascriptApplicationGraphInputSchema.parse(input),
|
||||
);
|
||||
const content = javascriptApplicationGraphContentSchema.parse(semantic);
|
||||
return {
|
||||
...content,
|
||||
@@ -498,6 +504,24 @@ export const createJavaScriptApplicationGraph = (
|
||||
};
|
||||
};
|
||||
|
||||
const normalizeGraphInput = (
|
||||
parsed: JavaScriptApplicationGraphInput,
|
||||
): JavaScriptApplicationGraphInput => ({
|
||||
...parsed,
|
||||
root_node_ids: uniqueSorted(parsed.root_node_ids),
|
||||
nodes: [...parsed.nodes].sort((left, right) =>
|
||||
compareUnicodeCodePoints(left.node_id, right.node_id),
|
||||
),
|
||||
edges: [...parsed.edges].sort((left, right) =>
|
||||
compareUnicodeCodePoints(left.edge_id, right.edge_id),
|
||||
),
|
||||
coverage: {
|
||||
...parsed.coverage,
|
||||
limits: normalizeLimits(parsed.coverage.limits),
|
||||
},
|
||||
limitations: uniqueSorted(parsed.limitations),
|
||||
});
|
||||
|
||||
const validatedImmutableApplicationGraphs = new WeakSet<object>();
|
||||
|
||||
/** Clone and validate synchronously, then return steps to seal the owned graph. */
|
||||
@@ -506,6 +530,65 @@ export const createImmutableJavaScriptApplicationGraphSteps = (
|
||||
): Generator<void, JavaScriptApplicationGraph> =>
|
||||
sealValidatedApplicationGraphSteps(createJavaScriptApplicationGraph(input));
|
||||
|
||||
const applicationGraphShape = javascriptApplicationGraphInputSchema.shape;
|
||||
const applicationGraphContainerSchema =
|
||||
javascriptApplicationGraphInputSchema.extend({
|
||||
nodes: arrayContainerSchema(applicationGraphShape.nodes),
|
||||
edges: arrayContainerSchema(applicationGraphShape.edges),
|
||||
});
|
||||
|
||||
/**
|
||||
* Seal a graph whose input the caller transfers: the caller neither retains nor
|
||||
* mutates any part of it afterwards. Only the graph's own fields are validated
|
||||
* synchronously; nodes and edges are validated, checked, hashed and frozen in
|
||||
* steps, because doing so at once held the event loop for large applications.
|
||||
*/
|
||||
export const sealTransferredJavaScriptApplicationGraphSteps = (
|
||||
input: unknown,
|
||||
): Generator<void, JavaScriptApplicationGraph> =>
|
||||
parseAndSealApplicationGraphSteps(
|
||||
applicationGraphContainerSchema.parse(input),
|
||||
);
|
||||
|
||||
function* parseAndSealApplicationGraphSteps(
|
||||
container: z.output<typeof applicationGraphContainerSchema>,
|
||||
): Generator<void, JavaScriptApplicationGraph> {
|
||||
const issues: z.core.$ZodIssue[] = [];
|
||||
const parsed: JavaScriptApplicationGraphInput = {
|
||||
...container,
|
||||
nodes: yield* parseElementSteps(
|
||||
applicationGraphShape.nodes.element,
|
||||
container.nodes,
|
||||
["nodes"],
|
||||
issues,
|
||||
),
|
||||
edges: yield* parseElementSteps(
|
||||
applicationGraphShape.edges.element,
|
||||
container.edges,
|
||||
["edges"],
|
||||
issues,
|
||||
),
|
||||
};
|
||||
if (issues.length > 0) throw new z.ZodError(issues);
|
||||
const content = normalizeGraphInput(parsed);
|
||||
let invalid = false;
|
||||
yield* checkGraphContentSteps(content, {
|
||||
addIssue: () => {
|
||||
invalid = true;
|
||||
},
|
||||
});
|
||||
// An invalid owned graph is a construction defect; report it with the exact
|
||||
// issues the content schema produces.
|
||||
if (invalid) {
|
||||
javascriptApplicationGraphContentSchema.parse(content);
|
||||
throw new TypeError("Application graph checks disagree with its schema");
|
||||
}
|
||||
return yield* sealValidatedApplicationGraphSteps({
|
||||
...content,
|
||||
graph_id: `jag_${yield* canonicalJsonDigestSteps(content)}`,
|
||||
});
|
||||
}
|
||||
|
||||
function* sealValidatedApplicationGraphSteps(
|
||||
graph: JavaScriptApplicationGraph,
|
||||
): Generator<void, JavaScriptApplicationGraph> {
|
||||
|
||||
Reference in New Issue
Block a user