From 5fdb0636b4734cabfa3147e8fcc94976c3bd8aa4 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:14:37 -0700 Subject: [PATCH] fix(javascript): yield during per-file projection and graph construction (#1525) analyze_javascript_application ran several long phases with no event-loop turn, so MCP ping and cancellation waited: each file's parse, analysis and semantic projection; building every application node and relationship before the application graph's first sealing step; and whole-graph schema parses inside both immutable graph factories. - Each file runs as steps (parse, static analysis, semantic analysis, then each projection pass), driven by the existing time-sliced checkpoint. - Application graph input construction yields between phases, per module link and per file of findings. - The pipeline transfers its privately built graph inputs to new sealing steps that validate the graph's own fields synchronously and elements, content checks and the digest in steps. The public copying factories keep their synchronous capture. Results are unchanged: graph IDs and the complete result digest match main on five inputs including Obsidian's 23 MB obsidian.asar. On the issue's 1,200-function fixture the longest event-loop block falls from about 290 ms to 90 ms, and ping after abort from up to 286 ms to at most 56 ms. On obsidian.asar it falls from 10.9 s to 5.4 s; the rest is one analyzer stage of a single very large file. Refs #1462 Co-authored-by: Claude Opus 5.5 Co-authored-by: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> --- .../JavaScriptApplicationCancellation.test.ts | 40 ++++++ .../javascript/JavaScriptArtifactAnalysis.ts | 54 +++++--- .../JavaScriptArtifactGraphBuilder.ts | 54 ++++++-- .../JavaScriptArtifactGraphFindings.ts | 7 +- .../JavaScriptArtifactReconstruction.ts | 3 +- .../JavaScriptModuleRelationships.ts | 8 +- .../JavaScriptSemanticGraphBuilder.ts | 69 ++++++---- ...javascriptApplicationAnalysisSteps.test.ts | 59 ++++++++ .../javascript/javascriptApplicationGraph.ts | 129 ++++++++++++++---- .../javascript/javascriptSemanticGraph.ts | 65 +++++++++ src/domain/zodCollectionSteps.ts | 33 +++++ 11 files changed, 438 insertions(+), 83 deletions(-) create mode 100644 src/domain/zodCollectionSteps.ts diff --git a/src/application/javascript/JavaScriptApplicationCancellation.test.ts b/src/application/javascript/JavaScriptApplicationCancellation.test.ts index 1457046ef2..5c2b2f0059 100644 --- a/src/application/javascript/JavaScriptApplicationCancellation.test.ts +++ b/src/application/javascript/JavaScriptApplicationCancellation.test.ts @@ -42,6 +42,46 @@ describe("JavaScript analysis cancellation before publication", () => { expect(phases).not.toContain("validate_javascript_application_result"); }); + it("handles control turns and cancellation inside one large source file (#1462)", async () => { + const root = await createTestTempDirectory("rea-js-cancel-in-file-"); + await writeFile( + join(root, "index.js"), + Array.from( + { length: 1200 }, + (_, index) => + `export function f${index}(){return {name:'f${index}',value:${index}};}`, + ).join("\n"), + ); + const controller = new AbortController(); + const phases: string[] = []; + let turns = 0; + // Count event-loop turns once the file starts; parsing, analysis and + // projection of this one file must leave room for them. + const tick = (): void => { + turns += 1; + if (turns === 3) controller.abort(); + else if (!controller.signal.aborted) setImmediate(tick); + }; + const result = await analyzeJavaScriptApplication( + { input_path: root, format: "directory" }, + { + signal: controller.signal, + progress: { + async report(event) { + phases.push(event.phase); + if (event.phase === "parse_javascript_source") setImmediate(tick); + }, + }, + }, + ); + if (result.ok) throw new Error("Cancelled analysis must not publish"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "cancelled", + }); + expect(phases).toContain("parse_javascript_source"); + expect(phases).not.toContain("build_javascript_application_graph"); + }); + it.each([ "parse_javascript_source", "build_javascript_application_graph", diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts index 84390b8738..3fac39dbb8 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysis.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts @@ -33,6 +33,7 @@ import type { JavaScriptSourceMapOriginal, } from "./JavaScriptArtifactAnalysisTypes.js"; import { analyzeJavaScriptJsonModule } from "./JavaScriptJsonModules.js"; +import { completeJavaScriptAnalysisSteps } from "./JavaScriptAnalysisControl.js"; import { htmlArtifactReferences } from "../../domain/javascript/htmlArtifactReferences.js"; interface MutableArtifactAnalysis< @@ -56,34 +57,44 @@ interface ArtifactAnalysisContext< readonly projectSemantics: ( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, - ) => SemanticIr; + ) => Generator; } /** Analyze every text source from the artifact without a result quota. */ export const analyzeJavaScriptArtifactFiles = ( fileSet: JavaScriptArtifactFileSet, ): JavaScriptArtifactAnalysis => - analyzeArtifactFiles(fileSet, (_file, ir) => ir); + analyzeArtifactFiles(fileSet, function* (_file, ir) { + // The full IR needs no projection; one step keeps the shared protocol. + yield; + return ir; + }); -/** Project each file's full IR immediately, retaining only later module facts. */ +/** + * Project each file's full IR immediately, retaining only later module facts. + * Parsing, analysis and each projection pass run as separate steps, so control + * messages and cancellation are handled within one large file as well. + */ export const analyzeAndProjectJavaScriptArtifactFiles = async ( fileSet: JavaScriptArtifactFileSet, projectSemantics: ( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, - ) => void, + ) => Iterator | void, beforeFile?: ( file: JavaScriptArtifactFile, completed: number, total: number, ) => Promise, + signal?: AbortSignal, ): Promise => { const state = emptyArtifactAnalysis(); - const projectFileSemantics = ( + const projectFileSemantics = function* ( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, - ): JavaScriptModuleSemanticIr => { - projectSemantics(file, ir); + ): Generator { + const projection = projectSemantics(file, ir); + if (projection !== undefined) while (projection.next().done !== true) yield; const programScopes = ir.scopes.filter(({ kind }) => kind === "program"); const programScopeId = programScopes[0]?.scopeId; const localNames = new Set( @@ -111,7 +122,10 @@ export const analyzeAndProjectJavaScriptArtifactFiles = async ( // Let progress, cancellation and garbage collection run between files. await setImmediate(); await beforeFile?.(file, index, fileSet.files.length); - analyzeArtifactFile(file, context); + await completeJavaScriptAnalysisSteps( + analyzeArtifactFileSteps(file, context), + signal, + ); } return finalizeArtifactAnalysis(state); }; @@ -122,7 +136,10 @@ const analyzeArtifactFiles = ( ): JavaScriptArtifactAnalysis => { const state = emptyArtifactAnalysis(); const context = { state, projectSemantics }; - for (const file of fileSet.files) analyzeArtifactFile(file, context); + for (const file of fileSet.files) { + const steps = analyzeArtifactFileSteps(file, context); + while (steps.next().done !== true); + } return finalizeArtifactAnalysis(state); }; @@ -148,10 +165,12 @@ const finalizeArtifactAnalysis = < }; }; -const analyzeArtifactFile = ( +function* analyzeArtifactFileSteps< + SemanticIr extends JavaScriptModuleSemanticIr, +>( file: JavaScriptArtifactFile, context: ArtifactAnalysisContext, -): void => { +): Generator { const { state } = context; addStructuredObservations(file, state); if (file.kind === "html" && file.text.included) @@ -162,6 +181,7 @@ const analyzeArtifactFile = ( return; } const parsed = parseJavaScriptSource(file.text.value, file.path); + yield; if (parsed === null) { const analysis = failedJavaScriptStaticAnalysis(); state.files.push({ file, javascript: analysis, semantic: null }); @@ -175,17 +195,19 @@ const analyzeArtifactFile = ( openReceiverFacts, ); const staticFindings = findingCount(analysis); + yield; const semantics = analysis.parse_status === "complete" || analysis.parse_status === "partial" ? analyzeParsedJavaScriptSemantics(parsed) : null; + const projected = + semantics === null + ? null + : yield* context.projectSemantics(file, semantics); state.files.push({ file, javascript: analysis, - semantic: - semantics === null - ? null - : { ir: context.projectSemantics(file, semantics) }, + semantic: projected === null ? null : { ir: projected }, }); state.visitedNodes += analysis.visited_ast_nodes; state.findings += staticFindings + (semantics?.moduleLinks.length ?? 0); @@ -194,7 +216,7 @@ const analyzeArtifactFile = ( 0, ); if (analysis.parse_status === "failed") state.parseFailures += 1; -}; +} const addSourceMap = ( file: JavaScriptArtifactFile, diff --git a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts index bf46a05577..023802a79a 100644 --- a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts +++ b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts @@ -1,7 +1,7 @@ import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js"; import { createJavaScriptApplicationGraph, - createImmutableJavaScriptApplicationGraphSteps, + sealTransferredJavaScriptApplicationGraphSteps, type JavaScriptApplicationGraph, } from "../../domain/javascript/javascriptApplicationGraph.js"; import type { JavaScriptModuleArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; @@ -15,9 +15,9 @@ import { addJavaScriptHtmlRoles, addJavaScriptSourceMapOriginals, } from "./JavaScriptArtifactGraphDocuments.js"; -import { addJavaScriptStaticFindings } from "./JavaScriptArtifactGraphFindings.js"; +import { addJavaScriptStaticFindingsSteps } from "./JavaScriptArtifactGraphFindings.js"; import { - addJavaScriptModuleRelationships, + addJavaScriptModuleRelationshipsSteps, type JavaScriptModuleRelationshipOmissions, addJavaScriptSourceModules, } from "./JavaScriptModuleRelationships.js"; @@ -51,24 +51,42 @@ export const buildJavaScriptArtifactGraph = ( analysis: JavaScriptModuleArtifactAnalysis, ): JavaScriptApplicationGraph => createJavaScriptApplicationGraph( - buildJavaScriptArtifactGraphInput(snapshot, fileSet, analysis), + completeSynchronously( + buildJavaScriptArtifactGraphInputSteps(snapshot, fileSet, analysis), + ), ); -/** Build a validated application graph with cooperative immutable ownership. */ -export const buildImmutableJavaScriptArtifactGraphSteps = ( +/** + * Build a validated application graph with cooperative immutable ownership. + * Constructing the graph input also runs in steps: projecting every module + * relationship and finding at once held the event loop for seconds. + */ +export function* buildImmutableJavaScriptArtifactGraphSteps( snapshot: ArtifactInventorySnapshot, fileSet: JavaScriptArtifactFileSet, analysis: JavaScriptModuleArtifactAnalysis, -): Generator => - createImmutableJavaScriptApplicationGraphSteps( - buildJavaScriptArtifactGraphInput(snapshot, fileSet, analysis), +): Generator { + const input = yield* buildJavaScriptArtifactGraphInputSteps( + snapshot, + fileSet, + analysis, ); + // The input was built here and is not retained, so it can be transferred. + return yield* sealTransferredJavaScriptApplicationGraphSteps(input); +} -const buildJavaScriptArtifactGraphInput = ( +const completeSynchronously = (steps: Iterator): Value => { + for (;;) { + const step = steps.next(); + if (step.done === true) return step.value; + } +}; + +function* buildJavaScriptArtifactGraphInputSteps( snapshot: ArtifactInventorySnapshot, fileSet: JavaScriptArtifactFileSet, analysis: JavaScriptModuleArtifactAnalysis, -): unknown => { +): Generator { const accumulator = new JavaScriptArtifactGraphAccumulator(); const root = createJavaScriptArtifactRootNode(accumulator, snapshot); const context: JavaScriptArtifactGraphContext = { @@ -86,15 +104,23 @@ const buildJavaScriptArtifactGraphInput = ( containerNodes: new Map([[snapshot.manifest.root_sha256, root]]), }; addJavaScriptArtifactContainers(context); + yield; addJavaScriptArtifactFiles(context); + yield; const packageRoots = addJavaScriptPackageNodes(context); + yield; addJavaScriptSourceModules(context); + yield; const bundlerLimitations = addJavaScriptBundlerNodes(context); - const relationshipOmissions = addJavaScriptModuleRelationships(context); - const findingLimitations = addJavaScriptStaticFindings(context); + const relationshipOmissions = + yield* addJavaScriptModuleRelationshipsSteps(context); + const findingLimitations = yield* addJavaScriptStaticFindingsSteps(context); addElectronBoundaries(context); + yield; addJavaScriptHtmlRoles(context); + yield; addJavaScriptSourceMapOriginals(context); + yield; const coverage = graphCoverage(context); return { schema: "JavaScriptApplicationGraph", @@ -111,7 +137,7 @@ const buildJavaScriptArtifactGraphInput = ( ...graphLimitations(context, coverage.status, relationshipOmissions), ], }; -}; +} const graphCoverage = (context: JavaScriptArtifactGraphContext) => { const resourceLimits = semanticResourceLimits(context); diff --git a/src/application/javascript/JavaScriptArtifactGraphFindings.ts b/src/application/javascript/JavaScriptArtifactGraphFindings.ts index 41f480daf3..e7a002a6ae 100644 --- a/src/application/javascript/JavaScriptArtifactGraphFindings.ts +++ b/src/application/javascript/JavaScriptArtifactGraphFindings.ts @@ -44,11 +44,12 @@ interface ResolvedReference { } /** Project imports, workers, roles, endpoints, storage, and source-map links. */ -export const addJavaScriptStaticFindings = ( +export function* addJavaScriptStaticFindingsSteps( context: JavaScriptArtifactGraphContext, -): readonly string[] => { +): Generator { const omissions = { selfReferences: 0 }; for (const analyzed of context.analysis.files) { + yield; const { file, javascript } = analyzed; const asset = context.assetNodes.get(file.path); if (javascript === null || asset === undefined) continue; @@ -69,7 +70,7 @@ export const addJavaScriptStaticFindings = ( ["static reference", "static references"], "the referencing module", ); -}; +} const addReference = ( context: JavaScriptArtifactGraphContext, diff --git a/src/application/javascript/JavaScriptArtifactReconstruction.ts b/src/application/javascript/JavaScriptArtifactReconstruction.ts index ab986608ae..25919923c9 100644 --- a/src/application/javascript/JavaScriptArtifactReconstruction.ts +++ b/src/application/javascript/JavaScriptArtifactReconstruction.ts @@ -90,7 +90,7 @@ export const reconstructJavaScriptArtifact = async ( const semanticProjection = createJavaScriptSemanticGraphProjection(); const analysis = await analyzeAndProjectJavaScriptArtifactFiles( files, - semanticProjection.projectFile, + semanticProjection.projectFileSteps, async (file, completed, total) => { abortIfNeeded(signal); await progress?.report({ @@ -101,6 +101,7 @@ export const reconstructJavaScriptArtifact = async ( }); abortIfNeeded(signal); }, + signal, ); await checkpointJavaScriptAnalysis(signal); abortIfNeeded(signal); diff --git a/src/application/javascript/JavaScriptModuleRelationships.ts b/src/application/javascript/JavaScriptModuleRelationships.ts index 8e12f3beb3..3784141a8f 100644 --- a/src/application/javascript/JavaScriptModuleRelationships.ts +++ b/src/application/javascript/JavaScriptModuleRelationships.ts @@ -136,15 +136,17 @@ export interface JavaScriptModuleRelationshipOmissions { type RelationshipOmissionCounter = { selfImports: number }; /** Compose bounded CommonJS and ESM binding relationships across artifact files. */ -export const addJavaScriptModuleRelationships = ( +export function* addJavaScriptModuleRelationshipsSteps( context: JavaScriptArtifactGraphContext, -): JavaScriptModuleRelationshipOmissions => { +): Generator { const omissions = { selfImports: 0 }; for (const analyzed of context.analysis.files) { const { file, semantic } = analyzed; const source = context.sourceModuleNodes.get(file.path); if (semantic === null || source === undefined) continue; for (const link of semantic.ir.moduleLinks) { + // One module can export thousands of bindings, each with return shapes. + yield; const input = { context, file, semantic, source, link }; if (isExportLink(link)) addExportRelationship(input, omissions); else if (link.specifier !== null) @@ -155,7 +157,7 @@ export const addJavaScriptModuleRelationships = ( } } return omissions; -}; +} const addExportRelationship = ( input: RelationshipInput, diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts index 45ded50518..770930de7a 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts @@ -1,6 +1,6 @@ import { createJavaScriptSemanticGraph, - createImmutableJavaScriptSemanticGraphSteps, + sealTransferredJavaScriptSemanticGraphSteps, type JavaScriptSemanticGraph, } from "../../domain/javascript/javascriptSemanticGraph.js"; import type { @@ -130,6 +130,11 @@ export interface JavaScriptSemanticGraphProjection { file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, ) => void; + /** Project one file in steps between its projection passes. */ + readonly projectFileSteps: ( + file: JavaScriptArtifactFile, + ir: JavaScriptSemanticIr, + ) => Generator; readonly finish: ( rootArtifactSha256: string, applicationGraph: BuilderInput["applicationGraph"], @@ -146,10 +151,10 @@ export const createJavaScriptSemanticGraphProjection = const state = emptyState({ nodes: [] }); const fingerprints: JavaScriptSemanticFingerprint[] = []; let truncatedFiles = 0; - const projectSource = ( + const projectSourceSteps = function* ( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, - ): void => { + ): Generator { if (state.nodes.size >= SEMANTIC_GRAPH_NODE_CEILING) { truncatedFiles += 1; return; @@ -161,7 +166,7 @@ export const createJavaScriptSemanticGraphProjection = remainingTreeBudget, ); state.fileNodesDropped = false; - fingerprints.push(...projectFile(file, ir, state)); + fingerprints.push(...(yield* projectFileSteps(file, ir, state))); // Nodes were dropped only if the budget actually blocked creation. A file // that exactly fills its share ends with a zero budget without dropping // anything, so the remaining budget alone cannot decide truncation. @@ -198,7 +203,8 @@ export const createJavaScriptSemanticGraphProjection = evidence_contexts: state.evidenceContexts.contexts, nodes: [...state.nodes.values()], relations, - fingerprints, + // The input is transferred to the factory; `fingerprints` is cleared below. + fingerprints: [...fingerprints], unknowns, coverage: { status: truncatedFiles > 0 ? "partial" : "unknown", @@ -246,7 +252,11 @@ export const createJavaScriptSemanticGraphProjection = return graph; }; return { - projectFile: projectSource, + projectFile: (file, ir) => { + const steps = projectSourceSteps(file, ir); + while (steps.next().done !== true); + }, + projectFileSteps: projectSourceSteps, finish: (rootArtifactSha256, applicationGraph) => finishWith( createJavaScriptSemanticGraph, @@ -255,18 +265,19 @@ export const createJavaScriptSemanticGraphProjection = ), finishImmutableSteps: (rootArtifactSha256, applicationGraph) => finishWith( - createImmutableJavaScriptSemanticGraphSteps, + sealTransferredJavaScriptSemanticGraphSteps, rootArtifactSha256, applicationGraph, ), }; }; -const projectFile = ( +function* projectFileSteps( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, state: BuilderState, -): JavaScriptSemanticFingerprint[] => { +): Generator { + yield; const moduleNode = retainNode(state, file, { kind: "module", roleKey: "module", @@ -328,6 +339,7 @@ const projectFile = ( callableOwnerAt, callSiteAt: createSemanticCallSiteLookup(ir, callSiteNodes), }; + yield; projectDefinitionsAndReferences(definitions); const context: FileContext = { ...definitions, @@ -335,26 +347,20 @@ const projectFile = ( definitions.referenceNodes, ), }; - projectSemanticValues(context); - projectSemanticObjects(context); - projectCalls(context); - projectSemanticPromises(context); - projectSemanticEvents(context); - projectSemanticTimers(context); - projectSemanticChildProcesses(context); - projectSemanticConfiguration(context); - projectSemanticRequests(context); - projectSemanticBoundaries(context); - projectSemanticResources(context); - projectSemanticClosureCaptures(context); - projectSemanticFrontiers(context); + // Each pass completes before the next starts, so yielding between passes + // keeps projection order and results unchanged. + for (const project of FILE_PROJECTION_PASSES) { + yield; + project(context); + } + yield; return projectSemanticFunctionFingerprints( file, ir, callableNodes, state.evidenceContexts, ); -}; +} const createReturnSiteNodes = ( file: JavaScriptArtifactFile, @@ -543,3 +549,20 @@ const createArgumentNode = ( : `argument ${String(argument.index)}`, functionNodeId: callNode.function_node_id, }); + +/** Per-file projection passes, run in this order with a yield between each. */ +const FILE_PROJECTION_PASSES: readonly ((context: FileContext) => void)[] = [ + projectSemanticValues, + projectSemanticObjects, + projectCalls, + projectSemanticPromises, + projectSemanticEvents, + projectSemanticTimers, + projectSemanticChildProcesses, + projectSemanticConfiguration, + projectSemanticRequests, + projectSemanticBoundaries, + projectSemanticResources, + projectSemanticClosureCaptures, + projectSemanticFrontiers, +]; diff --git a/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts b/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts index c96ccece77..a460de0775 100644 --- a/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts +++ b/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts @@ -11,11 +11,13 @@ import { createJavaScriptApplicationGraph, createImmutableJavaScriptApplicationGraphSteps, isValidatedImmutableJavaScriptApplicationGraph, + sealTransferredJavaScriptApplicationGraphSteps, } from "./javascriptApplicationGraph.js"; import { createJavaScriptSemanticGraph, createImmutableJavaScriptSemanticGraphSteps, isValidatedImmutableJavaScriptSemanticGraph, + sealTransferredJavaScriptSemanticGraphSteps, } from "./javascriptSemanticGraph.js"; const complete = (steps: Iterator): Value => { @@ -265,3 +267,60 @@ describe("owned JavaScript result bindings", () => { ); }); }); + +describe("graphs transferred to their sealing steps", () => { + it("seal the same validated graphs as the copying factories", () => { + const input = example(); + const { graph_id: _applicationId, ...application } = input.graph; + const { graph_id: _semanticId, ...semantic } = input.semantic_graph; + const expectedApplication = createJavaScriptApplicationGraph(application); + const expectedSemantic = createJavaScriptSemanticGraph(semantic); + const sealedApplication = complete( + sealTransferredJavaScriptApplicationGraphSteps( + structuredClone(application), + ), + ); + const sealedSemantic = complete( + sealTransferredJavaScriptSemanticGraphSteps(structuredClone(semantic)), + ); + expect(sealedApplication).toEqual(expectedApplication); + expect(sealedSemantic).toEqual(expectedSemantic); + expect( + isValidatedImmutableJavaScriptApplicationGraph(sealedApplication), + ).toBe(true); + expect(isValidatedImmutableJavaScriptSemanticGraph(sealedSemantic)).toBe( + true, + ); + }); + + it("report the issues a whole-graph parse reports", () => { + const input = example(); + const { graph_id: _applicationId, ...application } = input.graph; + const { graph_id: _semanticId, ...semantic } = input.semantic_graph; + const node = semantic.nodes[0]; + const applicationNode = application.nodes[0]; + if (node === undefined || applicationNode === undefined) + throw new Error("Expected semantic and application nodes"); + const malformed = { + ...semantic, + nodes: [{ ...node, label: 7 }, ...semantic.nodes.slice(1)], + }; + expect( + issuesFrom(() => + complete(sealTransferredJavaScriptSemanticGraphSteps(malformed)), + ), + ).toEqual(issuesFrom(() => createJavaScriptSemanticGraph(malformed))); + const stale = { + ...application, + nodes: [ + { ...applicationNode, node_id: `jag_node_${"f".repeat(64)}` }, + ...application.nodes.slice(1), + ], + }; + expect( + issuesFrom(() => + complete(sealTransferredJavaScriptApplicationGraphSteps(stale)), + ), + ).toEqual(issuesFrom(() => createJavaScriptApplicationGraph(stale))); + }); +}); diff --git a/src/domain/javascript/javascriptApplicationGraph.ts b/src/domain/javascript/javascriptApplicationGraph.ts index e16aded759..ea84585f10 100644 --- a/src/domain/javascript/javascriptApplicationGraph.ts +++ b/src/domain/javascript/javascriptApplicationGraph.ts @@ -1,4 +1,9 @@ import { z } from "zod"; +import { + arrayContainerSchema, + parseElementSteps, +} from "../zodCollectionSteps.js"; +import { canonicalJsonDigestSteps } from "../canonicalJsonDigestSteps.js"; import { canonicalJson } from "../comparisonSemantics.js"; import { digestCanonicalValue } from "../canonicalDigest.js"; import { uniqueSorted } from "../canonicalOrdering.js"; @@ -93,11 +98,14 @@ type EdgeSemantic = Omit; const edgeId = (edge: EdgeSemantic): string => `jag_edge_${digestCanonicalValue(edge, "JavaScript Application Graph")}`; +/** The issue sink shared by schema refinement and stepwise checking. */ +type GraphIssueContext = Pick; + const sortedUniqueIssue = ( values: readonly string[], path: PropertyKey[], label: string, - context: z.RefinementCtx, + context: GraphIssueContext, ): void => { for (let index = 1; index < values.length; index += 1) { if ( @@ -116,7 +124,7 @@ const sortedUniqueIssue = ( const checkEvidenceNormalization = ( evidence: ApplicationGraphEvidence, path: PropertyKey[], - context: z.RefinementCtx, + context: GraphIssueContext, ): void => { sortedUniqueIssue( evidence.limitations, @@ -226,7 +234,7 @@ const observationMatchesIdentity = ( const checkNode = ( node: ApplicationNode, index: number, - context: z.RefinementCtx, + context: GraphIssueContext, ): void => { if (node.node_id !== nodeId(node)) context.addIssue({ @@ -306,8 +314,16 @@ type GraphRecord = z.infer; const checkGraphContent = ( graph: JavaScriptApplicationGraphInput, - context: z.RefinementCtx, + context: GraphIssueContext, ): void => { + const steps = checkGraphContentSteps(graph, context); + while (steps.next().done !== true); +}; + +function* checkGraphContentSteps( + graph: JavaScriptApplicationGraphInput, + context: GraphIssueContext, +): Generator { sortedUniqueIssue( graph.root_node_ids, ["root_node_ids"], @@ -353,9 +369,12 @@ const checkGraphContent = ( message: "Root identifier must name a graph node", path: ["root_node_ids"], }); - for (const [index, node] of graph.nodes.entries()) + for (const [index, node] of graph.nodes.entries()) { + if (index % 64 === 0) yield; checkNode(node, index, context); + } for (const [index, edge] of graph.edges.entries()) { + if (index % 64 === 0) yield; const { edge_id: identifier, ...semantic } = edge; if (identifier !== edgeId(semantic)) context.addIssue({ @@ -394,11 +413,11 @@ const checkGraphContent = ( context, ); } -}; +} const checkGraphInvariants = ( graph: GraphRecord, - context: z.RefinementCtx, + context: GraphIssueContext, ): void => { checkGraphContent(graph, context); const { graph_id: identifier, ...semantic } = graph; @@ -475,22 +494,9 @@ export const createJavaScriptApplicationEdge = ( export const createJavaScriptApplicationGraph = ( input: unknown, ): JavaScriptApplicationGraph => { - const parsed = javascriptApplicationGraphInputSchema.parse(input); - const semantic: JavaScriptApplicationGraphInput = { - ...parsed, - root_node_ids: uniqueSorted(parsed.root_node_ids), - nodes: [...parsed.nodes].sort((left, right) => - compareUnicodeCodePoints(left.node_id, right.node_id), - ), - edges: [...parsed.edges].sort((left, right) => - compareUnicodeCodePoints(left.edge_id, right.edge_id), - ), - coverage: { - ...parsed.coverage, - limits: normalizeLimits(parsed.coverage.limits), - }, - limitations: uniqueSorted(parsed.limitations), - }; + const semantic = normalizeGraphInput( + javascriptApplicationGraphInputSchema.parse(input), + ); const content = javascriptApplicationGraphContentSchema.parse(semantic); return { ...content, @@ -498,6 +504,24 @@ export const createJavaScriptApplicationGraph = ( }; }; +const normalizeGraphInput = ( + parsed: JavaScriptApplicationGraphInput, +): JavaScriptApplicationGraphInput => ({ + ...parsed, + root_node_ids: uniqueSorted(parsed.root_node_ids), + nodes: [...parsed.nodes].sort((left, right) => + compareUnicodeCodePoints(left.node_id, right.node_id), + ), + edges: [...parsed.edges].sort((left, right) => + compareUnicodeCodePoints(left.edge_id, right.edge_id), + ), + coverage: { + ...parsed.coverage, + limits: normalizeLimits(parsed.coverage.limits), + }, + limitations: uniqueSorted(parsed.limitations), +}); + const validatedImmutableApplicationGraphs = new WeakSet(); /** Clone and validate synchronously, then return steps to seal the owned graph. */ @@ -506,6 +530,65 @@ export const createImmutableJavaScriptApplicationGraphSteps = ( ): Generator => sealValidatedApplicationGraphSteps(createJavaScriptApplicationGraph(input)); +const applicationGraphShape = javascriptApplicationGraphInputSchema.shape; +const applicationGraphContainerSchema = + javascriptApplicationGraphInputSchema.extend({ + nodes: arrayContainerSchema(applicationGraphShape.nodes), + edges: arrayContainerSchema(applicationGraphShape.edges), + }); + +/** + * Seal a graph whose input the caller transfers: the caller neither retains nor + * mutates any part of it afterwards. Only the graph's own fields are validated + * synchronously; nodes and edges are validated, checked, hashed and frozen in + * steps, because doing so at once held the event loop for large applications. + */ +export const sealTransferredJavaScriptApplicationGraphSteps = ( + input: unknown, +): Generator => + parseAndSealApplicationGraphSteps( + applicationGraphContainerSchema.parse(input), + ); + +function* parseAndSealApplicationGraphSteps( + container: z.output, +): Generator { + const issues: z.core.$ZodIssue[] = []; + const parsed: JavaScriptApplicationGraphInput = { + ...container, + nodes: yield* parseElementSteps( + applicationGraphShape.nodes.element, + container.nodes, + ["nodes"], + issues, + ), + edges: yield* parseElementSteps( + applicationGraphShape.edges.element, + container.edges, + ["edges"], + issues, + ), + }; + if (issues.length > 0) throw new z.ZodError(issues); + const content = normalizeGraphInput(parsed); + let invalid = false; + yield* checkGraphContentSteps(content, { + addIssue: () => { + invalid = true; + }, + }); + // An invalid owned graph is a construction defect; report it with the exact + // issues the content schema produces. + if (invalid) { + javascriptApplicationGraphContentSchema.parse(content); + throw new TypeError("Application graph checks disagree with its schema"); + } + return yield* sealValidatedApplicationGraphSteps({ + ...content, + graph_id: `jag_${yield* canonicalJsonDigestSteps(content)}`, + }); +} + function* sealValidatedApplicationGraphSteps( graph: JavaScriptApplicationGraph, ): Generator { diff --git a/src/domain/javascript/javascriptSemanticGraph.ts b/src/domain/javascript/javascriptSemanticGraph.ts index 9ab2dd485c..6908af2a19 100644 --- a/src/domain/javascript/javascriptSemanticGraph.ts +++ b/src/domain/javascript/javascriptSemanticGraph.ts @@ -6,6 +6,10 @@ import { uniqueSorted } from "../canonicalOrdering.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; import { canonicalJsonDigestSteps } from "../canonicalJsonDigestSteps.js"; import { freezeOwnedJsonSnapshotSteps } from "../immutableJson.js"; +import { + arrayContainerSchema, + parseElementSteps, +} from "../zodCollectionSteps.js"; import type { ApplicationGraphEvidence, ApplicationGraphEvidenceContext, @@ -765,6 +769,15 @@ export const createJavaScriptSemanticGraph = ( const validatedImmutableSemanticGraphs = new WeakSet(); +const semanticGraphShape = javaScriptSemanticGraphInputSchema.shape; +const semanticGraphContainerSchema = javaScriptSemanticGraphInputSchema.extend({ + evidence_contexts: arrayContainerSchema(semanticGraphShape.evidence_contexts), + nodes: arrayContainerSchema(semanticGraphShape.nodes), + relations: arrayContainerSchema(semanticGraphShape.relations), + fingerprints: arrayContainerSchema(semanticGraphShape.fingerprints), + unknowns: arrayContainerSchema(semanticGraphShape.unknowns), +}); + /** Clone input synchronously; commit, check and seal the owned graph in steps. */ export const createImmutableJavaScriptSemanticGraphSteps = ( input: unknown, @@ -773,6 +786,58 @@ export const createImmutableJavaScriptSemanticGraphSteps = ( normalizeGraphInput(javaScriptSemanticGraphInputSchema.parse(input)), ); +/** + * Seal a graph whose input the caller transfers: the caller neither retains nor + * mutates any part of it afterwards. Only the graph's own fields are validated + * synchronously; collection elements are validated and cloned in steps, because + * a whole-graph parse of a large application held the event loop for seconds. + */ +export const sealTransferredJavaScriptSemanticGraphSteps = ( + input: unknown, +): Generator => + parseAndSealSemanticGraphSteps(semanticGraphContainerSchema.parse(input)); + +function* parseAndSealSemanticGraphSteps( + container: z.output, +): Generator { + const issues: z.core.$ZodIssue[] = []; + const parsed: JavaScriptSemanticGraphInput = { + ...container, + evidence_contexts: yield* parseElementSteps( + semanticGraphShape.evidence_contexts.element, + container.evidence_contexts, + ["evidence_contexts"], + issues, + ), + nodes: yield* parseElementSteps( + semanticGraphShape.nodes.element, + container.nodes, + ["nodes"], + issues, + ), + relations: yield* parseElementSteps( + semanticGraphShape.relations.element, + container.relations, + ["relations"], + issues, + ), + fingerprints: yield* parseElementSteps( + semanticGraphShape.fingerprints.element, + container.fingerprints, + ["fingerprints"], + issues, + ), + unknowns: yield* parseElementSteps( + semanticGraphShape.unknowns.element, + container.unknowns, + ["unknowns"], + issues, + ), + }; + if (issues.length > 0) throw new z.ZodError(issues); + return yield* validateAndSealSemanticGraphSteps(normalizeGraphInput(parsed)); +} + function* validateAndSealSemanticGraphSteps( semantic: JavaScriptSemanticGraphInput, ): Generator { diff --git a/src/domain/zodCollectionSteps.ts b/src/domain/zodCollectionSteps.ts new file mode 100644 index 0000000000..a32e8c8044 --- /dev/null +++ b/src/domain/zodCollectionSteps.ts @@ -0,0 +1,33 @@ +import { z } from "zod"; + +/** + * An array schema's own constraints, such as a minimum length, without its + * element schema. Pair it with {@link parseElementSteps} to validate a large + * collection in cooperative steps. + */ +export const arrayContainerSchema = ( + array: z.ZodArray, +): z.ZodType => array.clone({ ...array.def, element: z.unknown() }); + +/** Elements validated between yields, in the issues and paths a whole-array parse reports. */ +export function* parseElementSteps( + element: Element, + values: readonly unknown[], + path: readonly PropertyKey[], + issues: z.core.$ZodIssue[], +): Generator[]> { + const parsed: z.output[] = []; + for (const [index, value] of values.entries()) { + if (index % 64 === 0) yield; + const result = element.safeParse(value); + if (result.success) parsed.push(result.data); + else + issues.push( + ...result.error.issues.map((issue) => ({ + ...issue, + path: [...path, index, ...issue.path], + })), + ); + } + return parsed; +}