diff --git a/src/application/javascript/JavaScriptApplicationCancellation.test.ts b/src/application/javascript/JavaScriptApplicationCancellation.test.ts index 1457046ef2..5c2b2f0059 100644 --- a/src/application/javascript/JavaScriptApplicationCancellation.test.ts +++ b/src/application/javascript/JavaScriptApplicationCancellation.test.ts @@ -42,6 +42,46 @@ describe("JavaScript analysis cancellation before publication", () => { expect(phases).not.toContain("validate_javascript_application_result"); }); + it("handles control turns and cancellation inside one large source file (#1462)", async () => { + const root = await createTestTempDirectory("rea-js-cancel-in-file-"); + await writeFile( + join(root, "index.js"), + Array.from( + { length: 1200 }, + (_, index) => + `export function f${index}(){return {name:'f${index}',value:${index}};}`, + ).join("\n"), + ); + const controller = new AbortController(); + const phases: string[] = []; + let turns = 0; + // Count event-loop turns once the file starts; parsing, analysis and + // projection of this one file must leave room for them. + const tick = (): void => { + turns += 1; + if (turns === 3) controller.abort(); + else if (!controller.signal.aborted) setImmediate(tick); + }; + const result = await analyzeJavaScriptApplication( + { input_path: root, format: "directory" }, + { + signal: controller.signal, + progress: { + async report(event) { + phases.push(event.phase); + if (event.phase === "parse_javascript_source") setImmediate(tick); + }, + }, + }, + ); + if (result.ok) throw new Error("Cancelled analysis must not publish"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "cancelled", + }); + expect(phases).toContain("parse_javascript_source"); + expect(phases).not.toContain("build_javascript_application_graph"); + }); + it.each([ "parse_javascript_source", "build_javascript_application_graph", diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts index 84390b8738..3fac39dbb8 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysis.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts @@ -33,6 +33,7 @@ import type { JavaScriptSourceMapOriginal, } from "./JavaScriptArtifactAnalysisTypes.js"; import { analyzeJavaScriptJsonModule } from "./JavaScriptJsonModules.js"; +import { completeJavaScriptAnalysisSteps } from "./JavaScriptAnalysisControl.js"; import { htmlArtifactReferences } from "../../domain/javascript/htmlArtifactReferences.js"; interface MutableArtifactAnalysis< @@ -56,34 +57,44 @@ interface ArtifactAnalysisContext< readonly projectSemantics: ( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, - ) => SemanticIr; + ) => Generator; } /** Analyze every text source from the artifact without a result quota. */ export const analyzeJavaScriptArtifactFiles = ( fileSet: JavaScriptArtifactFileSet, ): JavaScriptArtifactAnalysis => - analyzeArtifactFiles(fileSet, (_file, ir) => ir); + analyzeArtifactFiles(fileSet, function* (_file, ir) { + // The full IR needs no projection; one step keeps the shared protocol. + yield; + return ir; + }); -/** Project each file's full IR immediately, retaining only later module facts. */ +/** + * Project each file's full IR immediately, retaining only later module facts. + * Parsing, analysis and each projection pass run as separate steps, so control + * messages and cancellation are handled within one large file as well. + */ export const analyzeAndProjectJavaScriptArtifactFiles = async ( fileSet: JavaScriptArtifactFileSet, projectSemantics: ( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, - ) => void, + ) => Iterator | void, beforeFile?: ( file: JavaScriptArtifactFile, completed: number, total: number, ) => Promise, + signal?: AbortSignal, ): Promise => { const state = emptyArtifactAnalysis(); - const projectFileSemantics = ( + const projectFileSemantics = function* ( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, - ): JavaScriptModuleSemanticIr => { - projectSemantics(file, ir); + ): Generator { + const projection = projectSemantics(file, ir); + if (projection !== undefined) while (projection.next().done !== true) yield; const programScopes = ir.scopes.filter(({ kind }) => kind === "program"); const programScopeId = programScopes[0]?.scopeId; const localNames = new Set( @@ -111,7 +122,10 @@ export const analyzeAndProjectJavaScriptArtifactFiles = async ( // Let progress, cancellation and garbage collection run between files. await setImmediate(); await beforeFile?.(file, index, fileSet.files.length); - analyzeArtifactFile(file, context); + await completeJavaScriptAnalysisSteps( + analyzeArtifactFileSteps(file, context), + signal, + ); } return finalizeArtifactAnalysis(state); }; @@ -122,7 +136,10 @@ const analyzeArtifactFiles = ( ): JavaScriptArtifactAnalysis => { const state = emptyArtifactAnalysis(); const context = { state, projectSemantics }; - for (const file of fileSet.files) analyzeArtifactFile(file, context); + for (const file of fileSet.files) { + const steps = analyzeArtifactFileSteps(file, context); + while (steps.next().done !== true); + } return finalizeArtifactAnalysis(state); }; @@ -148,10 +165,12 @@ const finalizeArtifactAnalysis = < }; }; -const analyzeArtifactFile = ( +function* analyzeArtifactFileSteps< + SemanticIr extends JavaScriptModuleSemanticIr, +>( file: JavaScriptArtifactFile, context: ArtifactAnalysisContext, -): void => { +): Generator { const { state } = context; addStructuredObservations(file, state); if (file.kind === "html" && file.text.included) @@ -162,6 +181,7 @@ const analyzeArtifactFile = ( return; } const parsed = parseJavaScriptSource(file.text.value, file.path); + yield; if (parsed === null) { const analysis = failedJavaScriptStaticAnalysis(); state.files.push({ file, javascript: analysis, semantic: null }); @@ -175,17 +195,19 @@ const analyzeArtifactFile = ( openReceiverFacts, ); const staticFindings = findingCount(analysis); + yield; const semantics = analysis.parse_status === "complete" || analysis.parse_status === "partial" ? analyzeParsedJavaScriptSemantics(parsed) : null; + const projected = + semantics === null + ? null + : yield* context.projectSemantics(file, semantics); state.files.push({ file, javascript: analysis, - semantic: - semantics === null - ? null - : { ir: context.projectSemantics(file, semantics) }, + semantic: projected === null ? null : { ir: projected }, }); state.visitedNodes += analysis.visited_ast_nodes; state.findings += staticFindings + (semantics?.moduleLinks.length ?? 0); @@ -194,7 +216,7 @@ const analyzeArtifactFile = ( 0, ); if (analysis.parse_status === "failed") state.parseFailures += 1; -}; +} const addSourceMap = ( file: JavaScriptArtifactFile, diff --git a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts index bf46a05577..023802a79a 100644 --- a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts +++ b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts @@ -1,7 +1,7 @@ import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js"; import { createJavaScriptApplicationGraph, - createImmutableJavaScriptApplicationGraphSteps, + sealTransferredJavaScriptApplicationGraphSteps, type JavaScriptApplicationGraph, } from "../../domain/javascript/javascriptApplicationGraph.js"; import type { JavaScriptModuleArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; @@ -15,9 +15,9 @@ import { addJavaScriptHtmlRoles, addJavaScriptSourceMapOriginals, } from "./JavaScriptArtifactGraphDocuments.js"; -import { addJavaScriptStaticFindings } from "./JavaScriptArtifactGraphFindings.js"; +import { addJavaScriptStaticFindingsSteps } from "./JavaScriptArtifactGraphFindings.js"; import { - addJavaScriptModuleRelationships, + addJavaScriptModuleRelationshipsSteps, type JavaScriptModuleRelationshipOmissions, addJavaScriptSourceModules, } from "./JavaScriptModuleRelationships.js"; @@ -51,24 +51,42 @@ export const buildJavaScriptArtifactGraph = ( analysis: JavaScriptModuleArtifactAnalysis, ): JavaScriptApplicationGraph => createJavaScriptApplicationGraph( - buildJavaScriptArtifactGraphInput(snapshot, fileSet, analysis), + completeSynchronously( + buildJavaScriptArtifactGraphInputSteps(snapshot, fileSet, analysis), + ), ); -/** Build a validated application graph with cooperative immutable ownership. */ -export const buildImmutableJavaScriptArtifactGraphSteps = ( +/** + * Build a validated application graph with cooperative immutable ownership. + * Constructing the graph input also runs in steps: projecting every module + * relationship and finding at once held the event loop for seconds. + */ +export function* buildImmutableJavaScriptArtifactGraphSteps( snapshot: ArtifactInventorySnapshot, fileSet: JavaScriptArtifactFileSet, analysis: JavaScriptModuleArtifactAnalysis, -): Generator => - createImmutableJavaScriptApplicationGraphSteps( - buildJavaScriptArtifactGraphInput(snapshot, fileSet, analysis), +): Generator { + const input = yield* buildJavaScriptArtifactGraphInputSteps( + snapshot, + fileSet, + analysis, ); + // The input was built here and is not retained, so it can be transferred. + return yield* sealTransferredJavaScriptApplicationGraphSteps(input); +} -const buildJavaScriptArtifactGraphInput = ( +const completeSynchronously = (steps: Iterator): Value => { + for (;;) { + const step = steps.next(); + if (step.done === true) return step.value; + } +}; + +function* buildJavaScriptArtifactGraphInputSteps( snapshot: ArtifactInventorySnapshot, fileSet: JavaScriptArtifactFileSet, analysis: JavaScriptModuleArtifactAnalysis, -): unknown => { +): Generator { const accumulator = new JavaScriptArtifactGraphAccumulator(); const root = createJavaScriptArtifactRootNode(accumulator, snapshot); const context: JavaScriptArtifactGraphContext = { @@ -86,15 +104,23 @@ const buildJavaScriptArtifactGraphInput = ( containerNodes: new Map([[snapshot.manifest.root_sha256, root]]), }; addJavaScriptArtifactContainers(context); + yield; addJavaScriptArtifactFiles(context); + yield; const packageRoots = addJavaScriptPackageNodes(context); + yield; addJavaScriptSourceModules(context); + yield; const bundlerLimitations = addJavaScriptBundlerNodes(context); - const relationshipOmissions = addJavaScriptModuleRelationships(context); - const findingLimitations = addJavaScriptStaticFindings(context); + const relationshipOmissions = + yield* addJavaScriptModuleRelationshipsSteps(context); + const findingLimitations = yield* addJavaScriptStaticFindingsSteps(context); addElectronBoundaries(context); + yield; addJavaScriptHtmlRoles(context); + yield; addJavaScriptSourceMapOriginals(context); + yield; const coverage = graphCoverage(context); return { schema: "JavaScriptApplicationGraph", @@ -111,7 +137,7 @@ const buildJavaScriptArtifactGraphInput = ( ...graphLimitations(context, coverage.status, relationshipOmissions), ], }; -}; +} const graphCoverage = (context: JavaScriptArtifactGraphContext) => { const resourceLimits = semanticResourceLimits(context); diff --git a/src/application/javascript/JavaScriptArtifactGraphFindings.ts b/src/application/javascript/JavaScriptArtifactGraphFindings.ts index 41f480daf3..e7a002a6ae 100644 --- a/src/application/javascript/JavaScriptArtifactGraphFindings.ts +++ b/src/application/javascript/JavaScriptArtifactGraphFindings.ts @@ -44,11 +44,12 @@ interface ResolvedReference { } /** Project imports, workers, roles, endpoints, storage, and source-map links. */ -export const addJavaScriptStaticFindings = ( +export function* addJavaScriptStaticFindingsSteps( context: JavaScriptArtifactGraphContext, -): readonly string[] => { +): Generator { const omissions = { selfReferences: 0 }; for (const analyzed of context.analysis.files) { + yield; const { file, javascript } = analyzed; const asset = context.assetNodes.get(file.path); if (javascript === null || asset === undefined) continue; @@ -69,7 +70,7 @@ export const addJavaScriptStaticFindings = ( ["static reference", "static references"], "the referencing module", ); -}; +} const addReference = ( context: JavaScriptArtifactGraphContext, diff --git a/src/application/javascript/JavaScriptArtifactReconstruction.ts b/src/application/javascript/JavaScriptArtifactReconstruction.ts index ab986608ae..25919923c9 100644 --- a/src/application/javascript/JavaScriptArtifactReconstruction.ts +++ b/src/application/javascript/JavaScriptArtifactReconstruction.ts @@ -90,7 +90,7 @@ export const reconstructJavaScriptArtifact = async ( const semanticProjection = createJavaScriptSemanticGraphProjection(); const analysis = await analyzeAndProjectJavaScriptArtifactFiles( files, - semanticProjection.projectFile, + semanticProjection.projectFileSteps, async (file, completed, total) => { abortIfNeeded(signal); await progress?.report({ @@ -101,6 +101,7 @@ export const reconstructJavaScriptArtifact = async ( }); abortIfNeeded(signal); }, + signal, ); await checkpointJavaScriptAnalysis(signal); abortIfNeeded(signal); diff --git a/src/application/javascript/JavaScriptModuleRelationships.ts b/src/application/javascript/JavaScriptModuleRelationships.ts index 8e12f3beb3..3784141a8f 100644 --- a/src/application/javascript/JavaScriptModuleRelationships.ts +++ b/src/application/javascript/JavaScriptModuleRelationships.ts @@ -136,15 +136,17 @@ export interface JavaScriptModuleRelationshipOmissions { type RelationshipOmissionCounter = { selfImports: number }; /** Compose bounded CommonJS and ESM binding relationships across artifact files. */ -export const addJavaScriptModuleRelationships = ( +export function* addJavaScriptModuleRelationshipsSteps( context: JavaScriptArtifactGraphContext, -): JavaScriptModuleRelationshipOmissions => { +): Generator { const omissions = { selfImports: 0 }; for (const analyzed of context.analysis.files) { const { file, semantic } = analyzed; const source = context.sourceModuleNodes.get(file.path); if (semantic === null || source === undefined) continue; for (const link of semantic.ir.moduleLinks) { + // One module can export thousands of bindings, each with return shapes. + yield; const input = { context, file, semantic, source, link }; if (isExportLink(link)) addExportRelationship(input, omissions); else if (link.specifier !== null) @@ -155,7 +157,7 @@ export const addJavaScriptModuleRelationships = ( } } return omissions; -}; +} const addExportRelationship = ( input: RelationshipInput, diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts index 45ded50518..770930de7a 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts @@ -1,6 +1,6 @@ import { createJavaScriptSemanticGraph, - createImmutableJavaScriptSemanticGraphSteps, + sealTransferredJavaScriptSemanticGraphSteps, type JavaScriptSemanticGraph, } from "../../domain/javascript/javascriptSemanticGraph.js"; import type { @@ -130,6 +130,11 @@ export interface JavaScriptSemanticGraphProjection { file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, ) => void; + /** Project one file in steps between its projection passes. */ + readonly projectFileSteps: ( + file: JavaScriptArtifactFile, + ir: JavaScriptSemanticIr, + ) => Generator; readonly finish: ( rootArtifactSha256: string, applicationGraph: BuilderInput["applicationGraph"], @@ -146,10 +151,10 @@ export const createJavaScriptSemanticGraphProjection = const state = emptyState({ nodes: [] }); const fingerprints: JavaScriptSemanticFingerprint[] = []; let truncatedFiles = 0; - const projectSource = ( + const projectSourceSteps = function* ( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, - ): void => { + ): Generator { if (state.nodes.size >= SEMANTIC_GRAPH_NODE_CEILING) { truncatedFiles += 1; return; @@ -161,7 +166,7 @@ export const createJavaScriptSemanticGraphProjection = remainingTreeBudget, ); state.fileNodesDropped = false; - fingerprints.push(...projectFile(file, ir, state)); + fingerprints.push(...(yield* projectFileSteps(file, ir, state))); // Nodes were dropped only if the budget actually blocked creation. A file // that exactly fills its share ends with a zero budget without dropping // anything, so the remaining budget alone cannot decide truncation. @@ -198,7 +203,8 @@ export const createJavaScriptSemanticGraphProjection = evidence_contexts: state.evidenceContexts.contexts, nodes: [...state.nodes.values()], relations, - fingerprints, + // The input is transferred to the factory; `fingerprints` is cleared below. + fingerprints: [...fingerprints], unknowns, coverage: { status: truncatedFiles > 0 ? "partial" : "unknown", @@ -246,7 +252,11 @@ export const createJavaScriptSemanticGraphProjection = return graph; }; return { - projectFile: projectSource, + projectFile: (file, ir) => { + const steps = projectSourceSteps(file, ir); + while (steps.next().done !== true); + }, + projectFileSteps: projectSourceSteps, finish: (rootArtifactSha256, applicationGraph) => finishWith( createJavaScriptSemanticGraph, @@ -255,18 +265,19 @@ export const createJavaScriptSemanticGraphProjection = ), finishImmutableSteps: (rootArtifactSha256, applicationGraph) => finishWith( - createImmutableJavaScriptSemanticGraphSteps, + sealTransferredJavaScriptSemanticGraphSteps, rootArtifactSha256, applicationGraph, ), }; }; -const projectFile = ( +function* projectFileSteps( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, state: BuilderState, -): JavaScriptSemanticFingerprint[] => { +): Generator { + yield; const moduleNode = retainNode(state, file, { kind: "module", roleKey: "module", @@ -328,6 +339,7 @@ const projectFile = ( callableOwnerAt, callSiteAt: createSemanticCallSiteLookup(ir, callSiteNodes), }; + yield; projectDefinitionsAndReferences(definitions); const context: FileContext = { ...definitions, @@ -335,26 +347,20 @@ const projectFile = ( definitions.referenceNodes, ), }; - projectSemanticValues(context); - projectSemanticObjects(context); - projectCalls(context); - projectSemanticPromises(context); - projectSemanticEvents(context); - projectSemanticTimers(context); - projectSemanticChildProcesses(context); - projectSemanticConfiguration(context); - projectSemanticRequests(context); - projectSemanticBoundaries(context); - projectSemanticResources(context); - projectSemanticClosureCaptures(context); - projectSemanticFrontiers(context); + // Each pass completes before the next starts, so yielding between passes + // keeps projection order and results unchanged. + for (const project of FILE_PROJECTION_PASSES) { + yield; + project(context); + } + yield; return projectSemanticFunctionFingerprints( file, ir, callableNodes, state.evidenceContexts, ); -}; +} const createReturnSiteNodes = ( file: JavaScriptArtifactFile, @@ -543,3 +549,20 @@ const createArgumentNode = ( : `argument ${String(argument.index)}`, functionNodeId: callNode.function_node_id, }); + +/** Per-file projection passes, run in this order with a yield between each. */ +const FILE_PROJECTION_PASSES: readonly ((context: FileContext) => void)[] = [ + projectSemanticValues, + projectSemanticObjects, + projectCalls, + projectSemanticPromises, + projectSemanticEvents, + projectSemanticTimers, + projectSemanticChildProcesses, + projectSemanticConfiguration, + projectSemanticRequests, + projectSemanticBoundaries, + projectSemanticResources, + projectSemanticClosureCaptures, + projectSemanticFrontiers, +]; diff --git a/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts b/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts index c96ccece77..a460de0775 100644 --- a/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts +++ b/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts @@ -11,11 +11,13 @@ import { createJavaScriptApplicationGraph, createImmutableJavaScriptApplicationGraphSteps, isValidatedImmutableJavaScriptApplicationGraph, + sealTransferredJavaScriptApplicationGraphSteps, } from "./javascriptApplicationGraph.js"; import { createJavaScriptSemanticGraph, createImmutableJavaScriptSemanticGraphSteps, isValidatedImmutableJavaScriptSemanticGraph, + sealTransferredJavaScriptSemanticGraphSteps, } from "./javascriptSemanticGraph.js"; const complete = (steps: Iterator): Value => { @@ -265,3 +267,60 @@ describe("owned JavaScript result bindings", () => { ); }); }); + +describe("graphs transferred to their sealing steps", () => { + it("seal the same validated graphs as the copying factories", () => { + const input = example(); + const { graph_id: _applicationId, ...application } = input.graph; + const { graph_id: _semanticId, ...semantic } = input.semantic_graph; + const expectedApplication = createJavaScriptApplicationGraph(application); + const expectedSemantic = createJavaScriptSemanticGraph(semantic); + const sealedApplication = complete( + sealTransferredJavaScriptApplicationGraphSteps( + structuredClone(application), + ), + ); + const sealedSemantic = complete( + sealTransferredJavaScriptSemanticGraphSteps(structuredClone(semantic)), + ); + expect(sealedApplication).toEqual(expectedApplication); + expect(sealedSemantic).toEqual(expectedSemantic); + expect( + isValidatedImmutableJavaScriptApplicationGraph(sealedApplication), + ).toBe(true); + expect(isValidatedImmutableJavaScriptSemanticGraph(sealedSemantic)).toBe( + true, + ); + }); + + it("report the issues a whole-graph parse reports", () => { + const input = example(); + const { graph_id: _applicationId, ...application } = input.graph; + const { graph_id: _semanticId, ...semantic } = input.semantic_graph; + const node = semantic.nodes[0]; + const applicationNode = application.nodes[0]; + if (node === undefined || applicationNode === undefined) + throw new Error("Expected semantic and application nodes"); + const malformed = { + ...semantic, + nodes: [{ ...node, label: 7 }, ...semantic.nodes.slice(1)], + }; + expect( + issuesFrom(() => + complete(sealTransferredJavaScriptSemanticGraphSteps(malformed)), + ), + ).toEqual(issuesFrom(() => createJavaScriptSemanticGraph(malformed))); + const stale = { + ...application, + nodes: [ + { ...applicationNode, node_id: `jag_node_${"f".repeat(64)}` }, + ...application.nodes.slice(1), + ], + }; + expect( + issuesFrom(() => + complete(sealTransferredJavaScriptApplicationGraphSteps(stale)), + ), + ).toEqual(issuesFrom(() => createJavaScriptApplicationGraph(stale))); + }); +}); diff --git a/src/domain/javascript/javascriptApplicationGraph.ts b/src/domain/javascript/javascriptApplicationGraph.ts index e16aded759..ea84585f10 100644 --- a/src/domain/javascript/javascriptApplicationGraph.ts +++ b/src/domain/javascript/javascriptApplicationGraph.ts @@ -1,4 +1,9 @@ import { z } from "zod"; +import { + arrayContainerSchema, + parseElementSteps, +} from "../zodCollectionSteps.js"; +import { canonicalJsonDigestSteps } from "../canonicalJsonDigestSteps.js"; import { canonicalJson } from "../comparisonSemantics.js"; import { digestCanonicalValue } from "../canonicalDigest.js"; import { uniqueSorted } from "../canonicalOrdering.js"; @@ -93,11 +98,14 @@ type EdgeSemantic = Omit; const edgeId = (edge: EdgeSemantic): string => `jag_edge_${digestCanonicalValue(edge, "JavaScript Application Graph")}`; +/** The issue sink shared by schema refinement and stepwise checking. */ +type GraphIssueContext = Pick; + const sortedUniqueIssue = ( values: readonly string[], path: PropertyKey[], label: string, - context: z.RefinementCtx, + context: GraphIssueContext, ): void => { for (let index = 1; index < values.length; index += 1) { if ( @@ -116,7 +124,7 @@ const sortedUniqueIssue = ( const checkEvidenceNormalization = ( evidence: ApplicationGraphEvidence, path: PropertyKey[], - context: z.RefinementCtx, + context: GraphIssueContext, ): void => { sortedUniqueIssue( evidence.limitations, @@ -226,7 +234,7 @@ const observationMatchesIdentity = ( const checkNode = ( node: ApplicationNode, index: number, - context: z.RefinementCtx, + context: GraphIssueContext, ): void => { if (node.node_id !== nodeId(node)) context.addIssue({ @@ -306,8 +314,16 @@ type GraphRecord = z.infer; const checkGraphContent = ( graph: JavaScriptApplicationGraphInput, - context: z.RefinementCtx, + context: GraphIssueContext, ): void => { + const steps = checkGraphContentSteps(graph, context); + while (steps.next().done !== true); +}; + +function* checkGraphContentSteps( + graph: JavaScriptApplicationGraphInput, + context: GraphIssueContext, +): Generator { sortedUniqueIssue( graph.root_node_ids, ["root_node_ids"], @@ -353,9 +369,12 @@ const checkGraphContent = ( message: "Root identifier must name a graph node", path: ["root_node_ids"], }); - for (const [index, node] of graph.nodes.entries()) + for (const [index, node] of graph.nodes.entries()) { + if (index % 64 === 0) yield; checkNode(node, index, context); + } for (const [index, edge] of graph.edges.entries()) { + if (index % 64 === 0) yield; const { edge_id: identifier, ...semantic } = edge; if (identifier !== edgeId(semantic)) context.addIssue({ @@ -394,11 +413,11 @@ const checkGraphContent = ( context, ); } -}; +} const checkGraphInvariants = ( graph: GraphRecord, - context: z.RefinementCtx, + context: GraphIssueContext, ): void => { checkGraphContent(graph, context); const { graph_id: identifier, ...semantic } = graph; @@ -475,22 +494,9 @@ export const createJavaScriptApplicationEdge = ( export const createJavaScriptApplicationGraph = ( input: unknown, ): JavaScriptApplicationGraph => { - const parsed = javascriptApplicationGraphInputSchema.parse(input); - const semantic: JavaScriptApplicationGraphInput = { - ...parsed, - root_node_ids: uniqueSorted(parsed.root_node_ids), - nodes: [...parsed.nodes].sort((left, right) => - compareUnicodeCodePoints(left.node_id, right.node_id), - ), - edges: [...parsed.edges].sort((left, right) => - compareUnicodeCodePoints(left.edge_id, right.edge_id), - ), - coverage: { - ...parsed.coverage, - limits: normalizeLimits(parsed.coverage.limits), - }, - limitations: uniqueSorted(parsed.limitations), - }; + const semantic = normalizeGraphInput( + javascriptApplicationGraphInputSchema.parse(input), + ); const content = javascriptApplicationGraphContentSchema.parse(semantic); return { ...content, @@ -498,6 +504,24 @@ export const createJavaScriptApplicationGraph = ( }; }; +const normalizeGraphInput = ( + parsed: JavaScriptApplicationGraphInput, +): JavaScriptApplicationGraphInput => ({ + ...parsed, + root_node_ids: uniqueSorted(parsed.root_node_ids), + nodes: [...parsed.nodes].sort((left, right) => + compareUnicodeCodePoints(left.node_id, right.node_id), + ), + edges: [...parsed.edges].sort((left, right) => + compareUnicodeCodePoints(left.edge_id, right.edge_id), + ), + coverage: { + ...parsed.coverage, + limits: normalizeLimits(parsed.coverage.limits), + }, + limitations: uniqueSorted(parsed.limitations), +}); + const validatedImmutableApplicationGraphs = new WeakSet(); /** Clone and validate synchronously, then return steps to seal the owned graph. */ @@ -506,6 +530,65 @@ export const createImmutableJavaScriptApplicationGraphSteps = ( ): Generator => sealValidatedApplicationGraphSteps(createJavaScriptApplicationGraph(input)); +const applicationGraphShape = javascriptApplicationGraphInputSchema.shape; +const applicationGraphContainerSchema = + javascriptApplicationGraphInputSchema.extend({ + nodes: arrayContainerSchema(applicationGraphShape.nodes), + edges: arrayContainerSchema(applicationGraphShape.edges), + }); + +/** + * Seal a graph whose input the caller transfers: the caller neither retains nor + * mutates any part of it afterwards. Only the graph's own fields are validated + * synchronously; nodes and edges are validated, checked, hashed and frozen in + * steps, because doing so at once held the event loop for large applications. + */ +export const sealTransferredJavaScriptApplicationGraphSteps = ( + input: unknown, +): Generator => + parseAndSealApplicationGraphSteps( + applicationGraphContainerSchema.parse(input), + ); + +function* parseAndSealApplicationGraphSteps( + container: z.output, +): Generator { + const issues: z.core.$ZodIssue[] = []; + const parsed: JavaScriptApplicationGraphInput = { + ...container, + nodes: yield* parseElementSteps( + applicationGraphShape.nodes.element, + container.nodes, + ["nodes"], + issues, + ), + edges: yield* parseElementSteps( + applicationGraphShape.edges.element, + container.edges, + ["edges"], + issues, + ), + }; + if (issues.length > 0) throw new z.ZodError(issues); + const content = normalizeGraphInput(parsed); + let invalid = false; + yield* checkGraphContentSteps(content, { + addIssue: () => { + invalid = true; + }, + }); + // An invalid owned graph is a construction defect; report it with the exact + // issues the content schema produces. + if (invalid) { + javascriptApplicationGraphContentSchema.parse(content); + throw new TypeError("Application graph checks disagree with its schema"); + } + return yield* sealValidatedApplicationGraphSteps({ + ...content, + graph_id: `jag_${yield* canonicalJsonDigestSteps(content)}`, + }); +} + function* sealValidatedApplicationGraphSteps( graph: JavaScriptApplicationGraph, ): Generator { diff --git a/src/domain/javascript/javascriptSemanticGraph.ts b/src/domain/javascript/javascriptSemanticGraph.ts index 9ab2dd485c..6908af2a19 100644 --- a/src/domain/javascript/javascriptSemanticGraph.ts +++ b/src/domain/javascript/javascriptSemanticGraph.ts @@ -6,6 +6,10 @@ import { uniqueSorted } from "../canonicalOrdering.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; import { canonicalJsonDigestSteps } from "../canonicalJsonDigestSteps.js"; import { freezeOwnedJsonSnapshotSteps } from "../immutableJson.js"; +import { + arrayContainerSchema, + parseElementSteps, +} from "../zodCollectionSteps.js"; import type { ApplicationGraphEvidence, ApplicationGraphEvidenceContext, @@ -765,6 +769,15 @@ export const createJavaScriptSemanticGraph = ( const validatedImmutableSemanticGraphs = new WeakSet(); +const semanticGraphShape = javaScriptSemanticGraphInputSchema.shape; +const semanticGraphContainerSchema = javaScriptSemanticGraphInputSchema.extend({ + evidence_contexts: arrayContainerSchema(semanticGraphShape.evidence_contexts), + nodes: arrayContainerSchema(semanticGraphShape.nodes), + relations: arrayContainerSchema(semanticGraphShape.relations), + fingerprints: arrayContainerSchema(semanticGraphShape.fingerprints), + unknowns: arrayContainerSchema(semanticGraphShape.unknowns), +}); + /** Clone input synchronously; commit, check and seal the owned graph in steps. */ export const createImmutableJavaScriptSemanticGraphSteps = ( input: unknown, @@ -773,6 +786,58 @@ export const createImmutableJavaScriptSemanticGraphSteps = ( normalizeGraphInput(javaScriptSemanticGraphInputSchema.parse(input)), ); +/** + * Seal a graph whose input the caller transfers: the caller neither retains nor + * mutates any part of it afterwards. Only the graph's own fields are validated + * synchronously; collection elements are validated and cloned in steps, because + * a whole-graph parse of a large application held the event loop for seconds. + */ +export const sealTransferredJavaScriptSemanticGraphSteps = ( + input: unknown, +): Generator => + parseAndSealSemanticGraphSteps(semanticGraphContainerSchema.parse(input)); + +function* parseAndSealSemanticGraphSteps( + container: z.output, +): Generator { + const issues: z.core.$ZodIssue[] = []; + const parsed: JavaScriptSemanticGraphInput = { + ...container, + evidence_contexts: yield* parseElementSteps( + semanticGraphShape.evidence_contexts.element, + container.evidence_contexts, + ["evidence_contexts"], + issues, + ), + nodes: yield* parseElementSteps( + semanticGraphShape.nodes.element, + container.nodes, + ["nodes"], + issues, + ), + relations: yield* parseElementSteps( + semanticGraphShape.relations.element, + container.relations, + ["relations"], + issues, + ), + fingerprints: yield* parseElementSteps( + semanticGraphShape.fingerprints.element, + container.fingerprints, + ["fingerprints"], + issues, + ), + unknowns: yield* parseElementSteps( + semanticGraphShape.unknowns.element, + container.unknowns, + ["unknowns"], + issues, + ), + }; + if (issues.length > 0) throw new z.ZodError(issues); + return yield* validateAndSealSemanticGraphSteps(normalizeGraphInput(parsed)); +} + function* validateAndSealSemanticGraphSteps( semantic: JavaScriptSemanticGraphInput, ): Generator { diff --git a/src/domain/zodCollectionSteps.ts b/src/domain/zodCollectionSteps.ts new file mode 100644 index 0000000000..a32e8c8044 --- /dev/null +++ b/src/domain/zodCollectionSteps.ts @@ -0,0 +1,33 @@ +import { z } from "zod"; + +/** + * An array schema's own constraints, such as a minimum length, without its + * element schema. Pair it with {@link parseElementSteps} to validate a large + * collection in cooperative steps. + */ +export const arrayContainerSchema = ( + array: z.ZodArray, +): z.ZodType => array.clone({ ...array.def, element: z.unknown() }); + +/** Elements validated between yields, in the issues and paths a whole-array parse reports. */ +export function* parseElementSteps( + element: Element, + values: readonly unknown[], + path: readonly PropertyKey[], + issues: z.core.$ZodIssue[], +): Generator[]> { + const parsed: z.output[] = []; + for (const [index, value] of values.entries()) { + if (index % 64 === 0) yield; + const result = element.safeParse(value); + if (result.success) parsed.push(result.data); + else + issues.push( + ...result.error.issues.map((issue) => ({ + ...issue, + path: [...path, index, ...issue.path], + })), + ); + } + return parsed; +}