From 96a46ada0a36f0349f2b0fc20b4c932401b4ad84 Mon Sep 17 00:00:00 2001 From: DLME2024 Date: Wed, 7 Oct 2026 12:22:10 -0500 Subject: [PATCH] fix(javascript): bound semantic node projection for resource safety (#966) * fix(javascript): bound semantic node projection for resource safety analyze_javascript_application exhausted the heap or failed serialization on ordinary inputs. A small extracted tree crashed outright, and a larger application tree failed at both the default heap and an 8 GB heap. Cause: the semantic projection emits one node per AST expression, call site, binding and property slot, with no ceiling. A five-file tree already yields on the order of 170,000 nodes, and the whole result is serialized after the object graph is built and validated. Result ceilings were deliberately removed upstream, so nothing bounded this growth. Apply two resource-safety ceilings and report them through the graph's own coverage contract rather than dropping data silently: - SEMANTIC_GRAPH_NODE_CEILING bounds retained nodes per tree (100,000, roughly 300 MB serialized, below V8's 512 MB string limit). - SEMANTIC_GRAPH_FILE_NODE_CEILING bounds each source file (20,000) so one bundled library cannot starve the application's own modules. Exhausting either marks coverage truncated with a partial status and records the ceiling in coverage.limits. A small application stays complete and untruncated. Verified: a previously-crashing application tree now returns Evidence in under a minute, while a small tree stays complete. Existing tests and check:fast pass. * fix(javascript): report semantic graph truncation accurately --------- Co-authored-by: REA Contributor Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- .../JavaScriptApplicationService.test.ts | 35 ++++++ .../JavaScriptSemanticGraphBuilder.test.ts | 108 +++++++++++++++++- .../JavaScriptSemanticGraphBuilder.ts | 67 +++++++++-- .../JavaScriptSemanticGraphConstruction.ts | 26 +++++ 4 files changed, 228 insertions(+), 8 deletions(-) diff --git a/src/application/javascript/JavaScriptApplicationService.test.ts b/src/application/javascript/JavaScriptApplicationService.test.ts index 5ff423b09b..83e93e7132 100644 --- a/src/application/javascript/JavaScriptApplicationService.test.ts +++ b/src/application/javascript/JavaScriptApplicationService.test.ts @@ -47,6 +47,8 @@ describe("JavaScript application failure diagnostics", () => { const inputPath = await createTestTempDirectory("rea-js-failure-"); await writeFile(join(inputPath, "main.js"), "export const value = 1;\n"); const cause = new RangeError("Invalid string length"); + // Retain the original failure locally, but never serialize its object graph. + cause.cause = cause; const result = await analyzeJavaScriptApplication( { input_path: inputPath, format: "directory" }, { @@ -72,6 +74,8 @@ describe("JavaScript application failure diagnostics", () => { }, }, }); + const projection = projectAnalysisError(result.error); + expect(JSON.parse(JSON.stringify(projection))).toEqual(projection); }); it("keeps filesystem failures distinct from engine failures", async () => { @@ -119,4 +123,35 @@ describe("JavaScript application failure diagnostics", () => { }); }, ); + + it.each([new Error("x".repeat(10_000)), "y".repeat(10_000)])( + "preserves long failure messages through JSON projection (%#)", + async (cause) => { + const inputPath = await createTestTempDirectory("rea-js-oversized-"); + await writeFile(join(inputPath, "main.js"), "export const value = 1;\n"); + const message = cause instanceof Error ? cause.message : cause; + const result = await analyzeJavaScriptApplication( + { input_path: inputPath, format: "directory" }, + { + progress: { + report: async (event) => { + if (event.terminal) throw cause; + }, + }, + }, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected analysis failure"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "execution_failure", + details: { + diagnostics: { + error_message: message, + }, + }, + }); + const projection = projectAnalysisError(result.error); + expect(JSON.parse(JSON.stringify(projection))).toEqual(projection); + }, + ); }); diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts index e7d1c1240a..2707b06d21 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts @@ -1,6 +1,15 @@ import { expect, it } from "vitest"; -import { buildJavaScriptSemanticGraph } from "./JavaScriptSemanticGraphBuilder.js"; +import { + buildJavaScriptSemanticGraph, + SEMANTIC_GRAPH_FILE_NODE_CEILING, + SEMANTIC_GRAPH_NODE_CEILING, +} from "./JavaScriptSemanticGraphBuilder.js"; +import { + addSemanticGraphNode, + constructSemanticGraphNode, + createSemanticGraphProjectionState, +} from "./JavaScriptSemanticGraphConstruction.js"; import type { JavaScriptArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; import { queryJavaScriptSemanticGraph } from "../../domain/javascript/javascriptSemanticQuery.js"; @@ -287,6 +296,31 @@ const graphFor = (source: string) => { }); }; +const semanticNodeFor = (roleKey: string) => { + const file: JavaScriptArtifactFile = { + path: "budget.js", + container_sha256: SHA256, + sha256: SHA256, + bytes: 0, + inventory_artifact_id: `art_${SHA256}`, + kind: "javascript", + unpacked: false, + text: { included: true, value: "" }, + }; + const state = createSemanticGraphProjectionState({ nodes: [] }); + return constructSemanticGraphNode( + file, + { + kind: "expression", + roleKey, + location: null, + label: roleKey, + functionNodeId: null, + }, + state, + ); +}; + it.each([ "const value = consume(Promise.resolve(1));", "async function run() { await consume(Promise.resolve(1)); }", @@ -428,3 +462,75 @@ it.each([ expect(query.relations).toContainEqual(relation); expect(query.nodes.map(({ node_id }) => node_id)).toContain(promise?.node_id); }); + +it("bounds semantic node projection and reports the ceiling in coverage", () => { + // A bundled vendor library can emit tens of thousands of semantic nodes on its + // own (measured: 135,286 from one minified editor bundle). Without a ceiling the + // projection exhausts the heap and the result cannot be serialized. The bound + // must be reported through coverage rather than dropping data silently. + const statements = Array.from( + { length: 4_000 }, + (_, index) => + `const value${index} = { a: ${index}, b: [${index}, ${index + 1}] };`, + ).join("\n"); + const graph = graphFor(statements); + + expect(graph.nodes.length).toBeLessThanOrEqual(SEMANTIC_GRAPH_NODE_CEILING); + expect(graph.coverage.truncated).toBe(true); + expect(graph.coverage.status).toBe("partial"); + expect(graph.coverage.omitted_nodes).toBeNull(); + expect(graph.coverage.omitted_relations).toBeNull(); + expect( + graph.coverage.families.every( + ({ omitted_relations }) => omitted_relations === null, + ), + ).toBe(true); + expect( + graph.coverage.limits.some( + ({ name }) => name === "semantic_graph_node_ceiling", + ), + ).toBe(true); +}); + +it("leaves a small application complete and untruncated", () => { + const graph = graphFor("const answer = 40 + 2;"); + expect(graph.coverage.truncated).toBe(false); + expect(graph.coverage.omitted_nodes).toBe(0); + expect(graph.coverage.omitted_relations).toBe(0); + expect(graph.coverage.limits).toEqual([]); +}); + +it("does not mark a file truncated when it exactly fills its budget", () => { + // Each unbound reference projects one expression, plus the module root. + const graph = graphFor( + "external;\n".repeat(SEMANTIC_GRAPH_FILE_NODE_CEILING - 1), + ); + expect(graph.nodes).toHaveLength(SEMANTIC_GRAPH_FILE_NODE_CEILING); + expect(graph.coverage).toMatchObject({ + truncated: false, + omitted_nodes: 0, + omitted_relations: 0, + limits: [], + }); +}); + +it("only records node loss when the budget rejects a new identity", () => { + // The truncation signal must record blocked node creation, not a zero + // remaining budget: a file using exactly its share drops nothing. + const state = createSemanticGraphProjectionState({ nodes: [] }); + state.fileNodeBudget = 2; + state.fileNodesDropped = false; + const first = addSemanticGraphNode(state, semanticNodeFor("first")); + const second = addSemanticGraphNode(state, semanticNodeFor("second")); + expect(first).not.toBeNull(); + expect(second).not.toBeNull(); + expect(state.fileNodeBudget).toBe(0); + expect(state.fileNodesDropped).toBe(false); + + expect(addSemanticGraphNode(state, semanticNodeFor("first"))).toBe(first); + expect(state.fileNodesDropped).toBe(false); + + const blocked = addSemanticGraphNode(state, semanticNodeFor("third")); + expect(blocked).toBeNull(); + expect(state.fileNodesDropped).toBe(true); +}); diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts index d82edde74f..38a3522343 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts @@ -74,7 +74,35 @@ interface FileContext extends SemanticFlowProjectionContext { readonly callResolutions: ReadonlyMap; } -/** Project per-file semantic IR into an artifact-bound companion graph. */ +/** + * Resource-safety ceiling on retained semantic nodes. + * + * The semantic projection emits one node per AST expression, call site, binding + * and property slot. That is unbounded in input size. A small tree of five files + * already yields on the order of 170,000 nodes, and larger trees yield millions, + * exhausting the heap before the result can be serialized. + * + * This is a real resource-safety constraint, not a presentation preference, so + * the ceiling is applied and reported through the graph's own `coverage` fields + * (`truncated`, `omitted_nodes`, `omitted_relations`, `limits`) rather than + * silently dropping data. + * + * Sizing: a retained semantic node costs roughly 3 KB once serialized, and V8 + * additionally caps a single string at 512 MB. The result is serialized after the + * object graph is built and validated, so both phases must fit together. A single + * minified vendor bundle can account for most of a tree's nodes, so a whole-tree + * ceiling without a per-file ceiling does not bound the work. Both are applied. + */ +export const SEMANTIC_GRAPH_NODE_CEILING = 100_000; + +/** + * Per-file share of the semantic node budget. + * + * Bound each source file independently so one large bundled library cannot + * consume the whole tree's budget and starve the application's own modules. + */ +export const SEMANTIC_GRAPH_FILE_NODE_CEILING = 20_000; + export const buildJavaScriptSemanticGraph = ({ rootArtifactSha256, applicationGraph, @@ -82,11 +110,27 @@ export const buildJavaScriptSemanticGraph = ({ }: BuilderInput): JavaScriptSemanticGraph => { const state = emptyState(applicationGraph); const fingerprints: JavaScriptSemanticGraph["fingerprints"][number][] = []; + let truncatedFiles = 0; for (const analyzed of analysis.files) { if (analyzed.semantic === null) continue; + if (state.nodes.size >= SEMANTIC_GRAPH_NODE_CEILING) { + truncatedFiles += 1; + continue; + } + const remainingTreeBudget = SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size; + state.fileNodeBudget = Math.min( + SEMANTIC_GRAPH_FILE_NODE_CEILING, + remainingTreeBudget, + ); + state.fileNodesDropped = false; fingerprints.push( ...projectFile(analyzed.file, analyzed.semantic.ir, state), ); + // Nodes were dropped only if the budget actually blocked creation. A file + // that exactly fills its share ends with a zero budget without dropping + // anything, so the remaining budget alone cannot decide truncation. + if (state.fileNodesDropped) truncatedFiles += 1; + state.fileNodeBudget = null; } if (state.roots.size === 0) addFallbackRoot(rootArtifactSha256, state); const unknowns = [...state.unknowns.values()]; @@ -100,11 +144,20 @@ export const buildJavaScriptSemanticGraph = ({ fingerprints, unknowns, coverage: { - status: "unknown", - truncated: false, - omitted_nodes: 0, - omitted_relations: 0, - limits: [], + status: truncatedFiles > 0 ? "partial" : "unknown", + truncated: truncatedFiles > 0, + omitted_nodes: truncatedFiles > 0 ? null : 0, + omitted_relations: truncatedFiles > 0 ? null : 0, + limits: + truncatedFiles > 0 + ? [ + { + name: "semantic_graph_node_ceiling", + value: SEMANTIC_GRAPH_NODE_CEILING, + unit: "items" as const, + }, + ] + : [], families: JAVASCRIPT_SEMANTIC_RELATION_FAMILIES.map((family) => ({ family, status: semanticFamilyStatus(family, analysis), @@ -112,7 +165,7 @@ export const buildJavaScriptSemanticGraph = ({ (relation) => JAVASCRIPT_SEMANTIC_RELATION_FAMILY[relation.relation] === family, ).length, - omitted_relations: 0, + omitted_relations: truncatedFiles > 0 ? null : 0, unknown_ids: unknowns .filter((unknown) => unknown.family === family) .map(({ unknown_id: identifier }) => identifier), diff --git a/src/application/javascript/JavaScriptSemanticGraphConstruction.ts b/src/application/javascript/JavaScriptSemanticGraphConstruction.ts index a19f749826..d69d9e7367 100644 --- a/src/application/javascript/JavaScriptSemanticGraphConstruction.ts +++ b/src/application/javascript/JavaScriptSemanticGraphConstruction.ts @@ -26,6 +26,24 @@ export interface SemanticGraphProjectionState { readonly unknowns: Map; readonly roots: Set; readonly applicationNodeIdsByLocation: ReadonlyMap; + /** + * Remaining node budget for the file currently being projected. + * + * The projection emits one node per AST expression, call site, binding and + * property slot, which is unbounded in file size: one bundled vendor library + * measured 135,286 nodes on its own. `null` means unbounded; a number is the + * number of nodes this file may still add. Exhausting it stops node creation + * for that file and is reported through the graph's `coverage` fields. + */ + fileNodeBudget?: number | null; + /** + * Whether the file currently being projected lost any nodes to its budget. + * + * A file that exactly fills its share ends with a zero budget without + * dropping anything, so callers must consult this flag rather than the + * remaining budget to decide whether coverage is truncated. + */ + fileNodesDropped?: boolean; } /** Input for one exact artifact-version semantic node. */ @@ -90,6 +108,14 @@ export const addSemanticGraphNode = ( ): JavaScriptSemanticGraphNode | null => { const existing = state.nodes.get(node.node_id); if (existing !== undefined) return existing; + const budget = state.fileNodeBudget; + if (typeof budget === "number") { + if (budget <= 0) { + state.fileNodesDropped = true; + return null; + } + state.fileNodeBudget = budget - 1; + } state.nodes.set(node.node_id, node); return node; };