From 9f42495d11100adc8646de5c8b62762ab9715ae6 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:54:55 +0800 Subject: [PATCH] fix(javascript): hash graph and evidence JSON incrementally (#646) --- docs/javascript-artifact-reconstruction.md | 17 ++++ package.json | 1 + scripts/verify-large-javascript-digests.mjs | 38 ++++++++ .../JavaScriptApplicationService.test.ts | 87 +++++++++++++++++++ .../JavaScriptApplicationService.ts | 40 ++++++++- src/domain/canonicalDigest.test.ts | 83 ++++++++++++++++++ src/domain/canonicalDigest.ts | 75 ++++++++++++++++ src/domain/comparisonSemantics.ts | 8 +- src/domain/evidence.ts | 18 +--- 9 files changed, 346 insertions(+), 21 deletions(-) create mode 100644 scripts/verify-large-javascript-digests.mjs create mode 100644 src/application/JavaScriptApplicationService.test.ts create mode 100644 src/domain/canonicalDigest.test.ts create mode 100644 src/domain/canonicalDigest.ts diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md index a088629342..97d6cd41a8 100644 --- a/docs/javascript-artifact-reconstruction.md +++ b/docs/javascript-artifact-reconstruction.md @@ -52,6 +52,23 @@ manifest and graph commitments, JavaScript Application Graph, static Electron summary, reconstruction statistics, and explicit limitations. It does not require a live Hopper, Ghidra, browser, or Electron process. +## Large results + +Graph and Evidence identifiers hash canonical JSON incrementally, without +assembling a single string for the whole graph. The canonical bytes and existing +identifiers remain unchanged. The opt-in regression check is +`npm run verify:javascript:digests`; +it hashes a value larger than the running Node engine's single-string limit. + +Output formatters and MCP transport serialization still assemble whole strings. +For a large CLI result, select the needed fields +before formatting, for example `--format json --filter-output +evidence_id,normalized_result.statistics`. Complete serialization of a result +beyond the engine's string limit is not established by this digest check. +Client framing limits also apply: the pinned Node MCP SDK's stdio transport +defaults to a 10 MiB buffer. Its caller-selected `maxBufferSize` must accommodate +the complete response, including text and structured Evidence projections. + ## What is reconstructed The projector reuses the content-addressed artifact inventory and safe artifact diff --git a/package.json b/package.json index f9d6de563b..5caf041382 100644 --- a/package.json +++ b/package.json @@ -128,6 +128,7 @@ "verify:electron": "npm run build:cached && node scripts/verify-real-electron.mjs", "verify:inspector": "npm run build:cached && node scripts/verify-real-javascript-runtime-observation.mjs", "verify:javascript-runtime": "npm run build:cached && node scripts/verify-local-javascript-runtime.mjs", + "verify:javascript:digests": "npm run build:cached && node scripts/verify-large-javascript-digests.mjs", "verify:application-workflows": "npm run build:cached && node scripts/verify-local-application-workflows.mjs", "verify:managed": "npm run build:cached && node scripts/verify-managed-conformance.mjs", "verify:agent": "npm run build:cached && node scripts/verify-agent-experience.mjs", diff --git a/scripts/verify-large-javascript-digests.mjs b/scripts/verify-large-javascript-digests.mjs new file mode 100644 index 0000000000..14fa5977e1 --- /dev/null +++ b/scripts/verify-large-javascript-digests.mjs @@ -0,0 +1,38 @@ +#!/usr/bin/env node + +import assert from "node:assert/strict"; +import { constants } from "node:buffer"; +import { createHash } from "node:crypto"; + +import { canonicalDigest } from "../dist/domain/comparisonSemantics.js"; +import { createEvidence, parseEvidence } from "../dist/domain/evidence.js"; + +// This opt-in check crosses the actual engine string limit; keep it outside +// routine source tests. Build first, then run this script with the local Node. +const leaf = "x".repeat(1024 * 1024); +const encodedLeaf = JSON.stringify(leaf); +const count = Math.ceil(constants.MAX_STRING_LENGTH / encodedLeaf.length) + 1; +const value = { payload: Array.from({ length: count }, () => leaf) }; +const expected = createHash("sha256"); +expected.update('{"payload":['); +for (let index = 0; index < count; index += 1) { + if (index > 0) expected.update(","); + expected.update(encodedLeaf); +} +expected.update("]}"); +assert.equal(canonicalDigest(value), expected.digest("hex")); + +const evidence = createEvidence( + undefined, + { id: "large-json-fixture", name: "Large JSON fixture", version: "1" }, + { operation: "inspect_fixture", parameters: {}, result: value }, +); +assert.equal(parseEvidence(evidence).evidence_id, evidence.evidence_id); +process.stdout.write( + `${JSON.stringify({ + verified: true, + serialized_value_bytes: count * encodedLeaf.length + count - 1 + 14, + engine_string_limit: constants.MAX_STRING_LENGTH, + evidence_id: evidence.evidence_id, + })}\n`, +); diff --git a/src/application/JavaScriptApplicationService.test.ts b/src/application/JavaScriptApplicationService.test.ts new file mode 100644 index 0000000000..f43d3ce197 --- /dev/null +++ b/src/application/JavaScriptApplicationService.test.ts @@ -0,0 +1,87 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js"; + +describe("JavaScript application failure diagnostics", () => { + it("retains unexpected failures and their input identity in caller-visible diagnostics", async () => { + const inputPath = await createTestTempDirectory("rea-js-failure-"); + await writeFile(join(inputPath, "main.js"), "export const value = 1;\n"); + const cause = new RangeError("Invalid string length"); + const result = await analyzeJavaScriptApplication( + { input_path: inputPath, format: "directory" }, + { + progress: { + report: async (event) => { + if (event.terminal) throw cause; + }, + }, + }, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected analysis failure"); + expect(result.error.cause).toBe(cause); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "execution_failure", + details: { + provider_id: "rea-javascript-application", + operation: "analyze_javascript_application", + diagnostics: { + input_path: inputPath, + error_name: "RangeError", + error_message: "Invalid string length", + }, + }, + }); + }); + + it("keeps filesystem failures distinct from engine failures", async () => { + const root = await createTestTempDirectory("rea-js-missing-"); + const result = await analyzeJavaScriptApplication({ + input_path: join(root, "missing"), + format: "directory", + }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected missing artifact failure"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "artifact_operation_failed", + details: { operation: "analyze_javascript_application", reason: "io" }, + }); + }); + + it.each(["reported failure", null])( + "retains non-Error failures (%s)", + async (cause) => { + const inputPath = await createTestTempDirectory("rea-js-non-error-"); + await writeFile(join(inputPath, "main.js"), "export const value = 1;\n"); + const result = await analyzeJavaScriptApplication( + { input_path: inputPath, format: "directory" }, + { + progress: { + report: async (event) => { + if (event.terminal) throw cause; + }, + }, + }, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected analysis failure"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "execution_failure", + details: { + diagnostics: { + error_name: "UnknownError", + error_message: + typeof cause === "string" + ? cause + : "JavaScript analysis failed with a non-Error value", + }, + }, + }); + }, + ); +}); diff --git a/src/application/JavaScriptApplicationService.ts b/src/application/JavaScriptApplicationService.ts index 8e51a6d0f7..b2389cb685 100644 --- a/src/application/JavaScriptApplicationService.ts +++ b/src/application/JavaScriptApplicationService.ts @@ -14,9 +14,11 @@ import { type AnalysisError } from "../domain/analysisErrorBase.js"; import type { Evidence } from "../domain/evidence.js"; import { projectInputIssues } from "../domain/inputIssueProjection.js"; import { err, ok, type Result } from "../domain/result.js"; +import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import type { ExecutionOptions } from "./AnalysisProvider.js"; import { createJavaScriptApplicationEvidence } from "./JavaScriptApplicationEvidence.js"; import { reconstructJavaScriptArtifact } from "./JavaScriptArtifactReconstruction.js"; +import { JAVASCRIPT_APPLICATION_PROVIDER } from "./InvestigationProviders.js"; const OPERATION = "analyze_javascript_application" as const; @@ -82,6 +84,42 @@ export const analyzeJavaScriptApplicationValidated = async ( { cause }, ), ); - return err(new ArtifactOperationError(OPERATION, "io")); + if ( + cause instanceof Error && + "code" in cause && + typeof cause.code === "string" && + FILESYSTEM_ERROR_CODES.has(cause.code) + ) + return err(new ArtifactOperationError(OPERATION, "io")); + return err( + new ProviderAdapterError(JAVASCRIPT_APPLICATION_PROVIDER.id, OPERATION, { + cause, + diagnostics: { + input_path: input.input_path, + error_name: cause instanceof Error ? cause.name : "UnknownError", + error_message: + cause instanceof Error + ? cause.message + : typeof cause === "string" + ? cause + : "JavaScript analysis failed with a non-Error value", + }, + }), + ); } }; + +const FILESYSTEM_ERROR_CODES = new Set([ + "ENOENT", + "EACCES", + "EPERM", + "ENOTDIR", + "EISDIR", + "EIO", + "ELOOP", + "ENAMETOOLONG", + "EMFILE", + "ENFILE", + "ENOSPC", + "EROFS", +]); diff --git a/src/domain/canonicalDigest.test.ts b/src/domain/canonicalDigest.test.ts new file mode 100644 index 0000000000..02a641905a --- /dev/null +++ b/src/domain/canonicalDigest.test.ts @@ -0,0 +1,83 @@ +import { createHash } from "node:crypto"; + +import { fc, it } from "@fast-check/vitest"; +import canonicalize from "canonicalize"; +import { describe, expect } from "vitest"; + +import { digestCanonicalValue } from "./canonicalDigest.js"; + +const legacyDigest = (value: unknown): string => { + const encoded = canonicalize(value); + if (encoded === undefined) throw new TypeError("Unserializable fixture"); + return createHash("sha256").update(encoded).digest("hex"); +}; + +describe("incremental canonical digest", () => { + it.prop([fc.jsonValue()])( + "preserves canonicalize digests for JSON values", + (value) => { + expect(digestCanonicalValue(value)).toBe(legacyDigest(value)); + }, + ); + + it("preserves Unicode ordering, escaping, numeric forms and non-JSON normalization", () => { + const shared = { key: "value" }; + const sparse: unknown[] = [undefined, Symbol("omit"), () => undefined]; + sparse.length = 6; + const fixtures: unknown[] = [ + { + "€": "\ud800", + "😀": "\udc00", + "\r": '\n\t"\\', + number: -0, + exponent: 1e30, + }, + { omitted: undefined, symbol: Symbol("omit"), fn: () => undefined }, + sparse, + [shared, shared], + new Date("2026-10-06T00:00:00Z"), + { toJSON: () => ({ b: 2, a: [1, null] }) }, + { nested: { toJSON: () => undefined } }, + ]; + for (const value of fixtures) + expect(digestCanonicalValue(value)).toBe(legacyDigest(value)); + }); + + it("rejects unsupported roots, non-finite numbers, BigInt and ancestor cycles", () => { + for (const value of [undefined, Symbol("root"), () => undefined]) + expect(() => digestCanonicalValue(value, "Fixture")).toThrow( + "Fixture could not canonicalize data", + ); + const cyclic: Record = {}; + cyclic.self = cyclic; + const recursiveJson = { toJSON: (): unknown => recursiveJson }; + for (const value of [NaN, Infinity, -Infinity, 1n, cyclic, recursiveJson]) + expect(() => digestCanonicalValue(value)).toThrow(); + }); + + it("hashes repeated large leaves with the same byte sequence", () => { + const value = { + payload: Array.from({ length: 64 }, () => "x".repeat(16_384)), + }; + expect(digestCanonicalValue(value)).toBe(legacyDigest(value)); + }); + + it("preserves canonicalize property access and toJSON cycle behavior", () => { + const createAccessor = () => { + let reads = 0; + return { + get value() { + reads += 1; + return reads; + }, + }; + }; + expect(digestCanonicalValue(createAccessor())).toBe( + legacyDigest(createAccessor()), + ); + const shared = { toJSON: () => ({ answer: 42 }) }; + expect(digestCanonicalValue([shared, shared])).toBe( + legacyDigest([shared, shared]), + ); + }); +}); diff --git a/src/domain/canonicalDigest.ts b/src/domain/canonicalDigest.ts new file mode 100644 index 0000000000..c9f1c6260c --- /dev/null +++ b/src/domain/canonicalDigest.ts @@ -0,0 +1,75 @@ +import { createHash } from "node:crypto"; + +/** Hash canonicalize-compatible JSON without assembling an aggregate string. */ +export const digestCanonicalValue = ( + value: unknown, + context = "Comparison", +): string => { + const hash = createHash("sha256"); + const encoded = emitCanonical(value, (part) => hash.update(part), new Set()); + if (!encoded) throw new TypeError(`${context} could not canonicalize data`); + return hash.digest("hex"); +}; + +type Emit = (part: string) => void; + +const emitCanonical = ( + value: unknown, + emit: Emit, + ancestors: Set, +): boolean => { + if (typeof value === "number" && !Number.isFinite(value)) + throw new Error( + Number.isNaN(value) ? "NaN is not allowed" : "Infinity is not allowed", + ); + if (value === null || typeof value !== "object") { + const encoded = JSON.stringify(value); + if (encoded === undefined) return false; + emit(encoded); + return true; + } + if (ancestors.has(value)) throw new Error("Circular reference detected"); + ancestors.add(value); + try { + if ("toJSON" in value && typeof value.toJSON === "function") { + const normalized: unknown = value.toJSON(); + return emitCanonical(normalized, emit, ancestors); + } + if (Array.isArray(value)) { + emit("["); + const length = value.length; + for (let index = 0; index < length; index += 1) { + if (index > 0) emit(","); + // Preserve canonicalize's sparse-array and unsupported-value behavior. + if (!(index in value)) continue; + const item: unknown = value[index]; + emitCanonical( + item === undefined || typeof item === "symbol" ? null : item, + emit, + ancestors, + ); + } + emit("]"); + } else { + emit("{"); + let first = true; + for (const key of Object.keys(value).sort()) { + if ( + Reflect.get(value, key) === undefined || + typeof Reflect.get(value, key) === "symbol" + ) + continue; + const item: unknown = Reflect.get(value, key); + if (!first) emit(","); + first = false; + emit(JSON.stringify(key)); + emit(":"); + if (!emitCanonical(item, emit, ancestors)) emit("undefined"); + } + emit("}"); + } + return true; + } finally { + ancestors.delete(value); + } +}; diff --git a/src/domain/comparisonSemantics.ts b/src/domain/comparisonSemantics.ts index 804a446baa..9c005d1241 100644 --- a/src/domain/comparisonSemantics.ts +++ b/src/domain/comparisonSemantics.ts @@ -1,7 +1,7 @@ -import { createHash } from "node:crypto"; - import canonicalize from "canonicalize"; +import { digestCanonicalValue } from "./canonicalDigest.js"; + /** Whether an inventory can support a claim that an item is absent. */ export const absenceClaimable = (coverage: { readonly status: string; @@ -17,9 +17,7 @@ export const canonicalDigest = ( value: unknown, context = "Comparison", ): string => { - return createHash("sha256") - .update(canonicalJson(value, context)) - .digest("hex"); + return digestCanonicalValue(value, context); }; /** Canonical JSON used for deterministic tie breaking and stable digests. */ diff --git a/src/domain/evidence.ts b/src/domain/evidence.ts index c7ee5b8851..c4ebc71878 100644 --- a/src/domain/evidence.ts +++ b/src/domain/evidence.ts @@ -1,8 +1,7 @@ -import { createHash } from "node:crypto"; - -import canonicalize from "canonicalize"; import { z } from "zod"; +import { digestCanonicalValue } from "./canonicalDigest.js"; + import { analysisProfileSchema, type AnalysisProfileCommitment, @@ -173,22 +172,11 @@ export interface EvidenceObservation { readonly evidenceLinks?: readonly string[]; } -const sha256 = (value: string): string => - createHash("sha256").update(value).digest("hex"); - type WithoutEvidenceId = Record_ extends unknown ? Omit : never; type EvidenceWithoutId = WithoutEvidenceId; -/** Serialize JSON according to RFC 8785 JSON Canonicalization Scheme. */ -const canonicalJson = (value: JsonValue): string => { - const serialized = canonicalize(value); - if (serialized === undefined) - throw new TypeError("RFC 8785 canonicalization rejected a JSON value"); - return serialized; -}; - const semanticProjection = (evidence: EvidenceWithoutId): JsonValue => ({ subject: evidence.subject === null @@ -217,7 +205,7 @@ const semanticProjection = (evidence: EvidenceWithoutId): JsonValue => ({ /** Recompute the semantic identifier, excluding paths and raw payload bytes. */ const computeEvidenceId = (evidence: EvidenceWithoutId): string => - `ev_${sha256(canonicalJson(semanticProjection(evidence)))}`; + `ev_${digestCanonicalValue(semanticProjection(evidence), "Evidence")}`; /** Parse evidence and reject a syntactically valid but tampered semantic ID. */ export const parseEvidence = (input: unknown): Evidence => {