diff --git a/docs/firmware-analysis.md b/docs/firmware-analysis.md index aab58f5d47..b529ac6d0d 100644 --- a/docs/firmware-analysis.md +++ b/docs/firmware-analysis.md @@ -158,7 +158,17 @@ tree semantics, vulnerability scanning, emulation or hardware runtime capture. Supply a stable copy within the allowance. No in-place source modification. - One queued heavy operation per provider instance; Binwalk one thread and Unblob one worker. No automatic parallel extraction engines. -- **120-second** deadline; inherited **1 GiB per-process address-space** limit, +- **120-second** analysis deadline through inventory. Publication and final + cleanup start a separate **120-second** budget; select + `publication_timeout_ms` in MCP or `--publication-timeout-ms` in CLI for a slow + destination. Valid budgets are integers from 1 through 2,147,483,647 ms (the + timer representation limit). Publication timeouts identify their phase and + verified file progress; verified writes before atomic commit are not a claim + that output remains published after rollback. Caller cancellation applies to + both phases and final cleanup. Phase handoff, publication commit and final + completion also check monotonic elapsed time, so delayed timer callbacks + cannot revive an expired budget. +- Inherited **1 GiB per-process address-space** limit, 120 CPU seconds and zero core-dump allowance through `prlimit`. - JSON reports: at most **8 MiB** and valid UTF-8; malformed encoding and oversized reports fail without repairing or truncating producer data. diff --git a/src/application/firmware/FirmwareAnalysisService.ts b/src/application/firmware/FirmwareAnalysisService.ts index 774dd5ca65..6b553c6c4a 100644 --- a/src/application/firmware/FirmwareAnalysisService.ts +++ b/src/application/firmware/FirmwareAnalysisService.ts @@ -30,7 +30,9 @@ export class FirmwareAnalysisService { options?: ExecutionOptions, ): Promise> { if (options?.signal?.aborted === true) - return err(new AnalysisCancelledError(operation)); + return err( + new AnalysisCancelledError(operation, { cause: options.signal.reason }), + ); const parsed = firmwareRequestSchema.safeParse({ operation, input }); if (!parsed.success) return err(new AnalysisInputError(operation, { cause: parsed.error })); diff --git a/src/cli/firmwareCommands.ts b/src/cli/firmwareCommands.ts index a29cee323b..742dd92105 100644 --- a/src/cli/firmwareCommands.ts +++ b/src/cli/firmwareCommands.ts @@ -73,6 +73,7 @@ export const registerFirmwareCommands = ( maxDepth: extractInput.max_depth, maxOutputBytes: extractInput.max_output_bytes, maxOutputFiles: extractInput.max_output_files, + publicationTimeoutMs: extractInput.publication_timeout_ms, }), run: ({ args, options }) => execute(CLI_COMMANDS.extractFirmware, "extract_firmware", { @@ -84,6 +85,7 @@ export const registerFirmwareCommands = ( max_depth: options.maxDepth, max_output_bytes: options.maxOutputBytes, max_output_files: options.maxOutputFiles, + publication_timeout_ms: options.publicationTimeoutMs, }), }); }; diff --git a/src/domain/firmware/firmwareAnalysis.ts b/src/domain/firmware/firmwareAnalysis.ts index e25a378fbf..a434af3916 100644 --- a/src/domain/firmware/firmwareAnalysis.ts +++ b/src/domain/firmware/firmwareAnalysis.ts @@ -45,6 +45,16 @@ export const firmwareInputSchemas = { .max(100_000) .default(10_000) .describe("Staging entry budget, including directories and links"), + publication_timeout_ms: z + .number() + .int() + .min(1) + // Node timers cannot represent longer delays without overflow. + .max(2_147_483_647) + .default(120_000) + .describe( + "Publication and final cleanup budget in milliseconds, started after extraction and inventory; increase for a slow destination", + ), }), } as const; diff --git a/src/firmware/FirmwareDeadline.test.ts b/src/firmware/FirmwareDeadline.test.ts new file mode 100644 index 0000000000..5b953ba996 --- /dev/null +++ b/src/firmware/FirmwareDeadline.test.ts @@ -0,0 +1,111 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { firmwareRequestSchema } from "../domain/firmware/firmwareAnalysis.js"; +import { + firmwareFailure, + firmwarePhaseInterruption, +} from "./FirmwareFailures.js"; +import { startFirmwareDeadline } from "./FirmwareDeadline.js"; + +afterEach(() => vi.restoreAllMocks()); + +it("enforces absolute expiry before the timer callback while preserving observed caller cancellation", () => { + const now = vi.spyOn(performance, "now").mockReturnValue(100); + const deadline = startFirmwareDeadline("publication", 1_000); + expect(deadline.expiresAtMs).toBe(1_100); + now.mockReturnValue(1_099); + expect( + firmwarePhaseInterruption("extract_firmware", deadline.signal, deadline), + ).toBeUndefined(); + now.mockReturnValue(1_100); + expect(deadline.signal.aborted).toBe(false); + const expired = firmwarePhaseInterruption( + "extract_firmware", + deadline.signal, + deadline, + ); + if (expired === undefined) throw new Error("Expected elapsed deadline"); + expect(projectAnalysisError(expired)).toMatchObject({ + code: "provider_timeout", + details: { timeout_ms: 1_000, diagnostics: { phase: "publication" } }, + }); + const reason = { caller: "stop at expiry" }; + const cancelled = firmwarePhaseInterruption( + "extract_firmware", + AbortSignal.abort(reason), + deadline, + ); + expect(cancelled).toBeInstanceOf(AnalysisCancelledError); + expect(cancelled?.cause).toBe(reason); +}); + +it("defaults and preserves the caller's publication budget at the shared contract", () => { + const parse = (input: Record) => + firmwareRequestSchema.parse({ + operation: "extract_firmware", + input: { path: "/firmware.bin", output_directory: "/output", ...input }, + }); + expect(parse({}).input).toMatchObject({ publication_timeout_ms: 120_000 }); + expect(parse({ publication_timeout_ms: 345_678 }).input).toMatchObject({ + publication_timeout_ms: 345_678, + }); + for (const value of [0, -1, 0.5, 2_147_483_648, null, "120000"]) + expect(() => parse({ publication_timeout_ms: value })).toThrow(); +}); + +it("reports the publication phase, verified progress and its selected budget", () => { + const reason = new Error("deadline"); + const failure = firmwareFailure( + new AnalysisCancelledError("extract_firmware", { cause: reason }), + { + engine: "unblob", + operation: "extract_firmware", + deadline: { + phase: "publication", + signal: AbortSignal.abort(reason), + timeoutMs: 345_678, + expiresAtMs: 0, + }, + publication: { + output_directory: "/output", + verified_files: 2, + total_files: 10, + }, + }, + ); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "provider_timeout", + details: { + timeout_ms: 345_678, + diagnostics: { + phase: "publication", + publication: { + output_directory: "/output", + verified_files: 2, + total_files: 10, + }, + }, + }, + remediation: { action: expect.stringContaining("publication_timeout_ms") }, + }); +}); + +it("keeps a distinct caller cancellation when the phase deadline has also expired", () => { + const reason = { caller: "stop" }; + const original = new AnalysisCancelledError("extract_firmware", { + cause: reason, + }); + expect( + firmwareFailure(original, { + engine: "unblob", + operation: "extract_firmware", + deadline: { + phase: "publication", + signal: AbortSignal.abort(new Error("late timeout")), + timeoutMs: 345_678, + expiresAtMs: 0, + }, + }), + ).toBe(original); +}); diff --git a/src/firmware/FirmwareDeadline.ts b/src/firmware/FirmwareDeadline.ts new file mode 100644 index 0000000000..3415d8c1c3 --- /dev/null +++ b/src/firmware/FirmwareDeadline.ts @@ -0,0 +1,22 @@ +/** The phase and budget that own an observed firmware timeout. */ +export interface FirmwareDeadline { + readonly phase: "analysis" | "publication"; + readonly signal: AbortSignal; + readonly timeoutMs: number; + readonly expiresAtMs: number; +} + +/** Keep a phase's selected budget paired with its cancellation identity. */ +export const startFirmwareDeadline = ( + phase: FirmwareDeadline["phase"], + timeoutMs: number, +): FirmwareDeadline => ({ + phase, + expiresAtMs: performance.now() + timeoutMs, + signal: AbortSignal.timeout(timeoutMs), + timeoutMs, +}); + +/** Timers may not have run when synchronous inventory or result parsing returns. */ +export const firmwareDeadlineElapsed = (deadline: FirmwareDeadline): boolean => + performance.now() >= deadline.expiresAtMs; diff --git a/src/firmware/FirmwareFailures.ts b/src/firmware/FirmwareFailures.ts index 3b8bcda547..3aa66ec9b4 100644 --- a/src/firmware/FirmwareFailures.ts +++ b/src/firmware/FirmwareFailures.ts @@ -14,25 +14,66 @@ import { import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; -import { FIRMWARE_LIMITS } from "./FirmwareRelease.js"; +import type { FirmwareDeadline } from "./FirmwareDeadline.js"; +import { firmwareDeadlineElapsed } from "./FirmwareDeadline.js"; +import type { AnalysisErrorOptions } from "../domain/analysisErrorBase.js"; import { z } from "zod"; +type FirmwarePublicationProgress = { + readonly output_directory: string; + readonly verified_files: number; + readonly total_files: number; +}; + +interface FirmwareFailureContext { + readonly engine: string; + readonly operation: string; + readonly deadline?: FirmwareDeadline | undefined; + readonly treeCleanupLease?: AnalysisCleanupLease | undefined; + readonly publication?: FirmwarePublicationProgress | undefined; +} + +const timeoutFailure = ( + operation: string, + reason: unknown, + deadline: FirmwareDeadline, + options: Pick & { + readonly publication?: FirmwarePublicationProgress | undefined; + } = {}, +): AnalysisTimeoutError => + new AnalysisTimeoutError(operation, deadline.timeoutMs, { + cause: reason, + partialObservation: options.partialObservation, + diagnostics: { + phase: deadline.phase, + ...(options.publication === undefined + ? {} + : { publication: options.publication }), + }, + ...(deadline.phase === "publication" + ? { + userMessage: `Firmware publication exceeded its ${deadline.timeoutMs}ms budget. Extraction already completed.`, + remediationAction: + "Read any retained publication receipt first. For a retry, choose an absent local output directory or increase publication_timeout_ms (--publication-timeout-ms in the CLI).", + } + : {}), + }); + /** Translate the producing file owner's outcome without guessing from a later signal. */ export const firmwareFailure = ( cause: unknown, - engine: string, - operation: string, - deadline?: AbortSignal, - treeCleanupLease?: AnalysisCleanupLease, + context: FirmwareFailureContext, ): AnalysisError => { + const { engine, operation, deadline, treeCleanupLease, publication } = + context; if ( cause instanceof AnalysisCancelledError && - deadline?.aborted && - cause.cause === deadline.reason + deadline?.signal.aborted && + cause.cause === deadline.signal.reason ) - return new AnalysisTimeoutError(operation, FIRMWARE_LIMITS.timeoutMs, { - cause: cause.cause, + return timeoutFailure(operation, cause.cause, deadline, { partialObservation: cause.partialObservation, + publication, }); if (cause instanceof AnalysisError) return cause; if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined) { @@ -49,18 +90,13 @@ export const firmwareFailure = ( const outcome = cause.fileCleanupFailure.outcome; const primary = outcome.kind === "failed" - ? firmwareFailure( - outcome.cause, - engine, - operation, - deadline, - treeCleanupLease, - ) + ? firmwareFailure(outcome.cause, context) : cause.fileCleanupFailure.completionCancellation !== undefined ? firmwareCancellation( operation, cause.fileCleanupFailure.completionCancellation.reason, deadline, + publication, ) : undefined; if (primary !== undefined) @@ -76,6 +112,7 @@ export const firmwareFailure = ( operation, artifactFileCancellationCause(cause), deadline, + publication, ), cleanup, operation, @@ -87,6 +124,7 @@ export const firmwareFailure = ( operation, artifactFileCancellationCause(cause), deadline, + publication, ); if (cause instanceof z.ZodError) return new AnalysisOutputError( @@ -106,10 +144,22 @@ export const firmwareFailure = ( export const firmwareCancellation = ( operation: string, reason: unknown, - deadline?: AbortSignal, + deadline?: FirmwareDeadline, + publication?: FirmwarePublicationProgress, ): AnalysisError => - deadline?.aborted && reason === deadline.reason - ? new AnalysisTimeoutError(operation, FIRMWARE_LIMITS.timeoutMs, { - cause: reason, - }) + deadline?.signal.aborted && reason === deadline.signal.reason + ? timeoutFailure(operation, reason, deadline, { publication }) : new AnalysisCancelledError(operation, { cause: reason }); + +/** Preserve observed cancellation identity; otherwise enforce the monotonic budget. */ +export const firmwarePhaseInterruption = ( + operation: string, + signal: AbortSignal, + deadline: FirmwareDeadline, + publication?: FirmwarePublicationProgress, +): AnalysisError | undefined => + signal.aborted + ? firmwareCancellation(operation, signal.reason, deadline, publication) + : firmwareDeadlineElapsed(deadline) + ? timeoutFailure(operation, undefined, deadline, { publication }) + : undefined; diff --git a/src/firmware/FirmwareProvider.ts b/src/firmware/FirmwareProvider.ts index e8416122f1..07a6dfff07 100644 --- a/src/firmware/FirmwareProvider.ts +++ b/src/firmware/FirmwareProvider.ts @@ -30,6 +30,10 @@ import { BINWALK_PROVIDER_IDENTITY, UNBLOB_PROVIDER_IDENTITY, } from "./FirmwareRelease.js"; +import { + startFirmwareDeadline, + type FirmwareDeadline, +} from "./FirmwareDeadline.js"; import { admitFirmwareVersion } from "./FirmwareVersion.js"; import { firmwareToolUnavailable, @@ -47,7 +51,10 @@ import { normalizeUnblobReport, } from "./FirmwareReports.js"; -import { firmwareFailure, firmwareCancellation } from "./FirmwareFailures.js"; +import { + firmwareFailure, + firmwarePhaseInterruption, +} from "./FirmwareFailures.js"; import { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js"; type Outcome = Awaited>; @@ -186,7 +193,12 @@ export class FirmwareProvider implements FirmwareAnalysisPort { try { await this.#files.close(); } catch (cause: unknown) { - failures.push(firmwareFailure(cause, "firmware", "close_firmware")); + failures.push( + firmwareFailure(cause, { + engine: "firmware", + operation: "close_firmware", + }), + ); } const workspace = await this.#retryWorkspaceCleanup(); if (workspace.failure !== undefined) failures.push(workspace.failure); @@ -199,7 +211,10 @@ export class FirmwareProvider implements FirmwareAnalysisPort { try { await this.#files.retryCleanup(); } catch (cause: unknown) { - fileFailure = firmwareFailure(cause, "firmware", "close_firmware"); + fileFailure = firmwareFailure(cause, { + engine: "firmware", + operation: "close_firmware", + }); } const workspace = await this.#retryWorkspaceCleanup(); return { @@ -303,11 +318,14 @@ export class FirmwareProvider implements FirmwareAnalysisPort { let rootLease: AnalysisCleanupLease | undefined; const engineName = request.operation === "inspect_firmware_regions" ? "binwalk" : "unblob"; - const deadline = AbortSignal.timeout(FIRMWARE_LIMITS.timeoutMs); - const signal = + const withCaller = (phase: FirmwareDeadline) => options?.signal === undefined - ? deadline - : AbortSignal.any([options.signal, deadline]); + ? phase.signal + : AbortSignal.any([options.signal, phase.signal]); + // Extraction and publication have separate budgets: publication time depends on the + // destination filesystem, so it must not be charged to the extraction deadline. + let deadline = startFirmwareDeadline("analysis", FIRMWARE_LIMITS.timeoutMs); + let signal = withCaller(deadline); let outcome: Outcome; try { if (signal.aborted) @@ -465,6 +483,19 @@ export class FirmwareProvider implements FirmwareAnalysisPort { request.operation, signal, ); + // The inventory can finish after its deadline expired; an expired + // extraction must not publish under the fresh publication budget. + const interruption = firmwarePhaseInterruption( + request.operation, + signal, + deadline, + ); + if (interruption !== undefined) throw interruption; + deadline = startFirmwareDeadline( + "publication", + request.input.publication_timeout_ms, + ); + signal = withCaller(deadline); result = await publishFirmwareExtraction({ files: this.#files, workspacePath: root.path, @@ -520,12 +551,11 @@ export class FirmwareProvider implements FirmwareAnalysisPort { ), ); } catch (cause: unknown) { - const failure = firmwareFailure( - cause, - engineName, - request.operation, + const failure = firmwareFailure(cause, { + engine: engineName, + operation: request.operation, deadline, - ); + }); outcome = err(failure); } const finished = await finishFirmwareWorkspace( @@ -558,16 +588,19 @@ export class FirmwareProvider implements FirmwareAnalysisPort { finished.error.cleanupResources.includes(root.path) ) rootLease.bindCleanupError(finished.error); - if (finished.ok && signal.aborted) - return err( - firmwareCancellation( - request.operation, - signal.reason, - deadline, - ).retainPartialObservation( - completedFirmwareObservation(finished.value), - ), + if (finished.ok) { + const interruption = firmwarePhaseInterruption( + request.operation, + signal, + deadline, ); + if (interruption !== undefined) + return err( + interruption.retainPartialObservation( + completedFirmwareObservation(finished.value), + ), + ); + } return finished; } } @@ -580,7 +613,7 @@ const finishFirmwareWorkspace = async ( request: FirmwareRequest, rootLease: AnalysisCleanupLease | undefined, signal: AbortSignal, - deadline: AbortSignal, + deadline: FirmwareDeadline, ): Promise => { if ( !outcome.ok && @@ -613,12 +646,10 @@ const finishFirmwareWorkspace = async ( }, ); rootLease?.bindCleanupError(cleanup); - if (outcome.ok && signal.aborted) { - const primary = firmwareCancellation( - request.operation, - signal.reason, - deadline, - ); + const primary = outcome.ok + ? firmwarePhaseInterruption(request.operation, signal, deadline) + : undefined; + if (outcome.ok && primary !== undefined) { primary.retainPartialObservation( completedFirmwareObservation(outcome.value), ); diff --git a/src/firmware/FirmwarePublication.ts b/src/firmware/FirmwarePublication.ts index 92741c7e45..78d339daf1 100644 --- a/src/firmware/FirmwarePublication.ts +++ b/src/firmware/FirmwarePublication.ts @@ -1,16 +1,16 @@ import { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; -import { firmwareFailure } from "./FirmwareFailures.js"; +import { + firmwareFailure, + firmwarePhaseInterruption, +} from "./FirmwareFailures.js"; import { createHash } from "node:crypto"; import { join } from "node:path"; import type { z } from "zod"; import { SafeOutputTree } from "../artifacts/SafeOutputTree.js"; import { SafeOutputTreeCreationFailure } from "../artifacts/SafeOutputTreeCreationFailure.js"; -import { - AnalysisOutputError, - AnalysisCancelledError, -} from "../domain/analysisErrorCore.js"; +import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { jsonValueSchema } from "../domain/jsonValue.js"; import { firmwareResultSchemas } from "../domain/firmware/firmwareAnalysis.js"; @@ -40,6 +40,7 @@ const UNNAMED_DEPENDENCY_CONTINUATION = "The producer reported a missing extractor dependency without naming it; its identity is unavailable in this evidence. No specific installation or rerun can be prescribed from this report. Retain the gap as unextracted unless additional provider evidence establishes the dependency. REA installs nothing."; import { FIRMWARE_LIMITS } from "./FirmwareRelease.js"; +import type { FirmwareDeadline } from "./FirmwareDeadline.js"; /** The depth ceiling cannot be raised past the input schema maximum. */ const MAX_EXTRACTION_DEPTH = 10; @@ -89,7 +90,7 @@ const gapSortKey = (gap: ExtractionGap): string => export const publishFirmwareExtraction = async (context: { files: FirmwareFileAccess; workspacePath: string; - deadline: AbortSignal; + deadline: FirmwareDeadline; outputDirectory: string; entries: readonly FirmwareEntry[]; normalized: ReturnType; @@ -108,12 +109,20 @@ export const publishFirmwareExtraction = async (context: { let result: | z.infer | undefined; + const files: z.infer["files"] = + []; + let totalFiles = 0; + const progress = () => ({ + output_directory: context.outputDirectory, + verified_files: files.length, + total_files: totalFiles, + }); try { - const materialized = new Set( - context.entries - .filter((entry) => entry.kind === "file") - .map((entry) => `$output/${entry.relativePath}`), - ); + const materialized = new Set(); + for (const entry of context.entries) + if (entry.kind === "file") + materialized.add(`$output/${entry.relativePath}`); + totalFiles = materialized.size; for (const path of context.normalized.files.keys()) { if (path !== "$input" && !materialized.has(path)) throw new AnalysisOutputError( @@ -121,7 +130,6 @@ export const publishFirmwareExtraction = async (context: { `Reported regular file is absent from materialized output: ${path}`, ); } - const files = []; const unpublished = []; tree = await SafeOutputTree.create(context.outputDirectory); for (const entry of context.entries) { @@ -180,10 +188,13 @@ export const publishFirmwareExtraction = async (context: { ), published_bytes: files.reduce((sum, file) => sum + file.size, 0), }); - if (context.signal.aborted) - throw new AnalysisCancelledError("extract_firmware", { - cause: context.signal.reason, - }); + const interruption = firmwarePhaseInterruption( + "extract_firmware", + context.signal, + context.deadline, + progress(), + ); + if (interruption !== undefined) throw interruption; await tree.commit(context.signal); return result; } catch (cause: unknown) { @@ -195,13 +206,13 @@ export const publishFirmwareExtraction = async (context: { : new AnalysisCleanupLease(async () => { await ownedTree.rollback(); }); - const primary = firmwareFailure( - cause, - "unblob", - "extract_firmware", - context.deadline, - treeLease, - ); + const primary = firmwareFailure(cause, { + engine: "unblob", + operation: "extract_firmware", + deadline: context.deadline, + treeCleanupLease: treeLease, + publication: progress(), + }); try { await treeLease?.release(); } catch (cleanupCause: unknown) { diff --git a/tests/process-global/firmwareFileLifetime.test.ts b/tests/process-global/firmwareFileLifetime.test.ts index 00c3e53cdb..a0bc4a0047 100644 --- a/tests/process-global/firmwareFileLifetime.test.ts +++ b/tests/process-global/firmwareFileLifetime.test.ts @@ -125,6 +125,7 @@ import { AnalysisCancelledError } from "../../src/domain/analysisErrorCore.js"; import { PrivateRuntimeRoot } from "../../src/process/PrivateRuntimeRoot.js"; import { FIRMWARE_LIMITS } from "../../src/firmware/FirmwareRelease.js"; import { projectAnalysisError } from "../../src/domain/analysisErrorProjection.js"; +import { firmwareRequestSchema } from "../../src/domain/firmware/firmwareAnalysis.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; const providers: FirmwareProvider[] = []; @@ -168,7 +169,7 @@ it.each(["snapshot", "range"] as const)( try { const raw = await files .snapshot( - { + firmwareRequestSchema.parse({ operation: "extract_firmware", input: { path, @@ -178,7 +179,7 @@ it.each(["snapshot", "range"] as const)( max_output_files: 10, ...(stage === "range" ? { range: { offset: 1, length: 4 } } : {}), }, - }, + }), work, controller.signal, ) @@ -262,7 +263,7 @@ it.each( fault.onRead = undefined; await appendFile(path, Buffer.from([1])); }; - const request = + const request = firmwareRequestSchema.parse( operation === "inspect_firmware_regions" ? { operation, input: { path } } : { @@ -274,14 +275,15 @@ it.each( max_output_bytes: 1024, max_output_files: 10, }, - }; + }, + ); const files = new FirmwareFileAccess(); try { const raw = await files .snapshot(request, work, new AbortController().signal) .catch((cause: unknown) => cause); if (denyClose) expect(raw).toBeInstanceOf(ArtifactFileCleanupFailure); - const error = firmwareFailure(raw, "binwalk", operation); + const error = firmwareFailure(raw, { engine: "binwalk", operation }); const projected = projectAnalysisError(error); expect(projected.code).toBe( denyClose ? "cleanup_incomplete" : "invalid_request", @@ -374,16 +376,16 @@ it.each(["hash", "report"] as const)( ? createHash("sha256").update('{"value":"固件"}').digest("hex") : { value: "固件" }, }); - const original = firmwareFailure( - raw, - "binwalk", - "inspect_firmware_regions", + const original = firmwareFailure(raw, { + engine: "binwalk", + operation: "inspect_firmware_regions", + }); + const retry = await files.retryCleanup().catch((cause: unknown) => + firmwareFailure(cause, { + engine: "binwalk", + operation: "inspect_firmware_regions", + }), ); - const retry = await files - .retryCleanup() - .catch((cause: unknown) => - firmwareFailure(cause, "binwalk", "inspect_firmware_regions"), - ); expect(retry).toBeInstanceOf(AnalysisError); expect(fault.handles).toHaveLength(1); const handle = fault.handles[0]; @@ -409,7 +411,10 @@ it("preserves malformed-report failure when cancellation arrives during failed c const raw = await files .report(path, "inspect_firmware_regions", controller.signal) .catch((cause: unknown) => cause); - const error = firmwareFailure(raw, "binwalk", "inspect_firmware_regions"); + const error = firmwareFailure(raw, { + engine: "binwalk", + operation: "inspect_firmware_regions", + }); expect(error.diagnostics?.primary_error).toMatchObject({ code: "unreadable_output", }); @@ -434,7 +439,10 @@ it.each([null, "caller cancellation", new Error("caller cancellation")])( const raw = await files .report(path, "inspect_firmware_regions", controller.signal) .catch((cause: unknown) => cause); - const error = firmwareFailure(raw, "binwalk", "inspect_firmware_regions"); + const error = firmwareFailure(raw, { + engine: "binwalk", + operation: "inspect_firmware_regions", + }); expect(error).toBeInstanceOf(AnalysisCancelledError); expect(error.cause).toBe(reason); await files.close(); @@ -547,7 +555,10 @@ it.skipIf(process.platform === "win32").each(["snapshot", "report"] as const)( ) : files.report(path, "inspect_firmware_regions") ).catch((cause: unknown) => cause); - const error = firmwareFailure(raw, "binwalk", "inspect_firmware_regions"); + const error = firmwareFailure(raw, { + engine: "binwalk", + operation: "inspect_firmware_regions", + }); expect(error.diagnostics?.primary_error).toMatchObject({ code: kind === "snapshot" ? "invalid_request" : "unreadable_output", }); @@ -612,12 +623,16 @@ it("classifies a deadline before composing failed descriptor cleanup", async () const raw = await files .report(path, "inspect_firmware_regions", deadline.signal) .catch((cause: unknown) => cause); - const error = firmwareFailure( - raw, - "binwalk", - "inspect_firmware_regions", - deadline.signal, - ); + const error = firmwareFailure(raw, { + engine: "binwalk", + operation: "inspect_firmware_regions", + deadline: { + phase: "analysis", + signal: deadline.signal, + timeoutMs: 120_000, + expiresAtMs: performance.now() + 120_000, + }, + }); expect(error.diagnostics?.primary_error).toMatchObject({ code: "provider_timeout", }); @@ -628,7 +643,12 @@ it("classifies a deadline before composing failed descriptor cleanup", async () const publication = async ( signal: AbortSignal, - deadline = new AbortController().signal, + deadline = { + phase: "publication" as const, + signal: new AbortController().signal, + timeoutMs: 120_000, + expiresAtMs: performance.now() + 120_000, + }, ) => { const root = await createTestTempDirectory( "rea-firmware-publication-lifetime-", @@ -734,7 +754,14 @@ it.each(["caller", "deadline"])( : new DOMException("controlled deadline", "TimeoutError"); const fixture = await publication( controller.signal, - kind === "deadline" ? controller.signal : undefined, + kind === "deadline" + ? { + phase: "publication" as const, + signal: controller.signal, + timeoutMs: 120_000, + expiresAtMs: performance.now() + 120_000, + } + : undefined, ); const create = SafeOutputTree.create; vi.spyOn(SafeOutputTree, "create").mockImplementation(async (...args) => { @@ -842,7 +869,7 @@ it("waits through selected-range snapshot finalization when shutdown starts duri }; const files = new FirmwareFileAccess(); const pending = files.snapshot( - { + firmwareRequestSchema.parse({ operation: "extract_firmware", input: { path, @@ -852,7 +879,7 @@ it("waits through selected-range snapshot finalization when shutdown starts duri max_output_files: 10, range: { offset: 1, length: 4 }, }, - }, + }), work, new AbortController().signal, ); @@ -902,7 +929,10 @@ it("retains the exact publication source descriptor when its final close fails", ) .catch((cause: unknown) => cause); expect(raw).toBeInstanceOf(ArtifactFileCleanupFailure); - const error = firmwareFailure(raw, "unblob", "extract_firmware"); + const error = firmwareFailure(raw, { + engine: "unblob", + operation: "extract_firmware", + }); expect(error.cleanupResources).toContain(path); await tree.rollback(); expect(error.cleanupIncomplete).toBe(true); diff --git a/tests/process-global/firmwarePublicationDeadline.test.ts b/tests/process-global/firmwarePublicationDeadline.test.ts new file mode 100644 index 0000000000..cbd1692efe --- /dev/null +++ b/tests/process-global/firmwarePublicationDeadline.test.ts @@ -0,0 +1,345 @@ +import { access, readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import * as firmwareFiles from "../../src/firmware/FirmwareFiles.js"; +import { SafeOutputTree } from "../../src/artifacts/SafeOutputTree.js"; +import { PrivateRuntimeRoot } from "../../src/process/PrivateRuntimeRoot.js"; +import { FIRMWARE_LIMITS } from "../../src/firmware/FirmwareRelease.js"; +import { projectAnalysisError } from "../../src/domain/analysisErrorProjection.js"; +import { + firmwareFixture, + assertFirmwareCleanup, +} from "../fixtures/firmware/provider.js"; + +const test = it.skipIf(process.platform !== "linux" || process.arch !== "x64"); +afterEach(() => vi.restoreAllMocks()); + +test("preserves an already aborted caller reason without launching firmware work", async () => { + const fixture = await firmwareFixture(); + const reason = { stop: "before admission" }; + const result = await fixture.service.execute( + "extract_firmware", + { + path: fixture.path, + output_directory: fixture.output, + publication_timeout_ms: 654_321, + }, + { signal: AbortSignal.abort(reason) }, + ); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + if (result.ok) throw new Error("Expected caller cancellation"); + expect(result.error.cause).toBe(reason); + expect(fixture.launches).toEqual([]); + await fixture.close(); +}); + +const phases = (publicationMs: number = FIRMWARE_LIMITS.timeoutMs) => { + const analysis = new AbortController(); + const publication = new AbortController(); + const budgets: number[] = []; + const timeout = AbortSignal.timeout; + vi.spyOn(AbortSignal, "timeout").mockImplementation((ms) => { + if (budgets.length === 0 && ms === FIRMWARE_LIMITS.timeoutMs) { + budgets.push(ms); + return analysis.signal; + } + if (ms === publicationMs) { + budgets.push(ms); + return publication.signal; + } + return timeout(ms); + }); + return { analysis, publication, budgets }; +}; + +const phaseClock = () => { + const now = performance.now.bind(performance); + let offset = 0; + vi.spyOn(performance, "now").mockImplementation(() => now() + offset); + return (elapsedMs: number) => { + offset += elapsedMs; + }; +}; + +test("allows publication after the analysis phase expires", async () => { + const phase = phases(); + const fixture = await firmwareFixture(); + const publish = firmwareFiles.FirmwareFileAccess.prototype.publish; + let publicationSignal: AbortSignal | undefined; + vi.spyOn( + firmwareFiles.FirmwareFileAccess.prototype, + "publish", + ).mockImplementation(function ( + this: firmwareFiles.FirmwareFileAccess, + entry, + digest, + tree, + signal, + ) { + publicationSignal = signal; + phase.analysis.abort( + new DOMException("earlier analysis deadline", "TimeoutError"), + ); + return publish.call(this, entry, digest, tree, signal); + }); + const result = await fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: fixture.output, + }); + expect(result.ok, result.ok ? undefined : result.error.message).toBe(true); + expect(publicationSignal).toBe(phase.publication.signal); + expect(await readFile(join(fixture.output, "rootfs/config"), "utf8")).toBe( + "firmware=true\n", + ); + expect(phase.budgets).toEqual([120_000, 120_000]); + await fixture.close(); + await assertFirmwareCleanup(fixture.launches); +}); + +test.each(["signal", "clock"] as const)( + "does not revive expired analysis when inventory finishes: %s", + async (expiry) => { + const phase = phases(); + const advance = phaseClock(); + const fixture = await firmwareFixture(); + const inventory = firmwareFiles.inventoryFirmwareOutput; + vi.spyOn(firmwareFiles, "inventoryFirmwareOutput").mockImplementation( + async (...args) => { + const entries = await inventory(...args); + if (expiry === "clock") advance(120_001); + else + phase.analysis.abort( + new DOMException("inventory deadline", "TimeoutError"), + ); + return entries; + }, + ); + const result = await fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: fixture.output, + }); + if (result.ok) throw new Error("Expected expired analysis"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "provider_timeout", + details: { timeout_ms: 120_000, diagnostics: { phase: "analysis" } }, + }); + expect(phase.budgets).toEqual([120_000]); + expect(phase.analysis.signal.aborted).toBe(expiry === "signal"); + await expect(access(fixture.output)).rejects.toMatchObject({ + code: "ENOENT", + }); + await fixture.close(); + await assertFirmwareCleanup(fixture.launches); + }, +); + +test("keeps each queued request's publication budget", async () => { + const fixture = await firmwareFixture(); + const budgets: number[] = []; + const timeout = AbortSignal.timeout; + vi.spyOn(AbortSignal, "timeout").mockImplementation((ms) => { + budgets.push(ms); + return timeout(ms); + }); + let enter!: () => void; + let release!: () => void; + const entered = new Promise((resolve) => { + enter = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); + const publish = firmwareFiles.FirmwareFileAccess.prototype.publish; + let first = true; + vi.spyOn( + firmwareFiles.FirmwareFileAccess.prototype, + "publish", + ).mockImplementation(async function ( + this: firmwareFiles.FirmwareFileAccess, + ...args + ) { + if (first) { + first = false; + enter(); + await gate; + } + return publish.apply(this, args); + }); + const pending = fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: fixture.output, + publication_timeout_ms: 123_456, + }); + try { + await entered; + const queued = fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: join(fixture.root, "second-output"), + publication_timeout_ms: 654_321, + }); + expect(budgets).toEqual([120_000, 123_456]); + release(); + const results = await Promise.all([pending, queued]); + expect( + results.map((result) => + result.ok + ? result.value.parameters.publication_timeout_ms + : result.error.message, + ), + ).toEqual([123_456, 654_321]); + expect(budgets).toEqual([120_000, 123_456, 120_000, 654_321]); + } finally { + release(); + } + await fixture.close(); + await assertFirmwareCleanup(fixture.launches); +}); + +test.each(["none", "caller", "publication", "clock"] as const)( + "preserves the selected budget, verified progress and rollback at commit: %s", + async (interruption) => { + const phase = phases(654_321); + const advance = phaseClock(); + const caller = new AbortController(); + const reason = { stop: "before publication commit" }; + const fixture = await firmwareFixture(); + if (interruption === "clock") { + const publish = firmwareFiles.FirmwareFileAccess.prototype.publish; + vi.spyOn( + firmwareFiles.FirmwareFileAccess.prototype, + "publish", + ).mockImplementation(async function ( + this: firmwareFiles.FirmwareFileAccess, + ...args + ) { + const file = await publish.apply(this, args); + advance(654_322); + return file; + }); + } + const commit = SafeOutputTree.prototype.commit; + vi.spyOn(SafeOutputTree.prototype, "commit").mockImplementation( + async function (this: SafeOutputTree, signal) { + if (interruption === "caller") caller.abort(reason); + if (interruption === "publication") + phase.publication.abort( + new DOMException("publication deadline", "TimeoutError"), + ); + await commit.call(this, signal); + }, + ); + const result = await fixture.service.execute( + "extract_firmware", + { + path: fixture.path, + output_directory: fixture.output, + publication_timeout_ms: 654_321, + }, + { signal: caller.signal }, + ); + expect(phase.budgets).toEqual([120_000, 654_321]); + if (interruption === "none") { + if (!result.ok) throw result.error; + expect(result.value.parameters).toMatchObject({ + publication_timeout_ms: 654_321, + }); + } else { + if (result.ok) throw new Error("Expected interrupted publication"); + expect(result.error.cause).toBe( + interruption === "caller" ? reason : phase.publication.signal.reason, + ); + if (interruption === "clock") + expect(phase.publication.signal.aborted).toBe(false); + expect(projectAnalysisError(result.error)).toMatchObject( + interruption === "caller" + ? { code: "cancelled" } + : { + code: "provider_timeout", + details: { + timeout_ms: 654_321, + diagnostics: { + phase: "publication", + publication: { + output_directory: fixture.output, + verified_files: 1, + total_files: 1, + }, + }, + }, + }, + ); + await expect(access(fixture.output)).rejects.toMatchObject({ + code: "ENOENT", + }); + } + await fixture.close(); + await assertFirmwareCleanup(fixture.launches); + }, +); + +test.each([ + { expiry: "signal", denied: false }, + { expiry: "signal", denied: true }, + { expiry: "clock", denied: false }, + { expiry: "clock", denied: true }, +] as const)( + "retains committed output and late timeout through workspace cleanup: $expiry, denied: $denied", + async ({ expiry, denied }) => { + const phase = phases(654_321); + const advance = phaseClock(); + const fixture = await firmwareFixture(); + const commit = SafeOutputTree.prototype.commit; + const close = PrivateRuntimeRoot.prototype.close; + let committed = false; + let blocked = denied; + vi.spyOn(SafeOutputTree.prototype, "commit").mockImplementation( + async function (this: SafeOutputTree, signal) { + await commit.call(this, signal); + committed = true; + }, + ); + vi.spyOn(PrivateRuntimeRoot.prototype, "close").mockImplementation( + async function (this: PrivateRuntimeRoot) { + if (committed) { + if (expiry === "clock") advance(654_322); + else + phase.publication.abort( + new DOMException("final cleanup deadline", "TimeoutError"), + ); + if (blocked) + throw new Error("controlled firmware root cleanup denial"); + } + await close.call(this); + }, + ); + const result = await fixture.service.execute("extract_firmware", { + path: fixture.path, + output_directory: fixture.output, + publication_timeout_ms: 654_321, + }); + if (result.ok) throw new Error("Expected timeout during final cleanup"); + expect(phase.publication.signal.aborted).toBe(expiry === "signal"); + const projected = projectAnalysisError(result.error); + expect( + denied ? result.error.diagnostics?.primary_error : projected, + ).toMatchObject({ + code: "provider_timeout", + details: { timeout_ms: 654_321, diagnostics: { phase: "publication" } }, + }); + expect(result.error.partialObservation).toMatchObject({ + kind: "firmware", + result: { output_directory: fixture.output }, + }); + expect(await readFile(join(fixture.output, "rootfs/config"), "utf8")).toBe( + "firmware=true\n", + ); + expect(result.error.cleanupIncomplete).toBe(denied); + blocked = false; + await fixture.close(); + expect(result.error.cleanupIncomplete).toBe(false); + await assertFirmwareCleanup(fixture.launches); + }, +);