From 1b97b0f3707b1cf78ac98bd4ae81638b33cc286f Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:48:59 +0800 Subject: [PATCH] fix(lifecycle): preserve observations and cleanup ownership (#4274) * fix(application): retain completed evidence through cancellation Validate completed provider reports before classifying cancellation and preserve Evidence and captured diagnostics. Share MCP recording and delivery across custom observation handlers. Add source-owned completion and public SDK recording controls. Refs #3923, #3927, #4261. * fix(wasm): retain final observations through cancellation Keep completed Evidence after finalization and through CLI/MCP delivery. Preserve output and cleanup failure precedence. Refs #4254. * fix(browser): retain final observations and forward cancellation Classify final CDP detach and completed scenario cleanup at the provider boundary. Translate completed partials into canonical Evidence for MCP recording, and forward CLI cancellation to passive Electron tools. Refs #4255, #4256, #4257. * fix(ghidra): isolate cancellation of shared startup waiters Keep startup group-owned while each caller has an independent abortable wait. Await final-owner cleanup and preserve cleanup facts before retrying. Reuse the shared abortable promise helper. Refs #3983. * fix(mcp): keep shutdown registrations until cleanup succeeds Retain repeated event requests while unresolved owners need cleanup, and unregister only after success. Verify coalescing and failed-owner retry. Refs #4253. * fix(adb): own transfer snapshots and stage screen publication Bind transfer digests to the exact private input bytes and preserve caller APK names. Retain failed descriptors and private-root owners with typed cleanup evidence; publish captures exclusively after the cancellation cutoff. Refs #4251, #4252. * fix(verification): report ADB cleanup failures Always close after attempted verification and keep unexpected primary failures and cleanup details visible. Exercise the production script with controlled imports, including null and undefined throws. Refs #4260. * fix(runner): stop owned Windows commands on cancellation Use the retained ChildProcess handle for IPC cancellation and parent disconnection. Keep the SQLite lock until observed child close, and include native Windows regressions in the curated CI lane. Native Windows results remain pending. Refs #4258. * fix(website): canonicalize link containment boundaries Compare resolved targets with resolved repository and site roots. Accept directory aliases while preserving escape checks and keep fixtures within their temporary owner. Refs #4264. * docs(testing): cover completion races and cleanup retry Clarify validation-before-cancellation, independent startup waiter controls, producer and adapter coverage, and repeated shutdown events. * fix(filesystem): retain file owners and completed reads after close failure * fix(session): report retained snapshot staging in cleanup status Fixes #4277. Include both snapshot and profile file-resource scopes in the owning status fact, preserving publication receipts and cleanup retry. * docs(mcp): clarify recovery of raw partial observations Document the observed SDK cancellation boundary and the open design decision in #4276 without promoting raw errors to Evidence. * fix(kuna): own the complete installation inspection lifetime * test(kuna): cover session cleanup of shared cache resources * fix(hopper): isolate publisher operation resource admission * fix(hopper): preserve launcher cleanup failure classification * fix(adb): retain ordered cleanup and screen publication outcomes * fix(runner): retain Windows job ownership through cleanup retries * docs: clarify inspection admission and ordered cleanup regressions * fix(ghidra): preserve startup cleanup and request deadlines * fix(runner): supervise stdin errors and bootstrap native controls * docs: require queued lifecycle controls to survive admission * fix(cli): leave cleanup resolution with resource owners * test(adb): verify owned install snapshots through CLI * fix(errors): exclude resolved primary cleanup resources * fix(adb): resolve publication cleanup after its handle closes * fix(android): resolve JADX hash cleanup after owned close * refactor(android): share hash failure cause options * fix(kuna): gate commands on retained descriptor cleanup * docs: cover cleanup admission and owner-confirmed recovery * refactor(artifacts): bind cleanup errors to confirmed release * fix(adb): resolve recovered cleanup at retained owners * fix(kuna): resolve installation cleanup after owned close * fix(kuna): gate cache reads on retained hash cleanup * fix(setup): preserve publisher cleanup admission failures * fix(android): resolve the reported retirement cleanup error * test(runner): scope post-spawn fault control to POSIX * test(adb): cover persistent snapshot owner retirement * fix(kuna): preserve file outcomes with explicit cleanup ownership * fix(artifacts): resolve errors through retained cleanup owners * docs(lifecycle): carry cleanup ownership in typed outcomes * fix(adb): bind staged screen errors to their cleanup lease * fix(kuna): bind profile cleanup through a close capability * fix(artifacts): retain descriptor resources in cleanup errors * fix(pcap): bind existing errors to snapshot retirement * fix(windows): load source owner through its file URL * test(build): copy the command runner helper into fixtures * fix(mcp): own shared observation providers in the transport * fix(jadx): resolve every retained cleanup retry error * fix(kuna): bind retry errors to every retained cleanup owner * fix(adb): bind admission and shutdown errors to exact cleanup leases * fix(windows): resolve command paths before native ownership * test(publisher): assert exact descriptor recovery without release metadata * docs(lifecycle): require every retry error to follow its owners * fix(kuna): keep hash cleanup state stable across concurrent retries * test(windows): exercise surviving descendants with scoped shutdown deadlines * fix(uninstall): preserve primary read errors through cleanup failure * test(publisher): assert cleanup Result and exact owner recovery * test(windows): observe supervisor resources through IPC disconnect * fix(browser): retain typed cancellation after interrupted actions * test(publisher): assert ownership phases without retry-count coupling * fix(lifecycle): preserve abort reasons through shared provider execution * fix(jadx): preserve execution and late hash cancellation causes * fix(adb): compose late cancellation with exact cleanup failures * test(mcp): use the shared budget for session workflows --- .github/workflows/ci.yml | 4 +- .github/workflows/real-packet-capture.yml | 2 +- CONTRIBUTING.md | 5 +- docs/mcp-contracts.md | 30 +- docs/testing.md | 56 + docs/tool-design.md | 27 +- licenses/npm-dependencies.txt | 142 +- package-lock.json | 504 ++++++ package.json | 1 + scripts/ci/scopes.mjs | 4 + .../lib/kuna-native-reference-verifier.mjs | 14 +- scripts/lib/kuna-string-verifier.mjs | 15 +- scripts/lib/verifier-cleanup.mjs | 1 + scripts/lib/verifier-file-resources.d.mts | 7 + scripts/lib/verifier-file-resources.mjs | 10 + scripts/lib/windows-owned-command.d.mts | 51 + scripts/lib/windows-owned-command.mjs | 173 ++ scripts/run-exclusive-command.mjs | 67 +- scripts/test-website.py | 41 +- scripts/verify-adb.mjs | 52 +- scripts/verify-flutter.mjs | 70 +- scripts/verify-pcap.mjs | 65 +- scripts/verify-real-kuna.mjs | 20 +- src/adb/AdbInputSnapshot.ts | 191 ++ src/adb/AdbProvider.test.ts | 169 +- src/adb/AdbProvider.ts | 1571 ++++++++++++----- src/android/JadxProvider.ts | 219 ++- src/application/AnalysisProvider.ts | 11 +- src/application/BrowserObservationService.ts | 5 +- .../BrowserScenarioCaptureService.ts | 28 +- src/application/Doctor.ts | 27 +- src/application/LinuxHopper.test.ts | 39 +- src/application/LinuxHopper.ts | 16 +- src/application/LinuxHopperPublisher.ts | 463 +++-- src/application/ProviderOperationExecution.ts | 46 +- src/application/RegularFileRead.ts | 128 +- src/application/Uninstall.ts | 7 +- src/application/WebNetworkCaptureService.ts | 85 +- src/application/WebRuntimeService.test.ts | 24 +- src/application/WebRuntimeService.ts | 23 +- src/application/WebSourceLocationService.ts | 63 +- .../adb/AdbDeviceAnalysisService.test.ts | 7 +- .../android/AndroidAnalysisService.ts | 6 +- .../binary/AnalysisProviderEvaluation.ts | 11 +- .../binary/AnalysisProviderRegistry.ts | 23 +- src/application/binary/BinarySession.ts | 45 +- src/application/binary/BinarySessionOpen.ts | 10 +- .../binary/BinarySessionRecords.ts | 44 +- .../binary/SessionProviderRouter.ts | 10 +- .../BinaryLayoutService.test.ts | 4 + .../binaryDiagnostics/BinaryLayoutService.ts | 43 +- .../binaryDiagnostics/PeResourcesService.ts | 35 +- .../RecordedCrashService.test.ts | 4 + .../binaryDiagnostics/RecordedCrashService.ts | 49 +- .../electron/ElectronFuseService.ts | 37 +- .../evm/EvmInterfaceService.test.ts | 4 + src/application/evm/EvmInterfaceService.ts | 29 +- .../firmware/FirmwareAnalysisService.test.ts | 91 + .../firmware/FirmwareAnalysisService.ts | 6 +- src/application/go/GoBinaryService.ts | 59 +- .../ElectronObservationService.test.ts | 67 +- .../javascript/ElectronObservationService.ts | 55 +- ...avaScriptRuntimeObservationService.test.ts | 47 +- .../JavaScriptRuntimeObservationService.ts | 15 +- .../rust/RustBinaryService.test.ts | 45 + src/application/rust/RustBinaryService.ts | 5 +- .../wasm/WasmArtifactService.test.ts | 48 + src/application/wasm/WasmArtifactService.ts | 8 +- src/artifacts/AnalysisCleanupLease.test.ts | 106 ++ src/artifacts/AnalysisCleanupLease.ts | 70 + src/artifacts/ArtifactResourceScope.test.ts | 41 +- src/artifacts/ArtifactResourceScope.ts | 20 +- src/browser/CdpElectronProvider.ts | 15 +- .../PlaywrightBrowserScenarioProvider.test.ts | 88 +- .../PlaywrightBrowserScenarioProvider.ts | 37 +- src/cli/finishAnalysisCommand.ts | 4 +- src/doctorRuntime.ts | 6 +- src/domain/adb/adbDeviceAnalysis.ts | 20 +- src/domain/analysisErrorBase.ts | 4 +- src/domain/javascript/electronObservation.ts | 8 + src/domain/providerCleanupError.test.ts | 34 +- src/filesystem/OwnedRegularFile.ts | 97 + src/filesystem/RegularFile.ts | 25 +- src/filesystem/StableFileHash.ts | 78 +- src/kuna/KunaCommand.ts | 106 +- src/kuna/KunaDoctor.ts | 94 +- src/kuna/KunaInstallation.ts | 150 +- src/kuna/KunaProjectCache.ts | 402 ++++- src/kuna/KunaProjectExport.ts | 75 +- src/kuna/KunaProvider.ts | 169 +- src/kuna/KunaSessionClient.ts | 99 +- src/kuna/KunaTargetCommands.ts | 163 +- src/main/processShutdown.ts | 6 +- src/main/transport.ts | 31 +- src/pcap/PcapProvider.ts | 108 +- src/process/ArtifactAnalysisResources.test.ts | 251 ++- src/process/ArtifactAnalysisResources.ts | 108 +- src/server/contractToolHandler.ts | 42 +- src/server/createServer.ts | 12 +- src/server/observationEvidenceDelivery.ts | 35 - src/server/registerBrowserScenarioTool.ts | 6 +- src/server/registerBrowserTools.ts | 6 +- src/server/registerElectronTools.ts | 6 +- ...gisterJavaScriptRuntimeObservationTools.ts | 6 +- src/server/shutdownRegistration.test.ts | 178 ++ src/wasm/WabtArtifactProvider.test.ts | 10 +- .../browserScenarioCancellationCli.test.ts | 19 +- .../browser/cdpCleanupCancellation.test.ts | 8 +- .../browser/cdpElectronProvider.test.ts | 39 + .../cdpProviderCleanupOwnership.test.ts | 8 +- .../captures/captureDescriptorCleanup.test.ts | 27 +- tests/boundary/cli/adbCli.test.ts | 22 +- .../cli/finishAnalysisCommand.test.ts | 22 + .../cli/wasmLateCancellationCli.test.ts | 50 + .../filesystem/adbInputSnapshot.test.ts | 71 + .../filesystem/dependencyInstall.test.ts | 1 + .../stableFileHashOwnership.test.ts | 449 +++-- .../mcp/androidLateCompletionMcp.test.ts | 111 ++ tests/boundary/mcp/browserScenarioMcp.test.ts | 67 +- tests/boundary/mcp/electronMcp.test.ts | 117 ++ .../lateAbortApplicationEvidenceMcp.test.ts | 227 +++ tests/boundary/mcp/sessionMcp.test.ts | 2 +- tests/boundary/mcp/wasmArtifactMcp.test.ts | 67 + tests/boundary/pcap/pcapProvider.test.ts | 58 +- .../pcap/pcapProviderCrossPlatform.test.ts | 49 + .../process/adbVerifierCleanup.test.ts | 304 ++-- .../browserScenarioProbeLifecycle.test.ts | 60 +- .../boundary/process/jadxHashCleanup.test.ts | 14 +- .../boundary/process/jadxIntegration.test.ts | 236 ++- .../boundary/process/verifierCleanup.test.ts | 37 + .../process/verifierFileResources.test.ts | 86 + .../adb/adbBugreportPublication.test.ts | 115 ++ .../adb/adbHashCleanupConsumers.test.ts | 10 +- .../adb/adbInputSnapshotOwnership.test.ts | 588 ++++-- .../adbScreenPublicationCancellation.test.ts | 631 ++++++- .../android/androidLateCompletion.test.ts | 74 + .../providers/ghidra/ghidraClient.test.ts | 57 + .../kuna/kunaCommandFailures.test.ts | 2 +- .../kuna/kunaCommandLifetime.test.ts | 14 +- .../providers/kuna/kunaDoctor.test.ts | 2 +- .../providers/kuna/kunaFileCloseOwner.ts | 41 + .../kuna/kunaHashOwnerPropagation.test.ts | 493 +++++- .../providers/kuna/kunaInstallation.test.ts | 2 +- .../kuna/kunaInstallationLifetime.test.ts | 139 ++ .../providers/kuna/kunaProjectCache.test.ts | 380 +++- .../kuna/kunaProjectConcurrency.test.ts | 6 +- .../providers/kuna/kunaProvider.test.ts | 62 + .../providers/kuna/kunaRunOwnership.test.ts | 2 +- .../providers/kuna/kunaSessionFixture.ts | 5 +- .../linuxHopperPublisherVerification.test.ts | 226 +++ .../recordingDescriptorCleanup.test.ts | 27 +- .../signals/recordingProvider.test.ts | 3 + tests/conformance/providerCompletion.test.ts | 308 ++++ tests/fixtures/descriptorCloseFailure.ts | 3 +- .../process-global/curatedWindowsLane.test.ts | 37 +- .../process-global/runnerCancellation.test.ts | 628 ++++--- .../windowsOwnedCommand.test.ts | 422 +++++ 157 files changed, 11533 insertions(+), 2894 deletions(-) create mode 100644 scripts/lib/verifier-file-resources.d.mts create mode 100644 scripts/lib/verifier-file-resources.mjs create mode 100644 scripts/lib/windows-owned-command.d.mts create mode 100644 scripts/lib/windows-owned-command.mjs create mode 100644 src/adb/AdbInputSnapshot.ts create mode 100644 src/application/firmware/FirmwareAnalysisService.test.ts create mode 100644 src/artifacts/AnalysisCleanupLease.test.ts create mode 100644 src/artifacts/AnalysisCleanupLease.ts create mode 100644 src/filesystem/OwnedRegularFile.ts delete mode 100644 src/server/observationEvidenceDelivery.ts create mode 100644 src/server/shutdownRegistration.test.ts create mode 100644 tests/boundary/cli/wasmLateCancellationCli.test.ts create mode 100644 tests/boundary/filesystem/adbInputSnapshot.test.ts create mode 100644 tests/boundary/mcp/androidLateCompletionMcp.test.ts create mode 100644 tests/boundary/mcp/lateAbortApplicationEvidenceMcp.test.ts create mode 100644 tests/boundary/process/verifierFileResources.test.ts create mode 100644 tests/boundary/providers/android/androidLateCompletion.test.ts create mode 100644 tests/boundary/providers/kuna/kunaFileCloseOwner.ts create mode 100644 tests/boundary/providers/kuna/kunaInstallationLifetime.test.ts create mode 100644 tests/conformance/providerCompletion.test.ts create mode 100644 tests/process-global/windowsOwnedCommand.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bc5e26af4..82bf893175 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -710,12 +710,12 @@ jobs: with: name: rea-build-cache path: .turbo/cache/ - - run: npm run build:cached - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: windows-native-controls path: native/windows/build/ - run: npm run verify:windows-native:artifact + - run: npm run build:cached - name: Record named Windows host capabilities run: node scripts/windows-capabilities.mjs | Out-File -Encoding utf8NoBOM windows-capabilities.json - name: Validate Windows host capability report @@ -770,6 +770,8 @@ jobs: src/process/ProcessOwnership.validation.test.ts src/application/Update.test.ts tests/process-global/updateMaintenance.test.ts + tests/process-global/runnerCancellation.test.ts + tests/process-global/windowsOwnedCommand.test.ts tests/conformance/providers/ghidra/realGhidraWindowsWorkflow.test.ts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: diff --git a/.github/workflows/real-packet-capture.yml b/.github/workflows/real-packet-capture.yml index 12cb3d23db..ad7b62e9e0 100644 --- a/.github/workflows/real-packet-capture.yml +++ b/.github/workflows/real-packet-capture.yml @@ -53,7 +53,7 @@ jobs: - run: sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y tshark util-linux python3 - run: npm ci - run: npm run build:cached - - run: npm run test:focused -- tests/boundary/captures/fieldDiscovery.test.ts src/process/LinuxResourceLimits.test.ts tests/boundary/captures/captureProvider.test.ts tests/boundary/captures/captureDescriptorCleanup.test.ts tests/boundary/captures/resourceLimits.test.ts tests/boundary/captures/packetOutput.test.ts tests/boundary/mcp/packetCaptureMcp.test.ts tests/boundary/cli/captureCleanup.test.ts + - run: npm run test:focused -- tests/boundary/captures/fieldDiscovery.test.ts src/process/LinuxResourceLimits.test.ts tests/boundary/captures/captureProvider.test.ts tests/boundary/captures/captureDescriptorCleanup.test.ts tests/boundary/pcap/pcapProvider.test.ts tests/boundary/pcap/pcapProviderCrossPlatform.test.ts tests/boundary/captures/resourceLimits.test.ts tests/boundary/captures/packetOutput.test.ts tests/boundary/mcp/packetCaptureMcp.test.ts tests/boundary/cli/captureCleanup.test.ts - run: node scripts/verify-packet-capture.mjs env: REA_TSHARK_COMMAND: /usr/bin/tshark diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 10cede32b3..36714db43b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -225,7 +225,10 @@ manifest. A source checkout does not: `native/windows/build/` is ignored by Git, and neither `npm ci` nor `npm run build:cached` (or `npm run build`) produces `rea-windows-x64.node` or `manifest.json` in that directory. Windows workflows and tests that require native controls need these artifacts; -without them, those controls are unavailable even after a successful runtime build. +this includes `run-exclusive` commands, which use the native Job Object to keep +the artifact lock until every command descendant has exited. Without the +artifacts, those controls are unavailable even after a successful runtime +build. Follow the [Windows native build guide](native/windows/README.md#build) before running those workflows from source. The artifact build requires Linux, diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 8c7b4fa06a..677cd44371 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -404,14 +404,19 @@ at that bounded check. These are historical snapshots, not live process inventories, and do not claim that no short-lived descendant existed. `binary_session.cleanup_pending` reports whether the session retains a provider -client, session file resource, or provider-candidate resource whose cleanup has -not succeeded. It can be true with `open: false`; target retirement does not -establish resource release. Health, activity, and lineage remain available -under a retained client's original `analysis_run.run_id` where the provider -supplies them. Successful client cleanup clears the retained run; a replacement -client receives a fresh run ID. Follow the cleanup error's reported resources -and recovery action before reopening; a missing health observation does not -establish an idle provider. +client, session file resource, or provider-candidate resource whose cleanup +has not succeeded. It can be true with +`open: false`; target retirement does not establish resource release. For a +retained provider client, health, activity, and lineage remain available under +its original `analysis_run.run_id` where the provider supplies them. +Successful provider cleanup clears the retained run; +a replacement client receives a fresh run ID. Follow the cleanup error's +reported resources and recovery action before reopening; a missing health +observation does not establish an idle provider. +File cleanup can also remain after snapshot publication or after profile +inspection fails or is cancelled before a provider client exists. If no client +remains, no provider run or health observation is available; follow the cleanup +error's reported resources. ## Progress and cancellation @@ -449,6 +454,15 @@ attempt and its delivery result in `details.partial_observation`, with Derived comparisons and reconstruction verification yield before computation and before publication, so cancellation cannot race with successful Evidence. +Bundle recovery requires a recorded Evidence ID. Some provider errors carry +typed raw observations in `details.partial_observation` instead of Evidence. +Those raw observations are not recorded in the Evidence bundle. If the SDK +suppresses the error reply after client cancellation, Evidence tools cannot +recover them. An empty bundle does not prove that no partial work or side +effect occurred; output receipts and cleanup observations remain distinct. +The broader retention decision is tracked in +[#4276](https://github.com/morluto/rea/issues/4276). + `analyze_javascript_application` also yields between reconstruction phases and during graph/result sealing, cross-graph binding checks, Evidence JSON validation, and canonical hashing. Its final result validation reuses exact graphs whose diff --git a/docs/testing.md b/docs/testing.md index 7ce92cec6e..e30b150ed9 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -222,11 +222,67 @@ owned cleanup seams while preserving real cleanup where feasible. Check server settlement and retained resources even if the MCP SDK rejects the client promise before the handler finishes. A cancellation during active execution does not cover cancellation during final cleanup. +Use explicit abort reasons and assert their identity at admission and after +provider completion, alongside retained observations and cleanup state. For atomic writes, exercise cancellation on both sides of the documented commit point and preserve any publication receipt. A late interruption does not prove that no output was published. +For successful provider reports, test cancellation immediately before the report +reaches the application workflow as well as during provider cleanup. Validate +the report before classifying completion: a valid report must retain its Evidence +on cancellation, and an invalid report must keep its output error. Exercise both +the provider boundary and the shared CLI/MCP workflow; a downstream guard does +not repair the provider's own completion contract. When callers share startup, +cancel each waiter independently and verify that the last departure awaits owned +cleanup while interested peers can still reach readiness. Repeat the same +shutdown event after a cleanup failure to verify that its listener can retry. +Exercise non-default lifecycle deadlines through queued requests and their +consumers; default-only controls cannot detect dropped request options. + +For descriptor-backed reads, inject a close failure after a successful read and +after a primary read or cancellation failure. Verify that the exact descriptor +remains owned, both failures survive, and a later close retries it. Repeat +admission attempts while close still fails: no additional descriptor or +native command should start. Restore close and verify that the exact owner +resolves the originating cleanup marker and allows later work, while the earlier +failure remains available as a diagnostic. Retain errors returned by multiple +failed retries, including admission and shutdown aggregates, and assert that +each becomes complete after recovery. For an aggregate with multiple owners, +release one owner while another still fails and assert that it remains incomplete. +Cover recovery before the +caller-specific error is bound to its retained owner, as +another request can retry cleanup concurrently. Confirm that an already released +owner resolves a later binding too. Include +admission failure after opening the descriptor. If caller cancellation returns +before an inspection settles, gate its final read or close and verify that +session shutdown waits for the owned work and reports any retained resources. +Gate a later read in a multi-read inspection, not only its first read. Overlap +operations so one retains a failed descriptor while another reaches its next +read; both the operation and shutdown must settle without nested-admission +deadlock. Deny child close and verify its containing root survives until retry. + +Exercise publication on filesystems without hard links using an unsupported-link +seam. Check exclusive destination creation, stat/write/close failures, completed +output receipts, and cancellation at the documented cutoff. A streamed fallback +has different visibility from an atomic hard-link commit and must report it +accurately. For Windows command supervision, keep a descendant writing after its +leader exits, deny cleanup observation, and verify a competing writer remains +excluded until the exact Job Object closes. Portable doubles cover retry logic; +the native Windows lane must establish actual process-tree behavior. + +Supervisor tests must observe process exit, output-pipe closure, and IPC +disconnection separately. In the pinned Node runtime, a parent-initiated IPC +disconnect can omit the aggregate `ChildProcess` close event. Join the actual +resource observations before reporting completion; process exit alone does not +establish pipe or process-tree release. + +When changing an internal helper contract, check its JavaScript verifier and +subprocess-script consumers alongside TypeScript references. Source typecheck +does not check those calls; run their orchestration tests against the built +modules, including the verifier's failure and cleanup paths. + Representative regressions include [process cleanup-time SIGINT](https://github.com/morluto/rea/blob/e78455338fe2d06c4cd03d59cbf33473d754998b/tests/boundary/process/processCaptureCancellationAdapters.test.ts), [late authenticated cleanup](https://github.com/morluto/rea/blob/e78455338fe2d06c4cd03d59cbf33473d754998b/tests/boundary/providers/hopper/hopperClient.handshakeRecovery.test.ts), diff --git a/docs/tool-design.md b/docs/tool-design.md index 93e4af5878..237fb8bb06 100644 --- a/docs/tool-design.md +++ b/docs/tool-design.md @@ -93,9 +93,13 @@ providers need not expose identical state machines. last awaited finalization step. Retain collected observations under typed cancellation. Preserve an existing primary failure and let incomplete cleanup retain its own classification. + Preserve the caller's abort reason as the typed cancellation error's cause, + including when cancellation is observed only after provider completion. Client cancellation alone does not prove native work stopped. For atomic publication, document the cancellation cutoff and preserve any committed output's publication receipt. + Queued and retried work must retain the complete accepted execution controls, + including non-default lifecycle deadlines and cancellation signals. - Release only owned resources. Attaching owns a connection or session unless the provider contract explicitly establishes target ownership. A helper exit, missing reply, or missing PID does not establish target exit. @@ -106,12 +110,33 @@ providers need not expose identical state machines. Bind cleanup failures to their exact resource owner. Resolve current cleanup uncertainty only after that owner confirms release, while preserving the original failure and completed observations. Later failure composition must - report only resources whose cleanup remains unresolved. + report only resources whose cleanup remains unresolved. Carry cleanup + capabilities explicitly in typed outcomes so consumers can bind returned errors + to exact owners without reparsing diagnostic cause chains. + Bind every caller-visible retry error and aggregate, including an error already + produced by the provider. An aggregate remains incomplete until all owners it + reports are released; releasing one child cannot clear another child's status. - Await one-shot cleanup without discarding the completed result or primary diagnostic. Preserve partial observations, remaining resources, and the cleanup owner's recovery instructions. CLI adapters should reuse [`finishAnalysisCommand`](https://github.com/morluto/rea/blob/main/src/cli/finishAnalysisCommand.ts) where its contract applies rather than duplicate close/finally handling. +- Give descriptor-backed reads an owner before opening the file. A rejected + close must retain the exact handle and preserve any earlier failure. If an + abortable caller stops waiting, keep the underlying work in that owner's + lifetime so shutdown cannot report release before finalization settles. + Admit the entire inspection once, including later reads and finalization. + Helpers must borrow that admission or use an operation child scope; re-entering + the same draining scope can make cleanup wait for its own caller. + Retry retained cleanup before admitting new work and keep admission blocked + while it remains unresolved. Link the originating cleanup error to its exact + owner so confirmed release resolves its cleanup marker while preserving the + earlier diagnostic. Shared CLI/MCP adapters cannot infer that confirmation. +- Release child descriptors before removing their containing root. Retain a + composite owner when either stage fails so retry preserves that ordering. + A Windows command leader's exit does not establish Job Object closure. Keep + the process-tree observer and command lock alive through failed cleanup retries + until the owned tree's close event. Exercise the applicable [lifecycle regression scenarios](testing.md#lifecycle-regressions) through the producing owner and affected CLI/MCP consumers. diff --git a/licenses/npm-dependencies.txt b/licenses/npm-dependencies.txt index 31b00b3be0..19fb7e1ee6 100644 --- a/licenses/npm-dependencies.txt +++ b/licenses/npm-dependencies.txt @@ -2,7 +2,7 @@ REA npm dependency inventory Prepared from the root package-lock.json; refresh this inventory when locked dependencies change. License identifiers are upstream lockfile declarations, not a legal assessment. Runtime includes transitive and optional platform packages; development includes build, test and website tooling. -Locked package entries: 722; runtime: 178. +Locked package entries: 750; runtime: 178. Full runtime copyright/license/NOTICE texts: npm-runtime-notices.txt. @algolia/abtesting@1.25.0 | development | MIT @@ -390,6 +390,136 @@ commander@13.1.0 | runtime | MIT Package: https://www.npmjs.com/package/@emnapi/wasi-threads/v/1.2.2 Archive: https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz Integrity: sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA== +@esbuild/aix-ppc64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/aix-ppc64 + Package: https://www.npmjs.com/package/@esbuild/aix-ppc64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz + Integrity: sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ== +@esbuild/android-arm@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/android-arm + Package: https://www.npmjs.com/package/@esbuild/android-arm/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz + Integrity: sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg== +@esbuild/android-arm64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/android-arm64 + Package: https://www.npmjs.com/package/@esbuild/android-arm64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz + Integrity: sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A== +@esbuild/android-x64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/android-x64 + Package: https://www.npmjs.com/package/@esbuild/android-x64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz + Integrity: sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q== +@esbuild/darwin-arm64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/darwin-arm64 + Package: https://www.npmjs.com/package/@esbuild/darwin-arm64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz + Integrity: sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw== +@esbuild/darwin-x64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/darwin-x64 + Package: https://www.npmjs.com/package/@esbuild/darwin-x64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz + Integrity: sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw== +@esbuild/freebsd-arm64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/freebsd-arm64 + Package: https://www.npmjs.com/package/@esbuild/freebsd-arm64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz + Integrity: sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw== +@esbuild/freebsd-x64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/freebsd-x64 + Package: https://www.npmjs.com/package/@esbuild/freebsd-x64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz + Integrity: sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg== +@esbuild/linux-arm@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/linux-arm + Package: https://www.npmjs.com/package/@esbuild/linux-arm/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz + Integrity: sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w== +@esbuild/linux-arm64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/linux-arm64 + Package: https://www.npmjs.com/package/@esbuild/linux-arm64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz + Integrity: sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug== +@esbuild/linux-ia32@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/linux-ia32 + Package: https://www.npmjs.com/package/@esbuild/linux-ia32/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz + Integrity: sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ== +@esbuild/linux-loong64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/linux-loong64 + Package: https://www.npmjs.com/package/@esbuild/linux-loong64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz + Integrity: sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ== +@esbuild/linux-mips64el@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/linux-mips64el + Package: https://www.npmjs.com/package/@esbuild/linux-mips64el/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz + Integrity: sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA== +@esbuild/linux-ppc64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/linux-ppc64 + Package: https://www.npmjs.com/package/@esbuild/linux-ppc64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz + Integrity: sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ== +@esbuild/linux-riscv64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/linux-riscv64 + Package: https://www.npmjs.com/package/@esbuild/linux-riscv64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz + Integrity: sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA== +@esbuild/linux-s390x@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/linux-s390x + Package: https://www.npmjs.com/package/@esbuild/linux-s390x/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz + Integrity: sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg== +@esbuild/linux-x64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/linux-x64 + Package: https://www.npmjs.com/package/@esbuild/linux-x64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz + Integrity: sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ== +@esbuild/netbsd-arm64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/netbsd-arm64 + Package: https://www.npmjs.com/package/@esbuild/netbsd-arm64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz + Integrity: sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw== +@esbuild/netbsd-x64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/netbsd-x64 + Package: https://www.npmjs.com/package/@esbuild/netbsd-x64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz + Integrity: sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw== +@esbuild/openbsd-arm64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/openbsd-arm64 + Package: https://www.npmjs.com/package/@esbuild/openbsd-arm64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz + Integrity: sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ== +@esbuild/openbsd-x64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/openbsd-x64 + Package: https://www.npmjs.com/package/@esbuild/openbsd-x64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz + Integrity: sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw== +@esbuild/openharmony-arm64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/openharmony-arm64 + Package: https://www.npmjs.com/package/@esbuild/openharmony-arm64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz + Integrity: sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q== +@esbuild/sunos-x64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/sunos-x64 + Package: https://www.npmjs.com/package/@esbuild/sunos-x64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz + Integrity: sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g== +@esbuild/win32-arm64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/win32-arm64 + Package: https://www.npmjs.com/package/@esbuild/win32-arm64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz + Integrity: sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ== +@esbuild/win32-ia32@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/win32-ia32 + Package: https://www.npmjs.com/package/@esbuild/win32-ia32/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz + Integrity: sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA== +@esbuild/win32-x64@0.28.2 | development, optional | MIT + Lockfile path: node_modules/@esbuild/win32-x64 + Package: https://www.npmjs.com/package/@esbuild/win32-x64/v/0.28.2 + Archive: https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz + Integrity: sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g== @fast-check/vitest@0.5.0 | development | MIT Lockfile path: node_modules/@fast-check/vitest Package: https://www.npmjs.com/package/@fast-check/vitest/v/0.5.0 @@ -2010,6 +2140,11 @@ es-object-atoms@1.1.2 | runtime | MIT Package: https://www.npmjs.com/package/es-object-atoms/v/1.1.2 Archive: https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz Integrity: sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw== +esbuild@0.28.2 | development | MIT + Lockfile path: node_modules/esbuild + Package: https://www.npmjs.com/package/esbuild/v/0.28.2 + Archive: https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz + Integrity: sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA== escalade@3.2.0 | development | MIT Lockfile path: node_modules/escalade Package: https://www.npmjs.com/package/escalade/v/3.2.0 @@ -3255,6 +3390,11 @@ tslib@2.8.1 | development | 0BSD Package: https://www.npmjs.com/package/tslib/v/2.8.1 Archive: https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz Integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w== +tsx@4.23.15 | development | MIT + Lockfile path: node_modules/tsx + Package: https://www.npmjs.com/package/tsx/v/4.23.15 + Archive: https://registry.npmjs.org/tsx/-/tsx-4.23.15.tgz + Integrity: sha512-Yiex1Ovn8z2xPpOWckIiysV1SSyRMY9BkLF++q0yKiDxCqRhosKfMg3janKkiLBwZ5c/YryloKwGZcrEmtwxKw== tunnel@0.0.6 | development | MIT Lockfile path: node_modules/tunnel Package: https://www.npmjs.com/package/tunnel/v/0.0.6 diff --git a/package-lock.json b/package-lock.json index 1606244d90..30e043d696 100644 --- a/package-lock.json +++ b/package-lock.json @@ -72,6 +72,7 @@ "oxfmt": "0.72.0", "oxlint": "1.87.0", "rolldown": "1.2.12", + "tsx": "4.23.15", "turbo": "2.11.7", "typescript": "7.0.2", "typescript-termux": "npm:typescript@5.9.3", @@ -1234,6 +1235,448 @@ "tslib": "^2.4.0" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@fast-check/vitest": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/@fast-check/vitest/-/vitest-0.5.0.tgz", @@ -5909,6 +6352,48 @@ "node": ">= 0.4" } }, + "node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -9330,6 +9815,25 @@ "dev": true, "license": "0BSD" }, + "node_modules/tsx": { + "version": "4.23.15", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.15.tgz", + "integrity": "sha512-Yiex1Ovn8z2xPpOWckIiysV1SSyRMY9BkLF++q0yKiDxCqRhosKfMg3janKkiLBwZ5c/YryloKwGZcrEmtwxKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, "node_modules/tunnel": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", diff --git a/package.json b/package.json index 3845c7bf8b..2e2964a0e6 100644 --- a/package.json +++ b/package.json @@ -383,6 +383,7 @@ "oxfmt": "0.72.0", "oxlint": "1.87.0", "rolldown": "1.2.12", + "tsx": "4.23.15", "turbo": "2.11.7", "typescript": "7.0.2", "typescript-termux": "npm:typescript@5.9.3", diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index 515bc70b48..06c1356f73 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -104,6 +104,10 @@ export const SCOPES = { "tests/boundary/filesystem/npmRelease.test.ts", "tests/boundary/process/releaseArtifactCanary.test.ts", "src/windows/**", + "scripts/lib/windows-owned-command.mjs", + "scripts/run-exclusive-command.mjs", + "tests/process-global/runnerCancellation.test.ts", + "tests/process-global/windowsOwnedCommand.test.ts", "src/ghidra/GhidraBridgeSources.ts", "native/windows/**", "src/process/capture/FilesystemSnapshot.ts", diff --git a/scripts/lib/kuna-native-reference-verifier.mjs b/scripts/lib/kuna-native-reference-verifier.mjs index 37a8cee095..fad27f31e8 100644 --- a/scripts/lib/kuna-native-reference-verifier.mjs +++ b/scripts/lib/kuna-native-reference-verifier.mjs @@ -13,11 +13,15 @@ export const nativeKunaIncomingSites = async ({ binary, address, retain, + fileCloseResources, }) => { - const installed = await inspectKunaInstallation({ - REA_KUNA_COMMAND: executable, - REA_KUNA_CACHE_DIR: cacheRoot, - }); + const installed = await inspectKunaInstallation( + { + REA_KUNA_COMMAND: executable, + REA_KUNA_CACHE_DIR: cacheRoot, + }, + { fileCloseResources }, + ); assert.ok(installed.ok, "Native reference oracle could not inspect Kuna"); const input = await realpath(binary); const timeoutMs = installed.value.configuration.functionTimeoutMs; @@ -30,6 +34,8 @@ export const nativeKunaIncomingSites = async ({ reportedTimeoutMs: timeoutMs, outputBytes: KUNA_XREF_OUTPUT_BYTES, signal: new AbortController().signal, + retainFailedFileClose: (owner, resource) => + fileCloseResources.retain(owner, resource), }); if (outcome.kind === "retained") retain(outcome); if (outcome.kind !== "ok") throw outcome.error; diff --git a/scripts/lib/kuna-string-verifier.mjs b/scripts/lib/kuna-string-verifier.mjs index 58a8b60812..60f74e7489 100644 --- a/scripts/lib/kuna-string-verifier.mjs +++ b/scripts/lib/kuna-string-verifier.mjs @@ -18,11 +18,15 @@ export const nativeKunaStrings = async ({ workspace, binary, retain, + fileCloseResources, }) => { - const installed = await inspectKunaInstallation({ - REA_KUNA_COMMAND: executable, - REA_KUNA_CACHE_DIR: cacheRoot, - }); + const installed = await inspectKunaInstallation( + { + REA_KUNA_COMMAND: executable, + REA_KUNA_CACHE_DIR: cacheRoot, + }, + { fileCloseResources }, + ); assert.ok(installed.ok, "Native string oracle could not inspect Kuna"); const input = await realpath(binary); const outcome = await runKunaCommand({ @@ -45,6 +49,8 @@ export const nativeKunaStrings = async ({ reportedTimeoutMs: installed.value.configuration.projectTimeoutMs, outputBytes: 16 * 1024 * 1024, signal: new AbortController().signal, + retainFailedFileClose: (owner, resource) => + fileCloseResources.retain(owner, resource), }); if (outcome.kind === "retained") retain(outcome); if (outcome.kind !== "ok") throw outcome.error; @@ -80,6 +86,7 @@ export const verifyKunaStrings = async ({ cacheRoot, nativeStrings, expectedStrings = [], + fileCloseResources, }) => { const expected = await nativeStrings(); if (expectedStrings.length > 0) diff --git a/scripts/lib/verifier-cleanup.mjs b/scripts/lib/verifier-cleanup.mjs index 1a257f9ce7..2d4db196ca 100644 --- a/scripts/lib/verifier-cleanup.mjs +++ b/scripts/lib/verifier-cleanup.mjs @@ -8,6 +8,7 @@ export async function closeVerifierResources(tasks, primaryError) { failures.push(cause); } } + if (failures.length > 0) throw new AggregateError( primaryError === undefined ? failures : [primaryError, ...failures], diff --git a/scripts/lib/verifier-file-resources.d.mts b/scripts/lib/verifier-file-resources.d.mts new file mode 100644 index 0000000000..e47d043e4d --- /dev/null +++ b/scripts/lib/verifier-file-resources.d.mts @@ -0,0 +1,7 @@ +import type { ArtifactResourceScope } from "../../src/artifacts/ArtifactResourceScope.js"; + +/** Hash one verifier input and transfer failed-close ownership to its scope. */ +export function hashVerifierFile( + path: string, + resources: Pick, +): Promise<{ readonly bytes: number; readonly sha256: string }>; diff --git a/scripts/lib/verifier-file-resources.mjs b/scripts/lib/verifier-file-resources.mjs new file mode 100644 index 0000000000..42f4264fe2 --- /dev/null +++ b/scripts/lib/verifier-file-resources.mjs @@ -0,0 +1,10 @@ +import { hashStableFile } from "../../dist/filesystem/StableFileHash.js"; + +/** Hash one verifier input while retaining any descriptor whose close fails. */ +export function hashVerifierFile(path, resources) { + return hashStableFile(path, { + retainFailedClose(owner) { + resources.retain({ kind: "file-handle", resource: path, handle: owner }); + }, + }); +} diff --git a/scripts/lib/windows-owned-command.d.mts b/scripts/lib/windows-owned-command.d.mts new file mode 100644 index 0000000000..3288d742ed --- /dev/null +++ b/scripts/lib/windows-owned-command.d.mts @@ -0,0 +1,51 @@ +import type { EventEmitter } from "node:events"; + +import type { WindowsNativeAuthority } from "../../src/windows/WindowsNativeLoader.js"; +import type { WindowsOwnedProcess } from "../../src/windows/WindowsOwnedProcess.js"; + +export type WindowsOwnedProcessConstructor = typeof WindowsOwnedProcess; + +export interface WindowsOwnedCommandOptions { + readonly WindowsOwnedProcess: WindowsOwnedProcessConstructor; + readonly command: string; + readonly arguments: readonly string[]; + readonly cwd: string | undefined; + readonly environment: NodeJS.ProcessEnv; + readonly authority?: WindowsNativeAuthority; +} + +export interface WindowsOwnedProcessLifecycle extends Pick< + EventEmitter, + "on" | "once" | "off" +> { + readonly stdin?: Pick | null; + cleanup(): Promise<{ readonly cleaned: boolean; readonly reason?: string }>; +} + +export type WindowsOwnedCleanupDiagnostic = + | { readonly type: "cleanup-failed"; readonly reason: string } + | { readonly type: "cleanup-recovered" }; + +export interface WindowsOwnedProcessSupervisionOptions { + readonly retryIntervalMs?: number; + readonly onCleanupDiagnostic?: ( + diagnostic: WindowsOwnedCleanupDiagnostic, + ) => void; +} + +export function loadWindowsOwnedProcessClass(): Promise; + +export function createWindowsOwnedCommand( + options: WindowsOwnedCommandOptions, +): WindowsOwnedProcess; + +export function superviseWindowsOwnedProcess( + child: WindowsOwnedProcessLifecycle, + options?: WindowsOwnedProcessSupervisionOptions, +): { + readonly finished: Promise<{ + readonly code: number | null; + readonly signal: NodeJS.Signals | null; + }>; + requestCleanup(): void; +}; diff --git a/scripts/lib/windows-owned-command.mjs b/scripts/lib/windows-owned-command.mjs new file mode 100644 index 0000000000..b52e8c586e --- /dev/null +++ b/scripts/lib/windows-owned-command.mjs @@ -0,0 +1,173 @@ +import { createRequire } from "node:module"; + +const require = createRequire(import.meta.url); +const sourceWindowsProcess = new URL( + "../../src/windows/WindowsOwnedProcess.ts", + import.meta.url, +).href; + +/** Load the source owner without requiring a generated dist tree. */ +export async function loadWindowsOwnedProcessClass() { + const { tsImport } = await import("tsx/esm/api"); + const source = await tsImport(sourceWindowsProcess, import.meta.url); + return source.WindowsOwnedProcess; +} + +/** Start an owned Windows command while preserving cross-spawn's shim rules. */ +export function createWindowsOwnedCommand(options) { + const { + WindowsOwnedProcess, + command, + arguments: arguments_, + cwd, + environment, + authority, + } = options; + // This pinned private parser is cross-spawn's own resolution and cmd-shim + // boundary. Keep its compatibility coupling here instead of copying its rules. + const parse = require("cross-spawn/lib/parse"); + const parsed = parse(command, arguments_, { cwd, env: environment }); + if ( + typeof parsed !== "object" || + parsed === null || + typeof parsed.command !== "string" || + !Array.isArray(parsed.args) || + !parsed.args.every((argument) => typeof argument === "string") || + typeof parsed.options !== "object" || + parsed.options === null || + (parsed.options.windowsVerbatimArguments !== undefined && + typeof parsed.options.windowsVerbatimArguments !== "boolean") + ) + throw new TypeError( + "cross-spawn returned an unsupported Windows command shape", + ); + // cross-spawn keeps the user's spelling in `command` for a resolved .exe + // (for example bare `node`), while `file` is the PATH-resolved executable. + // The native boundary intentionally requires an absolute application path. + const executable = + parsed.command === command && typeof parsed.file === "string" + ? parsed.file + : parsed.command; + return new WindowsOwnedProcess( + executable, + parsed.args, + cwd, + environment, + parsed.options.windowsVerbatimArguments === true, + { + protocolStdin: true, + ...(authority === undefined ? {} : { authority }), + }, + ); +} + +/** Own a native command until its exact Job Object close event. */ +export function superviseWindowsOwnedProcess( + child, + { retryIntervalMs = 1_000, onCleanupDiagnostic = () => {} } = {}, +) { + const childStdin = child.stdin ?? undefined; + let jobClosed = false; + let stdinClosed = childStdin === undefined; + let settled = false; + let closeResult; + let hasFirstError = false; + let firstError; + let retryTimer; + let cleanupInFlight = false; + let lastCleanupFailure; + let reportedCleanupFailure = false; + + const finished = new Promise((resolveResult, reject) => { + child.on("error", handleError); + child.once("close", (code, signal) => { + jobClosed = true; + closeResult = { code, signal }; + clearTimeout(retryTimer); + child.off("error", handleError); + finishWhenClosed(resolveResult, reject); + }); + childStdin?.on("error", handleError); + childStdin?.once("close", () => { + stdinClosed = true; + childStdin.off("error", handleError); + finishWhenClosed(resolveResult, reject); + }); + }); + + function finishWhenClosed(resolveResult, reject) { + if (!jobClosed || !stdinClosed || settled) return; + settled = true; + if (reportedCleanupFailure) { + onCleanupDiagnostic({ type: "cleanup-recovered" }); + reportedCleanupFailure = false; + lastCleanupFailure = undefined; + } + if (hasFirstError) reject(firstError); + else resolveResult(closeResult); + } + + function handleError(cause) { + if (!hasFirstError) { + hasFirstError = true; + firstError = cause; + } + requestCleanup(); + } + + function scheduleRetry() { + if (jobClosed || retryTimer !== undefined) return; + // Keep this timer referenced: until Job Object close, it is the supervisor's + // live ownership observer even after IPC disconnect or stdin EOF. + retryTimer = setTimeout(() => { + retryTimer = undefined; + requestCleanup(); + }, retryIntervalMs); + } + + function reportFailure(reason) { + if (jobClosed) return; + if (reportedCleanupFailure && lastCleanupFailure === reason) return; + lastCleanupFailure = reason; + reportedCleanupFailure = true; + onCleanupDiagnostic({ type: "cleanup-failed", reason }); + } + + function requestCleanup() { + if (jobClosed || cleanupInFlight) return; + clearTimeout(retryTimer); + retryTimer = undefined; + cleanupInFlight = true; + let cleanup; + try { + cleanup = child.cleanup(); + } catch (cause) { + reportFailure(cause instanceof Error ? cause.message : String(cause)); + cleanupInFlight = false; + scheduleRetry(); + return; + } + void Promise.resolve(cleanup) + .then((result) => { + if (!result.cleaned) { + reportFailure( + result.reason ?? "Windows Job Object cleanup did not complete", + ); + return; + } + if (!jobClosed) + reportFailure( + "Windows Job Object cleanup reported success before its close event", + ); + }) + .catch((cause) => { + reportFailure(cause instanceof Error ? cause.message : String(cause)); + }) + .finally(() => { + cleanupInFlight = false; + scheduleRetry(); + }); + } + + return { finished, requestCleanup }; +} diff --git a/scripts/run-exclusive-command.mjs b/scripts/run-exclusive-command.mjs index b84b9fe9f3..6d7fc37bcf 100644 --- a/scripts/run-exclusive-command.mjs +++ b/scripts/run-exclusive-command.mjs @@ -4,6 +4,11 @@ import spawn from "cross-spawn"; import { mkdir } from "node:fs/promises"; import { join, resolve } from "node:path"; import { DatabaseSync } from "node:sqlite"; +import { + createWindowsOwnedCommand, + loadWindowsOwnedProcessClass, + superviseWindowsOwnedProcess, +} from "./lib/windows-owned-command.mjs"; const [, , lockName, ...command] = process.argv; const lockRoot = resolve(process.cwd(), ".cache", "rea-command-locks"); @@ -11,6 +16,7 @@ const signalExitCodes = { SIGINT: 130, SIGTERM: 143 }; let database; let child; +let windowsOwnership; let forceTimer; let shutdownSignal; @@ -69,16 +75,44 @@ async function run() { } } -function runCommand() { +async function runCommand() { + if (process.platform === "win32") { + const WindowsOwnedProcess = await loadWindowsOwnedProcessClass(); + if (shutdownSignal !== undefined) return { code: 1 }; + child = createWindowsOwnedCommand({ + WindowsOwnedProcess, + command: command[0], + arguments: command.slice(1), + cwd: process.cwd(), + environment: process.env, + }); + windowsOwnership = superviseWindowsOwnedProcess(child, { + onCleanupDiagnostic: reportCleanupDiagnostic, + }); + try { + child.stdout.pipe(process.stdout); + child.stderr.pipe(process.stderr); + process.stdin.pipe(child.stdin); + const { code } = await windowsOwnership.finished; + return { code }; + } catch (cause) { + windowsOwnership.requestCleanup(); + await windowsOwnership.finished.catch(() => {}); + throw cause; + } finally { + process.stdin.unpipe(child.stdin); + windowsOwnership = undefined; + } + } return new Promise((resolveResult, reject) => { + let spawned = false; + let hasChildError = false; + let childError; child = spawn(command[0], command.slice(1), { cwd: process.cwd(), env: process.env, stdio: "inherit", }); - let spawned = false; - let childError; - let hasChildError = false; child.once("spawn", () => { spawned = true; }); @@ -87,9 +121,7 @@ function runCommand() { hasChildError = true; childError = cause; } - if (!spawned) { - reject(childError); - } + if (!spawned) reject(childError); }; child.on("error", rememberChildError); child.once("close", (code) => { @@ -114,13 +146,14 @@ function handleDisconnect() { } function handleSignal(signal) { - if (shutdownSignal !== undefined) return; + if (shutdownSignal !== undefined) { + if (process.platform === "win32") windowsOwnership?.requestCleanup(); + return; + } shutdownSignal = signal; if (child === undefined) return; if (process.platform === "win32") { - // Windows has no POSIX process groups. Stop only the command owned by this - // supervisor, and keep the SQLite transaction until its close event. - child.kill(signal); + windowsOwnership?.requestCleanup(); return; } // Establish cancellation before signalling the group. This supervisor is @@ -129,6 +162,18 @@ function handleSignal(signal) { forceTimer = setTimeout(() => process.kill(-process.pid, "SIGKILL"), 1_000); } +function reportCleanupDiagnostic(diagnostic) { + if (diagnostic.type === "cleanup-recovered") { + console.error( + "Owned Windows command cleanup recovered; observing Job Object close.", + ); + return; + } + console.error( + `Could not clean up the owned Windows command: ${diagnostic.reason}; retrying while retaining the command lock.`, + ); +} + function isBusyError(cause) { return ( typeof cause === "object" && diff --git a/scripts/test-website.py b/scripts/test-website.py index b317ac345f..513139ff5f 100644 --- a/scripts/test-website.py +++ b/scripts/test-website.py @@ -308,19 +308,54 @@ class WebsiteMaintenanceChecks(unittest.TestCase): def test_repository_links_reject_missing_files_and_old_markdown_anchors(self): with tempfile.TemporaryDirectory() as directory: - root = Path(directory) + container = Path(directory) + root = container / "repository" (root / "website/public").mkdir(parents=True) (root / "docs").mkdir() (root / "docs/guide.md").write_text("# Guide\n## Set up REA\n") + (container / "outside.md").write_text("# Outside\n") page = root / "website/README.md" page.write_text("[Guide](../docs/guide.md#set-up-rea)\n") (root / "website/.venv").mkdir() (root / "website/.venv/vendor.md").write_text("[File](missing.md)") self.assertEqual(VERIFY.check_repository_links(root), []) - page.write_text("[Guide](../docs/guide.md#removed)\n[File](../docs/removed.md)\n") - errors = VERIFY.check_repository_links(root) + alias = container / "repository-alias" + alias.symlink_to(root.resolve(), target_is_directory=True) + self.assertEqual(VERIFY.check_repository_links(alias), []) + page.write_text( + "[Guide](../docs/guide.md#removed)\n" + "[File](../docs/removed.md)\n" + "[Escape](../../outside.md)\n" + ) + errors = VERIFY.check_repository_links(alias) self.assertTrue(any("missing Markdown fragment" in error for error in errors)) self.assertTrue(any("missing repository target" in error for error in errors)) + self.assertTrue( + any("missing repository target: ../../outside.md" in error for error in errors) + ) + + def test_site_links_accept_directory_aliases_and_still_reject_escape(self): + with tempfile.TemporaryDirectory() as directory: + site = Path(directory) / "public" + shutil.copytree(ROOT / "website/_site", site) + canonical_site = site.resolve() + alias = Path(directory) / "public-alias" + alias.symlink_to(canonical_site, target_is_directory=True) + self.assertEqual(VERIFY.check_site(canonical_site)[3], []) + self.assertEqual(VERIFY.check_site(alias)[3], []) + + page = canonical_site / "faq/index.html" + page.write_text( + page.read_text().replace( + "", + 'Escape', + 1, + ) + ) + errors = VERIFY.check_site(alias)[3] + self.assertTrue( + any("reference leaves the website" in error for error in errors) + ) def test_repository_links_accept_aliased_roots_and_reject_escaping_links(self): with tempfile.TemporaryDirectory() as directory: diff --git a/scripts/verify-adb.mjs b/scripts/verify-adb.mjs index cd162f00ac..916a9383ac 100644 --- a/scripts/verify-adb.mjs +++ b/scripts/verify-adb.mjs @@ -40,7 +40,8 @@ const run = async (operation, input) => { return result.value.normalized_result; }; -let primaryFailure; +let verifierFailed = false; +let verifierFailure; try { const client = await run("inspect_adb_client", {}); if (client !== null) { @@ -374,26 +375,27 @@ try { } } } catch (cause) { - primaryFailure = cause; + verifierFailed = true; + verifierFailure = cause; +} finally { + try { + await closeVerifierResources( + [() => provider.close()], + verifierFailed ? verifierFailure : undefined, + ); + } catch (cause) { + process.stderr.write(`[FAIL] verifier cleanup — ${formatFailure(cause)}\n`); + failures += 1; + } } - -let cleanupFailure; -try { - await closeVerifierResources([() => provider.close()], primaryFailure); -} catch (cause) { - cleanupFailure = cause; -} - -if (primaryFailure !== undefined || cleanupFailure !== undefined) { - const causes = - cleanupFailure instanceof AggregateError - ? cleanupFailure.errors - : [cleanupFailure ?? primaryFailure]; +if (verifierFailed) { process.stderr.write( - `ADB device verification failed: ${causes.map((cause) => (cause instanceof Error ? cause.message : String(cause))).join("; ")}\n`, + `[FAIL] verifier aborted — ${formatFailure(verifierFailure)}\n`, ); - process.exitCode = 1; -} else if (failures > 0) { + failures += 1; +} + +if (failures > 0) { process.stdout.write(`\n${failures} check(s) failed\n`); process.exitCode = 1; } else { @@ -421,3 +423,17 @@ function parseArguments(argv) { } return parsed; } + +function formatFailure(cause) { + if (cause instanceof AggregateError) + return `${cause.message}: ${cause.errors.map(formatFailure).join("; ")}`; + if (cause instanceof Error) { + const resources = cause.cleanupResources ?? cause.resources; + const details = Array.isArray(resources) + ? ` resources=${resources.join(", ")}` + : ""; + const reason = cause.diagnostics?.reason; + return `${cause.name}: ${cause.message}${details}${typeof reason === "string" ? ` reason=${reason}` : ""}`; + } + return String(cause); +} diff --git a/scripts/verify-flutter.mjs b/scripts/verify-flutter.mjs index 52213e71f8..3086442c81 100644 --- a/scripts/verify-flutter.mjs +++ b/scripts/verify-flutter.mjs @@ -1,16 +1,16 @@ import assert from "node:assert/strict"; import { execFile } from "node:child_process"; -import { mkdtemp, readdir, readFile, rmdir } from "node:fs/promises"; +import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { toolContract } from "../dist/contracts/toolContracts.js"; -import { hashStableFile } from "../dist/filesystem/StableFileHash.js"; -import { OwnedRegularFileCleanupFailure } from "../dist/filesystem/RegularFile.js"; +import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js"; import { mcpTextValue } from "./lib/mcp-verifier-results.mjs"; import { closeVerifierResources } from "./lib/verifier-cleanup.mjs"; +import { hashVerifierFile } from "./lib/verifier-file-resources.mjs"; const options = parseArguments(process.argv.slice(2)); if (options.help) { @@ -31,6 +31,7 @@ const expectations = : JSON.parse(await readFile(options.expectations, "utf8")); // This is a shared test parent, not a leased provider-owned Windows runtime. const root = await mkdtemp(join(tmpdir(), "rea-flutter-verifier-")); +const resources = new ArtifactResourceScope(); const entrypoint = resolve("scripts/rea.mjs"); const environment = Object.fromEntries( Object.entries(process.env).filter(([, value]) => typeof value === "string"), @@ -41,17 +42,6 @@ Object.assign(environment, { TEMP: root, }); const client = new Client({ name: "flutter-verifier", version: "1" }); -let primaryError; -const hashOwners = new Map(); -const hashOwned = async (...args) => { - try { - return await hashStableFile(...args); - } catch (cause) { - if (cause instanceof OwnedRegularFileCleanupFailure) - hashOwners.set(cause.owner, cause.path); - throw cause; - } -}; const transport = new StdioClientTransport({ command: process.execPath, args: [entrypoint, "mcp"], @@ -59,10 +49,11 @@ const transport = new StdioClientTransport({ stderr: "pipe", }); const execute = promisify(execFile); +let primaryError; try { await client.connect(transport); const apk = resolve(options.apk); - const identity = await hashOwned(apk); + const identity = await hashVerifierFile(apk, resources); const identified = await run("identify_flutter_build", { path: apk }); assert.deepEqual(identified.target, { path: apk, ...identity }); assert.equal(identified.flutter_detected, true); @@ -189,13 +180,13 @@ try { const negative = await run("identify_flutter_build", { path: plain }); assert.deepEqual(negative.target, { path: plain, - ...(await hashOwned(plain)), + ...(await hashVerifierFile(plain, resources)), }); assert.equal(negative.flutter_detected, false); assert.equal(negative.coverage, "complete"); } assert.deepEqual( - await hashOwned(apk), + await hashVerifierFile(apk, resources), identity, "Verification must preserve its input", ); @@ -203,32 +194,25 @@ try { primaryError = cause; throw cause; } finally { - const cleanup = []; - for (const [owner, path] of hashOwners) { - cleanup.push(async () => { - try { - await owner.close(); - hashOwners.delete(owner); - } catch (cause) { - throw new Error(`Could not close hash file handle for ${path}`, { - cause, - }); - } - }); - } - cleanup.push(async () => { - await closeVerifierResources([ - () => client.close(), - () => transport.close(), - ]); - assert.deepEqual( - await readdir(root), - [], - `Provider snapshots remain under ${root}`, - ); - await rmdir(root); - }); - await closeVerifierResources(cleanup, primaryError); + // Server shutdown retries retained provider cleanup and settles before success. + await closeVerifierResources( + [ + async () => { + await closeVerifierResources([ + () => client.close(), + () => transport.close(), + () => resources.close(), + ]); + assert.deepEqual( + await readdir(root), + [], + `Provider snapshots remain under ${root}`, + ); + await rm(root, { recursive: true, force: true }); + }, + ], + primaryError, + ); } process.stdout.write( "all asserted Flutter CLI/MCP checks passed; unverified facets are reported above\n", diff --git a/scripts/verify-pcap.mjs b/scripts/verify-pcap.mjs index 91d454ac16..b9aed11318 100644 --- a/scripts/verify-pcap.mjs +++ b/scripts/verify-pcap.mjs @@ -7,10 +7,10 @@ import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { toolContract } from "../dist/contracts/toolContracts.js"; -import { hashStableFile } from "../dist/filesystem/StableFileHash.js"; -import { OwnedRegularFileCleanupFailure } from "../dist/filesystem/RegularFile.js"; +import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js"; import { mcpTextValue } from "./lib/mcp-verifier-results.mjs"; import { closeVerifierResources } from "./lib/verifier-cleanup.mjs"; +import { hashVerifierFile } from "./lib/verifier-file-resources.mjs"; const args = process.argv.slice(2); if (args.includes("--help") || args.includes("-h")) { @@ -25,6 +25,7 @@ const path = args[1] === undefined ? null : resolve(args[1]); // This parent is shared by independent CLI/MCP owners. A leased Windows // runtime root would prevent their native admission of its directory handle. const root = await mkdtemp(join(tmpdir(), "rea-pcap-verifier-")); +const resources = new ArtifactResourceScope(); const environment = Object.fromEntries( Object.entries(process.env).filter(([, value]) => typeof value === "string"), ); @@ -36,24 +37,14 @@ Object.assign(environment, { const entrypoint = resolve("scripts/rea.mjs"); const execute = promisify(execFile); const client = new Client({ name: "pcap-real-verifier", version: "1" }); -let primaryError; const transport = new StdioClientTransport({ command: process.execPath, args: [entrypoint, "mcp"], env: environment, stderr: "pipe", }); -const hashOwners = new Map(); -const hashOwned = async (...args) => { - try { - return await hashStableFile(...args); - } catch (cause) { - if (cause instanceof OwnedRegularFileCleanupFailure) - hashOwners.set(cause.owner, cause.path); - throw cause; - } -}; transport.stderr?.resume(); +let primaryError; try { await client.connect(transport); const readiness = await run("inspect_pcap_client", {}); @@ -61,7 +52,7 @@ try { if (path === null) console.log("[unverified] Capture inspection: no --pcap was supplied"); else { - const identity = await hashOwned(path); + const identity = await hashVerifierFile(path, resources); const result = await run("inspect_pcap_capture", { path }); assert.deepEqual(result.target, { path, ...identity }); assert(result.packet_count >= 0); @@ -88,7 +79,7 @@ try { ); assert.equal(result.coverage, partial ? "partial" : "complete"); assert.deepEqual( - await hashOwned(path), + await hashVerifierFile(path, resources), identity, "Input capture must remain unchanged", ); @@ -100,32 +91,24 @@ try { primaryError = cause; throw cause; } finally { - const cleanup = []; - for (const [owner, path] of hashOwners) { - cleanup.push(async () => { - try { - await owner.close(); - hashOwners.delete(owner); - } catch (cause) { - throw new Error(`Could not close hash file handle for ${path}`, { - cause, - }); - } - }); - } - cleanup.push(async () => { - await closeVerifierResources([ - () => client.close(), - () => transport.close(), - ]); - assert.deepEqual( - await readdir(root), - [], - `Owned capture resources remain under ${root}`, - ); - await rmdir(root); - }); - await closeVerifierResources(cleanup, primaryError); + await closeVerifierResources( + [ + async () => { + await closeVerifierResources([ + () => client.close(), + () => transport.close(), + () => resources.close(), + ]); + assert.deepEqual( + await readdir(root), + [], + `Owned capture resources remain under ${root}`, + ); + await rmdir(root); + }, + ], + primaryError, + ); } console.log("All asserted pcap CLI/MCP checks passed"); diff --git a/scripts/verify-real-kuna.mjs b/scripts/verify-real-kuna.mjs index 289073c9ae..07a2681780 100644 --- a/scripts/verify-real-kuna.mjs +++ b/scripts/verify-real-kuna.mjs @@ -26,6 +26,7 @@ import { parseArgs, promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { execFileOutput } from "../dist/process/ExecFileOutput.js"; +import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js"; import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; import { verifyKunaReferences } from "./lib/kuna-reference-verifier.mjs"; import { @@ -70,6 +71,12 @@ if (values["compare-ghidra"]) { ); } const run = createVerifierRun(); +const fileCloseScope = new ArtifactResourceScope(); +const fileCloseResources = { + retain: (owner, resource) => + fileCloseScope.retain({ kind: "file-handle", resource, handle: owner }), + run: (operation) => fileCloseScope.run(operation), +}; const workspace = await mkdtemp(join(tmpdir(), "rea-kuna-lane-")); const cacheRoot = join(workspace, "cache"); // A server retains its workspace until both provider and transport cleanup @@ -285,6 +292,7 @@ const verifyKuna = async ( call: cold.call, cacheRoot, expectedStrings, + fileCloseResources, nativeStrings: () => nativeKunaStrings({ executable: kuna, @@ -292,6 +300,7 @@ const verifyKuna = async ( workspace, binary: path, retain: (owner) => retainedNative.add(owner), + fileCloseResources, }), }), }); @@ -315,6 +324,7 @@ const verifyKuna = async ( binary: path, address: targetAddress, retain: (outcome) => retainedNative.add(outcome), + fileCloseResources, }), }); const pseudocode = await cold.call("procedure_pseudo_code", { @@ -482,6 +492,7 @@ const verifyVariant = async ({ flavor, directory, symbols }, variant) => { const results = []; const failures = []; +let fileResourcesClean = true; try { kunaVersion = ( await execFileOutput(kuna, ["--version"], { @@ -554,8 +565,15 @@ for (const retained of [...retainedNative]) { failures.push(cause); } } +try { + await fileCloseScope.close(); +} catch (cause) { + fileResourcesClean = false; + failures.push(cause); +} const lifecycle = await completeVerifierRun(run); -let cleanupComplete = pendingServers.size === 0 && retainedNative.size === 0; +let cleanupComplete = + pendingServers.size === 0 && retainedNative.size === 0 && fileResourcesClean; try { assertVerifierCleanupComplete(lifecycle.process_lineage); } catch (cause) { diff --git a/src/adb/AdbInputSnapshot.ts b/src/adb/AdbInputSnapshot.ts new file mode 100644 index 0000000000..357174a95c --- /dev/null +++ b/src/adb/AdbInputSnapshot.ts @@ -0,0 +1,191 @@ +import { createHash } from "node:crypto"; +import { open, stat } from "node:fs/promises"; +import { basename, join } from "node:path"; + +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { + AnalysisArtifactChangedError, + AnalysisCancelledError, + AnalysisOutputError, +} from "../domain/analysisErrorCore.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; +import { + openRegularFile, + RegularFileAdmissionFailure, + sameRegularFileState, +} from "../filesystem/RegularFile.js"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import type { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; + +export interface AdbInputSnapshotDependencies { + readonly openSource?: typeof openRegularFile; + readonly openDestination?: typeof open; +} + +export interface AdbInputSnapshot { + readonly path: string; + readonly bytes: number; + readonly sha256: string; +} + +/** Snapshot failure with the handles that remain owned for a later retry. */ +export class AdbInputSnapshotFailure extends Error { + constructor( + readonly operation: string, + readonly sourcePath: string, + readonly primaryCause: unknown, + readonly cleanupResources: readonly string[], + readonly cleanupCause: unknown, + ) { + super(`ADB input snapshot failed for ${sourcePath}`, { + cause: primaryCause ?? cleanupCause, + }); + this.name = "AdbInputSnapshotFailure"; + } +} + +/** Copy one caller file into a private root, hashing the bytes adb will consume. */ +export const snapshotAdbInput = async ( + sourcePath: string, + root: Pick, + resources: ArtifactResourceScope, + operation: string, + signal?: AbortSignal, + dependencies: AdbInputSnapshotDependencies = {}, +): Promise => { + let source: OwnedFileHandle | undefined; + let destination: OwnedFileHandle | undefined; + const cleanupResources: string[] = []; + const cleanupCauses: unknown[] = []; + let snapshot: AdbInputSnapshot | undefined; + let primaryCause: unknown; + + try { + source = new OwnedFileHandle( + await (dependencies.openSource ?? openRegularFile)(sourcePath, { + symlinks: "follow", + signal, + }), + ); + const destinationPath = join(root.path, basename(sourcePath)); + destination = new OwnedFileHandle( + await (dependencies.openDestination ?? open)( + destinationPath, + "wx", + 0o600, + ), + ); + const initial = await source.handle.stat(); + if (!Number.isSafeInteger(initial.size) || initial.size < 0) + throw new RangeError( + `Regular file size cannot be represented safely: ${sourcePath}`, + ); + const hash = createHash("sha256"); + const chunk = Buffer.allocUnsafe(64 * 1024); + let position = 0; + while (position < initial.size) { + signal?.throwIfAborted(); + const read = await source.handle.read( + chunk, + 0, + Math.min(chunk.length, initial.size - position), + position, + ); + signal?.throwIfAborted(); + if (read.bytesRead === 0) + throw new AnalysisArtifactChangedError( + operation, + sourcePath, + "the file ended before its admitted size was copied", + ); + const bytes = chunk.subarray(0, read.bytesRead); + let written = 0; + while (written < bytes.length) { + signal?.throwIfAborted(); + const result = await destination.handle.write( + bytes, + written, + bytes.length - written, + position + written, + ); + if (result.bytesWritten === 0) + throw new Error("Private ADB input snapshot stopped accepting bytes"); + written += result.bytesWritten; + } + hash.update(bytes); + position += read.bytesRead; + } + signal?.throwIfAborted(); + const [opened, named] = await Promise.all([ + source.handle.stat(), + stat(sourcePath), + ]); + signal?.throwIfAborted(); + if ( + !sameRegularFileState(initial, opened) || + !sameRegularFileState(initial, named) + ) + throw new AnalysisArtifactChangedError( + operation, + sourcePath, + "the file changed while its private snapshot was copied", + ); + await destination.handle.sync(); + await destination.handle.chmod(0o400); + snapshot = { + path: destinationPath, + bytes: position, + sha256: hash.digest("hex"), + }; + } catch (cause: unknown) { + if (cause instanceof RegularFileAdmissionFailure) { + source = cause.owner; + primaryCause = cause.cause; + } else if (cause instanceof AnalysisError) { + primaryCause = cause; + } else { + primaryCause = + signal?.aborted === true + ? new AnalysisCancelledError(operation, { cause }) + : cause; + } + } + + for (const [owner, resource] of [ + [source, `${sourcePath} (ADB snapshot source handle)`], + [ + destination, + `${join(root.path, basename(sourcePath))} (ADB snapshot handle)`, + ], + ] as const) { + if (owner === undefined) continue; + const result = await resources.release({ + kind: "file-handle", + resource, + handle: owner, + }); + if (result.kind === "failed") { + cleanupResources.push(resource); + cleanupCauses.push(result.cause); + } + } + + if (cleanupCauses.length > 0) + throw new AdbInputSnapshotFailure( + operation, + sourcePath, + primaryCause, + cleanupResources, + new AggregateError(cleanupCauses, "ADB snapshot handles remain open"), + ); + if (primaryCause instanceof AnalysisError) throw primaryCause; + if (primaryCause !== undefined) + throw new AnalysisOutputError( + operation, + `The caller input could not be staged for adb: ${sourcePath}`, + { cause: primaryCause }, + ); + if (snapshot === undefined) + throw new Error("ADB input snapshot ended without a result"); + return snapshot; +}; diff --git a/src/adb/AdbProvider.test.ts b/src/adb/AdbProvider.test.ts index ec94bb9c7d..5485d8a192 100644 --- a/src/adb/AdbProvider.test.ts +++ b/src/adb/AdbProvider.test.ts @@ -9,10 +9,31 @@ import { writeFile, } from "node:fs/promises"; import { join } from "node:path"; -import { expect, onTestFinished, test } from "vitest"; +import { expect, onTestFinished, test, vi } from "vitest"; + +const adbTestHooks = vi.hoisted(() => ({ + afterScreencap: undefined as (() => void) | undefined, +})); + +vi.mock("../process/ExecFileOutput.js", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + execFileBytes: async ( + ...arguments_: Parameters + ) => { + const result = await actual.execFileBytes(...arguments_); + if (arguments_[1].includes("exec-out")) adbTestHooks.afterScreencap?.(); + return result; + }, + }; +}); import { AdbProvider } from "./AdbProvider.js"; +import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { parseEvidence } from "../domain/evidence.js"; import type { AdbRequest } from "../domain/adb/adbDeviceAnalysis.js"; const it = test.skipIf(process.platform === "win32"); @@ -880,21 +901,22 @@ exit 1`, }); }); -it("binds a push digest to the bytes adb consumes after caller-path replacement", async () => { +it("pushes the same private bytes that its reported digest identifies", async () => { const { binary, root } = await writeAdbStub( (stubRoot) => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi if [ "$1" = "-s" ] && [ "$4" = "test" ]; then exit 1; fi if [ "$1" = "-s" ] && [ "$3" = "push" ]; then - printf 'replacement-payload' > "${stubRoot}/payload.bin" - cat "$4" > "${stubRoot}/consumed.bin" || exit 1 + printf 'replacement' > "${join(stubRoot, "payload.bin")}" + cat "$4" > "${join(stubRoot, "consumed-push.bin")}" exit 0 fi if [ "$1" = "-s" ] && [ "$4" = "sha256sum" ]; then exit 1; fi exit 1`, ); const localFile = join(root, "payload.bin"); - await writeFile(localFile, "original-payload"); - const result = await evidenceResult(providerOver(binary), { + await writeFile(localFile, "original-push"); + const digest = createHash("sha256").update("original-push").digest("hex"); + const pushed = await evidenceResult(providerOver(binary), { operation: "push_adb_file", input: { serial: "emulator-5554", @@ -903,17 +925,62 @@ exit 1`, overwrite: false, }, }); - const consumedDigest = createHash("sha256") - .update(await readFile(join(root, "consumed.bin"))) - .digest("hex"); - expect(result).toMatchObject({ - local_path: localFile, - sha256: consumedDigest, - }); - expect(await readFile(localFile, "utf8")).toBe("replacement-payload"); - expect(await readFile(join(root, "consumed.bin"), "utf8")).toBe( - "original-payload", + expect(await readFile(join(root, "consumed-push.bin"), "utf8")).toBe( + "original-push", ); + expect(await readFile(localFile, "utf8")).toBe("replacement"); + expect(pushed).toMatchObject({ + local_path: localFile, + bytes: 13, + sha256: digest, + device_sha256: null, + device_digest_source: "unavailable", + }); +}); + +it("retains failed snapshot cleanup with completed push Evidence until close retries", async () => { + const { binary, root } = await writeAdbStub( + () => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi +if [ "$1" = "-s" ] && [ "$4" = "test" ]; then exit 1; fi +if [ "$1" = "-s" ] && [ "$3" = "push" ]; then exit 0; fi +if [ "$1" = "-s" ] && [ "$4" = "sha256sum" ]; then exit 1; fi +exit 1`, + ); + const localFile = join(root, "cleanup-payload.bin"); + await writeFile(localFile, "payload"); + let closeAttempts = 0; + const provider = new AdbProvider({ + environment: { REA_ADB_PATH: binary }, + createRuntimeRoot: async (options) => { + const runtime = await PrivateRuntimeRoot.create(options); + return { + path: runtime.path, + close: async () => { + closeAttempts += 1; + if (closeAttempts === 1) throw new Error("controlled cleanup fault"); + await runtime.close(); + }, + }; + }, + }); + const result = await provider.execute({ + operation: "push_adb_file", + input: { + serial: "emulator-5554", + local_path: localFile, + device_path: "/data/local/tmp/cleanup-payload.bin", + overwrite: false, + }, + }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected snapshot cleanup to be reported"); + expect(result.error.cleanupResources).toHaveLength(1); + expect(result.error.partialObservation).toMatchObject({ + operation: "push_adb_file", + normalized_result: { bytes: 7 }, + }); + await provider.close(); + expect(closeAttempts).toBe(2); }); it("reports a digest mismatch after push as a changed artifact", async () => { @@ -926,7 +993,7 @@ exit 1`, ); const localFile = join(root, "payload.bin"); await writeFile(localFile, "push-payload"); - const projected = await refusedBy(providerOver(binary), { + const result = await providerOver(binary).execute({ operation: "push_adb_file", input: { serial: "emulator-5554", @@ -935,6 +1002,20 @@ exit 1`, overwrite: false, }, }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected a changed-artifact result"); + const evidence = parseEvidence(result.error.partialObservation); + expect(evidence).toMatchObject({ + operation: "push_adb_file", + normalized_result: { + local_path: localFile, + device_path: "/data/local/tmp/payload.bin", + bytes: Buffer.byteLength("push-payload"), + sha256: createHash("sha256").update("push-payload").digest("hex"), + device_digest_source: "device_sha256sum", + }, + }); + const projected = projectAnalysisError(result.error); expect(projected).toMatchObject({ code: "artifact_changed" }); }); @@ -1033,6 +1114,37 @@ exit 1`, ).toBe(true); }); +it("stops a screen capture aborted after adb returns but before publication", async () => { + const { binary, root } = await writeAdbStub( + () => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi +if [ "$1" = "-s" ] && [ "$3" = "exec-out" ]; then + printf '\\211PNG\\r\\n\\032\\n\\0\\0\\0\\rIHDR\\0\\0\\0\\004\\0\\0\\0\\003' + exit 0 +fi +exit 1`, + ); + const output = join(root, "cancelled-capture"); + const controller = new AbortController(); + adbTestHooks.afterScreencap = () => controller.abort(); + try { + const result = await providerOver(binary).execute( + { + operation: "capture_adb_screen", + input: { serial: "emulator-5554", output_directory: output }, + }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (!result.ok) + expect(projectAnalysisError(result.error).code).toBe("cancelled"); + await expect(readFile(join(output, "screen.png"))).rejects.toMatchObject({ + code: "ENOENT", + }); + } finally { + adbTestHooks.afterScreencap = undefined; + } +}); + it("lists processes, features, and services through fixed commands", async () => { const { binary } = await writeAdbStub( shellStub({ @@ -1315,12 +1427,13 @@ exit 1`, expect(projected.message).toContain("INSTALL_FAILED_ALREADY_EXISTS"); }); -it("binds an install digest to the bytes adb consumes after caller-path replacement", async () => { +it("installs the same private APK bytes that its reported digest identifies", async () => { const { binary, root } = await writeAdbStub( (stubRoot) => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi if [ "$1" = "-s" ] && [ "$3" = "install" ]; then - printf 'replacement-apk' > "${stubRoot}/app.apk" - cat "$4" > "${stubRoot}/consumed.bin" || exit 1 + case "$4" in *.apk) ;; *) printf 'install input needs an APK suffix\\n' >&2; exit 1 ;; esac + printf 'replacement-apk' > "${join(stubRoot, "app.apk")}" + cat "$4" > "${join(stubRoot, "consumed-app.apk")}" printf 'Success\\n' exit 0 fi @@ -1328,18 +1441,20 @@ exit 1`, ); const apk = join(root, "app.apk"); await writeFile(apk, "original-apk"); - const result = await evidenceResult(providerOver(binary), { + const digest = createHash("sha256").update("original-apk").digest("hex"); + const installed = await evidenceResult(providerOver(binary), { operation: "install_adb_package", input: { serial: "emulator-5554", apk_path: apk, replace: false }, }); - const consumedDigest = createHash("sha256") - .update(await readFile(join(root, "consumed.bin"))) - .digest("hex"); - expect(result).toMatchObject({ apk_path: apk, sha256: consumedDigest }); - expect(await readFile(apk, "utf8")).toBe("replacement-apk"); - expect(await readFile(join(root, "consumed.bin"), "utf8")).toBe( + expect(await readFile(join(root, "consumed-app.apk"), "utf8")).toBe( "original-apk", ); + expect(await readFile(apk, "utf8")).toBe("replacement-apk"); + expect(installed).toMatchObject({ + apk_path: apk, + bytes: 12, + sha256: digest, + }); }); it("uninstalls a package and reports the device's refusal for absent ones", async () => { diff --git a/src/adb/AdbProvider.ts b/src/adb/AdbProvider.ts index 399f271a51..ff705ad3a3 100644 --- a/src/adb/AdbProvider.ts +++ b/src/adb/AdbProvider.ts @@ -4,8 +4,10 @@ import { access, copyFile, lstat, + link, mkdtemp, mkdir, + open, rmdir, rm, stat, @@ -25,12 +27,22 @@ import { import type { AdbActivityStartPartialObservation, AdbAcquiredFilePartialObservation, + AdbPackagePullPartialObservation, AdbRequest, } from "../domain/adb/adbDeviceAnalysis.js"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import { + ArtifactResourceRetryFailure, + ArtifactResourceScope, +} from "../artifacts/ArtifactResourceScope.js"; +import { + AnalysisCleanupLease, + AnalysisCleanupLeaseGroup, +} from "../artifacts/AnalysisCleanupLease.js"; import { adbResultSchemas } from "../domain/adb/adbDeviceAnalysis.js"; import { - AnalysisArtifactChangedError, AnalysisCapabilityUnavailableError, + AnalysisArtifactChangedError, AnalysisCancelledError, AnalysisInputError, AnalysisOutputError, @@ -39,20 +51,20 @@ import { AnalysisTimeoutError, AnalysisUnsupportedTargetError, } from "../domain/analysisErrorCore.js"; -import { createEvidence } from "../domain/evidence.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; -import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { err, ok, type Result } from "../domain/result.js"; +import type { JsonValue } from "../domain/jsonValue.js"; +import { jsonObjectSchema } from "../domain/jsonValue.js"; +import { hashStableFile } from "../filesystem/StableFileHash.js"; +import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { createEvidence } from "../domain/evidence.js"; import { - hashStableFile, - StableFileHashCleanupFailure, - type StableFileHashResult, -} from "../filesystem/StableFileHash.js"; -import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; -import { OwnedRegularFileCleanupFailure } from "../filesystem/RegularFile.js"; + PrivateRuntimeRoot, + type PrivateRuntimeRootOptions, +} from "../process/PrivateRuntimeRoot.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; -import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; import { PrivateRuntimeRootUnavailableError } from "../process/PrivateRuntimeRootUnavailableError.js"; import { execFileBytes, @@ -74,6 +86,10 @@ import { parsePmPathOutput, } from "./AdbDeviceOutput.js"; import { adbHostFileName } from "./AdbHostFileName.js"; +import { + AdbInputSnapshotFailure, + snapshotAdbInput, +} from "./AdbInputSnapshot.js"; import { ADB_DIRECTORY_BATCH_SCRIPT, parseAdbDirectoryListing, @@ -128,6 +144,207 @@ const BUGREPORT_MAX_OUTPUT_BYTES = 1024 * 1024; const STDERR_EXCERPT_BYTES = 512; const UNPARSED_LINES_REPORTED = 10; +/** Carry the exact digest owner through caller-specific partial-result mapping. */ +class AdbDigestCleanupFailure extends Error { + constructor( + readonly fileFailure: RegularFileCleanupFailure, + readonly bindCleanupError: (error: AnalysisError) => void, + ) { + super(fileFailure.message, { cause: fileFailure }); + } +} + +const hashAdbFile = async ( + path: string, + operation: string, + signal: AbortSignal | undefined, + resources: ArtifactResourceScope, +) => { + let closeLease: AnalysisCleanupLease | undefined; + try { + return await hashStableFile(path, { + ...(signal === undefined ? {} : { signal }), + retainFailedClose: (handle) => { + closeLease = new AnalysisCleanupLease(() => handle.close()); + resources.retain({ + kind: "lease", + resource: `${path} (ADB digest handle)`, + lease: closeLease, + }); + }, + }); + } catch (cause: unknown) { + if (cause instanceof RegularFileCleanupFailure) + throw new AdbDigestCleanupFailure( + cause as RegularFileCleanupFailure, + (error) => { + closeLease?.bindCleanupError(error); + }, + ); + if (signal?.aborted === true) + throw new AnalysisCancelledError(operation, { cause }); + throw cause; + } +}; + +const adbDigestCleanupError = ( + cause: unknown, + operation: string, + signal: AbortSignal | undefined, + partialObservation: + | AdbAcquiredFilePartialObservation + | AdbPackagePullPartialObservation, +): AnalysisError | undefined => { + if (!(cause instanceof AdbDigestCleanupFailure)) return undefined; + const digestFailure = cause.fileFailure; + const digestOutcome = digestFailure.outcome; + if (digestOutcome.kind === "completed") + partialObservation = partialObservationWithDigest( + partialObservation, + digestOutcome.value, + ); + const resource = `${digestFailure.path} (ADB digest handle)`; + const reason = + digestFailure.cleanupCause instanceof Error + ? digestFailure.cleanupCause.message + : String(digestFailure.cleanupCause); + const cleanup = new ProviderCleanupError( + ADB_PROVIDER_IDENTITY.id, + [resource], + { reason }, + { + operation, + cause: digestFailure, + partialObservation, + }, + ); + const primaryCause = + digestOutcome.kind === "failed" ? digestOutcome.cause : undefined; + if ( + signal?.aborted === true || + primaryCause instanceof AnalysisCancelledError + ) { + const error = new AnalysisCancelledError(operation, { + cause: digestFailure, + cleanup: { reason, resources: [resource] }, + partialObservation, + }); + cause.bindCleanupError(error); + return error; + } + + const primary = + primaryCause instanceof AnalysisError + ? primaryCause + : new AnalysisOutputError( + operation, + `The pulled file could not be fully digested or its handle could not be closed: ${digestFailure.path}`, + { + cause: primaryCause ?? digestFailure, + partialObservation, + }, + ); + primary.retainPartialObservation(partialObservation); + const error = analysisErrorWithCleanupFailure(primary, cleanup, operation); + cause.bindCleanupError(error); + return error; +}; + +type AdbFileDigest = Awaited>; + +const partialObservationWithDigest = ( + observation: + | AdbAcquiredFilePartialObservation + | AdbPackagePullPartialObservation, + digest: AdbFileDigest, +): AdbAcquiredFilePartialObservation | AdbPackagePullPartialObservation => { + if (observation.operation === "pull_adb_file") { + const { result } = observation; + if (!("digest_status" in result)) return observation; + return { + ...observation, + result: { + client: result.client, + serial: result.serial, + local_path: result.local_path, + device_path: result.device_path, + bytes: digest.bytes, + sha256: digest.sha256, + }, + }; + } + if (observation.operation === "collect_adb_bugreport") { + const { result } = observation; + if (!("digest_status" in result)) return observation; + return { + ...observation, + result: { + client: result.client, + serial: result.serial, + local_path: result.local_path, + adb_reported_path: result.adb_reported_path, + bytes: digest.bytes, + sha256: digest.sha256, + }, + }; + } + return observation; +}; + +const syncStagedAdbFile = async ( + path: string, + resources: ArtifactResourceScope, + operation: string, +): Promise => { + let owner: OwnedFileHandle | undefined; + let primary: unknown; + try { + owner = new OwnedFileHandle(await open(path, "r+")); + await owner.handle.sync(); + } catch (cause: unknown) { + primary = cause; + } + if (owner === undefined) { + if (primary !== undefined) throw primary; + return; + } + const resource = `${path} (ADB staged-output handle)`; + const stagedOwner = owner; + const closeLease = new AnalysisCleanupLease(() => stagedOwner.close()); + const released = await resources.release({ + kind: "lease", + resource, + lease: closeLease, + }); + if (released.kind === "failed") { + const cleanup = new ProviderCleanupError( + ADB_PROVIDER_IDENTITY.id, + [resource], + { + reason: + released.cause instanceof Error + ? released.cause.message + : String(released.cause), + }, + { operation, cause: released.cause }, + ); + let reported: AnalysisError; + if (primary !== undefined) { + const failure = new AnalysisOutputError( + operation, + `The staged ADB output could not be synchronized: ${path}`, + { cause: primary }, + ); + reported = analysisErrorWithCleanupFailure(failure, cleanup, operation); + } else { + reported = cleanup; + } + closeLease.bindCleanupError(reported); + throw reported; + } + if (primary !== undefined) throw primary; +}; + /** adb joins shell operands into a remote POSIX command; quote at that boundary. */ const quoteShellOperand = (value: string): string => /^[A-Za-z0-9_./:=@+,-]+$/u.test(value) @@ -160,7 +377,7 @@ const runAdbCapture = async ( signal: AbortSignal | undefined, ): Promise> => { if (signal?.aborted === true) - return err(new AnalysisCancelledError(operation)); + return err(new AnalysisCancelledError(operation, { cause: signal.reason })); try { return ok( await execFileBytes(binary, arguments_, { @@ -341,6 +558,202 @@ const refuseExistingTarget = async ( return undefined; }; +const hardLinksUnsupported = (cause: unknown): boolean => + cause instanceof Error && + "code" in cause && + ["EPERM", "EOPNOTSUPP", "ENOTSUP", "ENOSYS", "EXDEV"].includes( + String(cause.code), + ); + +const describeAdbFailure = (cause: unknown): string => + cause instanceof Error ? cause.message : String(cause); + +const cleanupLeasesForRetryFailure = ( + cause: unknown, +): readonly AnalysisCleanupLease[] | undefined => { + if (!(cause instanceof ArtifactResourceRetryFailure)) return undefined; + const leases = new Set(); + for (const owner of cause.failedOwners) { + if ( + owner.kind !== "lease" || + !(owner.lease instanceof AnalysisCleanupLease) + ) + return undefined; + leases.add(owner.lease); + } + return leases.size > 0 ? [...leases] : undefined; +}; + +interface ScreenPublicationCloseFailure { + readonly cause: unknown; + readonly resource: string; + bindCleanupError(error: ProviderCleanupError): void; +} + +/** Exclusive-create fallback for filesystems that cannot hard-link staged files. */ +const publishScreenExclusively = async ( + outputPath: string, + bytes: Buffer, + resources: ArtifactResourceScope, + operation: string, + signal: AbortSignal | undefined, +): Promise => { + if (signal?.aborted === true) + throw new AnalysisCancelledError(operation, { cause: signal.reason }); + const output = await open(outputPath, "wx", 0o600); + // Successful exclusive creation is this fallback's cutoff. Finish and sync + // the visible file despite later cancellation so its receipt remains known. + const ownedOutput = new OwnedFileHandle(output); + const handleResource = `${outputPath} (ADB publication handle)`; + const closeOnly = new AnalysisCleanupLease(() => ownedOutput.close()); + const removeOutput = async ( + expected: Awaited>, + ): Promise => { + let current: Awaited>; + try { + current = await lstat(outputPath); + } catch (cause: unknown) { + if (cause instanceof Error && Reflect.get(cause, "code") === "ENOENT") + return; + throw cause; + } + if ( + current.dev !== expected.dev || + current.ino !== expected.ino || + current.mode !== expected.mode + ) + throw new Error( + `The screen output changed before cleanup: ${outputPath}`, + ); + await rm(outputPath); + }; + let rollbackIdentity: Awaited> | undefined; + const inspectCloseAndRollback = new AnalysisCleanupLease(async () => { + rollbackIdentity ??= await output.stat(); + await ownedOutput.close(); + await removeOutput(rollbackIdentity); + }); + let identity: Awaited>; + try { + identity = await output.stat(); + } catch (cause: unknown) { + resources.retain({ + kind: "lease", + resource: `${outputPath} (screen output and descriptor)`, + lease: inspectCloseAndRollback, + }); + const error = new ProviderCleanupError( + ADB_PROVIDER_IDENTITY.id, + [handleResource, outputPath], + { + reason: + "The newly created screen output identity could not be established; its descriptor remains owned for cleanup retry", + }, + { operation, cause }, + ); + inspectCloseAndRollback.bindCleanupError(error); + throw error; + } + let writeFailure: { readonly cause: unknown } | undefined; + try { + await output.writeFile(bytes); + await output.sync(); + } catch (cause: unknown) { + writeFailure = { cause }; + } + if (writeFailure === undefined) { + try { + await ownedOutput.close(); + return undefined; + } catch (cause: unknown) { + resources.retain({ + kind: "lease", + resource: handleResource, + lease: closeOnly, + }); + return { + cause, + resource: handleResource, + bindCleanupError(error) { + closeOnly.bindCleanupError(error); + }, + }; + } + } + + const rollback = new AnalysisCleanupLease(async () => { + await ownedOutput.close(); + await removeOutput(identity); + }); + try { + await ownedOutput.close(); + } catch (closeCause: unknown) { + resources.retain({ kind: "lease", resource: outputPath, lease: rollback }); + const error = new ProviderCleanupError( + ADB_PROVIDER_IDENTITY.id, + [outputPath], + { reason: describeAdbFailure(closeCause) }, + { + operation, + cause: new AggregateError( + [writeFailure.cause, closeCause], + "Screen output write and descriptor cleanup failed", + { cause: writeFailure.cause }, + ), + }, + ); + rollback.bindCleanupError(error); + throw error; + } + const removed = await resources.release({ + kind: "lease", + resource: outputPath, + lease: rollback, + }); + if (removed.kind === "failed") { + const error = new ProviderCleanupError( + ADB_PROVIDER_IDENTITY.id, + [outputPath], + { reason: describeAdbFailure(removed.cause) }, + { + operation, + cause: new AggregateError( + [writeFailure.cause, removed.cause], + "Screen output write and rollback failed", + { cause: writeFailure.cause }, + ), + }, + ); + rollback.bindCleanupError(error); + throw error; + } + throw writeFailure.cause; +}; + +const cancellationWithCleanupFailure = ( + primary: AnalysisCancelledError, + cleanup: AnalysisError, + resources: readonly string[], +): AnalysisCancelledError => + new AnalysisCancelledError(primary.operation, { + cause: new AggregateError( + [primary, cleanup], + "ADB cancellation cleanup failed", + { cause: primary }, + ), + diagnostics: primary.diagnostics, + ...(primary.capturedOutput === undefined + ? {} + : { capturedOutput: primary.capturedOutput }), + cleanup: { + reason: [primary.cleanup?.reason, cleanup.message] + .filter((reason): reason is string => reason !== undefined) + .join("; "), + resources: [...new Set([...primary.cleanupResources, ...resources])], + }, + partialObservation: primary.partialObservation, + }); + /** Publish an ADB-produced archive without replacing an existing caller file. */ const publishBugreport = async ( source: string, @@ -368,41 +781,45 @@ const publishBugreport = async ( export interface AdbProviderOptions { readonly environment?: Readonly>; + /** Override runtime-root creation at the provider's owned-resource boundary. */ + readonly createRuntimeRoot?: ( + options?: PrivateRuntimeRootOptions, + ) => Promise; } -interface PendingAdbDirectoryCleanup { - readonly operation: - | "pull_adb_package" - | "collect_adb_bugreport" - | "push_adb_file" - | "install_adb_package"; +type AdbRuntimeRoot = Pick; + +/** A root lease releases its children before removing their containing root. */ +const runtimeRootLease = ( + root: AdbRuntimeRoot, + children: ArtifactResourceScope, +): AnalysisCleanupLease => + new AnalysisCleanupLease(async () => { + await children.close(); + await root.close(); + }); + +interface PendingOutputDirectoryCleanup { + readonly operation: "pull_adb_package" | "collect_adb_bugreport"; readonly path: string; readonly primary: AnalysisError; - readonly reportedError: AnalysisError; - readonly prerequisite?: AnalysisError; readonly remove: () => Promise; + readonly lease: AnalysisCleanupLease; } -interface PendingAdbFileCleanup { - readonly operation: string; - readonly path: string; - readonly owner: OwnedFileHandle; - readonly reportedError: AnalysisError; -} - -interface OwnedDirectoryIdentity { +interface OutputDirectoryIdentity { readonly dev: number; readonly ino: number; readonly mode: number; } -const verifyOwnedDirectory = async ( +const removeOwnedOutputDirectory = async ( path: string, - identity: OwnedDirectoryIdentity | undefined, + identity: OutputDirectoryIdentity | undefined, ): Promise => { if (identity === undefined) throw new Error( - `Cannot verify the created directory identity; refusing to remove ${path}`, + `Cannot verify the created output directory identity; refusing to remove ${path}`, ); let current: Awaited>; try { @@ -420,32 +837,11 @@ const verifyOwnedDirectory = async ( current.mode !== identity.mode ) throw new Error( - `The created directory was replaced or changed; refusing to remove ${path}`, + `The created output directory was replaced or changed; refusing to remove ${path}`, ); -}; - -const removeOwnedDirectory = async ( - path: string, - identity: OwnedDirectoryIdentity | undefined, -): Promise => { - await verifyOwnedDirectory(path, identity); await rm(path, { recursive: true, force: true }); }; -const executionEvidence = ( - execution: AnalysisExecution, - operation: "push_adb_file" | "install_adb_package" | "capture_adb_screen", - parameters: Readonly>, -) => - createEvidence(execution.subject ?? undefined, execution.provider, { - operation, - parameters, - result: execution.result, - confidence: "observed", - locations: execution.locations, - limitations: execution.limitations, - }); - const reservePackageDestinations = async ( operation: string, packageName: string, @@ -464,7 +860,7 @@ const reservePackageDestinations = async ( readonly path: string; readonly file_name: string; readonly device_path: string; - readonly identity: OwnedDirectoryIdentity; + readonly identity: OutputDirectoryIdentity; }[] = []; for (const entry of entries) { const path = join(packageDirectory, entry.file_name); @@ -563,78 +959,32 @@ const reservePackageDestinations = async ( */ export class AdbProvider { private readonly environment: Readonly>; - readonly #pendingDirectoryCleanup = new Set(); - readonly #pendingFileCleanup = new Set(); + readonly #createRuntimeRoot: ( + options?: PrivateRuntimeRootOptions, + ) => Promise; + readonly #resources = new ArtifactResourceScope(); + readonly #outputDirectoryRetryLeases = new WeakMap< + AnalysisError, + readonly AnalysisCleanupLease[] + >(); + readonly #pendingOutputDirectoryCleanup = + new Set(); readonly #activeOperations = new Set>(); #closing = false; #closePromise: Promise | undefined; - #directoryCleanupPromise: Promise | undefined; + #outputCleanupPromise: Promise | undefined; constructor(options: AdbProviderOptions = {}) { this.environment = options.environment ?? process.env; + this.#createRuntimeRoot = + options.createRuntimeRoot ?? + ((rootOptions) => PrivateRuntimeRoot.create(rootOptions)); } private selection(): AdbBinarySelection { return resolveAdbBinary(this.environment); } - private async hashAdbFile( - path: string, - operation: string, - signal: AbortSignal | undefined, - ) { - try { - return await hashStableFile(path, signal); - } catch (cause: unknown) { - if (cause instanceof OwnedRegularFileCleanupFailure) { - const primary = signal?.aborted - ? new AnalysisCancelledError(operation, { cause: cause.cause }) - : new ProviderAdapterError(ADB_PROVIDER_IDENTITY.id, operation, { - cause: cause.cause, - userMessage: - cause.cause instanceof Error - ? cause.cause.message - : String(cause.cause), - diagnostics: - cause instanceof StableFileHashCleanupFailure && - cause.completedHash !== undefined - ? { completed_hash: cause.completedHash } - : {}, - }); - const cleanup = new ProviderCleanupError( - ADB_PROVIDER_IDENTITY.id, - [path], - { - reason: - cause.cleanupCause instanceof Error - ? cause.cleanupCause.message - : String(cause.cleanupCause), - ...(cause instanceof StableFileHashCleanupFailure && - cause.completedHash !== undefined - ? { completed_hash: cause.completedHash } - : {}), - }, - { operation, cause: cause.cleanupCause }, - ); - const reported = analysisErrorWithCleanupFailure( - primary, - cleanup, - operation, - ); - this.#pendingFileCleanup.add({ - operation, - path, - owner: cause.owner, - reportedError: reported, - }); - throw reported; - } - if (signal?.aborted === true) - throw new AnalysisCancelledError(operation, { cause }); - throw cause; - } - } - /** Probe the selected adb binary; never contacts devices or the server. */ async inspectAvailability( signal?: AbortSignal, @@ -658,13 +1008,59 @@ export class AdbProvider { } } - /** Retry file and directory resources retained after failed cleanup. */ + /** Retry removal of owned output directories retained after failed cleanup. */ close(): Promise { this.#closing = true; if (this.#closePromise !== undefined) return this.#closePromise; const closing = (async () => { await Promise.all(this.#activeOperations); - await this.#retryDirectoryCleanup(); + const failures: unknown[] = []; + const failedLeases = new Set(); + let allFailuresHaveExactLeaseOwners = true; + // Directory errors may also depend on a retained digest descriptor. + // Settle those owners before deciding whether the directory error resolved. + for (const cleanup of [ + () => this.#resources.close(), + () => this.#retryOutputDirectoryCleanup(), + ]) { + try { + await cleanup(); + } catch (cause: unknown) { + failures.push(cause); + const leases = + (cause instanceof AnalysisError + ? this.#outputDirectoryRetryLeases.get(cause) + : undefined) ?? cleanupLeasesForRetryFailure(cause); + if (leases === undefined) allFailuresHaveExactLeaseOwners = false; + for (const lease of leases ?? []) failedLeases.add(lease); + } + } + if (failures.length > 0) { + const resources = failures.flatMap((cause) => + this.#cleanupResources(cause, "ADB provider-owned resource"), + ); + const error = new ProviderCleanupError( + ADB_PROVIDER_IDENTITY.id, + [...new Set(resources)], + { + reason: failures + .map((cause) => + cause instanceof Error ? cause.message : String(cause), + ) + .join("; "), + }, + { + operation: "close_binary", + cause: new AggregateError( + failures, + "ADB cleanup remains incomplete", + ), + }, + ); + if (allFailuresHaveExactLeaseOwners && failedLeases.size > 0) + new AnalysisCleanupLeaseGroup(error, [...failedLeases]); + throw error; + } })().finally(() => { if (this.#closePromise === closing) this.#closePromise = undefined; }); @@ -672,133 +1068,66 @@ export class AdbProvider { return this.#closePromise; } - #retryDirectoryCleanup(): Promise { - if (this.#directoryCleanupPromise !== undefined) - return this.#directoryCleanupPromise; - const cleanup = this.#performResourceCleanup().finally(() => { - if (this.#directoryCleanupPromise === cleanup) - this.#directoryCleanupPromise = undefined; - }); - this.#directoryCleanupPromise = cleanup; - return cleanup; + #cleanupResources(cause: unknown, resource: string): readonly string[] { + if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined) + return cause.cleanup.resources; + if (cause instanceof AnalysisError && cause.cleanupResources.length > 0) + return cause.cleanupResources; + return [resource]; } - async #performResourceCleanup(): Promise { - const failures: AnalysisError[] = []; - for (const owner of this.#pendingFileCleanup) { - try { - await owner.owner.close(); - this.#pendingFileCleanup.delete(owner); - owner.reportedError.resolveCleanup(); - } catch (cause: unknown) { - failures.push( - new ProviderCleanupError( - ADB_PROVIDER_IDENTITY.id, - [owner.path], - { reason: cause instanceof Error ? cause.message : String(cause) }, - { operation: owner.operation, cause }, - ), - ); - } - } - for (const owner of this.#pendingDirectoryCleanup) { - if (owner.prerequisite?.cleanupIncomplete === true) { - failures.push( - new ProviderCleanupError( - ADB_PROVIDER_IDENTITY.id, - [owner.path], - { reason: "A dependent owned resource still needs cleanup" }, - { operation: owner.operation, cause: owner.prerequisite }, - ), - ); - continue; - } - try { - await owner.remove(); - this.#pendingDirectoryCleanup.delete(owner); - owner.reportedError.resolveCleanup(); - } catch (cause: unknown) { - const cleanup = new ProviderCleanupError( - ADB_PROVIDER_IDENTITY.id, - [owner.path], - { - reason: cause instanceof Error ? cause.message : String(cause), - }, - { operation: owner.operation, cause }, - ); - failures.push( - analysisErrorWithCleanupFailure( - owner.primary, - cleanup, - owner.operation, - ), - ); - } - } - if (failures.length === 1) throw failures[0]; - if (failures.length > 1) - throw new ProviderCleanupError( - ADB_PROVIDER_IDENTITY.id, - [ - ...[...this.#pendingFileCleanup].map(({ path }) => path), - ...[...this.#pendingDirectoryCleanup].map(({ path }) => path), - ], - { - reason: "Failed ADB resources remain after cleanup retry", - operations: [ - ...[...this.#pendingFileCleanup].map(({ operation }) => operation), - ...[...this.#pendingDirectoryCleanup].map( - ({ operation }) => operation, - ), - ], - failures: failures.map(({ message, cleanupResources }) => ({ - message, - resources: [...cleanupResources], - })), - }, - { - operation: "close_binary", - cause: new AggregateError( - failures, - "Multiple ADB directory cleanups failed", - ), - }, - ); + #cleanupError( + operation: string, + parameters: Record, + causes: readonly unknown[], + resources: readonly string[], + execution?: AnalysisExecution, + ): ProviderCleanupError { + const partialObservation = + execution === undefined + ? undefined + : createEvidence(execution.subject ?? undefined, execution.provider, { + operation, + parameters, + result: execution.result, + rawResult: execution.rawResult, + limitations: execution.limitations, + locations: execution.locations, + confidence: "observed", + }); + return new ProviderCleanupError( + ADB_PROVIDER_IDENTITY.id, + resources, + { + reason: causes + .map((cause) => + cause instanceof Error ? cause.message : String(cause), + ) + .join("; "), + }, + { + operation, + cause: new AggregateError(causes, "ADB operation cleanup failed"), + ...(partialObservation === undefined ? {} : { partialObservation }), + }, + ); } - private async withInputSnapshot( - operation: "push_adb_file" | "install_adb_package", - sourcePath: string, - parameters: Readonly>, + async #withRuntimeRoot( + operation: string, + parameters: Record, + options: PrivateRuntimeRootOptions, signal: AbortSignal | undefined, - consume: ( - path: string, - digest: StableFileHashResult, + work: ( + root: AdbRuntimeRoot, + children: ArtifactResourceScope, ) => Promise>, ): Promise> { - let root: PrivateRuntimeRoot | undefined; - let identity: OwnedDirectoryIdentity | undefined; - let result: Result; + let root: AdbRuntimeRoot; try { - signal?.throwIfAborted(); - root = await PrivateRuntimeRoot.create({ prefix: "rea-adb-input-" }); - const created = await lstat(root.path); - if (!created.isDirectory() || created.isSymbolicLink()) - throw new Error( - `The created input snapshot root is not a directory: ${root.path}`, - ); - identity = { - dev: created.dev, - ino: created.ino, - mode: created.mode, - }; - const snapshotPath = join(root.path, basename(sourcePath)); - await copyFile(sourcePath, snapshotPath, fsConstants.COPYFILE_EXCL); - const digest = await this.hashAdbFile(snapshotPath, operation, signal); - signal?.throwIfAborted(); - result = await consume(snapshotPath, digest); + root = await this.#createRuntimeRoot(options); } catch (cause: unknown) { - result = err( + return err( cause instanceof AnalysisError ? cause : cause instanceof PrivateRuntimeRootUnavailableError @@ -808,96 +1137,222 @@ export class AdbProvider { cause.message, { cause, retryable: cause.retryable }, ) - : signal?.aborted - ? new AnalysisCancelledError(operation, { cause }) - : new AnalysisOutputError( - operation, - `The caller input could not be staged for adb: ${sourcePath}`, - { cause }, - ), + : new AnalysisOutputError( + operation, + "Could not create an ADB private workspace", + { + cause, + }, + ), ); } - if (root !== undefined) { - const prerequisite = - !result.ok && result.error.cleanupIncomplete ? result.error : undefined; - const remove = async () => { - if (prerequisite?.cleanupIncomplete === true) - throw new Error( - "A staged file still has an unresolved cleanup owner", - ); - await verifyOwnedDirectory(root!.path, identity); - await root!.close(); - }; - try { - await remove(); - } catch (cause: unknown) { - const cleanupReason = - cause instanceof Error ? cause.message : String(cause); - const cleanup = new ProviderCleanupError( - ADB_PROVIDER_IDENTITY.id, - [root.path], - { reason: cleanupReason }, - { operation, cause }, + let result: Result | undefined; + let primary: AnalysisError | undefined; + const children = new ArtifactResourceScope(); + try { + result = await work(root, children); + } catch (cause: unknown) { + if (cause instanceof AdbInputSnapshotFailure) { + primary = + cause.primaryCause instanceof AnalysisError + ? cause.primaryCause + : new AnalysisOutputError( + operation, + `The caller input could not be staged for adb: ${cause.sourcePath}`, + { cause: cause.primaryCause ?? cause }, + ); + // Snapshot handle cleanup may recover when the runtime-root lease is + // drained below. Keep that historical failure linked to the typed + // primary error while the lease owns and clears live cleanup state. + primary.retainSecondaryFailure( + new AnalysisOutputError( + operation, + `ADB input snapshot cleanup previously failed for ${cause.sourcePath}`, + { cause }, + ), + {}, ); - const partialObservation = result.ok - ? executionEvidence(result.value, operation, parameters) - : result.error.partialObservation; - const primary = - result.ok && signal?.aborted === true - ? new AnalysisCancelledError(operation, { - cause, - partialObservation, - cleanup: { - reason: cleanupReason, - resources: [root.path], - }, - }) - : result.ok - ? new AnalysisOutputError( - operation, - "The adb operation completed, but its private input snapshot could not be removed", - { cause, partialObservation }, - ) - : result.error instanceof AnalysisCancelledError - ? new AnalysisCancelledError(operation, { - cause: result.error, - partialObservation, - cleanup: { - reason: cleanupReason, - resources: [root.path], - }, - }) - : result.error; - const cleanupOwner: PendingAdbDirectoryCleanup = { - operation, - path: root.path, - primary, - reportedError: - primary instanceof AnalysisCancelledError - ? primary - : analysisErrorWithCleanupFailure(primary, cleanup, operation), - ...(prerequisite === undefined ? {} : { prerequisite }), - remove, - }; - this.#pendingDirectoryCleanup.add(cleanupOwner); - return err(cleanupOwner.reportedError); + } else { + primary = + cause instanceof AnalysisError + ? cause + : new AnalysisProtocolError(`ADB ${operation} failed`, { cause }); } } - if (result.ok && signal?.aborted === true) - return err( - new AnalysisCancelledError(operation, { - partialObservation: executionEvidence( - result.value, + const rootLease = runtimeRootLease(root, children); + const released = await this.#resources.release({ + kind: "lease", + resource: root.path, + lease: rootLease, + }); + if (primary === undefined && result?.ok && signal?.aborted === true) { + const completed = result.value; + primary = new AnalysisCancelledError(operation, { + cause: signal.reason, + partialObservation: createEvidence( + completed.subject ?? undefined, + completed.provider, + { operation, parameters, - ), - }), + result: completed.result, + rawResult: completed.rawResult, + limitations: completed.limitations, + locations: completed.locations, + confidence: "observed", + }, + ), + }); + } + if (released.kind === "failed") { + const cleanupResources = [ + root.path, + ...(released.cause instanceof ArtifactReaderFailure && + released.cause.cleanup !== undefined + ? released.cause.cleanup.resources + : []), + ]; + const cleanup = this.#cleanupError( + operation, + parameters, + [released.cause], + cleanupResources, + result?.ok ? result.value : undefined, + ); + const returnCleanupError = (error: AnalysisError) => { + rootLease.bindCleanupError(error); + return err(error); + }; + if (primary instanceof AnalysisCancelledError) { + return returnCleanupError( + cancellationWithCleanupFailure(primary, cleanup, cleanupResources), + ); + } + if (primary !== undefined) + return returnCleanupError( + analysisErrorWithCleanupFailure(primary, cleanup, operation), + ); + if (result !== undefined && !result.ok) { + if (result.error instanceof AnalysisCancelledError) { + return returnCleanupError( + cancellationWithCleanupFailure( + result.error, + cleanup, + cleanupResources, + ), + ); + } + return returnCleanupError( + analysisErrorWithCleanupFailure(result.error, cleanup, operation), + ); + } + return returnCleanupError(cleanup); + } + if (primary !== undefined) return err(primary); + if (result === undefined) + return err( + new AnalysisProtocolError(`ADB ${operation} returned no result`), ); return result; } + #retainOutputDirectoryCleanup( + owner: Omit, + cleanup: ProviderCleanupError, + ): AnalysisError { + const lease = new AnalysisCleanupLease(async () => { + await owner.remove(); + if (owner.primary.cleanupIncomplete) + throw new Error( + "ADB output directory primary cleanup remains incomplete", + ); + }); + const error = analysisErrorWithCleanupFailure( + owner.primary, + cleanup, + owner.operation, + ); + lease.bindCleanupError(error); + this.#pendingOutputDirectoryCleanup.add({ ...owner, lease }); + return error; + } + + #retryOutputDirectoryCleanup(): Promise { + if (this.#outputCleanupPromise !== undefined) + return this.#outputCleanupPromise; + const cleanup = this.#performOutputDirectoryCleanup().finally(() => { + if (this.#outputCleanupPromise === cleanup) + this.#outputCleanupPromise = undefined; + }); + this.#outputCleanupPromise = cleanup; + return cleanup; + } + + async #performOutputDirectoryCleanup(): Promise { + const failures: AnalysisError[] = []; + for (const owner of this.#pendingOutputDirectoryCleanup) { + try { + await owner.lease.release(); + this.#pendingOutputDirectoryCleanup.delete(owner); + } catch (cause: unknown) { + const cleanup = new ProviderCleanupError( + ADB_PROVIDER_IDENTITY.id, + [owner.path], + { + reason: cause instanceof Error ? cause.message : String(cause), + }, + { operation: owner.operation, cause }, + ); + const error = analysisErrorWithCleanupFailure( + owner.primary, + cleanup, + owner.operation, + ); + owner.lease.bindCleanupError(error); + failures.push(error); + } + } + if (failures.length > 0) { + const error = + failures.length === 1 + ? failures[0]! + : new ProviderCleanupError( + ADB_PROVIDER_IDENTITY.id, + [ + ...new Set( + failures.flatMap((failure) => failure.cleanupResources), + ), + ], + { + reason: + "Failed ADB output directories remain after cleanup retry", + operations: [...this.#pendingOutputDirectoryCleanup].map( + ({ operation }) => operation, + ), + failures: failures.map(({ message, cleanupResources }) => ({ + message, + resources: [...cleanupResources], + })), + }, + { + operation: "close_binary", + cause: new AggregateError( + failures, + "Multiple ADB output directory cleanups failed", + ), + }, + ); + const leases = [...this.#pendingOutputDirectoryCleanup].map( + ({ lease }) => lease, + ); + this.#outputDirectoryRetryLeases.set(error, leases); + if (failures.length > 1) new AnalysisCleanupLeaseGroup(error, leases); + throw error; + } + } + async execute( request: AdbRequest, options?: ExecutionOptions, @@ -910,24 +1365,31 @@ export class AdbProvider { }); this.#activeOperations.add(active); try { - if ( - this.#pendingDirectoryCleanup.size > 0 || - this.#pendingFileCleanup.size > 0 - ) { + try { + await this.#resources.retryCleanup(); + } catch (cause: unknown) { + const error = this.#cleanupError( + request.operation, + jsonObjectSchema.parse(request.input), + [cause], + this.#cleanupResources(cause, "ADB provider-owned resource"), + ); + const leases = cleanupLeasesForRetryFailure(cause); + if (leases !== undefined) new AnalysisCleanupLeaseGroup(error, leases); + return err(error); + } + if (this.#pendingOutputDirectoryCleanup.size > 0) { try { - await this.#retryDirectoryCleanup(); + await this.#retryOutputDirectoryCleanup(); } catch (cause: unknown) { return err( cause instanceof AnalysisError ? cause : new ProviderCleanupError( ADB_PROVIDER_IDENTITY.id, - [ - ...[...this.#pendingFileCleanup].map(({ path }) => path), - ...[...this.#pendingDirectoryCleanup].map( - ({ path }) => path, - ), - ], + [...this.#pendingOutputDirectoryCleanup].map( + ({ path }) => path, + ), { reason: cause instanceof Error ? cause.message : String(cause), @@ -950,7 +1412,9 @@ export class AdbProvider { ): Promise> { const signal = options?.signal; if (signal?.aborted === true) - return err(new AnalysisCancelledError(request.operation)); + return err( + new AnalysisCancelledError(request.operation, { cause: signal.reason }), + ); let selection: AdbBinarySelection; let identity: AdbClientIdentity; try { @@ -1653,7 +2117,7 @@ export class AdbProvider { if (!directory.ok) return directory; const packageDirectory = join(directory.value, packageName); let packageDirectoryCreated = false; - let packageDirectoryIdentity: OwnedDirectoryIdentity | undefined; + let packageDirectoryIdentity: OutputDirectoryIdentity | undefined; try { if ( await lstat(packageDirectory).then( @@ -1683,6 +2147,16 @@ export class AdbProvider { { cause }, ); if (!packageDirectoryCreated) return err(primary); + const owner = { + operation, + path: packageDirectory, + primary, + remove: () => + removeOwnedOutputDirectory( + packageDirectory, + packageDirectoryIdentity, + ), + }; const cleanup = new ProviderCleanupError( ADB_PROVIDER_IDENTITY.id, [packageDirectory], @@ -1691,28 +2165,23 @@ export class AdbProvider { }, { operation, cause }, ); - const reported = analysisErrorWithCleanupFailure( - primary, - cleanup, - operation, - ); - this.#pendingDirectoryCleanup.add({ - operation, - path: packageDirectory, - primary, - reportedError: reported, - remove: () => - removeOwnedDirectory(packageDirectory, packageDirectoryIdentity), - }); - return err(reported); + return err(this.#retainOutputDirectoryCleanup(owner, cleanup)); } const failOwnedPackage = async ( primary: AnalysisError, ): Promise> => { - const remove = () => - removeOwnedDirectory(packageDirectory, packageDirectoryIdentity); + const owner = { + operation, + path: packageDirectory, + primary, + remove: () => + removeOwnedOutputDirectory( + packageDirectory, + packageDirectoryIdentity, + ), + }; try { - await remove(); + await owner.remove(); } catch (cause: unknown) { const cleanup = new ProviderCleanupError( ADB_PROVIDER_IDENTITY.id, @@ -1720,20 +2189,7 @@ export class AdbProvider { { reason: cause instanceof Error ? cause.message : String(cause) }, { operation, cause }, ); - const reported = analysisErrorWithCleanupFailure( - primary, - cleanup, - operation, - ); - this.#pendingDirectoryCleanup.add({ - operation, - path: packageDirectory, - primary, - reportedError: reported, - prerequisite: primary, - remove, - }); - return err(reported); + return err(this.#retainOutputDirectoryCleanup(owner, cleanup)); } return err(primary); }; @@ -1760,8 +2216,8 @@ export class AdbProvider { stage: "resolve_paths" | "pull" | "digest"; message: string; }[] = []; - const partialPackagePull = ( - error: AnalysisError, + const cancelledPackagePull = ( + error: AnalysisCancelledError, localPath: string, devicePath: string, stage: "pull" | "digest", @@ -1782,8 +2238,8 @@ export class AdbProvider { stage, message: stage === "pull" - ? `${error instanceof AnalysisCancelledError ? "Pull cancelled" : "Pull failed"}; any incomplete file is retained at ${localPath}. Remaining APKs were not attempted.` - : `${error instanceof AnalysisCancelledError ? "Digesting the pulled file was cancelled" : "Digesting the pulled file failed"}; the file is retained at ${localPath}. Remaining APKs were not attempted.`, + ? `Pull cancelled; any incomplete file is retained at ${localPath}. Remaining APKs were not attempted.` + : `Digesting the pulled file was cancelled; the file is retained at ${localPath}. Remaining APKs were not attempted.`, }, ], coverage: "partial", @@ -1803,7 +2259,7 @@ export class AdbProvider { ); if (!pulled.ok) { if (pulled.error instanceof AnalysisCancelledError) - return partialPackagePull( + return cancelledPackagePull( pulled.error, localPath, entry.device_path, @@ -1817,7 +2273,12 @@ export class AdbProvider { continue; } try { - const digest = await this.hashAdbFile(localPath, operation, signal); + const digest = await hashAdbFile( + localPath, + operation, + signal, + this.#resources, + ); artifacts.push({ device_path: entry.device_path, file_name: entry.file_name, @@ -1828,11 +2289,60 @@ export class AdbProvider { role_basis: "file_name", }); } catch (cause: unknown) { - if ( - cause instanceof AnalysisCancelledError || - (cause instanceof AnalysisError && cause.cleanupIncomplete) - ) - return partialPackagePull( + const digestFailure = + cause instanceof AdbDigestCleanupFailure + ? cause.fileFailure + : undefined; + const completedDigest = + digestFailure?.outcome.kind === "completed" + ? digestFailure.outcome.value + : undefined; + const partialObservation: AdbPackagePullPartialObservation = { + provider_id: "adb", + operation, + result: { + client, + serial, + package_name: packageName, + output_directory: directory.value, + artifacts: + completedDigest === undefined + ? artifacts + : [ + ...artifacts, + { + device_path: entry.device_path, + file_name: entry.file_name, + local_path: localPath, + bytes: completedDigest.bytes, + sha256: completedDigest.sha256, + role: entry.role, + role_basis: "file_name", + }, + ], + failures: [ + ...failures, + { + device_path: entry.device_path, + stage: "digest", + message: + completedDigest === undefined + ? `The pulled file remains at ${localPath}, but its digest and descriptor cleanup are unconfirmed.` + : `The pulled file digest is known, but its descriptor cleanup is unconfirmed: ${localPath}`, + }, + ], + coverage: "partial", + }, + }; + const cleanupError = adbDigestCleanupError( + cause, + operation, + signal, + partialObservation, + ); + if (cleanupError !== undefined) return err(cleanupError); + if (cause instanceof AnalysisCancelledError) + return cancelledPackagePull( cause, localPath, entry.device_path, @@ -1914,7 +2424,12 @@ export class AdbProvider { ); if (!pulled.ok) return pulled; try { - const digest = await this.hashAdbFile(localPath, operation, signal); + const digest = await hashAdbFile( + localPath, + operation, + signal, + this.#resources, + ); return ok( createAnalysisExecution( { @@ -1934,21 +2449,26 @@ export class AdbProvider { ), ); } catch (cause) { - if ( - cause instanceof AnalysisCancelledError || - (cause instanceof AnalysisError && cause.cleanupIncomplete) - ) { - cause.retainPartialObservation({ - provider_id: "adb", - operation, - result: { - client, - serial: input.serial, - local_path: localPath, - device_path: input.device_path, - digest_status: "unknown", - }, - }); + const partialObservation: AdbAcquiredFilePartialObservation = { + provider_id: "adb", + operation, + result: { + client, + serial: input.serial, + local_path: localPath, + device_path: input.device_path, + digest_status: "unknown", + }, + }; + const cleanupError = adbDigestCleanupError( + cause, + operation, + signal, + partialObservation, + ); + if (cleanupError !== undefined) return err(cleanupError); + if (cause instanceof AnalysisCancelledError) { + cause.retainPartialObservation(partialObservation); return err(cause); } return err( @@ -1985,12 +2505,27 @@ export class AdbProvider { cause: new Error(`The local file does not exist: ${localPath}`), }), ); - return this.withInputSnapshot( + return this.#withRuntimeRoot( operation, - localPath, - input, + { + serial: input.serial, + local_path: localPath, + device_path: input.device_path, + overwrite: input.overwrite, + }, + { prefix: "rea-adb-input-" }, signal, - async (snapshotPath, digest) => { + async (root, resources) => { + const digest = await snapshotAdbInput( + localPath, + root, + resources, + operation, + signal, + ); + // `test -e` exits nonzero with no adb diagnosis when the path is absent. + // Any other probe failure leaves existence unknown, so it is reported as + // such rather than treated as absence or followed by a push. const existsProbe = await runAdb( operation, input.device_path, @@ -2004,9 +2539,6 @@ export class AdbProvider { { timeoutMs: SETTINGS_TIMEOUT_MS, maxOutputBytes: 1024 }, signal, ); - // `test -e` exits nonzero with no adb diagnosis when the path is absent. - // Any other probe failure leaves existence unknown, so it is reported as - // is rather than read as an existing path or followed by a push. if ( !existsProbe.ok && !(existsProbe.error instanceof AnalysisProtocolError) @@ -2024,15 +2556,15 @@ export class AdbProvider { operation, input.device_path, selection.binary, - adbArguments(input.serial, ["push", snapshotPath, input.device_path]), + adbArguments(input.serial, ["push", digest.path, input.device_path]), { timeoutMs: PULL_TIMEOUT_MS, maxOutputBytes: PULL_MAX_OUTPUT_BYTES }, signal, ); if (!pushed.ok) return pushed; - const createPushReceipt = ( + const receipt = ( deviceSha256: string | null, source: "device_sha256sum" | "unavailable", - additionalLimitations: readonly string[] = [], + extra: readonly string[] = [], ) => createAnalysisExecution( { @@ -2049,8 +2581,8 @@ export class AdbProvider { provider, { limitations: [ - "Push writes the caller's file to the device", - ...additionalLimitations, + "Push writes a private snapshot of the caller's file to the device", + ...extra, ], }, ); @@ -2065,46 +2597,57 @@ export class AdbProvider { if (!deviceDigest.ok) { const cancelled = deviceDigest.error instanceof AnalysisCancelledError; - const receipt = createPushReceipt(null, "unavailable", [ + const completed = receipt(null, "unavailable", [ cancelled ? "The device-side digest check was cancelled after push completed" : "The device did not answer sha256sum, so the transfer is not device-side verified", ]); if (cancelled) deviceDigest.error.retainPartialObservation( - executionEvidence(receipt, operation, input), + createEvidence( + completed.subject ?? undefined, + completed.provider, + { + operation, + parameters: input, + result: completed.result, + rawResult: completed.rawResult, + limitations: completed.limitations, + locations: completed.locations, + confidence: "observed", + }, + ), ); - return cancelled ? err(deviceDigest.error) : ok(receipt); + return cancelled ? err(deviceDigest.error) : ok(completed); } const digestMatch = /^([a-f0-9]{64})\s/u.exec( deviceDigest.value.stdout, ); - const receipt = createPushReceipt( - digestMatch?.[1] ?? null, - "device_sha256sum", - digestMatch !== null && digestMatch[1] !== digest.sha256 - ? [ - "The device-reported digest differs from the staged input digest", - ] - : [], - ); - if (digestMatch !== null && digestMatch[1] !== digest.sha256) { + const completed = receipt(digestMatch?.[1] ?? null, "device_sha256sum"); + if (digestMatch !== null && digestMatch[1] !== digest.sha256) return err( new AnalysisArtifactChangedError( operation, input.device_path, - `The device-reported digest does not match the pushed local file: ${localPath}`, + `The device-reported digest does not match the staged local file: ${localPath}`, { - partialObservation: executionEvidence( - receipt, - operation, - input, + partialObservation: createEvidence( + completed.subject ?? undefined, + completed.provider, + { + operation, + parameters: input, + result: completed.result, + rawResult: completed.rawResult, + limitations: completed.limitations, + locations: completed.locations, + confidence: "observed", + }, ), }, ), ); - } - return ok(receipt); + return ok(completed); }, ); } @@ -2132,12 +2675,23 @@ export class AdbProvider { cause: new Error(`The local APK does not exist: ${apkPath}`), }), ); - return this.withInputSnapshot( + return this.#withRuntimeRoot( operation, - apkPath, - input, + { + serial: input.serial, + apk_path: apkPath, + replace: input.replace, + }, + { prefix: "rea-adb-apk-" }, signal, - async (snapshotPath, digest) => { + async (root, resources) => { + const digest = await snapshotAdbInput( + apkPath, + root, + resources, + operation, + signal, + ); const installed = await runAdb( operation, input.serial, @@ -2145,7 +2699,7 @@ export class AdbProvider { adbArguments(input.serial, [ "install", ...(input.replace ? ["-r"] : []), - snapshotPath, + digest.path, ]), { timeoutMs: PULL_TIMEOUT_MS, maxOutputBytes: PULL_MAX_OUTPUT_BYTES }, signal, @@ -2178,7 +2732,7 @@ export class AdbProvider { provider, { limitations: [ - "Install changes device state; the installed APK's digest is the host file's digest", + "Install changes device state; the installed APK's digest identifies the private host snapshot", "Whether an existing package was replaced is unknown; replace_requested records only the adb install mode", ], }, @@ -2505,7 +3059,9 @@ export class AdbProvider { ); if (!captured.ok) return captured; if (signal?.aborted === true) - return err(new AnalysisCancelledError(operation)); + return err( + new AnalysisCancelledError(operation, { cause: signal.reason }), + ); const dimensions = parsePngDimensions(captured.value.stdout); if (dimensions.width === null) return err( @@ -2513,44 +3069,89 @@ export class AdbProvider { "screencap did not return a PNG payload; the device may not expose a display", ), ); - try { - await writeFile(localPath, captured.value.stdout, { flag: "wx" }); - } catch (cause) { - return err( - new AnalysisOutputError( - operation, - `The screen output could not be created exclusively: ${localPath}`, - { cause }, - ), - ); - } - const sha256 = createHash("sha256") - .update(captured.value.stdout) - .digest("hex"); - const execution = createAnalysisExecution( - { - client, - serial: input.serial, - local_path: localPath, - bytes: captured.value.stdout.byteLength, - sha256, - width: dimensions.width, - height: dimensions.height, - }, - provider, - { - limitations: [ - "The capture is one frame at read time; no recording or interaction occurred", - ], + return this.#withRuntimeRoot( + operation, + { serial: input.serial, output_directory: directory.value }, + { parent: directory.value, prefix: ".rea-screen-" }, + signal, + async (stage, resources) => { + let publicationCloseFailure: ScreenPublicationCloseFailure | undefined; + let usedExclusiveFallback = false; + try { + const stagedPath = join(stage.path, "screen.png"); + await writeFile(stagedPath, captured.value.stdout, { flag: "wx" }); + await syncStagedAdbFile(stagedPath, resources, operation); + if (signal?.aborted === true) + return err( + new AnalysisCancelledError(operation, { cause: signal.reason }), + ); + try { + // A same-filesystem hard link atomically publishes without replacing + // caller output. Its success is the publication cutoff. + await link(stagedPath, localPath); + } catch (cause: unknown) { + if (!hardLinksUnsupported(cause)) throw cause; + // Exclusive creation prevents replacement, but makes the path + // visible while the fallback writes the complete file. + usedExclusiveFallback = true; + publicationCloseFailure = + (await publishScreenExclusively( + localPath, + captured.value.stdout, + resources, + operation, + signal, + )) ?? undefined; + } + } catch (cause: unknown) { + if (cause instanceof AnalysisError) return err(cause); + return err( + new AnalysisOutputError( + operation, + `The screen output could not be published exclusively: ${localPath}`, + { cause }, + ), + ); + } + const sha256 = createHash("sha256") + .update(captured.value.stdout) + .digest("hex"); + const completed = createAnalysisExecution( + { + client, + serial: input.serial, + local_path: localPath, + bytes: captured.value.stdout.byteLength, + sha256, + width: dimensions.width, + height: dimensions.height, + }, + provider, + { + limitations: [ + "The capture is one frame at read time; no recording or interaction occurred", + ...(usedExclusiveFallback + ? [ + "The exclusive-create fallback makes the output path visible while its bytes are being written", + ] + : []), + ], + }, + ); + if (publicationCloseFailure !== undefined) { + const cleanupError = this.#cleanupError( + operation, + { serial: input.serial, output_directory: directory.value }, + [publicationCloseFailure.cause], + [publicationCloseFailure.resource], + completed, + ); + publicationCloseFailure.bindCleanupError(cleanupError); + return err(cleanupError); + } + return ok(completed); }, ); - if (signal?.aborted) - return err( - new AnalysisCancelledError(operation, { - partialObservation: executionEvidence(execution, operation, input), - }), - ); - return ok(execution); } private async collectBugreport( @@ -2567,7 +3168,7 @@ export class AdbProvider { ); if (!directory.ok) return directory; let stagingDirectory: string; - let stagingIdentity: OwnedDirectoryIdentity | undefined; + let stagingIdentity: OutputDirectoryIdentity | undefined; let retainedObservation: AdbAcquiredFilePartialObservation | undefined; let publishedPath: string | undefined; try { @@ -2656,7 +3257,12 @@ export class AdbProvider { ...retainedObservation, result: { ...retainedObservation.result, local_path: publishedPath }, }; - const digest = await this.hashAdbFile(publishedPath, operation, signal); + const digest = await hashAdbFile( + publishedPath, + operation, + signal, + this.#resources, + ); const report = { client, serial: input.serial, @@ -2678,10 +3284,17 @@ export class AdbProvider { }), ); } catch (cause) { - if ( - cause instanceof AnalysisCancelledError || - (cause instanceof AnalysisError && cause.cleanupIncomplete) - ) { + const cleanupError = + retainedObservation === undefined + ? undefined + : adbDigestCleanupError( + cause, + operation, + signal, + retainedObservation, + ); + if (cleanupError !== undefined) return err(cleanupError); + if (cause instanceof AnalysisCancelledError) { if (retainedObservation !== undefined) cause.retainPartialObservation(retainedObservation); return err(cause); @@ -2700,7 +3313,7 @@ export class AdbProvider { if (retainedObservation !== undefined && publishedPath === undefined) return collectedResult; try { - await removeOwnedDirectory(stagingDirectory, stagingIdentity); + await removeOwnedOutputDirectory(stagingDirectory, stagingIdentity); } catch (cause) { const primary = collectedResult.ok ? new AnalysisOutputError( @@ -2710,25 +3323,23 @@ export class AdbProvider { : collectedResult.error; if (retainedObservation !== undefined) primary.retainPartialObservation(retainedObservation); - const reported = analysisErrorWithCleanupFailure( - primary, - new ProviderCleanupError( - ADB_PROVIDER_IDENTITY.id, - [stagingDirectory], - { reason: cause instanceof Error ? cause.message : String(cause) }, - { operation, cause }, + return err( + this.#retainOutputDirectoryCleanup( + { + operation, + path: stagingDirectory, + primary, + remove: () => + removeOwnedOutputDirectory(stagingDirectory, stagingIdentity), + }, + new ProviderCleanupError( + ADB_PROVIDER_IDENTITY.id, + [stagingDirectory], + { reason: cause instanceof Error ? cause.message : String(cause) }, + { operation, cause }, + ), ), - operation, ); - this.#pendingDirectoryCleanup.add({ - operation, - path: stagingDirectory, - primary, - reportedError: reported, - prerequisite: primary, - remove: () => removeOwnedDirectory(stagingDirectory, stagingIdentity), - }); - return err(reported); } return collectedResult; } diff --git a/src/android/JadxProvider.ts b/src/android/JadxProvider.ts index 2de555964d..39b13871b5 100644 --- a/src/android/JadxProvider.ts +++ b/src/android/JadxProvider.ts @@ -27,15 +27,10 @@ import { snapshotAndroidEngine, snapshotAndroidTarget, } from "./AndroidTargetSnapshot.js"; -import { - hashStableFile, - StableFileHashCleanupFailure, -} from "../filesystem/StableFileHash.js"; -import { - OwnedRegularFileCleanupFailure, - RegularFileChangedError, -} from "../filesystem/RegularFile.js"; +import { hashStableFile } from "../filesystem/StableFileHash.js"; import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { RegularFileChangedError } from "../filesystem/RegularFile.js"; +import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js"; import { inspectJadxAvailability, resolveJadxConfigurationInputs, @@ -74,19 +69,35 @@ const executionError = (context: { session: JadxSession | undefined; }): AnalysisError => { const { cause, request, timeout, signal, session } = context; - if (cause instanceof OwnedRegularFileCleanupFailure) { + if (cause instanceof RegularFileCleanupFailure) { + const primaryCause = + cause.outcome.kind === "failed" + ? cause.outcome.cause + : signal?.aborted + ? signal.reason + : timeout.aborted + ? timeout.reason + : undefined; + const primaryOptions = + primaryCause === undefined ? {} : { cause: primaryCause }; const primary = signal?.aborted === true - ? new AnalysisCancelledError(request.operation, { cause: cause.cause }) + ? new AnalysisCancelledError(request.operation, primaryOptions) : timeout.aborted - ? new AnalysisTimeoutError(request.operation, OPERATION_TIMEOUT_MS) - : new ProviderAdapterError("jadx", request.operation, { - cause: cause.cause, - userMessage: - cause.cause instanceof Error - ? cause.cause.message - : String(cause.cause), - }); + ? new AnalysisTimeoutError( + request.operation, + OPERATION_TIMEOUT_MS, + primaryOptions, + ) + : primaryCause instanceof AnalysisError + ? primaryCause + : new ProviderAdapterError("jadx", request.operation, { + cause: primaryCause ?? cause, + userMessage: + primaryCause instanceof Error + ? primaryCause.message + : cause.message, + }); const cleanup = new ProviderCleanupError( "jadx", [cause.path], @@ -95,12 +106,8 @@ const executionError = (context: { cause.cleanupCause instanceof Error ? cause.cleanupCause.message : String(cause.cleanupCause), - ...(cause instanceof StableFileHashCleanupFailure && - cause.completedHash !== undefined - ? { completed_hash: cause.completedHash } - : {}), }, - { operation: request.operation, cause: cause.cleanupCause }, + { operation: request.operation, cause }, ); return analysisErrorWithCleanupFailure(primary, cleanup, request.operation); } @@ -180,9 +187,10 @@ const hashAndroidSource = async ( path: string, operation: string, signal: AbortSignal, + retainFailedClose: (owner: OwnedFileHandle) => void, ): Promise => { try { - return (await hashStableFile(path, signal)).sha256; + return (await hashStableFile(path, { signal, retainFailedClose })).sha256; } catch (cause: unknown) { if (signal.aborted) throw cause; if (cause instanceof RegularFileChangedError) @@ -194,73 +202,112 @@ const hashAndroidSource = async ( interface PendingCleanup { session: JadxSession | undefined; root: JadxWorkspaceRoot | undefined; - readonly fileHandles: Map< - OwnedFileHandle, - { readonly path: string; readonly error: AnalysisError } - >; + readonly fileHandles: Map; readonly previousError: AnalysisError | undefined; + readonly operationCause: unknown; + reportedErrors: WeakRef[]; } const cleanup = async ( resources: PendingCleanup, ): Promise> => { - const { session, root, previousError, fileHandles } = resources; + const { session, root, previousError } = resources; const diagnostics = { previous_error: previousError === undefined ? null : projectAnalysisError(previousError), }; - const failures: { readonly resource: string; readonly cause: unknown }[] = []; - for (const [owner, { path, error }] of fileHandles) { - try { - await owner.close(); - error.resolveCleanup(); - fileHandles.delete(owner); - } catch (cause: unknown) { - failures.push({ resource: path, cause }); - } - } + let sessionCloseFailed = false; + let sessionCloseCause: unknown; try { await session?.close(); resources.session = undefined; } catch (cause) { - failures.push({ resource: session?.transport.runId ?? "unknown", cause }); + // Still drain independent hash descriptors, but keep the workspace until + // this session's process ownership is proven stopped. + sessionCloseFailed = true; + sessionCloseCause = cause; } - if (resources.session === undefined && failures.length === 0) { + const failedHandles: { + readonly path: string; + readonly owner: OwnedFileHandle; + readonly cause: unknown; + }[] = []; + for (const [owner, path] of resources.fileHandles) { try { - await root?.close(); - resources.root = undefined; + await owner.close(); + if ( + resources.operationCause instanceof RegularFileCleanupFailure && + resources.operationCause.owner === owner + ) + previousError?.resolveCleanup(); + resources.fileHandles.delete(owner); } catch (cause: unknown) { - failures.push({ resource: root?.path ?? "unknown", cause }); + failedHandles.push({ path, owner, cause }); } } - if (failures.length > 0) { - const cleanupCause = new AggregateError( - failures.map(({ cause }) => cause), - "JADX file, session, or workspace cleanup failed", - ); - const error = new ProviderCleanupError( + if (sessionCloseFailed || failedHandles.length > 0) { + const cleanupCauses = [ + ...(sessionCloseFailed ? [sessionCloseCause] : []), + ...failedHandles.map(({ cause }) => cause), + ]; + const cleanupResources = [ + ...(!sessionCloseFailed + ? [] + : [ + session?.transport.runId ?? "unknown", + ...(root ? [root.path] : []), + ]), + ...failedHandles.map(({ path }) => path), + ]; + const cleanupError = new ProviderCleanupError( "jadx", - failures.map(({ resource }) => resource), + cleanupResources, { ...diagnostics, - reason: failures - .map(({ cause }) => - cause instanceof Error ? cause.message : String(cause), + reason: cleanupCauses + .map((failure) => + failure instanceof Error ? failure.message : String(failure), ) .join("; "), - provider_cleanup: failures.map(({ cause }) => - cause instanceof ProviderCleanupError - ? (cause.diagnostics ?? null) + cleanup_causes: cleanupCauses.map((failure) => + failure instanceof Error ? failure.message : String(failure), + ), + provider_cleanup: cleanupCauses.map((failure) => + failure instanceof ProviderCleanupError + ? (failure.diagnostics ?? null) : null, ), }, { - cause: previousError ?? cleanupCause, - ...(previousError?.partialObservation === undefined - ? {} - : { partialObservation: previousError.partialObservation }), + cause: new AggregateError( + [ + ...(previousError === undefined ? [] : [previousError]), + resources.operationCause, + ...cleanupCauses, + ], + "JADX operation and resource cleanup failed", + ), }, ); + if (previousError?.partialObservation !== undefined) + cleanupError.retainPartialObservation(previousError.partialObservation); + return err(cleanupError); + } + try { + await root?.close(); + resources.root = undefined; + } catch (cause) { + const error = new ProviderCleanupError( + "jadx", + [root?.path ?? "unknown"], + { + ...diagnostics, + reason: cause instanceof Error ? cause.message : String(cause), + }, + { cause: previousError ?? cause }, + ); + if (previousError?.partialObservation !== undefined) + error.retainPartialObservation(previousError.partialObservation); return err(error); } return ok(undefined); @@ -355,12 +402,27 @@ export class JadxProvider implements AndroidAnalysisPort { root: this.#retained.root, fileHandles: new Map(), previousError: undefined, + operationCause: undefined, + reportedErrors: [], }; this.#retained = undefined; } if (this.#pendingCleanup === undefined) return ok(undefined); - const result = await cleanup(this.#pendingCleanup); - if (result.ok) this.#pendingCleanup = undefined; + const pending = this.#pendingCleanup; + const result = await cleanup(pending); + if (result.ok) { + for (const reportedError of pending.reportedErrors) + reportedError.deref()?.resolveCleanup(); + pending.reportedErrors = []; + this.#pendingCleanup = undefined; + } else { + pending.reportedErrors = [ + ...pending.reportedErrors.filter( + (reportedError) => reportedError.deref() !== undefined, + ), + new WeakRef(result.error), + ]; + } return result; } @@ -385,6 +447,12 @@ export class JadxProvider implements AndroidAnalysisPort { : AbortSignal.any([options.signal, timeout]); let root: JadxWorkspaceRoot | undefined; let session: JadxSession | undefined; + const fileHandles: PendingCleanup["fileHandles"] = new Map(); + const retainFailedClose = + (path: string) => + (owner: OwnedFileHandle): void => { + fileHandles.set(owner, path); + }; try { if (target.format !== "apk") throw new AnalysisCapabilityUnavailableError( @@ -402,15 +470,22 @@ export class JadxProvider implements AndroidAnalysisPort { configuration.jar, request.operation, signal, + retainFailedClose(configuration.jar), ); const bridgeHash = await hashAndroidSource( BRIDGE_SOURCE, request.operation, signal, + retainFailedClose(BRIDGE_SOURCE), ); let targetHash: string; try { - targetHash = (await hashStableFile(target.path, signal)).sha256; + targetHash = ( + await hashStableFile(target.path, { + signal, + retainFailedClose: retainFailedClose(target.path), + }) + ).sha256; } catch (cause: unknown) { if (signal.aborted) throw cause; if (cause instanceof RegularFileChangedError) @@ -473,6 +548,7 @@ export class JadxProvider implements AndroidAnalysisPort { bridge, request.operation, workspaceSignal, + retainFailedClose(bridge), ); if (copiedBridgeHash !== bridgeHash) throw androidSourceChanged( @@ -533,12 +609,6 @@ export class JadxProvider implements AndroidAnalysisPort { signal: options?.signal, session, }); - const fileHandles: PendingCleanup["fileHandles"] = new Map(); - if (cause instanceof OwnedRegularFileCleanupFailure) - fileHandles.set(cause.owner, { - path: cause.path, - error: previousError, - }); const partialObservation = session?.partialObservation(); if (partialObservation !== undefined) previousError.retainPartialObservation(partialObservation); @@ -546,7 +616,14 @@ export class JadxProvider implements AndroidAnalysisPort { root ??= this.#retained?.root; session ??= this.#retained?.session; this.#retained = undefined; - this.#pendingCleanup = { session, root, fileHandles, previousError }; + this.#pendingCleanup = { + session, + root, + fileHandles, + previousError, + operationCause: cause, + reportedErrors: [], + }; const retired = await this.#retire(); return retired.ok ? err(previousError) : retired; } diff --git a/src/application/AnalysisProvider.ts b/src/application/AnalysisProvider.ts index 335018d74c..668b5ca5fa 100644 --- a/src/application/AnalysisProvider.ts +++ b/src/application/AnalysisProvider.ts @@ -173,6 +173,12 @@ export interface AnalysisClientContext { readonly runId: string; } +/** One lifecycle owner for descriptor cleanup and abortable inspection work. */ +export interface FileCloseResourceOwner { + retain(owner: { close(): Promise }, resource: string): void; + run(operation: () => Promise): Promise; +} + export interface ProviderIdentity { readonly id: string; readonly name: string; @@ -187,6 +193,7 @@ export interface AnalysisProfileResolution { /** Cancellation context for side-effect-free provider profile resolution. */ export interface AnalysisProfileResolutionOptions { readonly signal?: AbortSignal; + readonly fileCloseResources?: FileCloseResourceOwner; } interface CapabilityEffects { @@ -290,8 +297,8 @@ export interface AnalysisProviderCandidate extends AnalysisProvider { inspectAvailability(): ProviderAvailability; inspectReadinessSnapshot?(): AnalysisProviderReadinessSnapshot; inspectTargetSupport(target: BinaryTarget): ProviderTargetSupport; - /** Report retained candidate-owned cleanup without starting work or retrying it. */ + /** Observe retained candidate cleanup without starting or retrying work. */ hasPendingOwnedResources?(): boolean; - /** Release candidate-owned resources that may exist without a client. */ + /** Release candidate resources that are independent of any active client. */ closeOwnedResources?(): Promise>; } diff --git a/src/application/BrowserObservationService.ts b/src/application/BrowserObservationService.ts index 87dfc8db37..53f44360a0 100644 --- a/src/application/BrowserObservationService.ts +++ b/src/application/BrowserObservationService.ts @@ -242,9 +242,8 @@ const browserEvidence = ( ), ); return completeProviderEvidence( - ok(createBrowserEvidence(operation, input, result, provider)), - operation, - options, + createBrowserEvidence(operation, input, result, provider), + options.signal, ); }; diff --git a/src/application/BrowserScenarioCaptureService.ts b/src/application/BrowserScenarioCaptureService.ts index 7f912ca5c4..9573c53fd4 100644 --- a/src/application/BrowserScenarioCaptureService.ts +++ b/src/application/BrowserScenarioCaptureService.ts @@ -41,7 +41,16 @@ export const captureBrowserScenario = async ( partial.kind === "browser-scenario-observation" ) { const report = browserScenarioCaptureSchema.safeParse(partial.capture); - if (!report.success) { + if ( + report.success && + report.data.steps.every(({ status }) => status === "completed") + ) { + captured.error.partialObservation = createBrowserScenarioEvidence( + scenario, + report.data, + provider.identity(), + ); + } else if (!report.success) { if (captured.error instanceof AnalysisCancelledError) return err( new AnalysisOutputError( @@ -52,11 +61,6 @@ export const captureBrowserScenario = async ( ); return captured; } - captured.error.partialObservation = createBrowserScenarioEvidence( - scenario, - report.data, - provider.identity(), - ); } return captured; } @@ -70,15 +74,7 @@ export const captureBrowserScenario = async ( ), ); return completeProviderEvidence( - { - ok: true, - value: createBrowserScenarioEvidence( - scenario, - report.data, - provider.identity(), - ), - }, - OPERATION, - options, + createBrowserScenarioEvidence(scenario, report.data, provider.identity()), + options.signal, ); }; diff --git a/src/application/Doctor.ts b/src/application/Doctor.ts index 7b4d31f57d..fcfc9c1f7e 100644 --- a/src/application/Doctor.ts +++ b/src/application/Doctor.ts @@ -1,7 +1,7 @@ import { constants } from "node:fs"; import { access, readdir, stat } from "node:fs/promises"; import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; -import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import type { FileCloseResourceOwner } from "./AnalysisProvider.js"; import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js"; import { join } from "node:path"; @@ -435,10 +435,7 @@ export interface SystemDoctorHostOptions { readonly environment?: NodeJS.ProcessEnv; readonly execFileOutput?: typeof execFileOutput; readonly providerInspections?: ( - adoptFileHandle: ( - resource: string, - handle: OwnedFileHandle, - ) => Promise, + fileCloseResources: FileCloseResourceOwner, ) => Promise; readonly providerRegistrationEnvironment?: () => Readonly< Record @@ -458,6 +455,15 @@ export const systemDoctorHost = ( const hostExecFileOutput = options.execFileOutput ?? execFileOutput; const homeDirectory = homeDirectoryFromEnvironment(environment, platform); const commandEnvironment = { env: environment }; + const fileCloseResources: FileCloseResourceOwner = { + retain: (owner, resource) => + resources.retain({ + kind: "lease", + resource, + lease: { release: () => owner.close() }, + }), + run: (operation) => resources.run(operation), + }; return { platform, architecture, @@ -555,16 +561,7 @@ export const systemDoctorHost = ( ? {} : { providerInspections: () => - options.providerInspections!( - async (resource, handle) => - ( - await resources.release({ - kind: "file-handle", - resource, - handle, - }) - ).kind === "released", - ), + options.providerInspections!(fileCloseResources), }), ...(options.providerRegistrationEnvironment === undefined ? {} diff --git a/src/application/LinuxHopper.test.ts b/src/application/LinuxHopper.test.ts index e0ca5eef65..df2c83aa70 100644 --- a/src/application/LinuxHopper.test.ts +++ b/src/application/LinuxHopper.test.ts @@ -1,5 +1,7 @@ import { createHash } from "node:crypto"; -import { describe, expect, it } from "vitest"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; import { formatCommandForDisplay, @@ -9,6 +11,7 @@ import { parseLinuxDistribution, supportsReaHost, LinuxHopperTemporaryCleanupOwner, + systemLinuxHopperInstallHost, type LinuxDistribution, type LinuxHopperDownload, type LinuxHopperInstallHost, @@ -23,6 +26,9 @@ import { type LinuxHopperRelease, } from "./HopperReleaseFeed.js"; import type { VerifiedLinuxHopperPackage } from "./LinuxHopperPublisher.js"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import * as nativeCommands from "../process/ExecFileOutput.js"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; describe("Linux package-manager command display", () => { it("preserves argument boundaries for paths with spaces and quotes", () => { @@ -331,6 +337,37 @@ describe("Linux Hopper installation failure classification", () => { }); describe("Linux Hopper cleanup and failure details", () => { + it("preserves a publisher descriptor cleanup failure during launcher status inspection", async () => { + const directory = await createTestTempDirectory("rea-hopper-status-"); + const launcher = join(directory, "Hopper"); + await writeFile(launcher, "fixture launcher"); + const host = systemLinuxHopperInstallHost({ + ...process.env, + HOME: directory, + XDG_CACHE_HOME: join(directory, "cache"), + }); + const command = vi + .spyOn(nativeCommands, "execFileOutput") + .mockResolvedValue({ + stdout: "libc.so.6 => /lib/libc.so.6", + stderr: "", + stdoutBytes: Buffer.byteLength("libc.so.6 => /lib/libc.so.6"), + stderrBytes: 0, + }); + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockRejectedValue(new Error("Fixture descriptor close unavailable")); + try { + await expect(host.launcherStatus(launcher, "0".repeat(64))).resolves.toBe( + "cleanup_failed", + ); + } finally { + close.mockRestore(); + command.mockRestore(); + await host.close?.(); + } + }); + it("preserves the failing package-manager command and its output", async () => { const host = new RecordingLinuxHost(); host.installSucceeds = false; diff --git a/src/application/LinuxHopper.ts b/src/application/LinuxHopper.ts index 5130a29bbd..53bd7a06d1 100644 --- a/src/application/LinuxHopper.ts +++ b/src/application/LinuxHopper.ts @@ -4,7 +4,9 @@ import { join } from "node:path"; import { execFileBoundedOutput } from "../process/ExecFileBoundedOutput.js"; import { execFileOutput } from "../process/ExecFileOutput.js"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import { downloadHopperPackage } from "./HopperDownload.js"; +import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js"; import { err, type Result } from "../domain/result.js"; import { HOPPER_RELEASES_URL, @@ -219,7 +221,8 @@ export type LinuxHopperLauncherStatus = | "ready" | "missing" | "runtime_dependencies" - | "unsupported_hopper_build"; + | "unsupported_hopper_build" + | "cleanup_failed"; /** Parse an os-release document without executing its shell syntax. */ export const parseLinuxDistribution = (text: string): LinuxDistribution => { @@ -431,15 +434,18 @@ export const systemLinuxHopperInstallHost = ( !linuxSharedLibrariesAvailable(`${linked.stdout}\n${linked.stderr}`) ) return "runtime_dependencies"; - return (await publisher.hashLauncher(path)).sha256 === expectedSha256 + return (await publisher.hashLauncher(path)) === expectedSha256 ? "ready" : "unsupported_hopper_build"; } catch (cause: unknown) { if ( - cause instanceof HopperReleaseError && - cause.reason === "cleanup_failed" + cause instanceof RegularFileCleanupFailure || + (cause instanceof HopperReleaseError && + cause.reason === "cleanup_failed") || + (cause instanceof ArtifactReaderFailure && + cause.cleanup !== undefined) ) - throw cause; + return "cleanup_failed"; // best-effort cleanup: optional launcher probing; unreadable means missing. void cause; return "missing"; diff --git a/src/application/LinuxHopperPublisher.ts b/src/application/LinuxHopperPublisher.ts index f267d0d623..7fa3085888 100644 --- a/src/application/LinuxHopperPublisher.ts +++ b/src/application/LinuxHopperPublisher.ts @@ -11,11 +11,8 @@ import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js"; import { err, ok, type Result } from "../domain/result.js"; import { hashStableFile } from "../filesystem/StableFileHash.js"; -import { - StableFileHashCleanupFailure, - type StableFileHashResult, -} from "../filesystem/StableFileHash.js"; -import { OwnedRegularFileCleanupFailure } from "../filesystem/RegularFile.js"; +import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js"; import { readBoundedFileBytes } from "../process/BoundedFileBytes.js"; import { verifyDetachedPgpSignature } from "../process/DetachedPgpSignature.js"; import { execFileOutput } from "../process/ExecFileOutput.js"; @@ -39,7 +36,10 @@ import { type LinuxHopperRelease, type LinuxPackageFamily, } from "./HopperReleaseFeed.js"; -import { withRegularFile } from "./RegularFileRead.js"; +import { + RegularFileCleanupFailure as RegularFileReadCleanupFailure, + withRegularFile, +} from "./RegularFileRead.js"; /** Metadata/key inspection budget, independent of the compressed package budget. */ const RELEASE_METADATA_BYTES = 1024 * 1024; @@ -129,46 +129,6 @@ export class LinuxHopperPublisher { readonly packageFamily: () => Promise, ) {} - /** Hash a selected launcher while retaining any descriptor cleanup owner. */ - hashLauncher(path: string): Promise { - return this.#hashStableFile(path); - } - - async #hashStableFile( - path: string, - signal?: AbortSignal, - ): Promise { - try { - return await hashStableFile(path, signal); - } catch (cause: unknown) { - if (!(cause instanceof OwnedRegularFileCleanupFailure)) throw cause; - const cleanup = await this.#resources.release({ - kind: "file-handle", - resource: path, - handle: cause.owner, - }); - if (cleanup.kind === "failed") { - pendingCleanup.add(this.#resources); - throw new HopperReleaseError( - "cleanup_failed", - `Hopper publisher retained file descriptor cleanup for ${path}.`, - { - cause: new AggregateError( - [cause, cleanup.cause], - "Stable file hash and descriptor cleanup failed", - { cause }, - ), - }, - ); - } - if (cause instanceof StableFileHashCleanupFailure) { - if (cause.hasPrimaryCause) throw cause.cause; - if (cause.completedHash !== undefined) return cause.completedHash; - } - throw cause; - } - } - get cacheRoot(): string { const location = linuxHopperReleaseCacheDirectory(this.environment); if (!location.ok) throw location.error; @@ -209,9 +169,34 @@ export class LinuxHopperPublisher { archive: string, signal?: AbortSignal, suppliedPublicKey?: string, + ): Promise> { + return this.#runOperation(signal, (resources) => + this.#verifyArchive( + family, + release, + archive, + signal, + suppliedPublicKey, + resources, + ), + ); + } + + async #verifyArchive( + family: LinuxPackageFamily, + release: LinuxHopperRelease, + archive: string, + signal: AbortSignal | undefined, + suppliedPublicKey: string | undefined, + resources: ArtifactResourceScope, ): Promise> { try { - const bytes = await this.#read(archive, HOPPER_DOWNLOAD_BYTES, signal); + const bytes = await this.#read( + archive, + HOPPER_DOWNLOAD_BYTES, + signal, + resources, + ); if ( bytes.length !== Number(release.file_length) || createHash("sha1").update(bytes).digest("hex") !== release.file_hash @@ -277,12 +262,20 @@ export class LinuxHopperPublisher { { cause: verified.error }, ), ); - const mainExeHash = await this.#withTemporary(async (root) => { - // Extract only after authentication, from this operation's private byte snapshot. - const snapshot = join(root, "package"); - await writeFile(snapshot, bytes, { flag: "wx", mode: 0o600 }); - return this.#packageLauncherHash(family, snapshot, root, signal); - }); + const mainExeHash = await this.#withTemporary( + async (root, temporaryResources) => { + // Extract only after authentication, from this operation's private byte snapshot. + const snapshot = join(root, "package"); + await writeFile(snapshot, bytes, { flag: "wx", mode: 0o600 }); + return this.#packageLauncherHash( + family, + snapshot, + root, + signal, + temporaryResources, + ); + }, + ); if (mainExeHash !== release.main_exe_hash) return err( new HopperReleaseError( @@ -308,6 +301,17 @@ export class LinuxHopperPublisher { verified: VerifiedLinuxHopperPackage, archive: string, signal?: AbortSignal, + ): Promise> { + return this.#runOperation(signal, (resources) => + this.#retainVerifiedArchive(verified, archive, signal, resources), + ); + } + + async #retainVerifiedArchive( + verified: VerifiedLinuxHopperPackage, + archive: string, + signal: AbortSignal | undefined, + resources: ArtifactResourceScope, ): Promise> { try { signal?.throwIfAborted(); @@ -335,11 +339,16 @@ export class LinuxHopperPublisher { }, async () => { signal?.throwIfAborted(); - await this.#withTemporary(async (stage) => { + await this.#withTemporary(async (stage, temporaryResources) => { await copyFile(archive, join(stage, "archive")); if ( - (await this.#hashStableFile(join(stage, "archive"), signal)) - .sha256 !== verified.packageSha256 + ( + await this.#hashFileWithResources( + join(stage, "archive"), + signal, + temporaryResources, + ) + ).sha256 !== verified.packageSha256 ) throw new HopperReleaseError( "integrity", @@ -363,6 +372,7 @@ export class LinuxHopperPublisher { target, verified.release.main_exe_hash, signal, + resources, ); }, this.cacheRoot); }, @@ -377,9 +387,10 @@ export class LinuxHopperPublisher { stage: string, target: string, digest: string, - signal?: AbortSignal, + signal: AbortSignal | undefined, + resources: ArtifactResourceScope, ): Promise { - const existing = await this.#cachedPackage(digest, signal); + const existing = await this.#cachedPackage(digest, signal, resources); if (existing.ok) return; if (existing.error.reason === "cache_missing") { await rename(stage, target); @@ -411,7 +422,7 @@ export class LinuxHopperPublisher { throw new Error("The Hopper cache entry changed during repair."); await rename(stage, target); } catch (cause: unknown) { - const replacement = await this.#cachedPackage(digest, signal); + const replacement = await this.#cachedPackage(digest, signal, resources); if (replacement.ok) return; await this.#failedCachePublication( target, @@ -478,13 +489,32 @@ export class LinuxHopperPublisher { async inspectLauncher( path: string, signal?: AbortSignal, + ): Promise> { + return this.#runOperation(signal, (resources) => + this.#inspectLauncher(path, signal, resources), + ); + } + + async #inspectLauncher( + path: string, + signal: AbortSignal | undefined, + resources: ArtifactResourceScope, ): Promise> { try { - const observed = await this.#hashStableFile(path, signal); - const cached = await this.#cachedPackage(observed.sha256, signal); + const observed = await this.#hashFileWithResources( + path, + signal, + resources, + ); + const cached = await this.#cachedPackage( + observed.sha256, + signal, + resources, + ); if (cached.ok) { if ( - (await this.#hashStableFile(path, signal)).sha256 !== observed.sha256 + (await this.#hashFileWithResources(path, signal, resources)) + .sha256 !== observed.sha256 ) return err( new HopperReleaseError( @@ -516,28 +546,35 @@ export class LinuxHopperPublisher { "The selected Hopper launcher does not match an authenticated cached release or the official current Linux demo.", ), ); - const verified = await this.#withTemporary(async (root) => { - const response = await downloadHopperPackage( - selected.value.filename, - signal === undefined ? {} : { signal }, - ); - if (!response.ok) - throw new HopperReleaseError( - "download", - "The official Hopper package is unavailable for publisher verification.", + const verified = await this.#withTemporary( + async (root, temporaryResources) => { + const response = await downloadHopperPackage( + selected.value.filename, + signal === undefined ? {} : { signal }, ); - const archive = join(root, "archive"); - await writeFile(archive, response.bytes, { flag: "wx", mode: 0o600 }); - const verified = await this.verifyArchive( - family, - selected.value, - archive, - signal, - ); - if (!verified.ok) throw verified.error; - return verified.value; - }); - if ((await this.#hashStableFile(path, signal)).sha256 !== observed.sha256) + if (!response.ok) + throw new HopperReleaseError( + "download", + "The official Hopper package is unavailable for publisher verification.", + ); + const archive = join(root, "archive"); + await writeFile(archive, response.bytes, { flag: "wx", mode: 0o600 }); + const verified = await this.#verifyArchive( + family, + selected.value, + archive, + signal, + undefined, + temporaryResources, + ); + if (!verified.ok) throw verified.error; + return verified.value; + }, + ); + if ( + (await this.#hashFileWithResources(path, signal, resources)).sha256 !== + observed.sha256 + ) return err( new HopperReleaseError( "integrity", @@ -562,9 +599,122 @@ export class LinuxHopperPublisher { } } + /** Admit once at the publisher boundary; file reads use this operation's child scope. */ + async #runOperation( + signal: AbortSignal | undefined, + operation: ( + resources: ArtifactResourceScope, + ) => Promise>, + ): Promise> { + try { + return await this.#resources.run(async () => { + const resources = new ArtifactResourceScope(); + let result: Result; + try { + result = await operation(resources); + } catch (cause: unknown) { + result = err( + releaseFailure(cause, signal, "verification_unavailable"), + ); + } + + const cleanup = await resources.close().then( + () => ({ ok: true as const }), + (cause: unknown) => ({ ok: false as const, cause }), + ); + if (!cleanup.ok) { + const owner: ArtifactResourceOwner = { + kind: "lease", + resource: "hopper-publisher-operation-resources", + lease: { release: () => resources.close() }, + }; + this.#resources.retain(owner); + pendingCleanup.add(this.#resources); + throw new HopperReleaseError( + "cleanup_failed", + "Hopper publisher retained operation resource cleanup.", + { + cause: new AggregateError( + result.ok ? [cleanup.cause] : [result.error, cleanup.cause], + "Publisher operation and resource cleanup outcomes", + ), + }, + ); + } + return result; + }); + } catch (cause: unknown) { + return err(releaseFailure(cause, signal, "verification_unavailable")); + } + } + + /** Hash an installed launcher under this publisher's cleanup lifetime. */ + async hashLauncher(path: string): Promise { + const result = await this.#runOperation(undefined, async (resources) => + ok( + (await this.#hashFileWithResources(path, undefined, resources)).sha256, + ), + ); + if (!result.ok) throw result.error; + return result.value; + } + + async #hashFileWithResources( + path: string, + signal: AbortSignal | undefined, + resources: ArtifactResourceScope, + ): Promise>> { + let retainedOwner: ArtifactResourceOwner | undefined; + try { + return await hashStableFile(path, { + ...(signal === undefined ? {} : { signal }), + retainFailedClose: (handle: OwnedFileHandle) => { + retainedOwner = { + kind: "file-handle", + resource: path, + handle, + }; + resources.retain(retainedOwner); + if (resources === this.#resources) pendingCleanup.add(resources); + }, + }); + } catch (cause: unknown) { + if (!(cause instanceof RegularFileCleanupFailure)) throw cause; + const hashFailure = cause as RegularFileCleanupFailure< + Awaited> + >; + const owner = + retainedOwner ?? + ({ + kind: "file-handle", + resource: path, + handle: hashFailure.owner, + } satisfies ArtifactResourceOwner); + const released = await resources.release(owner); + if (released.kind === "failed") { + if (resources === this.#resources) pendingCleanup.add(resources); + throw new HopperReleaseError( + "cleanup_failed", + `Hopper publisher retained file descriptor cleanup for ${path}.`, + { + cause: new AggregateError( + [cause, released.cause], + "Stable file hash and descriptor cleanup failed", + { cause }, + ), + }, + ); + } + if (hashFailure.outcome.kind === "failed") + throw hashFailure.outcome.cause; + return hashFailure.outcome.value; + } + } + async #cachedPackage( digest: string, - signal?: AbortSignal, + signal: AbortSignal | undefined, + resources: ArtifactResourceScope, ): Promise> { try { const root = join(this.cacheRoot, digest); @@ -595,6 +745,7 @@ export class LinuxHopperPublisher { join(root, "release.json"), RELEASE_METADATA_BYTES, signal, + resources, ); let value: unknown; try { @@ -626,6 +777,7 @@ export class LinuxHopperPublisher { join(root, "public-key.asc"), RELEASE_METADATA_BYTES, signal, + resources, ); let key: string; try { @@ -637,12 +789,13 @@ export class LinuxHopperPublisher { { cause }, ); } - return this.verifyArchive( + return this.#verifyArchive( parsed.data.family, release.value, join(root, "archive"), signal, key, + resources, ); } catch (cause: unknown) { return err(releaseFailure(cause, signal, "verification_unavailable")); @@ -672,7 +825,8 @@ export class LinuxHopperPublisher { family: LinuxPackageFamily, archive: string, root: string, - signal?: AbortSignal, + signal: AbortSignal | undefined, + resources: ArtifactResourceScope, ): Promise { const commandOptions = { env: this.environment, @@ -730,14 +884,20 @@ export class LinuxHopperPublisher { commandOptions, ); return ( - await this.#hashStableFile( + await this.#hashFileWithResources( join(extracted, "opt", "hopper", "bin", "Hopper"), signal, + resources, ) ).sha256; } - #read(path: string, limit: number, signal?: AbortSignal): Promise { + #read( + path: string, + limit: number, + signal: AbortSignal | undefined, + resources: ArtifactResourceScope, + ): Promise { return withRegularFile( path, async (handle, stats) => { @@ -754,26 +914,83 @@ export class LinuxHopperPublisher { ); return bytes; }, - { signal, resources: this.#resources, symlinks: "reject" }, - ); + { signal, symlinks: "reject", resources }, + ).catch((cause: unknown) => { + if ( + resources === this.#resources && + cause instanceof RegularFileReadCleanupFailure + ) + pendingCleanup.add(resources); + throw cause; + }); } async #withTemporary( - operation: (path: string) => Promise, + operation: (path: string, resources: ArtifactResourceScope) => Promise, parent?: string, ): Promise { const root = await PrivateRuntimeRoot.create({ prefix: "rea-hopper-publisher-", ...(parent === undefined ? {} : { parent }), }); - return this.#withOwnedResource( - { - kind: "private-runtime-root", - resource: root.path, - root, + const temporaryResources = new ArtifactResourceScope(); + const dependency = { + kind: "lease" as const, + resource: root.path, + lease: { + release: async () => { + await temporaryResources.close(); + await root.close(); + }, }, - () => operation(root.path), + }; + let outcome: + | { readonly ok: true; readonly value: T } + | { readonly ok: false; readonly cause: unknown }; + try { + outcome = { + ok: true, + value: await operation(root.path, temporaryResources), + }; + } catch (cause: unknown) { + outcome = { ok: false, cause }; + } + const released = await temporaryResources.close().then( + () => ({ ok: true as const }), + (cause: unknown) => ({ ok: false as const, cause }), ); + if (!released.ok) { + this.#resources.retain(dependency); + pendingCleanup.add(this.#resources); + throw new HopperReleaseError( + "cleanup_failed", + `Hopper publisher verification retained temporary cleanup ownership for ${root.path}.`, + { + cause: new AggregateError( + outcome.ok ? [released.cause] : [outcome.cause, released.cause], + "Temporary operation and descendant cleanup outcomes", + ), + }, + ); + } + try { + await root.close(); + } catch (cleanupCause: unknown) { + this.#resources.retain(dependency); + pendingCleanup.add(this.#resources); + throw new HopperReleaseError( + "cleanup_failed", + `Hopper publisher verification retained temporary cleanup ownership for ${root.path}.`, + { + cause: new AggregateError( + outcome.ok ? [cleanupCause] : [outcome.cause, cleanupCause], + "Temporary operation and cleanup outcomes", + ), + }, + ); + } + if (!outcome.ok) throw outcome.cause; + return outcome.value; } async #withOwnedResource( @@ -788,6 +1005,20 @@ export class LinuxHopperPublisher { } catch (cause: unknown) { outcome = { ok: false, cause }; } + if (this.#resources.hasPending) { + this.#resources.retain(owner); + pendingCleanup.add(this.#resources); + throw new HopperReleaseError( + "cleanup_failed", + `Hopper publisher verification retained cleanup ownership for ${owner.resource}.`, + { + cause: new AggregateError( + outcome.ok ? [] : [outcome.cause], + "Publisher operation and descendant cleanup outcomes", + ), + }, + ); + } const cleanup = await this.#resources.release(owner); if (cleanup.kind === "failed") { pendingCleanup.add(this.#resources); @@ -852,22 +1083,28 @@ const releaseFailure = ( cause: unknown, signal: AbortSignal | undefined, fallback: HopperReleaseError["reason"], -): HopperReleaseError => - cause instanceof HopperReleaseError && cause.reason === "cleanup_failed" - ? cause - : signal?.aborted === true - ? new HopperReleaseError( - "cancelled", - "Hopper publisher verification was cancelled.", - { cause }, - ) - : cause instanceof HopperReleaseError - ? cause - : new HopperReleaseError( - fallback, - "Hopper publisher verification could not complete.", - { cause }, - ); +): HopperReleaseError => { + if (cause instanceof HopperReleaseError && cause.reason === "cleanup_failed") + return cause; + if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined) + return new HopperReleaseError( + "cleanup_failed", + "Hopper publisher cleanup remains incomplete.", + { cause }, + ); + if (signal?.aborted === true) + return new HopperReleaseError( + "cancelled", + "Hopper publisher verification was cancelled.", + { cause }, + ); + if (cause instanceof HopperReleaseError) return cause; + return new HopperReleaseError( + fallback, + "Hopper publisher verification could not complete.", + { cause }, + ); +}; const launcherIdentity = ( verified: VerifiedLinuxHopperPackage, source: VerifiedLinuxHopperLauncher["source"], diff --git a/src/application/ProviderOperationExecution.ts b/src/application/ProviderOperationExecution.ts index 928366e1ac..b9d20f38ef 100644 --- a/src/application/ProviderOperationExecution.ts +++ b/src/application/ProviderOperationExecution.ts @@ -30,24 +30,6 @@ export interface ProviderOperationPort< ): Promise>; } -/** Preserve valid completed Evidence when cancellation arrives at completion. */ -export const completeProviderEvidence = ( - completed: Result, - operation: string, - options?: ExecutionOptions, - errorOptions?: Omit, -): Result => { - if (!completed.ok) return completed; - return options?.signal?.aborted - ? err( - new AnalysisCancelledError(operation, { - ...errorOptions, - partialObservation: completed.value, - }), - ) - : completed; -}; - /** * Shared CLI/MCP admission, execution, and Evidence composition for * single-request providers. The caller retains its operation vocabulary and @@ -65,7 +47,9 @@ export const executeProviderOperation = async < confidence: "observed" | "derived", ): Promise> => { if (options?.signal?.aborted) - return err(new AnalysisCancelledError(operation)); + return err( + new AnalysisCancelledError(operation, { cause: options.signal.reason }), + ); const request = schema.safeParse({ operation, input }); if (!request.success) return err(new AnalysisInputError(operation, { cause: request.error })); @@ -85,10 +69,22 @@ export const executeProviderOperation = async < confidence, }, ); - // A provider that cannot observe the signal may finish its work after the - // caller cancelled, so re-check rather than reporting a completed operation - // the caller no longer wants. The observation it completed is still this - // provider's Evidence, so the cancellation carries it as the partial - // observation instead of discarding a finished result. - return completeProviderEvidence(ok(evidence), operation, options); + return completeProviderEvidence(evidence, options?.signal); +}; + +/** Classify completion after Evidence assembly, preserving facts on late abort. */ +export const completeProviderEvidence = ( + evidence: Evidence, + signal?: AbortSignal, + capturedOutput?: AnalysisErrorOptions["capturedOutput"], +): Result => { + if (signal?.aborted) + return err( + new AnalysisCancelledError(evidence.operation, { + cause: signal.reason, + partialObservation: evidence, + ...(capturedOutput === undefined ? {} : { capturedOutput }), + }), + ); + return ok(evidence); }; diff --git a/src/application/RegularFileRead.ts b/src/application/RegularFileRead.ts index ee4d6170a2..85588aed1d 100644 --- a/src/application/RegularFileRead.ts +++ b/src/application/RegularFileRead.ts @@ -2,12 +2,14 @@ import type { Stats } from "node:fs"; import { lstat, stat, type FileHandle } from "node:fs/promises"; import { - RegularFileAdmissionFailure, - openRegularFile, sameRegularFileState, RegularFileChangedError, } from "../filesystem/RegularFile.js"; -import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { + RegularFileCleanupFailure, + withOwnedRegularFile, + type RegularFileOperationOutcome, +} from "../filesystem/OwnedRegularFile.js"; import { readBoundedFileBytes } from "../process/BoundedFileBytes.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import type { AnalysisCleanupObservation } from "../domain/analysisErrorBase.js"; @@ -30,24 +32,9 @@ export class InvalidUtf8FileError extends TypeError { } } -export type RegularFileReadOutcome = - | { readonly kind: "completed"; readonly value: Value } - | { readonly kind: "failed"; readonly cause: unknown }; +export { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js"; -/** A close failed after reading; this error transfers the exact handle owner. */ -export class RegularFileCleanupFailure extends Error { - constructor( - readonly path: string, - readonly owner: OwnedFileHandle, - readonly outcome: RegularFileReadOutcome, - readonly cleanupCause: unknown, - ) { - super(`Regular file cleanup failed: ${path}`, { - cause: outcome.kind === "failed" ? outcome.cause : cleanupCause, - }); - this.name = "RegularFileCleanupFailure"; - } -} +export type RegularFileReadOutcome = RegularFileOperationOutcome; /** Retry a failed close through its retained owner and report any remaining cleanup. */ export const retryRegularFileCleanup = async ( @@ -80,73 +67,52 @@ export const withRegularFile = async ( path: string, read: (handle: FileHandle, stats: Stats) => Promise, options: RegularFileReadOptions = {}, -): Promise => - options.resources === undefined - ? withRegularFileOwned(path, read, options) - : options.resources.run(() => withRegularFileOwned(path, read, options)); +): Promise => withRegularFileOwned(path, read, options); const withRegularFileOwned = async ( path: string, read: (handle: FileHandle, stats: Stats) => Promise, options: RegularFileReadOptions, ): Promise => { - const { signal } = options; - let handle: FileHandle; - try { - handle = await openRegularFile(path, { - symlinks: options.symlinks ?? "follow", - signal, - }); - } catch (cause: unknown) { - if (!(cause instanceof RegularFileAdmissionFailure)) throw cause; - const failure = new RegularFileCleanupFailure( - path, - cause.owner, - { kind: "failed", cause: cause.cause }, - cause.cleanupCause, - ); - if (options.resources === undefined) throw failure; - const retry = await retryRegularFileCleanup(failure, options.resources); - if (retry.cleanup !== undefined) throw failure; - throw cause.cause; - } - const owner = new OwnedFileHandle(handle); - let outcome: RegularFileReadOutcome; - try { - const stats = await handle.stat(); - signal?.throwIfAborted(); - const value = await read(handle, stats); - signal?.throwIfAborted(); - const [opened, currentPath] = await Promise.all([ - handle.stat(), - options.symlinks === "reject" ? lstat(path) : stat(path), - ]); - if ( - !sameRegularFileState(stats, opened) || - !sameRegularFileState(stats, currentPath) - ) - throw new RegularFileChangedError(path); - outcome = { kind: "completed", value }; - } catch (cause: unknown) { - outcome = { kind: "failed", cause }; - } - try { - await owner.close(); - } catch (cleanupCause: unknown) { - const failure = new RegularFileCleanupFailure( - path, - owner, - outcome, - cleanupCause, - ); - if (options.resources === undefined) throw failure; - const retry = await retryRegularFileCleanup(failure, options.resources); - if (retry.cleanup !== undefined) throw failure; - if (retry.outcome.kind === "failed") throw retry.outcome.cause; - return retry.outcome.value; - } - if (outcome.kind === "failed") throw outcome.cause; - return outcome.value; + const { signal, resources } = options; + const operation = async () => { + try { + return await withOwnedRegularFile( + path, + { + ...(signal === undefined ? {} : { signal }), + symlinks: options.symlinks ?? "follow", + }, + async (handle) => { + const stats = await handle.stat(); + signal?.throwIfAborted(); + const value = await read(handle, stats); + signal?.throwIfAborted(); + const [opened, currentPath] = await Promise.all([ + handle.stat(), + options.symlinks === "reject" ? lstat(path) : stat(path), + ]); + if ( + !sameRegularFileState(stats, opened) || + !sameRegularFileState(stats, currentPath) + ) + throw new RegularFileChangedError(path); + return value; + }, + ); + } catch (cause: unknown) { + if ( + !(cause instanceof RegularFileCleanupFailure) || + resources === undefined + ) + throw cause; + const retry = await retryRegularFileCleanup(cause, resources); + if (retry.cleanup !== undefined) throw cause; + if (retry.outcome.kind === "failed") throw retry.outcome.cause; + return retry.outcome.value; + } + }; + return resources === undefined ? operation() : resources.run(operation); }; /** Admit a regular file without waiting for a pipe, then read its verified handle. */ diff --git a/src/application/Uninstall.ts b/src/application/Uninstall.ts index 2c8b948cc2..5173431ed5 100644 --- a/src/application/Uninstall.ts +++ b/src/application/Uninstall.ts @@ -366,9 +366,12 @@ const readClientConfiguration = async ( original = await fileSystem.readText(transactionPath); } catch (cause: unknown) { const readFailure = - cause instanceof RegularFileCleanupFailure ? cause.cause : cause; + cause instanceof RegularFileCleanupFailure && + cause.outcome.kind === "failed" + ? cause.outcome.cause + : cause; return itemRead( - isMissing(cause) + isMissing(readFailure) ? item(client.name, "skipped", "Configuration does not exist.") : item( client.name, diff --git a/src/application/WebNetworkCaptureService.ts b/src/application/WebNetworkCaptureService.ts index 6e6724ff30..b755ae570f 100644 --- a/src/application/WebNetworkCaptureService.ts +++ b/src/application/WebNetworkCaptureService.ts @@ -159,54 +159,49 @@ export class WebNetworkCaptureService { sensitive_value_count: input.sensitive_values.length, }); return completeProviderEvidence( - ok( - createEvidence( - safeSha256 === null - ? undefined - : { - path: safePath, - sha256: safeSha256, - format: "file", - }, - value.decoder, - { - operation: OPERATION, - parameters, - result: { - ...safeValue, - artifact: { - ...value.artifact, - path: safePath, - sha256: safeSha256, - }, - records, - limitations: - safeSha256 === null - ? [...limitations, subjectUnavailableReason] - : limitations, + createEvidence( + safeSha256 === null + ? undefined + : { + path: safePath, + sha256: safeSha256, + format: "file", }, - ...(safeValue.format !== "qlog" - ? {} - : { - analysisProfile: createAnalysisProfile(value.decoder, { - qlog_profile: safeValue.qlog.profile, - clock: safeValue.qlog.clock, - scope: safeValue.qlog.scope, - }), - }), - confidence: "observed", - authority: "historical-reference", - limitations, - ...(safeSha256 === null ? { subjectUnavailableReason } : {}), - locations: - safePath === "" - ? [] - : [{ kind: "artifact-path", path: safePath }], + value.decoder, + { + operation: OPERATION, + parameters, + result: { + ...safeValue, + artifact: { + ...value.artifact, + path: safePath, + sha256: safeSha256, + }, + records, + limitations: + safeSha256 === null + ? [...limitations, subjectUnavailableReason] + : limitations, }, - ), + ...(safeValue.format !== "qlog" + ? {} + : { + analysisProfile: createAnalysisProfile(value.decoder, { + qlog_profile: safeValue.qlog.profile, + clock: safeValue.qlog.clock, + scope: safeValue.qlog.scope, + }), + }), + confidence: "observed", + authority: "historical-reference", + limitations, + ...(safeSha256 === null ? { subjectUnavailableReason } : {}), + locations: + safePath === "" ? [] : [{ kind: "artifact-path", path: safePath }], + }, ), - OPERATION, - options, + options?.signal, ); } } diff --git a/src/application/WebRuntimeService.test.ts b/src/application/WebRuntimeService.test.ts index b9709f2eb4..b0160d4d0c 100644 --- a/src/application/WebRuntimeService.test.ts +++ b/src/application/WebRuntimeService.test.ts @@ -108,6 +108,9 @@ it("retains completed execution Evidence on late abort and keeps invalid output 20_000, ); const controller = new AbortController(); + const cancellationReason = new Error( + "caller cancelled the runtime observation", + ); const completed = await new WebRuntimeService({ identity: () => ({ id: "web-runtime", @@ -115,20 +118,21 @@ it("retains completed execution Evidence on late abort and keeps invalid output version: "1", }), observeExecution: async () => { - controller.abort(); + controller.abort(cancellationReason); return ok(webExecutionFixture(input.target_id, input.observation_ms)); }, inspectEventListeners: async () => err(unused), }).observe(input, { signal: controller.signal }); - expect(completed.ok).toBe(false); - if (!completed.ok) - expect(parseEvidence(completed.error.partialObservation)).toMatchObject({ - operation: "observe_web_execution", - normalized_result: webExecutionFixture( - input.target_id, - input.observation_ms, - ), - }); + if (completed.ok) throw new Error("Expected late cancellation"); + expect(completed.error._tag).toBe("AnalysisCancelledError"); + expect(completed.error.cause).toBe(cancellationReason); + expect(parseEvidence(completed.error.partialObservation)).toMatchObject({ + operation: "observe_web_execution", + normalized_result: webExecutionFixture( + input.target_id, + input.observation_ms, + ), + }); const late = new AbortController(); const invalid = await new WebRuntimeService({ diff --git a/src/application/WebRuntimeService.ts b/src/application/WebRuntimeService.ts index f924b557bd..d7180b397f 100644 --- a/src/application/WebRuntimeService.ts +++ b/src/application/WebRuntimeService.ts @@ -166,19 +166,16 @@ export class WebRuntimeService { ), ); return completeProviderEvidence( - ok( - createEvidence(undefined, this.provider.identity(), { - predicateType: `rea.${operation.replaceAll("_", "-")}`, - operation, - parameters: jsonObjectSchema.parse(input), - result, - confidence: "observed", - authority: "external-service", - limitations: result.limitations, - }), - ), - operation, - options, + createEvidence(undefined, this.provider.identity(), { + predicateType: `rea.${operation.replaceAll("_", "-")}`, + operation, + parameters: jsonObjectSchema.parse(input), + result, + confidence: "observed", + authority: "external-service", + limitations: result.limitations, + }), + options?.signal, ); } } diff --git a/src/application/WebSourceLocationService.ts b/src/application/WebSourceLocationService.ts index 563a7b81ad..55731d5ac7 100644 --- a/src/application/WebSourceLocationService.ts +++ b/src/application/WebSourceLocationService.ts @@ -147,40 +147,37 @@ export class WebSourceLocationService { }); if (!normalized.success) return invalidOutput(normalized.error.message); return completeProviderEvidence( - ok( - createEvidence( - { - path: data.sourceFile.path, - sha256: data.sourceFile.sha256, - format: "file", - }, - { - id: "rea-web-source-location", - name: "REA captured source-map point trace", - version: "1", - }, - { - operation: OPERATION, - parameters: jsonObjectSchema.parse(input), - result: normalized.data, - rawResult: { source_map_text: map.text }, - confidence: "derived", - authority: "historical-reference", - limitations, - locations: [ - data.sourceFile.path, - data.manifestFile.path, - map.file.path, - ].map((path) => ({ kind: "artifact-path", path })), - evidenceLinks: - data.manifest.source_evidence_id === null - ? [] - : [data.manifest.source_evidence_id], - }, - ), + createEvidence( + { + path: data.sourceFile.path, + sha256: data.sourceFile.sha256, + format: "file", + }, + { + id: "rea-web-source-location", + name: "REA captured source-map point trace", + version: "1", + }, + { + operation: OPERATION, + parameters: jsonObjectSchema.parse(input), + result: normalized.data, + rawResult: { source_map_text: map.text }, + confidence: "derived", + authority: "historical-reference", + limitations, + locations: [ + data.sourceFile.path, + data.manifestFile.path, + map.file.path, + ].map((path) => ({ kind: "artifact-path", path })), + evidenceLinks: + data.manifest.source_evidence_id === null + ? [] + : [data.manifest.source_evidence_id], + }, ), - OPERATION, - options, + options?.signal, ); } } diff --git a/src/application/adb/AdbDeviceAnalysisService.test.ts b/src/application/adb/AdbDeviceAnalysisService.test.ts index 6e3f62641b..9cd84d3f2e 100644 --- a/src/application/adb/AdbDeviceAnalysisService.test.ts +++ b/src/application/adb/AdbDeviceAnalysisService.test.ts @@ -81,7 +81,8 @@ it("composes observed Evidence with the caller parameters inline", async () => { it("preserves cancellation before execution", async () => { const controller = new AbortController(); - controller.abort(); + const cancellationReason = { source: "caller", operation: "shutdown" }; + controller.abort(cancellationReason); const service = new AdbDeviceAnalysisService(refusedPort); const result = await service.execute( "list_adb_devices", @@ -90,7 +91,7 @@ it("preserves cancellation before execution", async () => { signal: controller.signal, }, ); - expect(result.ok).toBe(false); - if (result.ok) return; + if (result.ok) throw new Error("Expected cancellation before execution"); expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(cancellationReason); }); diff --git a/src/application/android/AndroidAnalysisService.ts b/src/application/android/AndroidAnalysisService.ts index eae2efb8d8..d642764811 100644 --- a/src/application/android/AndroidAnalysisService.ts +++ b/src/application/android/AndroidAnalysisService.ts @@ -12,7 +12,8 @@ import { import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import { createEvidence, type Evidence } from "../../domain/evidence.js"; import { jsonObjectSchema } from "../../domain/jsonValue.js"; -import { err, ok, type Result } from "../../domain/result.js"; +import { err, type Result } from "../../domain/result.js"; +import { completeProviderEvidence } from "../ProviderOperationExecution.js"; /** Shared CLI/MCP admission, execution and Evidence composition for Android. */ export class AndroidAnalysisService { @@ -41,7 +42,7 @@ export class AndroidAnalysisService { ); if (!executed.ok) return executed; const execution = executed.value; - return ok( + return completeProviderEvidence( createEvidence(execution.subject ?? target.value, execution.provider, { operation, parameters: jsonObjectSchema.parse(request.data.input), @@ -52,6 +53,7 @@ export class AndroidAnalysisService { confidence: operation === "inspect_android_method" ? "derived" : "observed", }), + options?.signal, ); } } diff --git a/src/application/binary/AnalysisProviderEvaluation.ts b/src/application/binary/AnalysisProviderEvaluation.ts index 27292ed31b..559e18b084 100644 --- a/src/application/binary/AnalysisProviderEvaluation.ts +++ b/src/application/binary/AnalysisProviderEvaluation.ts @@ -13,6 +13,7 @@ import { err, ok, type Result } from "../../domain/result.js"; import type { AnalysisProviderCandidate } from "../AnalysisProvider.js"; import type { AnalysisProviderCandidateStatus } from "./AnalysisProviderRegistry.js"; import { ABORTED, waitForAbortable } from "../../process/AbortablePromise.js"; +import type { FileCloseResourceOwner } from "../AnalysisProvider.js"; /** Profile-resolution truth for one discovered provider candidate. */ export interface AnalysisProviderCandidateEvaluation { @@ -28,6 +29,7 @@ export const evaluateAnalysisProviderCandidate = async ( status: AnalysisProviderCandidateStatus, target: BinaryTarget, signal: AbortSignal | undefined, + fileCloseResources?: FileCloseResourceOwner, ): Promise< Result > => { @@ -38,7 +40,14 @@ export const evaluateAnalysisProviderCandidate = async ( const resolution = await waitForAbortable( candidate.resolveAnalysisProfile( target, - signal === undefined ? undefined : { signal }, + signal === undefined && fileCloseResources === undefined + ? undefined + : { + ...(signal === undefined ? {} : { signal }), + ...(fileCloseResources === undefined + ? {} + : { fileCloseResources }), + }, ), signal, ); diff --git a/src/application/binary/AnalysisProviderRegistry.ts b/src/application/binary/AnalysisProviderRegistry.ts index deac9d4372..427ceee16d 100644 --- a/src/application/binary/AnalysisProviderRegistry.ts +++ b/src/application/binary/AnalysisProviderRegistry.ts @@ -1,3 +1,4 @@ +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import { analysisProviderIdSchema, analysisProviderSelectorSchema, @@ -7,7 +8,6 @@ import type { AnalysisProfileCommitment } from "../../domain/analysisProfile.js" import type { BinaryTarget } from "../../domain/binaryTargetTypes.js"; import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; -import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import { ProviderSelectionError, type ProviderSelectionFailureReason, @@ -27,6 +27,7 @@ import { type AnalysisProviderCandidateEvaluation, } from "./AnalysisProviderEvaluation.js"; import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; +import type { FileCloseResourceOwner } from "../AnalysisProvider.js"; /** Deterministic discovery truth for one configured deep provider. */ export interface AnalysisProviderCandidateStatus { @@ -66,6 +67,7 @@ interface TargetSelectionContext { readonly target: BinaryTarget; readonly baseline: readonly AnalysisProviderCandidateStatus[]; readonly signal: AbortSignal | undefined; + readonly fileCloseResources?: FileCloseResourceOwner; } /** @@ -181,7 +183,10 @@ export class AnalysisProviderRegistry { async select( target: BinaryTarget, requestedSelector?: AnalysisProviderSelector, - options: { readonly signal?: AbortSignal } = {}, + options: { + readonly signal?: AbortSignal; + readonly fileCloseResources?: FileCloseResourceOwner; + } = {}, ): Promise> { if (signalIsAborted(options.signal)) return err(new AnalysisCancelledError("open_binary")); @@ -219,7 +224,14 @@ export class AnalysisProviderRegistry { }); if (selector !== "auto") return this.#selectConcrete( - { target, baseline, signal: options.signal }, + { + target, + baseline, + signal: options.signal, + ...(options.fileCloseResources === undefined + ? {} + : { fileCloseResources: options.fileCloseResources }), + }, selector, request === undefined ? "environment" : "request", ); @@ -227,6 +239,9 @@ export class AnalysisProviderRegistry { target, baseline, signal: options.signal, + ...(options.fileCloseResources === undefined + ? {} + : { fileCloseResources: options.fileCloseResources }), }); } @@ -275,6 +290,7 @@ export class AnalysisProviderRegistry { initial, target, signal, + context.fileCloseResources, ); if (!evaluation.ok) return evaluation; const evaluated = evaluation.value; @@ -321,6 +337,7 @@ export class AnalysisProviderRegistry { status, target, signal, + context.fileCloseResources, ) : ok({ candidate: provider, diff --git a/src/application/binary/BinarySession.ts b/src/application/binary/BinarySession.ts index 9bb34760bd..fd02c8c12f 100644 --- a/src/application/binary/BinarySession.ts +++ b/src/application/binary/BinarySession.ts @@ -200,6 +200,7 @@ export class BinarySession current: this.#active, target, options, + fileCloseResources: this.fileCloseResourceOwner, stagedSnapshotMatches: (target, profile) => this.matchesSnapshot(target, profile), }).then((resolved) => @@ -224,6 +225,7 @@ export class BinarySession current: this.#active, path, options: admitted, + fileCloseResources: this.fileCloseResourceOwner, stagedSnapshotMatches: (target, profile) => this.matchesSnapshot(target, profile), }), @@ -337,7 +339,9 @@ export class BinarySession ), ); const runId = randomUUID(); - const client = route.createClient(target, { runId }); + const client = route.createClient(target, { + runId, + }); const run: SessionRun = { client, route, runId }; const started = await client.execute("health", {}, options); if (!started.ok) { @@ -499,26 +503,31 @@ export class BinarySession if (clearRecords) this.clearSessionRecords(); this.#clearRuntimeAvailability(); const fileResources = await this.closeWriteResources(); - const candidates = await this.#providerRouter.closeOwnedResources(); - let lifecycle: Result; - if (!closed.ok && !fileResources.ok) - lifecycle = err( - analysisErrorWithCleanupFailure( - closed.error, - fileResources.error, + const hashResources = await this.closeFailedFileCloses(); + const candidateResources = await this.#providerRouter.closeOwnedResources(); + const resourceFailures = [ + ...(fileResources.ok ? [] : [fileResources.error]), + ...(hashResources.ok ? [] : [hashResources.error]), + ...(candidateResources.ok ? [] : [candidateResources.error]), + ]; + let resourceFailure = resourceFailures[0]; + for (const next of resourceFailures.slice(1)) + if (resourceFailure !== undefined) + resourceFailure = analysisErrorWithCleanupFailure( + resourceFailure, + next, "close_binary", - ), - ); - else lifecycle = closed.ok ? fileResources : closed; - if (!lifecycle.ok && !candidates.ok) + ); + if (!closed.ok && resourceFailure !== undefined) return err( analysisErrorWithCleanupFailure( - lifecycle.error, - candidates.error, + closed.error, + resourceFailure, "close_binary", ), ); - return lifecycle.ok ? candidates : lifecycle; + if (!closed.ok) return closed; + return resourceFailure === undefined ? ok(null) : err(resourceFailure); } async #retireClient( @@ -596,7 +605,7 @@ export class BinarySession providerOperationHealth: client?.operationHealthSnapshot?.() ?? null, cleanupPending: this.#pendingCleanup !== undefined || - this.hasPendingWriteResources || + this.hasPendingFileResources || this.#providerRouter.hasPendingOwnedResources(), }); } @@ -932,7 +941,9 @@ export class BinarySession ): Promise> { if (previous === undefined) return ok(null); const runId = randomUUID(); - const client = previous.route.createClient(previous.target, { runId }); + const client = previous.route.createClient(previous.target, { + runId, + }); const run: SessionRun = { client, route: previous.route, runId }; const started = await client.execute("health", {}); if (started.ok) { diff --git a/src/application/binary/BinarySessionOpen.ts b/src/application/binary/BinarySessionOpen.ts index bb1631f350..263501aa63 100644 --- a/src/application/binary/BinarySessionOpen.ts +++ b/src/application/binary/BinarySessionOpen.ts @@ -15,6 +15,7 @@ import { AnalysisSnapshotMismatchError, } from "../../domain/evidenceErrors.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import type { FileCloseResourceOwner } from "../AnalysisProvider.js"; import { err, type Result } from "../../domain/result.js"; import type { SessionProviderRoute } from "./SessionProviderRouter.js"; import { SessionProviderRouter } from "./SessionProviderRouter.js"; @@ -60,6 +61,7 @@ interface ResolveSessionOpenInput { target: BinaryTarget, profile: AnalysisProfileCommitment | null, ) => boolean; + readonly fileCloseResources?: FileCloseResourceOwner; } interface ResolveSessionTargetInput { @@ -71,6 +73,7 @@ interface ResolveSessionTargetInput { target: BinaryTarget, profile: AnalysisProfileCommitment | null, ) => boolean; + readonly fileCloseResources?: FileCloseResourceOwner; } /** Parse a target, resolve its provider route, and validate snapshot binding. */ @@ -106,7 +109,12 @@ export const resolveSessionTarget = async ( const resolvedRoute = sameTarget && options.providerId === undefined && current !== undefined ? ({ ok: true, value: current.route } as const) - : await router.resolve(target, options.providerId, options.signal); + : await router.resolve( + target, + options.providerId, + options.signal, + input.fileCloseResources, + ); if (!resolvedRoute.ok) return resolvedRoute; return validateResolvedSessionTarget({ ...input, diff --git a/src/application/binary/BinarySessionRecords.ts b/src/application/binary/BinarySessionRecords.ts index 2ea612cb42..f88b27fd74 100644 --- a/src/application/binary/BinarySessionRecords.ts +++ b/src/application/binary/BinarySessionRecords.ts @@ -32,6 +32,8 @@ import { AnalysisSnapshotCache } from "./AnalysisSnapshotCache.js"; import { InvestigationRecords } from "../investigation/InvestigationRecords.js"; import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; +import type { FileCloseResourceOwner } from "../AnalysisProvider.js"; import { writeAnalysisSnapshot as writeSnapshotFile } from "./AnalysisSnapshotFiles.js"; const SNAPSHOT_MUTATION_RECOVERY = @@ -55,6 +57,22 @@ export interface WorkflowSnapshotRecordInput { export abstract class BinarySessionRecords { readonly #records: InvestigationRecords; readonly #writeResources = new ArtifactResourceScope(); + readonly #fileCloseResources = new ArtifactResourceScope(); + protected readonly fileCloseResourceOwner: FileCloseResourceOwner = { + retain: (owner, resource) => + this.#fileCloseResources.retain({ + kind: "lease", + resource, + lease: { release: () => owner.close() }, + }), + run: (operation) => this.#fileCloseResources.run(operation), + }; + + protected get hasPendingFileResources(): boolean { + return ( + this.#writeResources.hasPending || this.#fileCloseResources.hasPending + ); + } readonly #snapshot = new AnalysisSnapshotCache(); #snapshotInvalidated = false; readonly #snapshotListeners = new Set<() => void | Promise>(); @@ -63,11 +81,6 @@ export abstract class BinarySessionRecords { this.#records = records; } - /** Snapshot staging retained after a failed resource cleanup attempt. */ - protected get hasPendingWriteResources(): boolean { - return this.#writeResources.hasPending; - } - /** Observe changes to the mutable current analysis snapshot resource. */ onAnalysisSnapshotChanged(listener: () => void | Promise): () => void { this.#snapshotListeners.add(listener); @@ -137,6 +150,27 @@ export abstract class BinarySessionRecords { } } + /** Retry retained descriptors owned by this composed binary session. */ + async closeFailedFileCloses(): Promise> { + try { + await this.#fileCloseResources.retryCleanup(); + return ok(null); + } catch (cause: unknown) { + const observation = ArtifactReaderFailure.cleanupObservation( + cause, + "provider file descriptor", + ); + return err( + new ProviderCleanupError( + "rea", + observation.resources, + { reason: observation.reason }, + { cause, operation: "close_binary" }, + ), + ); + } + } + importEvidenceBundle( bundle: unknown, ): Result { diff --git a/src/application/binary/SessionProviderRouter.ts b/src/application/binary/SessionProviderRouter.ts index e5babdf669..9101a6b786 100644 --- a/src/application/binary/SessionProviderRouter.ts +++ b/src/application/binary/SessionProviderRouter.ts @@ -12,6 +12,7 @@ import type { CapabilityDescriptor, ProviderIdentity, } from "../AnalysisProvider.js"; +import type { FileCloseResourceOwner } from "../AnalysisProvider.js"; import { AnalysisProviderRegistry, type AnalysisProviderBinding, @@ -85,8 +86,15 @@ export class SessionProviderRouter { target: BinaryTarget, providerId?: AnalysisProviderSelector, signal?: AbortSignal, + fileCloseResources?: FileCloseResourceOwner, ): Promise> { - const resolutionOptions = signal === undefined ? undefined : { signal }; + const resolutionOptions = + signal === undefined && fileCloseResources === undefined + ? undefined + : { + ...(signal === undefined ? {} : { signal }), + ...(fileCloseResources === undefined ? {} : { fileCloseResources }), + }; const selected = await this.registry.select( target, providerId, diff --git a/src/application/binaryDiagnostics/BinaryLayoutService.test.ts b/src/application/binaryDiagnostics/BinaryLayoutService.test.ts index 7e0a02e69e..76532d5e28 100644 --- a/src/application/binaryDiagnostics/BinaryLayoutService.test.ts +++ b/src/application/binaryDiagnostics/BinaryLayoutService.test.ts @@ -85,6 +85,10 @@ it("cancels before invocation and after an acquired observation", async () => { error: { _tag: "AnalysisCancelledError", capturedOutput: binaryLayoutFixture().diagnostics, + partialObservation: { + operation: "inspect_binary_layout", + normalized_result: binaryLayoutFixture(), + }, }, }); if (!late.ok) { diff --git a/src/application/binaryDiagnostics/BinaryLayoutService.ts b/src/application/binaryDiagnostics/BinaryLayoutService.ts index ae2d729e1f..291bc1fee1 100644 --- a/src/application/binaryDiagnostics/BinaryLayoutService.ts +++ b/src/application/binaryDiagnostics/BinaryLayoutService.ts @@ -91,30 +91,27 @@ export class BinaryLayoutService { ); const value = report.data; return completeProviderEvidence( - ok( - createEvidence( - { - path: value.artifact.path, - format: "elf", - architecture: "x86_64", - sha256: value.artifact.sha256, - }, - this.provider.identity, - { - operation: OPERATION, - parameters: input.data, - result: value, - // The decoded report is the only representation; do not repeat it. - rawResult: null, - confidence: "observed", - limitations: value.limitations, - locations: [{ kind: "artifact-path", path: value.artifact.path }], - }, - ), + createEvidence( + { + path: value.artifact.path, + format: "elf", + architecture: "x86_64", + sha256: value.artifact.sha256, + }, + this.provider.identity, + { + operation: OPERATION, + parameters: input.data, + result: value, + // The decoded report is the only representation; do not repeat it. + rawResult: null, + confidence: "observed", + limitations: value.limitations, + locations: [{ kind: "artifact-path", path: value.artifact.path }], + }, ), - OPERATION, - options, - outputOptions, + options?.signal, + outputOptions?.capturedOutput, ); } } diff --git a/src/application/binaryDiagnostics/PeResourcesService.ts b/src/application/binaryDiagnostics/PeResourcesService.ts index b119a7cac3..c64838e572 100644 --- a/src/application/binaryDiagnostics/PeResourcesService.ts +++ b/src/application/binaryDiagnostics/PeResourcesService.ts @@ -70,26 +70,23 @@ export class PeResourcesService { ); const value = report.data; return completeProviderEvidence( - ok( - createEvidence( - { - path: value.artifact.path, - format: "pe", - sha256: value.artifact.sha256, - }, - this.provider.identity, - { - operation, - parameters: input.data, - result: value, - confidence: "observed", - limitations: value.limitations, - locations: [{ kind: "artifact-path", path: value.artifact.path }], - }, - ), + createEvidence( + { + path: value.artifact.path, + format: "pe", + sha256: value.artifact.sha256, + }, + this.provider.identity, + { + operation, + parameters: input.data, + result: value, + confidence: "observed", + limitations: value.limitations, + locations: [{ kind: "artifact-path", path: value.artifact.path }], + }, ), - operation, - options, + options?.signal, ); } } diff --git a/src/application/binaryDiagnostics/RecordedCrashService.test.ts b/src/application/binaryDiagnostics/RecordedCrashService.test.ts index 5e1bf9bf79..7e225b795e 100644 --- a/src/application/binaryDiagnostics/RecordedCrashService.test.ts +++ b/src/application/binaryDiagnostics/RecordedCrashService.test.ts @@ -114,6 +114,10 @@ it("cancels before invocation and after an acquired observation", async () => { error: { _tag: "AnalysisCancelledError", capturedOutput: recordedCrashFixture().diagnostics, + partialObservation: { + operation: "inspect_recorded_crash", + normalized_result: recordedCrashFixture(), + }, }, }); if (!late.ok) { diff --git a/src/application/binaryDiagnostics/RecordedCrashService.ts b/src/application/binaryDiagnostics/RecordedCrashService.ts index 25872d9e3f..5c08075651 100644 --- a/src/application/binaryDiagnostics/RecordedCrashService.ts +++ b/src/application/binaryDiagnostics/RecordedCrashService.ts @@ -97,33 +97,30 @@ export class RecordedCrashService { ); const value = report.data; return completeProviderEvidence( - ok( - createEvidence( - { - path: value.artifact.path, - format: "elf", - architecture: - RECORDED_LINUX_CORE_PROFILES[value.architecture] - .evidenceArchitecture, - sha256: value.artifact.sha256, - }, - this.provider.identity, - { - operation: OPERATION, - parameters: input.data, - result: value, - // The decoded report is the only representation; do not repeat it. - rawResult: null, - confidence: - value.debugger.status === "available" ? "derived" : "observed", - limitations: value.limitations, - locations: [{ kind: "artifact-path", path: value.artifact.path }], - }, - ), + createEvidence( + { + path: value.artifact.path, + format: "elf", + architecture: + RECORDED_LINUX_CORE_PROFILES[value.architecture] + .evidenceArchitecture, + sha256: value.artifact.sha256, + }, + this.provider.identity, + { + operation: OPERATION, + parameters: input.data, + result: value, + // The decoded report is the only representation; do not repeat it. + rawResult: null, + confidence: + value.debugger.status === "available" ? "derived" : "observed", + limitations: value.limitations, + locations: [{ kind: "artifact-path", path: value.artifact.path }], + }, ), - OPERATION, - options, - outputOptions, + options?.signal, + outputOptions?.capturedOutput, ); } } diff --git a/src/application/electron/ElectronFuseService.ts b/src/application/electron/ElectronFuseService.ts index eac540bcd2..769a6ab90d 100644 --- a/src/application/electron/ElectronFuseService.ts +++ b/src/application/electron/ElectronFuseService.ts @@ -64,27 +64,24 @@ export class ElectronFuseService { const value = report.data; const architecture = subjectArchitecture(value); return completeProviderEvidence( - ok( - createEvidence( - { - path: value.artifact.resolved_path, - sha256: value.artifact.sha256, - format: value.artifact.format, - ...(architecture === undefined ? {} : { architecture }), - }, - this.provider.identity, - { - operation: OPERATION, - parameters: input.data, - result: value, - confidence: "observed", - limitations: value.limitations, - locations: locationsFor(value), - }, - ), + createEvidence( + { + path: value.artifact.resolved_path, + sha256: value.artifact.sha256, + format: value.artifact.format, + ...(architecture === undefined ? {} : { architecture }), + }, + this.provider.identity, + { + operation: OPERATION, + parameters: input.data, + result: value, + confidence: "observed", + limitations: value.limitations, + locations: locationsFor(value), + }, ), - OPERATION, - options, + options?.signal, ); } } diff --git a/src/application/evm/EvmInterfaceService.test.ts b/src/application/evm/EvmInterfaceService.test.ts index f07537d45e..1f89258023 100644 --- a/src/application/evm/EvmInterfaceService.test.ts +++ b/src/application/evm/EvmInterfaceService.test.ts @@ -97,6 +97,10 @@ it("preserves meaningful provider failure and cancellation before/after executio error: { _tag: "AnalysisCancelledError", capturedOutput: observed.diagnostics, + partialObservation: { + operation: "inspect_evm_interface", + normalized_result: observed, + }, }, }); if (!late.ok) { diff --git a/src/application/evm/EvmInterfaceService.ts b/src/application/evm/EvmInterfaceService.ts index c0812ca2de..01e001a9b0 100644 --- a/src/application/evm/EvmInterfaceService.ts +++ b/src/application/evm/EvmInterfaceService.ts @@ -78,22 +78,19 @@ export class EvmInterfaceService { ), ); return completeProviderEvidence( - ok( - createEvidence(execution.subject, execution.provider, { - operation: OPERATION, - parameters: input.data, - result: report.data, - ...(execution.rawResult === null - ? {} - : { rawResult: execution.rawResult }), - confidence: "inferred", - locations: execution.locations, - limitations: execution.limitations, - }), - ), - OPERATION, - options, - outputOptions, + createEvidence(execution.subject, execution.provider, { + operation: OPERATION, + parameters: input.data, + result: report.data, + ...(execution.rawResult === null + ? {} + : { rawResult: execution.rawResult }), + confidence: "inferred", + locations: execution.locations, + limitations: execution.limitations, + }), + options?.signal, + outputOptions?.capturedOutput, ); } } diff --git a/src/application/firmware/FirmwareAnalysisService.test.ts b/src/application/firmware/FirmwareAnalysisService.test.ts new file mode 100644 index 0000000000..91520fd9d0 --- /dev/null +++ b/src/application/firmware/FirmwareAnalysisService.test.ts @@ -0,0 +1,91 @@ +import { expect, it } from "vitest"; + +import { FirmwareAnalysisService } from "./FirmwareAnalysisService.js"; +import { createAnalysisExecution } from "../AnalysisProvider.js"; +import { firmwareResultSchemas } from "../../domain/firmware/firmwareAnalysis.js"; +import { AnalysisOutputError } from "../../domain/analysisErrorCore.js"; +import { err, ok } from "../../domain/result.js"; + +it.each(["inspect_firmware_regions", "extract_firmware"] as const)( + "retains %s Evidence after late cancellation and preserves provider failure precedence", + async (operation) => { + const path = "/owned-fixture/input.bin"; + const input = { + path, + ...(operation === "extract_firmware" + ? { output_directory: "/owned-fixture/output" } + : {}), + }; + const engine = { + name: "fixture", + version: "1", + executable_path: "/fixture/engine", + executable_sha256: "b".repeat(64), + source_revision: null, + worker_count: 1, + }; + const normalized = firmwareResultSchemas[operation].parse( + operation === "inspect_firmware_regions" + ? { engine, input_size: 1, regions: [], coverage: "signature_scan" } + : { + engine, + selection: { offset: 0, length: 1, sha256: "a".repeat(64) }, + output_directory: "/owned-fixture/output", + coverage: "complete", + files: [], + unpublished_entries: [], + chunks: [], + derivations: [], + depth_limited_paths: [], + diagnostics: [], + extraction_gaps: [], + extraction_continuations: [], + published_bytes: 0, + }, + ); + const execution = createAnalysisExecution( + normalized, + { id: "fixture", name: "Fixture firmware", version: "1" }, + { + subject: { + path, + format: "file", + sha256: "a".repeat(64), + }, + rawResult: { source: "controlled producer" }, + }, + ); + const control = await new FirmwareAnalysisService({ + execute: async () => ok(execution), + }).execute(operation, input); + if (!control.ok) throw control.error; + const controller = new AbortController(); + const result = await new FirmwareAnalysisService({ + execute: async () => { + controller.abort(); + return ok(execution); + }, + }).execute(operation, input, { signal: controller.signal }); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + operation, + partialObservation: control.value, + }, + }); + const failedController = new AbortController(); + const failure = new AnalysisOutputError( + operation, + "controlled producer failure", + ); + expect( + await new FirmwareAnalysisService({ + execute: async () => { + failedController.abort(); + return err(failure); + }, + }).execute(operation, input, { signal: failedController.signal }), + ).toEqual(err(failure)); + }, +); diff --git a/src/application/firmware/FirmwareAnalysisService.ts b/src/application/firmware/FirmwareAnalysisService.ts index 50e7ee7bb2..774dd5ca65 100644 --- a/src/application/firmware/FirmwareAnalysisService.ts +++ b/src/application/firmware/FirmwareAnalysisService.ts @@ -12,7 +12,8 @@ import { import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import { createEvidence, type Evidence } from "../../domain/evidence.js"; import { jsonObjectSchema } from "../../domain/jsonValue.js"; -import { err, ok, type Result } from "../../domain/result.js"; +import { err, type Result } from "../../domain/result.js"; +import { completeProviderEvidence } from "../ProviderOperationExecution.js"; /** Shared CLI/MCP validation and Evidence composition for arbitrary firmware files. */ export class FirmwareAnalysisService { @@ -43,7 +44,7 @@ export class FirmwareAnalysisService { "Firmware execution did not bind its source artifact", ), ); - return ok( + return completeProviderEvidence( createEvidence(execution.subject, execution.provider, { operation, parameters: jsonObjectSchema.parse(parsed.data.input), @@ -53,6 +54,7 @@ export class FirmwareAnalysisService { limitations: execution.limitations, confidence: "observed", }), + options?.signal, ); } } diff --git a/src/application/go/GoBinaryService.ts b/src/application/go/GoBinaryService.ts index 818526b687..92c288c33d 100644 --- a/src/application/go/GoBinaryService.ts +++ b/src/application/go/GoBinaryService.ts @@ -76,38 +76,35 @@ export class GoBinaryService { const value = report.data; const architecture = architectureForSubject(value.architecture); return completeProviderEvidence( - ok( - createEvidence( - { - path: value.artifact.path, - sha256: value.artifact.sha256, - format: value.format === "macho" ? "mach-o" : value.format, - ...(architecture === undefined ? {} : { architecture }), - }, - this.provider.identity, - { - operation: OPERATION, - parameters: input.data, - result: value, - confidence: "observed", - limitations: value.limitations, - locations: [ - { kind: "artifact-path", path: value.artifact.path }, - ...(value.build_info === null - ? [] - : [ - { - kind: "file-offset-range" as const, - start: value.build_info.header_offset, - end: value.build_info.header_offset + 32, - }, - ]), - ], - }, - ), + createEvidence( + { + path: value.artifact.path, + sha256: value.artifact.sha256, + format: value.format === "macho" ? "mach-o" : value.format, + ...(architecture === undefined ? {} : { architecture }), + }, + this.provider.identity, + { + operation: OPERATION, + parameters: input.data, + result: value, + confidence: "observed", + limitations: value.limitations, + locations: [ + { kind: "artifact-path", path: value.artifact.path }, + ...(value.build_info === null + ? [] + : [ + { + kind: "file-offset-range" as const, + start: value.build_info.header_offset, + end: value.build_info.header_offset + 32, + }, + ]), + ], + }, ), - OPERATION, - options, + options?.signal, ); } } diff --git a/src/application/javascript/ElectronObservationService.test.ts b/src/application/javascript/ElectronObservationService.test.ts index 8fbf6fef92..4222e62027 100644 --- a/src/application/javascript/ElectronObservationService.test.ts +++ b/src/application/javascript/ElectronObservationService.test.ts @@ -12,7 +12,7 @@ import { electronTargetListSchema, inspectElectronPageInputSchema, } from "../../domain/javascript/electronObservation.js"; -import { err } from "../../domain/result.js"; +import { err, ok } from "../../domain/result.js"; import type { ElectronObservationPort } from "./ElectronObservationPort.js"; import { inspectElectronPage, @@ -71,7 +71,11 @@ const inspection = electronPageInspectionSchema.parse({ it("converts a retained raw Electron inspection into partial Evidence", async () => { const failure = new AnalysisCancelledError("inspect_electron_page", { - partialObservation: inspection, + partialObservation: { + kind: "electron-page-observation", + inspection, + cleanup: "transport-closed", + }, }); const provider: ElectronObservationPort = { identity: () => identity, @@ -85,7 +89,15 @@ it("converts a retained raw Electron inspection into partial Evidence", async () expect(result.error).toBe(failure); expect(parseEvidence(result.error.partialObservation)).toMatchObject({ operation: "inspect_electron_page", - normalized_result: inspection, + normalized_result: { + ...inspection, + limitations: [ + "REA closed its CDP transport; this does not imply target shutdown.", + ], + }, + limitations: expect.arrayContaining([ + "REA closed its CDP transport; this does not imply target shutdown.", + ]), }); } }); @@ -95,7 +107,14 @@ it("converts a raw inspection retained by provider cleanup failure", async () => "rea-cdp-electron", ["electron-page"], { reason: "target detach failed" }, - { operation: "inspect_electron_page", partialObservation: inspection }, + { + operation: "inspect_electron_page", + partialObservation: { + kind: "electron-page-observation", + inspection, + cleanup: "incomplete", + }, + }, ); const provider: ElectronObservationPort = { identity: () => identity, @@ -110,7 +129,13 @@ it("converts a raw inspection retained by provider cleanup failure", async () => expect(result.error.cleanupIncomplete).toBe(true); expect(parseEvidence(result.error.partialObservation)).toMatchObject({ operation: "inspect_electron_page", - normalized_result: inspection, + normalized_result: { + ...inspection, + limitations: ["CDP transport cleanup remains incomplete."], + }, + limitations: expect.arrayContaining([ + "CDP transport cleanup remains incomplete.", + ]), }); } }); @@ -148,3 +173,35 @@ it("returns a typed output error for malformed Electron target output after abor expect(result.ok).toBe(false); if (!result.ok) expect(result.error).toBeInstanceOf(AnalysisOutputError); }); + +it("retains completed target facts when cancellation arrives before Evidence publication", async () => { + const controller = new AbortController(); + const targets = electronTargetListSchema.parse({ + browser: inspection.browser, + targets: [], + excluded: { unsupported_url: 0, non_page: 0 }, + limitations: [], + }); + const provider: ElectronObservationPort = { + identity: () => identity, + listTargets: async () => { + controller.abort(); + return ok(targets); + }, + inspectPage: async () => + err(new AnalysisCancelledError("inspect_electron_page")), + }; + const result = await listElectronTargets( + provider, + { cdp_endpoint: input.cdp_endpoint }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(parseEvidence(result.error.partialObservation)).toMatchObject({ + operation: "list_electron_targets", + normalized_result: targets, + }); + } +}); diff --git a/src/application/javascript/ElectronObservationService.ts b/src/application/javascript/ElectronObservationService.ts index 3923c6b661..f658bf7ccb 100644 --- a/src/application/javascript/ElectronObservationService.ts +++ b/src/application/javascript/ElectronObservationService.ts @@ -2,6 +2,7 @@ import type { ExecutionOptions } from "../AnalysisProvider.js"; import type { ElectronObservationPort } from "./ElectronObservationPort.js"; import type { Evidence } from "../../domain/evidence.js"; import type { + ElectronPageInspection, InspectElectronPageInput, ListElectronTargetsInput, } from "../../domain/javascript/electronObservation.js"; @@ -41,16 +42,13 @@ export const listElectronTargets = async ( ), ); return completeProviderEvidence( - ok( - createElectronEvidence( - "list_electron_targets", - input, - report.data, - ready.value.identity(), - ), + createElectronEvidence( + "list_electron_targets", + input, + report.data, + ready.value.identity(), ), - "list_electron_targets", - options, + options.signal, ); }; @@ -66,16 +64,13 @@ export const inspectElectronPage = async ( return err(new AnalysisCancelledError("inspect_electron_page")); const result = await ready.value.inspectPage(input, options); if (!result.ok) { + const partial = result.error.partialObservation; if ( - result.error.partialObservation !== undefined && - !("evidence_id" in result.error.partialObservation) && - "target" in result.error.partialObservation && - "dom" in result.error.partialObservation && - "capture_window" in result.error.partialObservation + partial !== undefined && + "kind" in partial && + partial.kind === "electron-page-observation" ) { - const report = electronPageInspectionSchema.safeParse( - result.error.partialObservation, - ); + const report = electronPageInspectionSchema.safeParse(partial.inspection); if (!report.success) { if (result.error instanceof AnalysisCancelledError) return err( @@ -87,10 +82,19 @@ export const inspectElectronPage = async ( ); return result; } + const inspection = { + ...report.data, + limitations: [ + ...report.data.limitations, + partial.cleanup === "transport-closed" + ? "REA closed its CDP transport; this does not imply target shutdown." + : "CDP transport cleanup remains incomplete.", + ], + } satisfies ElectronPageInspection; result.error.partialObservation = createElectronEvidence( "inspect_electron_page", input, - report.data, + inspection, ready.value.identity(), ); } @@ -106,16 +110,13 @@ export const inspectElectronPage = async ( ), ); return completeProviderEvidence( - ok( - createElectronEvidence( - "inspect_electron_page", - input, - report.data, - ready.value.identity(), - ), + createElectronEvidence( + "inspect_electron_page", + input, + report.data, + ready.value.identity(), ), - "inspect_electron_page", - options, + options.signal, ); }; diff --git a/src/application/javascript/JavaScriptRuntimeObservationService.test.ts b/src/application/javascript/JavaScriptRuntimeObservationService.test.ts index eff4acf394..770c210e8d 100644 --- a/src/application/javascript/JavaScriptRuntimeObservationService.test.ts +++ b/src/application/javascript/JavaScriptRuntimeObservationService.test.ts @@ -54,7 +54,7 @@ it("preserves failed Inspector cleanup when cancellation occurs during observati expect(result.error.userCategory).toBe("cancelled"); }); -it("does not publish targets when cancellation arrives during provider cleanup", async () => { +it("retains target Evidence when cancellation arrives during provider cleanup", async () => { const controller = new AbortController(); const provider: JavaScriptRuntimeObservationPort = { identity: () => ({ @@ -89,6 +89,12 @@ it("does not publish targets when cancellation arrives during provider cleanup", error: { _tag: "AnalysisCancelledError", operation: "list_javascript_runtime_targets", + partialObservation: { + normalized_result: { + runtime: { product: "node.js/v24" }, + targets: [], + }, + }, }, }); if (!result.ok) { @@ -226,3 +232,42 @@ it("converts a raw runtime observation retained by cleanup failure", async () => }); } }); + +it("reports non-JSON Inspector output before late cancellation", async () => { + const controller = new AbortController(); + const value = { + runtime: { + product: "node.js/v24", + protocol_version: "1.3", + v8_version: "13", + }, + targets: [], + excluded: { unsupported_location: 0, unconnectable: 0 }, + limitations: [] as string[], + }; + Reflect.set(value, "limitations", [undefined]); + const provider: JavaScriptRuntimeObservationPort = { + identity: () => ({ + id: "test-inspector", + name: "Test Inspector", + version: "1", + }), + listTargets: async () => { + controller.abort(); + return ok(value); + }, + observe: async () => + err(new AnalysisCancelledError("observe_javascript_runtime")), + }; + + await expect( + listJavaScriptRuntimeTargets( + provider, + { inspector_endpoint: "http://127.0.0.1:9229" }, + { signal: controller.signal }, + ), + ).resolves.toMatchObject({ + ok: false, + error: { _tag: "AnalysisOutputError", partialObservation: undefined }, + }); +}); diff --git a/src/application/javascript/JavaScriptRuntimeObservationService.ts b/src/application/javascript/JavaScriptRuntimeObservationService.ts index a56dae2b26..bdb3dcd988 100644 --- a/src/application/javascript/JavaScriptRuntimeObservationService.ts +++ b/src/application/javascript/JavaScriptRuntimeObservationService.ts @@ -127,15 +127,12 @@ const runtimeEvidence = ( options: ExecutionOptions, ): Result => { return completeProviderEvidence( - ok( - createJavaScriptRuntimeObservationEvidence( - operation, - input, - result, - provider, - ), + createJavaScriptRuntimeObservationEvidence( + operation, + input, + result, + provider, ), - operation, - options, + options.signal, ); }; diff --git a/src/application/rust/RustBinaryService.test.ts b/src/application/rust/RustBinaryService.test.ts index d7c04de696..c6c47df0f1 100644 --- a/src/application/rust/RustBinaryService.test.ts +++ b/src/application/rust/RustBinaryService.test.ts @@ -113,6 +113,51 @@ it("rejects a cycle claim the package dependencies do not contain", async () => if (!result.ok) expect(result.error).toBeInstanceOf(AnalysisOutputError); }); +it("retains completed Rust Evidence on late cancellation and keeps output failure precedence", async () => { + const controller = new AbortController(); + const service = new RustBinaryService({ + identity, + inspect: () => { + controller.abort(); + return Promise.resolve(ok(report)); + }, + }); + const cancelled = await service.inspect( + { path: report.artifact.path }, + { signal: controller.signal }, + ); + expect(cancelled).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + partialObservation: { normalized_result: report }, + }, + }); + + const invalidController = new AbortController(); + const mismatched: RustBinary = { + ...report, + artifact: { ...report.artifact, path: "/artifacts/other" }, + }; + const invalid = await new RustBinaryService({ + identity, + inspect: () => { + invalidController.abort(); + return Promise.resolve(ok(mismatched)); + }, + }).inspect( + { path: report.artifact.path }, + { signal: invalidController.signal }, + ); + expect(invalid).toMatchObject({ + ok: false, + error: { _tag: "AnalysisOutputError" }, + }); + expect(cancelled.ok).toBe(false); + if (!cancelled.ok) + expect(cancelled.error).toBeInstanceOf(AnalysisCancelledError); +}); + it.each([ { from_index: 1, to_index: 0 }, { from_index: 0, to_index: 1 }, diff --git a/src/application/rust/RustBinaryService.ts b/src/application/rust/RustBinaryService.ts index 20a2e49cff..2732cac768 100644 --- a/src/application/rust/RustBinaryService.ts +++ b/src/application/rust/RustBinaryService.ts @@ -71,9 +71,8 @@ export class RustBinaryService { ), ); return completeProviderEvidence( - ok(evidenceFor(this.provider.identity, input.data, report.data)), - OPERATION, - options, + evidenceFor(this.provider.identity, input.data, report.data), + options?.signal, ); } } diff --git a/src/application/wasm/WasmArtifactService.test.ts b/src/application/wasm/WasmArtifactService.test.ts index 570fc0a2b6..afb1e80101 100644 --- a/src/application/wasm/WasmArtifactService.test.ts +++ b/src/application/wasm/WasmArtifactService.test.ts @@ -77,3 +77,51 @@ it("cancels before provider work and delivers observed Evidence on success", asy value: { confidence: "observed", normalized_result: value }, }); }); + +it("retains completed Evidence when cancellation arrives after provider return", async () => { + const value = wasmArtifactFixture(); + const controller = new AbortController(); + const service = new WasmArtifactService({ + close: () => Promise.resolve(), + inspect: async () => { + controller.abort(); + return ok(wasmArtifactExecution(value)); + }, + }); + + expect( + await service.inspect( + { path: value.artifact.path }, + { signal: controller.signal }, + ), + ).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + partialObservation: { normalized_result: value }, + }, + }); +}); + +it("preserves malformed-output failure when cancellation arrives with it", async () => { + const value = wasmArtifactFixture(); + const controller = new AbortController(); + const malformed = { ...wasmArtifactExecution(value), result: {} }; + const service = new WasmArtifactService({ + close: () => Promise.resolve(), + inspect: async () => { + controller.abort(); + return ok(malformed); + }, + }); + + expect( + await service.inspect( + { path: value.artifact.path }, + { signal: controller.signal }, + ), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisOutputError" }, + }); +}); diff --git a/src/application/wasm/WasmArtifactService.ts b/src/application/wasm/WasmArtifactService.ts index fc68acc909..07aed7bd0d 100644 --- a/src/application/wasm/WasmArtifactService.ts +++ b/src/application/wasm/WasmArtifactService.ts @@ -15,7 +15,8 @@ import { wasmArtifactMatchesSubject, } from "../../domain/wasm/wasmArtifact.js"; import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; -import { err, ok, type Result } from "../../domain/result.js"; +import { err, type Result } from "../../domain/result.js"; +import { completeProviderEvidence } from "../ProviderOperationExecution.js"; const OPERATION = "inspect_wasm_artifact"; @@ -67,8 +68,6 @@ export class WasmArtifactService { return err(new AnalysisInputError(OPERATION, undefined, pathIssues)); const inspected = await this.provider.inspect(input.data, options); if (!inspected.ok) return inspected; - if (options?.signal?.aborted) - return err(new AnalysisCancelledError(OPERATION)); const execution = inspected.value; const report = wasmArtifactSchema.safeParse(execution.result); if ( @@ -88,7 +87,7 @@ export class WasmArtifactService { "WABT adapter returned malformed data or changed the selected artifact identity.", ), ); - return ok( + return completeProviderEvidence( createEvidence(execution.subject, execution.provider, { operation: OPERATION, parameters: input.data, @@ -100,6 +99,7 @@ export class WasmArtifactService { locations: execution.locations, limitations: execution.limitations, }), + options?.signal, ); } } diff --git a/src/artifacts/AnalysisCleanupLease.test.ts b/src/artifacts/AnalysisCleanupLease.test.ts new file mode 100644 index 0000000000..214237f3e9 --- /dev/null +++ b/src/artifacts/AnalysisCleanupLease.test.ts @@ -0,0 +1,106 @@ +import { expect, it } from "vitest"; + +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { + AnalysisCleanupLease, + AnalysisCleanupLeaseGroup, +} from "./AnalysisCleanupLease.js"; + +it.each(["before release", "after release"])( + "resolves its originating error bound %s without erasing the cause", + async (binding) => { + const cause = new Error("first close failed"); + const error = new ProviderCleanupError( + "fixture", + ["owned resource"], + { reason: cause.message }, + { cause }, + ); + let releases = 0; + const lease = new AnalysisCleanupLease(async () => { + releases += 1; + }); + if (binding === "before release") lease.bindCleanupError(error); + await lease.release(); + if (binding === "after release") lease.bindCleanupError(error); + expect(error.cleanupIncomplete).toBe(false); + expect(error.cause).toBe(cause); + await lease.release(); + expect(releases).toBe(1); + }, +); + +it("retains uncertainty across a failed retry until the same lease releases", async () => { + const cause = new Error("close still denied"); + const error = new ProviderCleanupError("fixture", ["owned resource"], {}); + let denied = true; + const lease = new AnalysisCleanupLease(async () => { + if (denied) throw cause; + }); + lease.bindCleanupError(error); + await expect(lease.release()).rejects.toBe(cause); + expect(error.cleanupIncomplete).toBe(true); + denied = false; + await lease.release(); + expect(error.cleanupIncomplete).toBe(false); +}); + +it("resolves an aggregate only after every bound owner releases", async () => { + const error = new ProviderCleanupError( + "fixture", + ["first resource", "second resource"], + {}, + ); + const first = new AnalysisCleanupLease(async () => undefined); + const second = new AnalysisCleanupLease(async () => undefined); + new AnalysisCleanupLeaseGroup(error, [first, second]); + + await first.release(); + expect(error.cleanupIncomplete).toBe(true); + await second.release(); + expect(error.cleanupIncomplete).toBe(false); +}); + +it("deduplicates owners and accounts for owners already released at binding time", async () => { + const error = new ProviderCleanupError("fixture", ["owned resource"], {}); + const alreadyReleased = new AnalysisCleanupLease(async () => undefined); + const pending = new AnalysisCleanupLease(async () => undefined); + await alreadyReleased.release(); + + new AnalysisCleanupLeaseGroup(error, [alreadyReleased, pending, pending]); + expect(error.cleanupIncomplete).toBe(true); + await pending.release(); + expect(error.cleanupIncomplete).toBe(false); +}); + +it("does not resolve a cleanup group without exact owners", () => { + const error = new ProviderCleanupError("fixture", ["unknown owner"], {}); + new AnalysisCleanupLeaseGroup(error, []); + expect(error.cleanupIncomplete).toBe(true); +}); + +it("resolves every live retry error bound to one lease", async () => { + const firstCause = new Error("first admission retry failed"); + const secondCause = new Error("second admission retry failed"); + const firstError = new ProviderCleanupError( + "fixture", + ["owned resource"], + {}, + { cause: firstCause }, + ); + const secondError = new ProviderCleanupError( + "fixture", + ["owned resource"], + {}, + { cause: secondCause }, + ); + const lease = new AnalysisCleanupLease(async () => undefined); + lease.bindCleanupError(firstError); + lease.bindCleanupError(secondError); + + await lease.release(); + expect(firstError.cleanupIncomplete).toBe(false); + expect(secondError.cleanupIncomplete).toBe(false); + expect(firstError.cause).toBe(firstCause); + expect(secondError.cause).toBe(secondCause); +}); diff --git a/src/artifacts/AnalysisCleanupLease.ts b/src/artifacts/AnalysisCleanupLease.ts new file mode 100644 index 0000000000..42ff774255 --- /dev/null +++ b/src/artifacts/AnalysisCleanupLease.ts @@ -0,0 +1,70 @@ +import type { AnalysisError } from "../domain/analysisErrorBase.js"; + +/** Link one resource's confirmed release to its originating analysis error. */ +export class AnalysisCleanupLease { + #released = false; + #errors: WeakRef[] = []; + readonly #groups = new Set(); + + constructor(private readonly cleanup: () => Promise) {} + + async release(): Promise { + if (this.#released) return; + await this.cleanup(); + this.#released = true; + this.#resolveBoundErrors(); + for (const group of this.#groups) group.ownerReleased(this); + this.#groups.clear(); + } + + /** Binding can follow a concurrent retry that already confirmed release. */ + bindCleanupError(error: AnalysisError): void { + if (this.#released) { + error.resolveCleanup(); + return; + } + this.#errors = [ + ...this.#errors.filter((reference) => reference.deref() !== undefined), + new WeakRef(error), + ]; + } + + /** Bind one aggregate error to every exact lease in a failed retry set. */ + bindCleanupGroup(group: AnalysisCleanupLeaseGroup): void { + if (this.#released) { + group.ownerReleased(this); + return; + } + for (const retained of this.#groups) + if (!retained.hasLiveError) this.#groups.delete(retained); + this.#groups.add(group); + } + + #resolveBoundErrors(): void { + for (const reference of this.#errors) reference.deref()?.resolveCleanup(); + this.#errors = []; + } +} + +/** Conjunctive cleanup binding for an error covering multiple exact owners. */ +export class AnalysisCleanupLeaseGroup { + readonly #error: WeakRef; + readonly #pending: Set; + + constructor(error: AnalysisError, owners: readonly AnalysisCleanupLease[]) { + this.#error = new WeakRef(error); + this.#pending = new Set(owners); + if (this.#pending.size === 0) return; + for (const owner of this.#pending) owner.bindCleanupGroup(this); + } + + get hasLiveError(): boolean { + return this.#error.deref() !== undefined; + } + + ownerReleased(owner: AnalysisCleanupLease): void { + if (!this.#pending.delete(owner) || this.#pending.size > 0) return; + this.#error.deref()?.resolveCleanup(); + this.#pending.clear(); + } +} diff --git a/src/artifacts/ArtifactResourceScope.test.ts b/src/artifacts/ArtifactResourceScope.test.ts index 395f113b38..1763ffdbcc 100644 --- a/src/artifacts/ArtifactResourceScope.test.ts +++ b/src/artifacts/ArtifactResourceScope.test.ts @@ -42,7 +42,7 @@ it("waits for admitted operations and shares concurrent close", async () => { expect(closed).toBe(true); }); -it("retains the same failed reader owner and retries before admitting work", async () => { +it("retains the same failed reader owner in a retry failure and retries before admitting work", async () => { const scope = new ArtifactResourceScope(); let closeAttempts = 0; const reader = { @@ -59,6 +59,7 @@ it("retains the same failed reader owner and retries before admitting work", asy await expect(scope.run(async () => "must not run")).rejects.toMatchObject({ reason: "unavailable", message: "Artifact cleanup remains incomplete: fixture reader", + failedOwners: [owner], }); expect(closeAttempts).toBe(2); await expect(scope.close()).rejects.toMatchObject({ @@ -69,6 +70,44 @@ it("retains the same failed reader owner and retries before admitting work", asy expect(closeAttempts).toBe(4); }); +it("reports only owners whose cleanup failed during a mixed retry", async () => { + const scope = new ArtifactResourceScope(); + let firstAttempts = 0; + let secondAttempts = 0; + const first = { + kind: "lease" as const, + resource: "first lease", + lease: { + async release() { + firstAttempts += 1; + if (firstAttempts === 1) throw new Error("first initial close"); + }, + }, + }; + const second = { + kind: "lease" as const, + resource: "second lease", + lease: { + async release() { + secondAttempts += 1; + if (secondAttempts < 3) throw new Error("second close still denied"); + }, + }, + }; + expect(await scope.release(first)).toMatchObject({ kind: "failed" }); + expect(await scope.release(second)).toMatchObject({ kind: "failed" }); + + await expect(scope.retryCleanup()).rejects.toMatchObject({ + failedOwners: [second], + cleanup: { resources: ["second lease"] }, + }); + expect(firstAttempts).toBe(2); + expect(secondAttempts).toBe(2); + await scope.close(); + expect(firstAttempts).toBe(2); + expect(secondAttempts).toBe(3); +}); + it("retains a failed source analyzer until shutdown can verify its release", async () => { const scope = new ArtifactResourceScope(); let releaseAllowed = false; diff --git a/src/artifacts/ArtifactResourceScope.ts b/src/artifacts/ArtifactResourceScope.ts index f069845357..648521bf98 100644 --- a/src/artifacts/ArtifactResourceScope.ts +++ b/src/artifacts/ArtifactResourceScope.ts @@ -55,6 +55,17 @@ export type ArtifactCleanupAttempt = | { readonly kind: "released" } | { readonly kind: "failed"; readonly cause: unknown }; +/** Retry failure carrying the exact retained owners that did not release. */ +export class ArtifactResourceRetryFailure extends ArtifactReaderFailure { + constructor( + readonly failedOwners: readonly ArtifactResourceOwner[], + message: string, + options: ConstructorParameters[2], + ) { + super("unavailable", message, options); + } +} + /** Caller-owned scope that retains failed artifact cleanup for a later close. */ export class ArtifactResourceScope { readonly #pending = new Set(); @@ -106,6 +117,11 @@ export class ArtifactResourceScope { } } + /** Adopt an owner after another boundary's cleanup attempt failed. */ + retain(owner: ArtifactResourceOwner): void { + this.#pending.add(owner); + } + /** Retry retained owners without closing resource admission. */ retryCleanup(): Promise { if (this.#draining !== undefined) return this.#draining; @@ -142,8 +158,8 @@ export class ArtifactResourceScope { const resources = [ ...new Set(observations.flatMap(({ resources }) => resources)), ]; - throw new ArtifactReaderFailure( - "unavailable", + throw new ArtifactResourceRetryFailure( + failures.map(({ owner }) => owner), `Artifact cleanup remains incomplete: ${resources.join(", ")}`, { cause: new AggregateError( diff --git a/src/browser/CdpElectronProvider.ts b/src/browser/CdpElectronProvider.ts index fbfc7ebd61..86e2bbfb2c 100644 --- a/src/browser/CdpElectronProvider.ts +++ b/src/browser/CdpElectronProvider.ts @@ -181,7 +181,14 @@ export class CdpElectronProvider implements ElectronObservationPort { result.ok && options.signal?.aborted === true ? new AnalysisCancelledError("inspect_electron_page", { cause: options.signal.reason, - partialObservation: result.value, + partialObservation: { + kind: "electron-page-observation", + inspection: result.value, + cleanup: + cleanupFailure === undefined + ? "transport-closed" + : "incomplete", + }, }) : undefined; if (cleanupFailure !== undefined) { @@ -209,7 +216,11 @@ export class CdpElectronProvider implements ElectronObservationPort { { operation: "inspect_electron_page", cause: cleanupFailure, - partialObservation: result.value, + partialObservation: { + kind: "electron-page-observation", + inspection: result.value, + cleanup: "incomplete", + }, }, ), ); diff --git a/src/browser/PlaywrightBrowserScenarioProvider.test.ts b/src/browser/PlaywrightBrowserScenarioProvider.test.ts index 300592beb8..97ed96da05 100644 --- a/src/browser/PlaywrightBrowserScenarioProvider.test.ts +++ b/src/browser/PlaywrightBrowserScenarioProvider.test.ts @@ -121,6 +121,7 @@ class FakeSession implements BrowserScenarioSessionPort { performCalls = 0; captureCalls = 0; onPerform: (() => void) | undefined; + onClose: (() => void) | undefined; eventGaps: readonly string[] = []; eventItems: readonly BrowserScenarioEvent[] = []; @@ -210,6 +211,7 @@ class FakeSession implements BrowserScenarioSessionPort { close() { this.closeCalls += 1; + this.onClose?.(); if (this.failures.close !== undefined) return Promise.reject(this.failures.close); return Promise.resolve( @@ -695,18 +697,98 @@ describe("PlaywrightBrowserScenarioProvider request cancellation", () => { it("cancels an active action and still closes the session", async () => { const controller = new AbortController(); + const cancellationReason = new Error("cancel active scenario"); const session = new FakeSession("launch"); - session.onPerform = () => controller.abort(); + session.onPerform = () => controller.abort(cancellationReason); const provider = new PlaywrightBrowserScenarioProvider(() => Promise.resolve(session), ); const result = await provider.captureScenario(scenario(), { signal: controller.signal, }); - if (!result.ok) throw result.error; - expect(result.value.steps[1]?.status).toBe("cancelled"); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + partialObservation: { + kind: "browser-scenario-observation", + capture: { + browser: { cleanup: "terminated-owned-process" }, + steps: [{ status: "completed" }, { status: "cancelled" }], + }, + }, + }, + }); + if (result.ok) throw new Error("Expected active scenario cancellation"); + expect(result.error.cause).toBe(cancellationReason); expect(session.closeCalls).toBe(1); }); + + it("retains the completed capture when cancellation arrives during final close", async () => { + const controller = new AbortController(); + const cancellationReason = new Error("cancel final close"); + const session = new FakeSession("launch"); + session.onClose = () => controller.abort(cancellationReason); + const provider = new PlaywrightBrowserScenarioProvider(() => + Promise.resolve(session), + ); + + const result = await provider.captureScenario(scenario(), { + signal: controller.signal, + }); + + expect(session.closeCalls).toBe(1); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + partialObservation: { + kind: "browser-scenario-observation", + capture: { + browser: { cleanup: "terminated-owned-process" }, + steps: [{ status: "completed" }, { status: "completed" }], + }, + }, + }, + }); + if (result.ok) throw new Error("Expected final-close cancellation"); + expect(result.error.cause).toBe(cancellationReason); + }); + + it("keeps cleanup failure primary when cancellation arrives during a failed close", async () => { + const controller = new AbortController(); + const session = new FakeSession("launch", { + close: new Error("fixture close failed"), + }); + session.onClose = () => controller.abort(); + const provider = new PlaywrightBrowserScenarioProvider(() => + Promise.resolve(session), + ); + + const result = await provider.captureScenario(scenario(), { + signal: controller.signal, + }); + + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "BrowserObservationError", + reason: "cleanup_failed", + partialObservation: { + kind: "browser-scenario-observation", + capture: { + browser: { cleanup: "incomplete" }, + steps: [{ status: "completed" }, { status: "completed" }], + }, + }, + }, + }); + await expect(provider.close()).rejects.toMatchObject({ + _tag: "ProviderAdapterError", + operation: "capture_browser_scenario", + }); + expect(session.closeCalls).toBe(2); + }); }); it("shares concurrent close and retries browser cleanup after failure", async () => { diff --git a/src/browser/PlaywrightBrowserScenarioProvider.ts b/src/browser/PlaywrightBrowserScenarioProvider.ts index 8a5cc71cbb..a4150263e1 100644 --- a/src/browser/PlaywrightBrowserScenarioProvider.ts +++ b/src/browser/PlaywrightBrowserScenarioProvider.ts @@ -38,6 +38,9 @@ import { const OPERATION = "capture_browser_scenario" as const; import { PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY } from "./providerIdentities.js"; +const signalIsAborted = (signal: AbortSignal | undefined): boolean => + signal?.aborted === true; + type SnapshotKind = BrowserScenario["capture"]["after_each_step"][number]; const SNAPSHOT_KINDS = [ "screenshot", @@ -325,25 +328,8 @@ const runScenario = async ( ], }), }); - if (options.signal?.aborted) - throw new AnalysisCancelledError(OPERATION, { - cause: options.signal?.reason, - partialObservation: { - kind: "browser-scenario-observation", - capture: browserScenarioCaptureData({ - session, - scenario, - startedAt, - steps, - cleanup: cleanupOutcome.cleanup, - limitations: [ - "Scenario capture completed, but the request was cancelled during final browser cleanup.", - ], - }), - }, - }); const completeness = globalCompleteness(scenario, session, steps); - return browserScenarioCaptureSchema.parse({ + const capture = browserScenarioCaptureSchema.parse({ ...browserScenarioCaptureData({ session, scenario, @@ -353,6 +339,21 @@ const runScenario = async ( }), completeness, }); + if (signalIsAborted(options.signal)) + throw new AnalysisCancelledError(OPERATION, { + cause: options.signal?.reason, + partialObservation: { + kind: "browser-scenario-observation", + capture: { + ...capture, + limitations: [ + ...capture.limitations, + "Scenario capture was cancelled; collected steps, observations, and cleanup state are retained.", + ], + }, + }, + }); + return capture; }; /** Controlled Playwright/CDP scenario driver with exact process ownership. */ diff --git a/src/cli/finishAnalysisCommand.ts b/src/cli/finishAnalysisCommand.ts index d370ff52ca..a279464399 100644 --- a/src/cli/finishAnalysisCommand.ts +++ b/src/cli/finishAnalysisCommand.ts @@ -34,9 +34,9 @@ export const finishAnalysisCommand = async < }; } try { + // Resource owners may resolve their retained errors during close. This + // adapter cannot infer resolution of a producer's earlier cleanup failure. await service.close(); - if (!result.ok && result.error.cleanupIncomplete) - result.error.resolveCleanup(); } catch (cause) { let cleanup: AnalysisError; if (cause instanceof AnalysisError) cleanup = cause; diff --git a/src/doctorRuntime.ts b/src/doctorRuntime.ts index 0384937f0b..dffa46f970 100644 --- a/src/doctorRuntime.ts +++ b/src/doctorRuntime.ts @@ -42,11 +42,13 @@ export const createSystemDoctorHost = ( // parser-owned error when IDA itself was explicitly selected. return analysisProvider === "ida" ? registration : ok(undefined); }, - providerInspections: async (adoptFileHandle) => [ + providerInspections: async (fileCloseResources) => [ await inspectSystemGhidraProvider(environment), inspectIdaRegistration(environment.REA_IDA_MCP_CONFIG), await inspectJadxProvider(environment), - await inspectKunaProvider(environment, process.platform, adoptFileHandle), + await inspectKunaProvider(environment, process.platform, { + fileCloseResources, + }), ], linuxDemoRuntimeCheck: () => inspectLinuxPrivateDisplay(environment), }); diff --git a/src/domain/adb/adbDeviceAnalysis.ts b/src/domain/adb/adbDeviceAnalysis.ts index e5b14b8ee1..7798b78fde 100644 --- a/src/domain/adb/adbDeviceAnalysis.ts +++ b/src/domain/adb/adbDeviceAnalysis.ts @@ -757,15 +757,17 @@ export type AdbAcquiredFilePartialObservation = | { readonly provider_id: "adb"; readonly operation: "pull_adb_file"; - readonly result: { - readonly client: z.infer< - typeof adbResultSchemas.pull_adb_file - >["client"]; - readonly serial: string; - readonly local_path: string; - readonly device_path: string; - readonly digest_status: "unknown"; - }; + readonly result: + | { + readonly client: z.infer< + typeof adbResultSchemas.pull_adb_file + >["client"]; + readonly serial: string; + readonly local_path: string; + readonly device_path: string; + readonly digest_status: "unknown"; + } + | z.infer; } | { readonly provider_id: "adb"; diff --git a/src/domain/analysisErrorBase.ts b/src/domain/analysisErrorBase.ts index 9574fdce52..a2a52c1d74 100644 --- a/src/domain/analysisErrorBase.ts +++ b/src/domain/analysisErrorBase.ts @@ -126,7 +126,6 @@ import type { import type { FileOffsetPartialObservation } from "./native/fileOffsetPartialObservation.js"; import type { NativeCallPartialObservation } from "./native/nativeCallPartialObservation.js"; import type { ElectronActivePartialObservation } from "./javascript/electronActiveObservation.js"; -import type { ElectronPageInspection } from "./javascript/electronObservation.js"; import type { JavaScriptRuntimeObservation } from "./javascript/javascriptRuntimeObservation.js"; import type { ArtifactInventoryPartialObservation, @@ -136,6 +135,7 @@ import type { WebScriptExportPartialObservation } from "./webScriptExport.js"; import type { JavaScriptRecoveryPartialObservation } from "./javascript/javascriptRecovery.js"; import type { BrowserScenarioPartialObservation } from "./browserScenarioCapture.js"; import type { Evidence } from "./evidence.js"; +import type { ElectronPagePartialObservation } from "./javascript/electronObservation.js"; import type { AndroidPartialObservation } from "./android/androidPartialObservation.js"; import type { FirmwarePartialObservation } from "./firmware/firmwareAnalysis.js"; import type { @@ -159,7 +159,6 @@ export type AnalysisPartialObservation = | NativeCallPartialObservation | FileOffsetPartialObservation | ElectronActivePartialObservation - | ElectronPageInspection | BrowserScenarioPartialObservation | WebPageInspection | WebExecution @@ -176,6 +175,7 @@ export type AnalysisPartialObservation = | AndroidPartialObservation | FirmwarePartialObservation | NativeUiObservationResult + | ElectronPagePartialObservation | AdbActivityStartPartialObservation | AdbAcquiredFilePartialObservation | AdbPackagePullPartialObservation; diff --git a/src/domain/javascript/electronObservation.ts b/src/domain/javascript/electronObservation.ts index 6258c90c3e..929690454c 100644 --- a/src/domain/javascript/electronObservation.ts +++ b/src/domain/javascript/electronObservation.ts @@ -152,3 +152,11 @@ export const electronPageInspectionSchema = z.object({ export type ElectronPageInspection = z.infer< typeof electronPageInspectionSchema >; + +/** Completed Electron facts retained when later request finalization fails. */ +export interface ElectronPagePartialObservation { + readonly kind: "electron-page-observation"; + readonly inspection: ElectronPageInspection; + /** REA's CDP transport closed; target shutdown is never implied. */ + readonly cleanup: "transport-closed" | "incomplete"; +} diff --git a/src/domain/providerCleanupError.test.ts b/src/domain/providerCleanupError.test.ts index b3e24861dd..6363fddba5 100644 --- a/src/domain/providerCleanupError.test.ts +++ b/src/domain/providerCleanupError.test.ts @@ -20,28 +20,34 @@ it("preserves the binary cleanup default and an explicitly owned nonbinary opera }); it.each([false, true])( - "reports only live primary resources after cleanup recovery is %s", - (recovered) => { - const primary = new ProviderCleanupError("adb", ["descriptor"], { - reason: "descriptor close failed", - }); - if (recovered) primary.resolveCleanup(); - const cleanup = new ProviderCleanupError("adb", ["runtime-root"], { - reason: "root removal failed", - }); - const combined = analysisErrorWithCleanupFailure(primary, cleanup); - expect(combined.cleanupResources).toEqual( - recovered ? ["runtime-root"] : ["descriptor", "runtime-root"], + "combines only live cleanup resources when primary resolution is %s", + (resolved) => { + const primary = new ProviderCleanupError( + "adb", + ["publication-handle"], + { reason: "first descriptor close failed" }, + { operation: "capture_adb_screen" }, ); + if (resolved) primary.resolveCleanup(); + const cleanup = new ProviderCleanupError("adb", ["runtime-root"], { + reason: "workspace removal failed", + }); + + const combined = analysisErrorWithCleanupFailure(primary, cleanup); expect(projectAnalysisError(combined)).toMatchObject({ code: "cleanup_incomplete", details: { + resources: resolved + ? ["runtime-root"] + : ["publication-handle", "runtime-root"], diagnostics: { primary_error: { - details: { diagnostics: { reason: "descriptor close failed" } }, + details: { + diagnostics: { reason: "first descriptor close failed" }, + }, }, cleanup_error: { - details: { diagnostics: { reason: "root removal failed" } }, + details: { diagnostics: { reason: "workspace removal failed" } }, }, }, }, diff --git a/src/filesystem/OwnedRegularFile.ts b/src/filesystem/OwnedRegularFile.ts new file mode 100644 index 0000000000..dc100c6efa --- /dev/null +++ b/src/filesystem/OwnedRegularFile.ts @@ -0,0 +1,97 @@ +import type { FileHandle } from "node:fs/promises"; + +import { OwnedFileHandle } from "./OwnedFileHandle.js"; +import { openRegularFile, RegularFileAdmissionFailure } from "./RegularFile.js"; + +export type RegularFileOperationOutcome = + | { readonly kind: "completed"; readonly value: Value } + | { readonly kind: "failed"; readonly cause: unknown }; + +/** A failed close transfers its exact descriptor owner and operation outcome. */ +export class RegularFileCleanupFailure extends Error { + constructor( + readonly path: string, + readonly owner: OwnedFileHandle, + readonly outcome: RegularFileOperationOutcome, + readonly cleanupCause: unknown, + readonly retention: + | { readonly kind: "not_requested" } + | { readonly kind: "callback_returned" } + | { readonly kind: "callback_threw"; readonly cause: unknown } = { + kind: "not_requested", + }, + ) { + super(`Regular file cleanup failed for ${path}`, { + cause: new AggregateError( + [ + ...(outcome.kind === "failed" ? [outcome.cause] : []), + cleanupCause, + ...(retention.kind === "callback_threw" ? [retention.cause] : []), + ], + "File operation and cleanup failures", + ), + }); + this.name = "RegularFileCleanupFailure"; + } +} + +export interface OwnedRegularFileOptions { + readonly signal?: AbortSignal; + readonly symlinks?: "follow" | "reject"; + readonly retainFailedClose?: (owner: OwnedFileHandle) => void; +} + +/** Run an operation while one admitted descriptor and its cleanup owner stay together. */ +export const withOwnedRegularFile = async ( + path: string, + options: OwnedRegularFileOptions, + operation: (handle: FileHandle) => Promise, +): Promise => { + const handle = await openRegularFile(path, { + symlinks: options.symlinks ?? "follow", + signal: options.signal, + }).catch((cause: unknown) => { + if (!(cause instanceof RegularFileAdmissionFailure)) throw cause; + const retention = retainOwner(options.retainFailedClose, cause.owner); + throw new RegularFileCleanupFailure( + path, + cause.owner, + { kind: "failed", cause: cause.cause }, + cause.cleanupCause, + retention, + ); + }); + const owner = new OwnedFileHandle(handle); + let outcome: RegularFileOperationOutcome; + try { + outcome = { kind: "completed", value: await operation(handle) }; + } catch (cause: unknown) { + outcome = { kind: "failed", cause }; + } + try { + await owner.close(); + } catch (cleanupCause: unknown) { + throw new RegularFileCleanupFailure( + path, + owner, + outcome, + cleanupCause, + retainOwner(options.retainFailedClose, owner), + ); + } + if (outcome.kind === "failed") throw outcome.cause; + return outcome.value; +}; + +const retainOwner = ( + retain: OwnedRegularFileOptions["retainFailedClose"], + owner: OwnedFileHandle, +): RegularFileCleanupFailure["retention"] => { + if (retain === undefined) return { kind: "not_requested" }; + try { + retain(owner); + return { kind: "callback_returned" }; + } catch (cause: unknown) { + return { kind: "callback_threw", cause }; + } +}; diff --git a/src/filesystem/RegularFile.ts b/src/filesystem/RegularFile.ts index 9b416f7924..47fdfa1d85 100644 --- a/src/filesystem/RegularFile.ts +++ b/src/filesystem/RegularFile.ts @@ -19,38 +19,21 @@ export class NonRegularFileReadError extends Error { } } -/** One regular-file operation failed while its opened descriptor could not be confirmed closed. */ -export class OwnedRegularFileCleanupFailure extends Error { +/** Admission failed and the opened descriptor could not be confirmed closed. */ +export class RegularFileAdmissionFailure extends Error { override readonly cause: unknown; constructor( readonly path: string, - cause: unknown, + admissionCause: unknown, readonly cleanupCause: unknown, readonly owner: OwnedFileHandle, - message: string, - ) { - super(message, { cause }); - this.cause = cause; - } -} - -/** Admission failed and the opened descriptor could not be confirmed closed. */ -export class RegularFileAdmissionFailure extends OwnedRegularFileCleanupFailure { - constructor( - path: string, - admissionCause: unknown, - cleanupCause: unknown, - owner: OwnedFileHandle, ) { super( - path, - admissionCause, - cleanupCause, - owner, `Selected input failed admission and its descriptor close failed: ${path}`, ); this.name = "RegularFileAdmissionFailure"; + this.cause = admissionCause; } } diff --git a/src/filesystem/StableFileHash.ts b/src/filesystem/StableFileHash.ts index 066d44e03d..657020efa9 100644 --- a/src/filesystem/StableFileHash.ts +++ b/src/filesystem/StableFileHash.ts @@ -1,54 +1,24 @@ import { createHash } from "node:crypto"; import { stat } from "node:fs/promises"; +import { withOwnedRegularFile } from "./OwnedRegularFile.js"; import { - OwnedRegularFileCleanupFailure, - openRegularFile, RegularFileChangedError, sameRegularFileState, } from "./RegularFile.js"; -import { OwnedFileHandle } from "./OwnedFileHandle.js"; +import type { OwnedFileHandle } from "./OwnedFileHandle.js"; -export type StableFileHashResult = { - readonly sha256: string; - readonly bytes: number; -}; - -/** Hashing finished or failed, but the owned descriptor close was rejected. */ -export class StableFileHashCleanupFailure extends OwnedRegularFileCleanupFailure { - constructor( - path: string, - primaryCause: unknown, - readonly hasPrimaryCause: boolean, - cleanupCause: unknown, - owner: OwnedFileHandle, - readonly completedHash: StableFileHashResult | undefined, - ) { - super( - path, - hasPrimaryCause ? primaryCause : cleanupCause, - cleanupCause, - owner, - `Stable file hashing could not confirm descriptor close: ${path}`, - ); - this.name = "StableFileHashCleanupFailure"; - } +export interface StableFileHashOptions { + readonly signal?: AbortSignal; + readonly retainFailedClose: (owner: OwnedFileHandle) => void; } /** Hash one regular file from its admitted descriptor and verify its path state. */ export const hashStableFile = async ( path: string, - signal?: AbortSignal, -): Promise => { - const handle = await openRegularFile(path, { - symlinks: "follow", - signal, - }); - const owner = new OwnedFileHandle(handle); - let primaryCause: unknown; - let hasPrimaryCause = false; - let completedHash: StableFileHashResult | undefined; - try { + options: StableFileHashOptions, +): Promise<{ readonly sha256: string; readonly bytes: number }> => + withOwnedRegularFile(path, options, async (handle) => { const initial = await handle.stat(); if (!Number.isSafeInteger(initial.size) || initial.size < 0) throw new RangeError( @@ -58,47 +28,27 @@ export const hashStableFile = async ( const chunk = Buffer.allocUnsafe(64 * 1024); let position = 0; while (position <= initial.size) { - signal?.throwIfAborted(); + options.signal?.throwIfAborted(); const length = Math.min(chunk.length, initial.size - position + 1); if (length <= 0) break; const read = await handle.read(chunk, 0, length, position); - signal?.throwIfAborted(); + options.signal?.throwIfAborted(); if (read.bytesRead === 0) break; hash.update(chunk.subarray(0, read.bytesRead)); position += read.bytesRead; if (position > initial.size) throw new RegularFileChangedError(path); } - signal?.throwIfAborted(); + options.signal?.throwIfAborted(); const [opened, currentPath] = await Promise.all([ handle.stat(), stat(path), ]); - signal?.throwIfAborted(); + options.signal?.throwIfAborted(); if ( position !== initial.size || !sameRegularFileState(initial, opened) || !sameRegularFileState(initial, currentPath) ) throw new RegularFileChangedError(path); - completedHash = { sha256: hash.digest("hex"), bytes: position }; - } catch (cause: unknown) { - primaryCause = cause; - hasPrimaryCause = true; - } - try { - await owner.close(); - } catch (cleanupCause: unknown) { - throw new StableFileHashCleanupFailure( - path, - primaryCause, - hasPrimaryCause, - cleanupCause, - owner, - completedHash, - ); - } - if (hasPrimaryCause) throw primaryCause; - if (completedHash === undefined) - throw new Error("Stable file hashing completed without a digest"); - return completedHash; -}; + return { sha256: hash.digest("hex"), bytes: position }; + }); diff --git a/src/kuna/KunaCommand.ts b/src/kuna/KunaCommand.ts index 0061059992..46f7af5d5c 100644 --- a/src/kuna/KunaCommand.ts +++ b/src/kuna/KunaCommand.ts @@ -4,7 +4,8 @@ import { stat } from "node:fs/promises"; import { Transform } from "node:stream"; import { pipeline } from "node:stream/promises"; -import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; +import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; import { AnalysisCancelledError, AnalysisResourceConstraintError, @@ -16,12 +17,9 @@ import { OwnedCommandFailure, runOwnedCommand, } from "../process/OwnedCommand.js"; -import { - hashStableFile, - StableFileHashCleanupFailure, -} from "../filesystem/StableFileHash.js"; -import { OwnedRegularFileCleanupFailure } from "../filesystem/RegularFile.js"; -import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; +import { hashStableFile } from "../filesystem/StableFileHash.js"; +import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js"; import type { OwnedProviderProcessSpawnOptions, ProviderProcessSnapshot, @@ -58,6 +56,10 @@ export interface KunaCommandRequest { readonly reportedTimeoutMs: number; readonly outputBytes: number; readonly signal: AbortSignal; + readonly retainFailedFileClose: ( + owner: OwnedFileHandle, + resource: string, + ) => void; readonly launcher?: KunaProcessLauncher | undefined; } @@ -71,7 +73,7 @@ export type KunaCommandOutcome = | { readonly kind: "failed"; readonly error: AnalysisError; - readonly fileCleanup?: OwnedRegularFileCleanupFailure; + readonly fileCleanupFailure?: RegularFileCleanupFailure; } | { readonly kind: "retained"; @@ -83,6 +85,48 @@ export type KunaCommandOutcome = const message = (cause: unknown): string => cause instanceof Error ? cause.message : String(cause); +/** Preserve the operation's typed failure while attaching descriptor cleanup. */ +export const kunaFileCleanupError = ( + failure: RegularFileCleanupFailure, + operation: string, + signal?: AbortSignal, +): AnalysisError => { + const cleanup = new ProviderCleanupError( + KUNA_PROVIDER_ID, + [failure.path], + { reason: message(failure.cleanupCause) }, + { cause: failure, operation }, + ); + if (failure.outcome.kind === "completed") return cleanup; + const primaryCause = failure.outcome.cause; + const primary = + primaryCause instanceof AnalysisError + ? primaryCause + : signal?.aborted === true + ? new AnalysisCancelledError(operation, { cause: primaryCause }) + : new ProviderAdapterError(KUNA_PROVIDER_ID, operation, { + cause: primaryCause, + userMessage: `Kuna ${operation} failed: ${message(primaryCause)}`, + }); + if (primary instanceof AnalysisCancelledError) + return new AnalysisCancelledError(operation, { + cause: new AggregateError( + [primary, cleanup], + "Kuna operation was cancelled and descriptor cleanup failed", + { cause: primary }, + ), + cleanup: { + reason: cleanup.message, + resources: [failure.path], + }, + ...(primary.capturedOutput === undefined + ? {} + : { capturedOutput: primary.capturedOutput }), + partialObservation: primary.partialObservation, + }); + return analysisErrorWithCleanupFailure(primary, cleanup, operation); +}; + /** File metadata that, unchanged, lets a verified digest stand without re-hashing. */ const verifiedExecutables = new Map< string, @@ -96,6 +140,7 @@ const verifiedExecutables = new Map< const verifyExecutable = async ( installation: KunaInstallation, signal: AbortSignal, + retainFailedFileClose: KunaCommandRequest["retainFailedFileClose"], ): Promise => { const status = await stat(installation.executable, { bigint: true }); const identity = [ @@ -108,7 +153,13 @@ const verifyExecutable = async ( const known = verifiedExecutables.get(installation.executable); let sha256 = known?.identity === identity ? known.sha256 : undefined; if (sha256 === undefined) { - sha256 = (await hashStableFile(installation.executable, signal)).sha256; + sha256 = ( + await hashStableFile(installation.executable, { + signal, + retainFailedClose: (owner) => + retainFailedFileClose(owner, installation.executable), + }) + ).sha256; verifiedExecutables.set(installation.executable, { identity, sha256 }); } return sha256 === installation.sha256 ? undefined : sha256; @@ -120,7 +171,11 @@ export const runKunaCommand = async ( ): Promise => { const { installation } = request; try { - const changed = await verifyExecutable(installation, request.signal); + const changed = await verifyExecutable( + installation, + request.signal, + request.retainFailedFileClose, + ); if (changed !== undefined) return { kind: "failed", @@ -133,36 +188,11 @@ export const runKunaCommand = async ( }), }; } catch (cause: unknown) { - if (cause instanceof OwnedRegularFileCleanupFailure) { - const primary = request.signal.aborted - ? new AnalysisCancelledError(request.operation, { cause: cause.cause }) - : new ProviderAdapterError(KUNA_PROVIDER_ID, request.operation, { - cause: cause.cause, - userMessage: `Cannot verify ${installation.executable}: ${message(cause.cause)}`, - }); - const cleanup = new ProviderCleanupError( - KUNA_PROVIDER_ID, - [cause.path], - { - reason: - cause.cleanupCause instanceof Error - ? cause.cleanupCause.message - : String(cause.cleanupCause), - ...(cause instanceof StableFileHashCleanupFailure && - cause.completedHash !== undefined - ? { completed_hash: cause.completedHash } - : {}), - }, - { operation: request.operation, cause: cause.cleanupCause }, - ); + if (cause instanceof RegularFileCleanupFailure) { return { kind: "failed", - error: analysisErrorWithCleanupFailure( - primary, - cleanup, - request.operation, - ), - fileCleanup: cause, + error: kunaFileCleanupError(cause, request.operation, request.signal), + fileCleanupFailure: cause, }; } return { diff --git a/src/kuna/KunaDoctor.ts b/src/kuna/KunaDoctor.ts index 049f083e16..8a89b4e4a4 100644 --- a/src/kuna/KunaDoctor.ts +++ b/src/kuna/KunaDoctor.ts @@ -6,8 +6,9 @@ import type { DoctorProviderCheck, DoctorProviderInspection, } from "../application/Doctor.js"; -import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; import { readKunaConfiguration } from "./KunaConfiguration.js"; +import type { FileCloseResourceOwner } from "../application/AnalysisProvider.js"; +import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js"; import { inspectKunaInstallation, type KunaReadinessFailure, @@ -30,74 +31,53 @@ const REGISTRATION_SETTINGS = [ "REA_KUNA_JOBS", ] as const; -/** Readiness failed and doctor has no lifecycle owner for the hash descriptor. */ -export class KunaDoctorHashCleanupFailure extends Error { - constructor( - readonly readinessFailure: KunaReadinessFailure, - readonly owner: OwnedFileHandle, - readonly cleanupCause: unknown, - ) { - super( - `${readinessFailure.reason}; the executable hash descriptor could not be closed`, - { - cause: - readinessFailure.code === "hash_cleanup_failed" - ? readinessFailure.hashCleanupFailure - : cleanupCause, - }, - ); - this.name = "KunaDoctorHashCleanupFailure"; - } -} - /** Project Kuna readiness into the doctor contract without writing anything. */ export const inspectKunaProvider = async ( environment: Readonly, platform: NodeJS.Platform = process.platform, - adoptFileHandle?: ( - resource: string, - handle: OwnedFileHandle, - ) => Promise, + options: { + readonly fileCloseResources: FileCloseResourceOwner; + }, ): Promise => { - const installation = await inspectKunaInstallation(environment, { - platform, - }); + let installation: Awaited>; + try { + installation = await inspectKunaInstallation(environment, { + platform, + fileCloseResources: options.fileCloseResources, + }); + } catch (cause: unknown) { + if (!(cause instanceof RegularFileCleanupFailure)) throw cause; + return { + id: KUNA_PROVIDER_ID, + configured: environment.REA_KUNA_COMMAND !== undefined, + available: false, + providerVersion: null, + registrationEnvironment: {}, + checks: [ + { + name: "installation", + ok: false, + code: "cleanup_incomplete", + detail: `Kuna installation inspection could not confirm file cleanup: ${cause instanceof Error ? cause.message : String(cause)}`, + remediation: null, + classification: "config_drift", + details: { cleanup_incomplete: true }, + }, + ], + }; + } const configured = environment.REA_KUNA_COMMAND !== undefined && environment.REA_KUNA_COMMAND.length > 0; - if (!installation.ok) { - let cleanupDetail: string | undefined; - const hashFailure = - installation.error.code === "hash_cleanup_failed" - ? installation.error.hashCleanupFailure - : undefined; - if (hashFailure !== undefined) { - if (adoptFileHandle === undefined) { - throw new KunaDoctorHashCleanupFailure( - installation.error, - hashFailure.owner, - hashFailure.cleanupCause, - ); - } else { - const closed = await adoptFileHandle( - hashFailure.path, - hashFailure.owner, - ); - if (!closed) - cleanupDetail = "Executable hash cleanup is still pending."; - } - } + if (!installation.ok) return { id: KUNA_PROVIDER_ID, configured, available: false, providerVersion: null, registrationEnvironment: {}, - checks: [ - installationFailure(installation.error, configured, cleanupDetail), - ], + checks: [installationFailure(installation.error, configured)], }; - } const { value } = installation; return { id: KUNA_PROVIDER_ID, @@ -137,15 +117,11 @@ export const inspectKunaProvider = async ( const installationFailure = ( failure: KunaReadinessFailure, configured: boolean, - cleanupDetail?: string, ): DoctorProviderCheck => ({ name: "installation", ok: false, code: failure.code, - detail: - cleanupDetail === undefined - ? failure.reason - : `${failure.reason} ${cleanupDetail}`, + detail: failure.reason, remediation: failure.code === "unsupported_host" ? KUNA_UNSUPPORTED_HOST_REMEDIATION diff --git a/src/kuna/KunaInstallation.ts b/src/kuna/KunaInstallation.ts index 9d62dc8e2d..bde43dcc64 100644 --- a/src/kuna/KunaInstallation.ts +++ b/src/kuna/KunaInstallation.ts @@ -1,14 +1,17 @@ import { constants } from "node:fs"; -import { access, open, realpath, stat } from "node:fs/promises"; +import { access, realpath, stat } from "node:fs/promises"; import { join } from "node:path"; import type { JsonValue } from "../domain/jsonValue.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; import { err, ok, type Result } from "../domain/result.js"; +import { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js"; +import { hashStableFile } from "../filesystem/StableFileHash.js"; +import type { FileCloseResourceOwner } from "../application/AnalysisProvider.js"; import { - hashStableFile, - StableFileHashCleanupFailure, -} from "../filesystem/StableFileHash.js"; -import { OwnedRegularFileCleanupFailure } from "../filesystem/RegularFile.js"; + RegularFileCleanupFailure, + withOwnedRegularFile, +} from "../filesystem/OwnedRegularFile.js"; import { execFileOutput, execFileOutputFailure, @@ -50,7 +53,7 @@ export interface KunaInstallation { } /** Why the selected Kuna installation cannot be used. */ -interface KunaReadinessFailureBase { +export interface KunaReadinessFailure { readonly code: | KunaConfigurationFailure["code"] | "executable_missing" @@ -59,18 +62,41 @@ interface KunaReadinessFailureBase { readonly diagnostics: Readonly>; } -/** Hash close failure with one owner and any already-completed installation facts. */ -export interface KunaInstallationHashCleanupFailure { - readonly code: "hash_cleanup_failed"; - readonly reason: string; - readonly diagnostics: Readonly>; - readonly hashCleanupFailure: OwnedRegularFileCleanupFailure; - readonly completedInstallation?: KunaInstallation; +export interface KunaInstallationOptions { + readonly platform?: NodeJS.Platform; + readonly signal?: AbortSignal; + /** Probe deadline; tests shorten it. */ + readonly probeTimeoutMs?: number; + readonly fileCloseResources: FileCloseResourceOwner; } -export type KunaReadinessFailure = - | KunaReadinessFailureBase - | KunaInstallationHashCleanupFailure; +/** One installation inspection's file owners and their originating errors. */ +export interface KunaFileCloseScope { + readonly owner: FileCloseResourceOwner; + bindFailure(failure: RegularFileCleanupFailure, error: AnalysisError): void; +} + +/** Adapt retained descriptor owners into leases and bind errors to exact owners. */ +export const createKunaFileCloseScope = ( + parent: FileCloseResourceOwner, +): KunaFileCloseScope => { + const leases = new Map< + Parameters[0], + AnalysisCleanupLease + >(); + return { + owner: { + retain: (owner, resource) => { + const lease = new AnalysisCleanupLease(() => owner.close()); + leases.set(owner, lease); + parent.retain({ close: () => lease.release() }, resource); + }, + run: (operation) => parent.run(operation), + }, + bindFailure: (failure, error) => + leases.get(failure.owner)?.bindCleanupError(error), + }; +}; /** * Environment for every Kuna child. Kuna unpacks its built-in SLEIGH tree into @@ -89,11 +115,10 @@ export const kunaChildEnvironment = ( }; const failure = ( - code: KunaReadinessFailureBase["code"], + code: KunaReadinessFailure["code"], reason: string, diagnostics: Readonly>, -): Result => - err({ code, reason, diagnostics }); +): Result => err({ code, reason, diagnostics }); const message = (cause: unknown): string => cause instanceof Error ? cause.message : String(cause); @@ -101,12 +126,7 @@ const message = (cause: unknown): string => /** Resolve, version-probe and fingerprint the configured executable. */ export const inspectKunaInstallation = async ( environment: Readonly, - options: { - readonly platform?: NodeJS.Platform; - readonly signal?: AbortSignal; - /** Probe deadline; tests shorten it. */ - readonly probeTimeoutMs?: number; - } = {}, + options: KunaInstallationOptions, ): Promise> => { const configuration = readKunaConfiguration(environment, options.platform); if (!configuration.ok) @@ -174,38 +194,25 @@ export const inspectKunaInstallation = async ( ); let fingerprint: { readonly sha256: string; readonly bytes: number }; - let script: boolean | undefined; + let script: boolean; try { - script = await startsWithShebang(executable); - fingerprint = await hashStableFile(executable, options.signal); - } catch (cause: unknown) { - if (cause instanceof OwnedRegularFileCleanupFailure) - return err({ - code: "hash_cleanup_failed", - reason: `Cannot fingerprint ${executable}: ${message(cause)}`, - diagnostics: { - command, + ({ fingerprint, script } = await options.fileCloseResources.run( + async () => { + const fingerprint = await hashStableFile(executable, { + ...(options.signal === undefined ? {} : { signal: options.signal }), + retainFailedClose: (owner) => + options.fileCloseResources.retain(owner, executable), + }); + const script = await startsWithShebang( executable, - version, - ...(typeof script === "boolean" ? { script } : {}), - }, - hashCleanupFailure: cause, - ...(cause instanceof StableFileHashCleanupFailure && - !cause.hasPrimaryCause && - cause.completedHash !== undefined && - script !== undefined - ? { - completedInstallation: { - configuration: configuration.value, - executable, - version, - verified: version === KUNA_VERIFIED_VERSION, - ...cause.completedHash, - script, - }, - } - : {}), - }); + options.fileCloseResources, + ); + options.signal?.throwIfAborted(); + return { fingerprint, script }; + }, + )); + } catch (cause: unknown) { + if (cause instanceof RegularFileCleanupFailure) throw cause; options.signal?.throwIfAborted(); return failure( "executable_missing", @@ -214,13 +221,6 @@ export const inspectKunaInstallation = async ( ); } - if (script === undefined) - return failure( - "executable_missing", - `Cannot inspect Kuna executable ${executable}.`, - { command, executable }, - ); - return ok({ configuration: configuration.value, executable, @@ -231,13 +231,19 @@ export const inspectKunaInstallation = async ( }); }; -const startsWithShebang = async (path: string): Promise => { - const handle = await open(path, "r"); - try { - const prefix = Buffer.alloc(2); - const { bytesRead } = await handle.read(prefix, 0, 2, 0); - return bytesRead === 2 && prefix.toString("latin1") === "#!"; - } finally { - await handle.close(); - } -}; +const startsWithShebang = ( + path: string, + fileCloseResources: FileCloseResourceOwner, +): Promise => + withOwnedRegularFile( + path, + { + symlinks: "follow", + retainFailedClose: (owner) => fileCloseResources.retain(owner, path), + }, + async (handle) => { + const prefix = Buffer.alloc(2); + const { bytesRead } = await handle.read(prefix, 0, 2, 0); + return bytesRead === 2 && prefix.toString("latin1") === "#!"; + }, + ); diff --git a/src/kuna/KunaProjectCache.ts b/src/kuna/KunaProjectCache.ts index 5d522e7a5d..56485e2998 100644 --- a/src/kuna/KunaProjectCache.ts +++ b/src/kuna/KunaProjectCache.ts @@ -15,6 +15,7 @@ import canonicalize from "canonicalize"; import writeFileAtomic from "write-file-atomic"; import { z } from "zod"; +import { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import { AnalysisCancelledError, @@ -23,8 +24,8 @@ import { import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { err, ok, type Result } from "../domain/result.js"; import { hashStableFile } from "../filesystem/StableFileHash.js"; +import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js"; import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; -import { OwnedRegularFileCleanupFailure } from "../filesystem/RegularFile.js"; import type { KunaInstallation } from "./KunaInstallation.js"; import { exportKunaProject, @@ -141,8 +142,30 @@ interface RetainedProcess { readonly staging: string; } +interface RetainedHashFile { + readonly key: string; + readonly staging: string | null; + readonly resource: string; + readonly lease: AnalysisCleanupLease; + /** The first failure tied to this owner; kept stable across retry attempts. */ + originatingFailure?: AnalysisError; + /** Most recent close failure, reported without wrapping prior retry errors. */ + lastCloseFailure?: ProviderCleanupError; +} + +interface CleanupErrorReport { + readonly error: WeakRef; + readonly pendingOwners: Set; +} + +interface CleanupFailureOwner { + readonly error: AnalysisError; + readonly owners: readonly object[]; +} + interface StagedResult { readonly result: Result; + readonly fileCleanupFailure?: RegularFileCleanupFailure; /** Set when an owned process may still write into staging. */ readonly retainStaging: boolean; } @@ -224,17 +247,21 @@ export class KunaProjectCache { /** Only resources whose process cleanup is proven may be retried as files. */ readonly #retainedFiles = new Map< string, - { readonly key: string | null; readonly error: ProviderCleanupError } - >(); - /** Hash descriptors remain owned here until the exact handle closes. */ - readonly #retainedHashHandles = new Map< - OwnedFileHandle, { - readonly key: string; - readonly path: string; + readonly key: string | null; readonly error: ProviderCleanupError; + readonly cleanupOwner: object; } >(); + /** Hash descriptors whose close was not confirmed; staging stays until retry. */ + readonly #retainedHashFiles = new Map(); + /** Keep the exact close lease available for errors bound after a retry ends. */ + readonly #hashRecords = new WeakMap(); + /** Serialize owner release and staging retirement across concurrent lookups. */ + #hashRetry: Promise = Promise.resolve(); + /** Weakly track caller errors against every resource owner they report. */ + readonly #cleanupErrorReports: CleanupErrorReport[] = []; + readonly #stagingCleanupOwners = new Map(); constructor( root: string, @@ -253,7 +280,7 @@ export class KunaProjectCache { return ( this.#retainedProcesses.length > 0 || this.#retainedFiles.size > 0 || - this.#retainedHashHandles.size > 0 + this.#retainedHashFiles.size > 0 ); } @@ -280,8 +307,9 @@ export class KunaProjectCache { key: string, signal?: AbortSignal, ): Promise { - const pendingHashCleanup = await this.#retryHashHandles(key); - if (pendingHashCleanup !== undefined) throw pendingHashCleanup; + await this.#retryHashFiles(key); + const cleanup = this.#cleanupResult(this.#hashCleanupFailures(key)); + if (!cleanup.ok) throw cleanup.error; const verified = this.#verified.get(key); if (verified !== undefined) { // Another process may have evicted it since this one verified it. @@ -308,17 +336,20 @@ export class KunaProjectCache { for (const file of Object.values(manifest.files)) { let observed: { readonly sha256: string; readonly bytes: number }; try { - observed = await hashStableFile(join(directory, file.path), signal); + const path = join(directory, file.path); + observed = await hashStableFile(path, { + ...(signal === undefined ? {} : { signal }), + retainFailedClose: (owner) => + this.#retainHashFile(owner, key, null, path), + }); } catch (cause: unknown) { - if (cause instanceof OwnedRegularFileCleanupFailure) { - this.#retainHashHandle( + if (cause instanceof RegularFileCleanupFailure) + throw this.#replaceHashCloseError( key, - cause.path, - cause.owner, - cause.cleanupCause, + null, + kunaExportFailure(cause, signal), + cause, ); - throw cause; - } signal?.throwIfAborted(); if (!isDamaged(cause)) throw cause; await this.#discard(key); @@ -404,15 +435,7 @@ export class KunaProjectCache { ); await this.#retryProcesses((retained) => retained.key === key); await this.#retryFiles(key); - await this.#retryHashHandles(key); - return cleanupResult([ - ...[...this.#retainedProcesses, ...this.#retainedFiles.values()] - .filter((retained) => retained.key === key) - .map(({ error }) => error), - ...[...this.#retainedHashHandles.values()] - .filter((retained) => retained.key === key) - .map(({ error }) => error), - ]); + return this.#cleanupResult(this.#retainedCleanupFailures(key)); } /** Wait for builds, retry safe file removal, and report unresolved cleanup. */ @@ -423,12 +446,7 @@ export class KunaProjectCache { while (this.#maintenance !== undefined) await this.#maintenance; await this.#retryProcesses(() => true); await this.#retryFiles(); - await this.#retryHashHandles(); - return cleanupResult([ - ...this.#retainedProcesses.map(({ error }) => error), - ...[...this.#retainedFiles.values()].map(({ error }) => error), - ...[...this.#retainedHashHandles.values()].map(({ error }) => error), - ]); + return this.#cleanupResult(this.#retainedCleanupFailures()); } /** Retry stopping retained process trees; retire the staging of each one proven stopped. */ @@ -442,58 +460,188 @@ export class KunaProjectCache { this.#retainedProcesses.indexOf(retained), 1, ); + this.#resolveCleanupOwner(retained); await this.#retireStaging(retained.staging, retained.key); } } /** Retry only stopped file resources belonging to the closing task, or all. */ async #retryFiles(key?: string): Promise { + await this.#retryHashFiles(key); for (const [path, retained] of [...this.#retainedFiles]) if (key === undefined || retained.key === key) await this.#removeRetired(path).catch(() => undefined); } - async #retryHashHandles( - key?: string, - ): Promise { - for (const [owner, retained] of [...this.#retainedHashHandles]) { - if (key !== undefined && retained.key !== key) continue; - try { - await owner.close(); - this.#retainedHashHandles.delete(owner); - } catch { - // Preserve the same owner and its original cleanup failure for callers. + /** Retry retained hash descriptors before admitting another read for a key. */ + async #retryHashFiles(key?: string): Promise { + const previous = this.#hashRetry; + let release!: () => void; + const current = new Promise((resolve) => { + release = resolve; + }); + this.#hashRetry = previous.then(() => current); + await previous; + try { + const staging = new Map(); + for (const [owner, retained] of [...this.#retainedHashFiles]) { + if (key !== undefined && retained.key !== key) continue; + try { + await retained.lease.release(); + this.#retainedHashFiles.delete(owner); + this.#resolveCleanupOwner(owner); + if (retained.staging !== null) + staging.set(retained.staging, retained.key); + } catch (cause: unknown) { + const original = retained.originatingFailure; + const closeFailure = new ProviderCleanupError( + KUNA_PROVIDER_ID, + [retained.resource], + { reason: cause instanceof Error ? cause.message : String(cause) }, + { + operation: KUNA_EXPORT_OPERATION, + cause: + original === undefined + ? cause + : new AggregateError( + [original, cause], + "Hash operation and descriptor cleanup both failed", + ), + }, + ); + retained.lastCloseFailure = closeFailure; + this.#bindCleanupError(closeFailure, [ + owner, + ...(retained.staging === null + ? [] + : [this.#stagingCleanupOwner(retained.staging)]), + ]); + } + } + for (const [path, retainedKey] of staging) + if ( + ![...this.#retainedHashFiles.values()].some( + ({ staging: ownerStaging }) => ownerStaging === path, + ) + ) + await this.#retireStaging(path, retainedKey); + } finally { + release(); + } + } + + #hashCleanupFailures(key?: string): CleanupFailureOwner[] { + return [...this.#retainedHashFiles.entries()] + .filter(([, retained]) => key === undefined || retained.key === key) + .flatMap(([owner, retained]) => { + const failure = + retained.lastCloseFailure ?? retained.originatingFailure; + return failure === undefined + ? [] + : [ + { + error: failure, + owners: [ + owner, + ...(retained.staging === null + ? [] + : [this.#stagingCleanupOwner(retained.staging)]), + ], + }, + ]; + }); + } + + #retainedCleanupFailures(key?: string): CleanupFailureOwner[] { + return [ + ...this.#retainedProcesses + .filter((retained) => key === undefined || retained.key === key) + .map((retained) => ({ + error: retained.error, + owners: [retained, this.#stagingCleanupOwner(retained.staging)], + })), + ...[...this.#retainedFiles.values()] + .filter((retained) => key === undefined || retained.key === key) + .map((retained) => ({ + error: retained.error, + owners: [retained.cleanupOwner], + })), + ...this.#hashCleanupFailures(key), + ]; + } + + #cleanupResult( + failures: readonly CleanupFailureOwner[], + ): Result { + const result = cleanupResult(failures.map(({ error }) => error)); + if (result.ok) return result; + const owners = new Set(failures.flatMap(({ owners }) => owners)); + this.#bindCleanupError(result.error, [...owners]); + return result; + } + + #bindCleanupError(error: AnalysisError, owners: readonly object[]): void { + this.#pruneCleanupErrorReports(); + const report = this.#cleanupErrorReports.find( + (candidate) => candidate.error.deref() === error, + ); + const pending = report?.pendingOwners ?? new Set(); + for (const owner of owners) pending.add(owner); + if (report === undefined && pending.size > 0) + this.#cleanupErrorReports.push({ + error: new WeakRef(error), + pendingOwners: pending, + }); + } + + #fileCleanupOwner(path: string): object { + return this.#retainedFiles.get(path)?.cleanupOwner ?? {}; + } + + #stagingCleanupOwner(path: string): object { + let owner = this.#stagingCleanupOwners.get(path); + if (owner === undefined) { + owner = {}; + this.#stagingCleanupOwners.set(path, owner); + } + return owner; + } + + #resolveCleanupOwner(owner: object): void { + for (const report of [...this.#cleanupErrorReports]) { + const error = report.error.deref(); + if (error === undefined) { + this.#cleanupErrorReports.splice( + this.#cleanupErrorReports.indexOf(report), + 1, + ); + continue; + } + report.pendingOwners.delete(owner); + if (report.pendingOwners.size === 0) { + error.resolveCleanup(); + this.#cleanupErrorReports.splice( + this.#cleanupErrorReports.indexOf(report), + 1, + ); } } - const remaining = [...this.#retainedHashHandles.values()].filter( - (retained) => key === undefined || retained.key === key, - ); - if (remaining.length === 0) return undefined; - const result = cleanupResult(remaining.map(({ error }) => error)); - return result.ok ? undefined : result.error; } - #retainHashHandle( - key: string, - path: string, - owner: OwnedFileHandle, - cause: unknown, - ): void { - this.#retainedHashHandles.set(owner, { - key, - path, - error: new ProviderCleanupError( - KUNA_PROVIDER_ID, - [path], - { reason: cause instanceof Error ? cause.message : String(cause) }, - { operation: KUNA_EXPORT_OPERATION, cause }, - ), - }); + #transferCleanupOwner(from: object, to: object): void { + this.#pruneCleanupErrorReports(); + for (const report of this.#cleanupErrorReports) + if (report.pendingOwners.delete(from)) report.pendingOwners.add(to); } - #adoptHashFailure(key: string, cause: unknown): void { - if (cause instanceof OwnedRegularFileCleanupFailure) - this.#retainHashHandle(key, cause.path, cause.owner, cause.cleanupCause); + #pruneCleanupErrorReports(): void { + for ( + let index = this.#cleanupErrorReports.length - 1; + index >= 0; + index -= 1 + ) + if (this.#cleanupErrorReports[index]?.error.deref() === undefined) + this.#cleanupErrorReports.splice(index, 1); } /** @@ -505,15 +653,27 @@ export class KunaProjectCache { key: string, ): Promise { let cleanupPath = staging; + const stagingOwner = this.#stagingCleanupOwners.get(staging); try { const retired = join(this.#root, "trash", `build-${randomUUID()}`); await rename(staging, retired); cleanupPath = retired; await this.#removeRetired(retired, key); + if (stagingOwner !== undefined) this.#resolveCleanupOwner(stagingOwner); return undefined; } catch (cause: unknown) { - if (cause instanceof ProviderCleanupError) return cause; - return this.#retainFileCleanup(cleanupPath, cause, key); + const failure = + cause instanceof ProviderCleanupError + ? cause + : this.#retainFileCleanup(cleanupPath, cause, key); + if (stagingOwner !== undefined) + this.#transferCleanupOwner( + stagingOwner, + this.#fileCleanupOwner(cleanupPath), + ); + return failure; + } finally { + this.#stagingCleanupOwners.delete(staging); } } @@ -574,7 +734,6 @@ export class KunaProjectCache { await this.#sweep(signal); staging = await mkdtemp(join(this.#root, "staging", "build-")); } catch (cause: unknown) { - this.#adoptHashFailure(key, cause); return err(kunaExportFailure(cause, signal)); } const built = await this.#exportAndPublish( @@ -583,14 +742,31 @@ export class KunaProjectCache { request, signal, onStatus, - ).catch((cause: unknown): StagedResult => { - this.#adoptHashFailure(key, cause); - return { - result: err(kunaExportFailure(cause, signal)), - retainStaging: false, - }; - }); + ).catch((cause: unknown): StagedResult => ({ + result: err(kunaExportFailure(cause, signal)), + retainStaging: false, + ...(cause instanceof RegularFileCleanupFailure + ? { fileCleanupFailure: cause } + : {}), + })); if (built.retainStaging) return built.result; + const retainedStaging = [...this.#retainedHashFiles.values()].some( + (retained) => retained.staging === staging, + ); + if (retainedStaging) { + if ( + !built.result.ok && + built.result.error.cleanupIncomplete && + built.fileCleanupFailure !== undefined + ) + this.#replaceHashCloseError( + key, + staging, + built.result.error, + built.fileCleanupFailure, + ); + return built.result; + } // Only a proven-stopped build may enter trash. A failed write, crash or // lost cleanup identity therefore never makes live staging reclaimable. const retired = await this.#retireStaging(staging, key); @@ -617,18 +793,23 @@ export class KunaProjectCache { signal, onStatus, launcher: this.#launcher, + retainFailedFileClose: (owner, resource) => + this.#retainHashFile(owner, key, staging, resource), }); if (outcome.kind === "retained") { - this.#retainedProcesses.push({ + const retained: RetainedProcess = { key, error: outcome.error, owner: outcome.owner, staging, - }); + }; + this.#retainedProcesses.push(retained); + this.#bindCleanupError(outcome.error, [ + retained, + this.#stagingCleanupOwner(staging), + ]); return { result: err(outcome.error), retainStaging: true }; } - if (outcome.fileCleanup !== undefined) - this.#adoptHashFailure(key, outcome.fileCleanup); return { result: outcome.result.ok ? await this.#publish( @@ -640,6 +821,9 @@ export class KunaProjectCache { ) : outcome.result, retainStaging: false, + ...(outcome.fileCleanupFailure === undefined + ? {} + : { fileCleanupFailure: outcome.fileCleanupFailure }), }; } @@ -803,7 +987,10 @@ export class KunaProjectCache { async #removeRetired(path: string, key?: string): Promise { try { await rm(path, { recursive: true, force: true }); + const retained = this.#retainedFiles.get(path); this.#retainedFiles.delete(path); + if (retained !== undefined) + this.#resolveCleanupOwner(retained.cleanupOwner); } catch (cause: unknown) { throw this.#retainFileCleanup(path, cause, key); } @@ -841,17 +1028,52 @@ export class KunaProjectCache { { reason: cause instanceof Error ? cause.message : String(cause) }, { operation: KUNA_EXPORT_OPERATION, cause }, ); - this.#retainedFiles.set(path, { + const retained = { key: key ?? this.#retainedFiles.get(path)?.key ?? null, error, - }); + cleanupOwner: this.#retainedFiles.get(path)?.cleanupOwner ?? {}, + }; + this.#retainedFiles.set(path, retained); + this.#bindCleanupError(error, [retained.cleanupOwner]); return error; } + + #retainHashFile( + owner: OwnedFileHandle, + key: string, + staging: string | null, + resource: string, + ): void { + const retained: RetainedHashFile = { + key, + staging, + resource, + lease: new AnalysisCleanupLease(async () => owner.close()), + }; + this.#hashRecords.set(owner, retained); + this.#retainedHashFiles.set(owner, retained); + } + + #replaceHashCloseError( + key: string, + staging: string | null, + failure: AnalysisError, + fileCleanupFailure: RegularFileCleanupFailure, + ): AnalysisError { + const owner = fileCleanupFailure.owner; + const retained = this.#hashRecords.get(owner); + if (retained !== undefined) { + retained.lease.bindCleanupError(failure); + if (retained.key === key && retained.staging === staging) + retained.originatingFailure ??= failure; + } + return failure; + } } /** One cleanup error aggregating what remains owned, or success when nothing does. */ const cleanupResult = ( - remaining: readonly ProviderCleanupError[], + remaining: readonly AnalysisError[], ): Result => { if (remaining.length === 0) return ok(null); return err( @@ -865,7 +1087,13 @@ const cleanupResult = ( ) .join("; "), }, - { operation: KUNA_EXPORT_OPERATION }, + { + operation: KUNA_EXPORT_OPERATION, + cause: new AggregateError( + remaining, + "Kuna cache cleanup is incomplete", + ), + }, ), ); }; diff --git a/src/kuna/KunaProjectExport.ts b/src/kuna/KunaProjectExport.ts index 9a2d28b270..b6077ae4c2 100644 --- a/src/kuna/KunaProjectExport.ts +++ b/src/kuna/KunaProjectExport.ts @@ -9,16 +9,15 @@ import { } from "../domain/analysisErrorCore.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; -import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; import { err, ok, type Result } from "../domain/result.js"; import { hashStableFile } from "../filesystem/StableFileHash.js"; -import { StableFileHashCleanupFailure } from "../filesystem/StableFileHash.js"; -import { OwnedRegularFileCleanupFailure } from "../filesystem/RegularFile.js"; +import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; import { prepareProcessOwnershipInspection } from "../process/ProcessOwnershipObservation.js"; import type { ProviderProcessSupervisor } from "../process/ProviderProcess.js"; import { copyAndHash, KUNA_INVENTORY_OUTPUT_BYTES, + kunaFileCleanupError, runKunaCommand, type KunaProcessLauncher, } from "./KunaCommand.js"; @@ -32,6 +31,7 @@ import { type KunaProjectFunction, } from "./KunaProjectFiles.js"; import { KUNA_PROVIDER_ID } from "./KunaRelease.js"; +import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js"; /** Operation name every export and cache failure reports. */ export const KUNA_EXPORT_OPERATION = "kuna:decompile-project"; @@ -86,7 +86,7 @@ export type KunaExportOutcome = | { readonly kind: "done"; readonly result: Result; - readonly fileCleanup?: OwnedRegularFileCleanupFailure; + readonly fileCleanupFailure?: RegularFileCleanupFailure; } | { readonly kind: "retained"; @@ -105,6 +105,10 @@ export interface KunaExportRequest { /** Directory, inside the workspace, that receives the export's files. */ readonly output: string; readonly signal: AbortSignal; + readonly retainFailedFileClose: ( + owner: OwnedFileHandle, + resource: string, + ) => void; readonly onStatus: (status: KunaExportStatus) => void; readonly launcher?: KunaProcessLauncher | undefined; } @@ -120,39 +124,11 @@ const isMissing = (cause: unknown): boolean => /** Map an unexpected failure to a typed one, preferring cancellation. */ export const kunaExportFailure = ( cause: unknown, - signal: AbortSignal, + signal: AbortSignal | undefined, ): AnalysisError => { - if (cause instanceof OwnedRegularFileCleanupFailure) { - const primary = signal.aborted - ? new AnalysisCancelledError(KUNA_EXPORT_OPERATION, { - cause: cause.cause, - }) - : new ProviderAdapterError(KUNA_PROVIDER_ID, KUNA_EXPORT_OPERATION, { - cause: cause.cause, - userMessage: `Kuna project cache failed: ${message(cause.cause)}`, - }); - const cleanup = new ProviderCleanupError( - KUNA_PROVIDER_ID, - [cause.path], - { - reason: - cause.cleanupCause instanceof Error - ? cause.cleanupCause.message - : String(cause.cleanupCause), - ...(cause instanceof StableFileHashCleanupFailure && - cause.completedHash !== undefined - ? { completed_hash: cause.completedHash } - : {}), - }, - { operation: KUNA_EXPORT_OPERATION, cause: cause.cleanupCause }, - ); - return analysisErrorWithCleanupFailure( - primary, - cleanup, - KUNA_EXPORT_OPERATION, - ); - } - if (signal.aborted) + if (cause instanceof RegularFileCleanupFailure) + return kunaFileCleanupError(cause, KUNA_EXPORT_OPERATION, signal); + if (signal?.aborted === true) return new AnalysisCancelledError(KUNA_EXPORT_OPERATION, { cause }); if (cause instanceof AnalysisError) return cause; return new ProviderAdapterError(KUNA_PROVIDER_ID, KUNA_EXPORT_OPERATION, { @@ -174,8 +150,13 @@ const unusable = (reason: string): KunaExportOutcome => ({ const verifyKunaExecutable = async ( installation: KunaInstallation, signal: AbortSignal, + retainFailedFileClose: KunaExportRequest["retainFailedFileClose"], ): Promise> => { - const executable = await hashStableFile(installation.executable, signal); + const executable = await hashStableFile(installation.executable, { + signal, + retainFailedClose: (owner) => + retainFailedFileClose(owner, installation.executable), + }); if (executable.sha256 === installation.sha256) return ok(null); return err( new ProviderAdapterError(KUNA_PROVIDER_ID, KUNA_EXPORT_OPERATION, { @@ -228,6 +209,7 @@ export const exportKunaProject = async ( outputBytes, signal, launcher: request.launcher, + retainFailedFileClose: request.retainFailedFileClose, }); if (outcome.kind === "ok") return { ok: true as const, snapshot: outcome.snapshot }; @@ -243,9 +225,9 @@ export const exportKunaProject = async ( : ({ kind: "done", result: err(outcome.error), - ...(outcome.fileCleanup === undefined + ...(outcome.fileCleanupFailure === undefined ? {} - : { fileCleanup: outcome.fileCleanup }), + : { fileCleanupFailure: outcome.fileCleanupFailure }), } as const), }; }; @@ -306,7 +288,14 @@ export const exportKunaProject = async ( }); if (status === undefined || !status.isFile()) return unusable(`Kuna's export has no regular ${name}.`); - files[role] = { path: name, ...(await hashStableFile(path, signal)) }; + files[role] = { + path: name, + ...(await hashStableFile(path, { + signal, + retainFailedClose: (owner) => + request.retainFailedFileClose(owner, path), + })), + }; } const complete = files as KunaExportFiles; const functions = await readKunaExportSource( @@ -316,7 +305,11 @@ export const exportKunaProject = async ( signal, ); if (!functions.ok) return unusable(functions.error); - const identity = await verifyKunaExecutable(installation, signal); + const identity = await verifyKunaExecutable( + installation, + signal, + request.retainFailedFileClose, + ); if (!identity.ok) return { kind: "done", result: identity }; // `functions.json` holds exactly the text this inventory was parsed from, so // publishing reuses it rather than holding a second parsed copy. diff --git a/src/kuna/KunaProvider.ts b/src/kuna/KunaProvider.ts index 978ce2abf4..48f4d35c65 100644 --- a/src/kuna/KunaProvider.ts +++ b/src/kuna/KunaProvider.ts @@ -9,26 +9,25 @@ import type { ProviderTargetSupport, } from "../application/AnalysisProvider.js"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; -import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { createAnalysisProfile, type AnalysisProfileCommitment, } from "../domain/analysisProfile.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { err, ok } from "../domain/result.js"; -import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; -import { ProviderCleanupError } from "../domain/providerCleanupError.js"; -import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; -import { StableFileHashCleanupFailure } from "../filesystem/StableFileHash.js"; +import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js"; import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; -import type { KunaProcessLauncher } from "./KunaCommand.js"; +import { + kunaFileCleanupError, + type KunaProcessLauncher, +} from "./KunaCommand.js"; import { readKunaConfiguration, type KunaConfiguration, } from "./KunaConfiguration.js"; import { + createKunaFileCloseScope, inspectKunaInstallation, - type KunaInstallation, } from "./KunaInstallation.js"; import { KunaProjectCache } from "./KunaProjectCache.js"; import { @@ -53,7 +52,6 @@ const VERIFIED_ARCHITECTURES: ReadonlySet = new Set([ */ export class KunaProvider implements AnalysisProviderCandidate { readonly #environment: Readonly; - readonly #readinessFileOwners = new Map(); readonly #platform: NodeJS.Platform; readonly #launcher: KunaProcessLauncher | undefined; readonly #singleFunctionRuns: number | undefined; @@ -171,83 +169,35 @@ export class KunaProvider implements AnalysisProviderCandidate { ) { if (options?.signal?.aborted === true) return err(new AnalysisCancelledError("kuna:profile")); - const pending = await this.#closeReadinessOwners(); - if (!pending.ok) return pending; - const installation = await inspectKunaInstallation(this.#environment, { - platform: this.#platform, - ...(options?.signal === undefined ? {} : { signal: options.signal }), - }); - let value: KunaInstallation; - if (!installation.ok) { - const readiness = installation.error; - const hashFailure = - readiness.code === "hash_cleanup_failed" - ? readiness.hashCleanupFailure - : undefined; - if (hashFailure !== undefined) { - this.#readinessFileOwners.set(hashFailure.owner, hashFailure.path); - try { - await hashFailure.owner.close(); - this.#readinessFileOwners.delete(hashFailure.owner); - } catch (cleanupCause: unknown) { - const cleanup = new ProviderCleanupError( - KUNA_PROVIDER_ID, - [hashFailure.path], - { - reason: - cleanupCause instanceof Error - ? cleanupCause.message - : String(cleanupCause), - ...(hashFailure instanceof StableFileHashCleanupFailure && - hashFailure.completedHash !== undefined - ? { completed_hash: hashFailure.completedHash } - : {}), - }, - { operation: "open_binary", cause: cleanupCause }, - ); - const primary = options?.signal?.aborted - ? new AnalysisCancelledError("kuna:profile", { - cause: hashFailure.cause, - }) - : hashFailure instanceof StableFileHashCleanupFailure && - hashFailure.hasPrimaryCause - ? new ProviderAdapterError(KUNA_PROVIDER_ID, "open_binary", { - cause: hashFailure.cause, - userMessage: readiness.reason, - }) - : kunaUnavailable("open_binary", readiness.reason); - return err( - analysisErrorWithCleanupFailure(primary, cleanup, "open_binary"), - ); - } - if ( - hashFailure instanceof StableFileHashCleanupFailure && - hashFailure.hasPrimaryCause - ) { - if (options?.signal?.aborted) - return err( - new AnalysisCancelledError("kuna:profile", { - cause: hashFailure.cause, - }), - ); - return err( - new ProviderAdapterError(KUNA_PROVIDER_ID, "open_binary", { - cause: hashFailure.cause, - userMessage: readiness.reason, - }), - ); - } - const completedInstallation = - readiness.code === "hash_cleanup_failed" - ? readiness.completedInstallation - : undefined; - if (completedInstallation === undefined) - return err(kunaUnavailable("open_binary", installation.error.reason)); - value = completedInstallation; - } else { - return err(kunaUnavailable("open_binary", installation.error.reason)); - } - } else value = installation.value; + let installation: Awaited>; + const fileCloseResources = options?.fileCloseResources; + if (fileCloseResources === undefined) + return err( + kunaUnavailable( + "kuna:profile", + "Kuna profile inspection requires a durable file resource owner", + ), + ); + const fileCloseScope = createKunaFileCloseScope(fileCloseResources); + try { + installation = await inspectKunaInstallation(this.#environment, { + platform: this.#platform, + ...(options?.signal === undefined ? {} : { signal: options.signal }), + fileCloseResources: fileCloseScope.owner, + }); + } catch (cause: unknown) { + if (!(cause instanceof RegularFileCleanupFailure)) throw cause; + const error = kunaFileCleanupError( + cause, + "kuna:profile", + options?.signal, + ); + fileCloseScope.bindFailure(cause, error); + return err(error); + } + if (!installation.ok) + return err(kunaUnavailable("open_binary", installation.error.reason)); + const { value } = installation; return ok({ profile: createAnalysisProfile( { ...KUNA_PROVIDER_IDENTITY, version: value.version }, @@ -277,54 +227,13 @@ export class KunaProvider implements AnalysisProviderCandidate { }); } - /** Release candidate resources even when no target client was opened. */ - async closeOwnedResources() { - const readiness = await this.#closeReadinessOwners(); - const cache = await this.#cache?.close(); - if (readiness.ok && (cache === undefined || cache.ok)) return ok(null); - if (!readiness.ok && (cache === undefined || cache.ok)) return readiness; - if (readiness.ok && cache !== undefined && !cache.ok) return cache; - const failures = [ - ...(readiness.ok ? [] : [readiness.error]), - ...(cache === undefined || cache.ok ? [] : [cache.error]), - ]; - return err( - new ProviderCleanupError( - KUNA_PROVIDER_ID, - failures.flatMap((failure) => failure.cleanupResources), - { reason: failures.map((failure) => failure.message).join("; ") }, - { cause: new AggregateError(failures) }, - ), - ); + /** Shared cache cleanup belongs to the candidate, not a retired client. */ + closeOwnedResources() { + return this.#cache?.close() ?? Promise.resolve(ok(null)); } - /** Report retained file cleanup without probing readiness or retrying owners. */ hasPendingOwnedResources(): boolean { - return ( - this.#readinessFileOwners.size > 0 || - this.#cache?.hasPendingCleanup === true - ); - } - - async #closeReadinessOwners() { - const failures: { readonly path: string; readonly cause: unknown }[] = []; - for (const [owner, path] of this.#readinessFileOwners) { - try { - await owner.close(); - this.#readinessFileOwners.delete(owner); - } catch (cause: unknown) { - failures.push({ path, cause }); - } - } - if (failures.length === 0) return ok(null); - return err( - new ProviderCleanupError( - KUNA_PROVIDER_ID, - failures.map(({ path }) => path), - { reason: "Kuna executable hash descriptors remain open" }, - { operation: "open_binary", cause: new AggregateError(failures) }, - ), - ); + return this.#cache?.hasPendingCleanup === true; } createClient( diff --git a/src/kuna/KunaSessionClient.ts b/src/kuna/KunaSessionClient.ts index e11a62bce1..ae8478459e 100644 --- a/src/kuna/KunaSessionClient.ts +++ b/src/kuna/KunaSessionClient.ts @@ -6,6 +6,7 @@ import { type AnalysisExecution, type AnalysisOperation, type ExecutionOptions, + type FileCloseResourceOwner, } from "../application/AnalysisProvider.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; import { @@ -30,15 +31,15 @@ import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import type { RelationshipCoverage } from "../domain/relationshipCoverage.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; -import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; -import { StableFileHashCleanupFailure } from "../filesystem/StableFileHash.js"; import { err, ok, type Result } from "../domain/result.js"; import { ABORTED, waitForAbortable } from "../process/AbortablePromise.js"; import { KUNA_INVENTORY_OUTPUT_BYTES, + kunaFileCleanupError, type KunaProcessLauncher, } from "./KunaCommand.js"; import { + createKunaFileCloseScope, inspectKunaInstallation, type KunaInstallation, } from "./KunaInstallation.js"; @@ -72,6 +73,9 @@ import { type KunaProjectFileSpan, } from "./KunaProjectReader.js"; import { kunaUnavailable } from "./KunaUnavailable.js"; +import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js"; +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; /** How often a request waiting on the project export reports that it still waits. */ const WAIT_PROGRESS_INTERVAL_MS = 10_000; @@ -227,6 +231,7 @@ export class KunaSessionClient implements AnalysisClient { readonly #cache: KunaProjectCache; readonly #environment: Readonly; readonly #runSlots: KunaRunSlots; + readonly #fileCloseResources = new ArtifactResourceScope(); /** Single-function runs in flight, by entry address. */ readonly #singleFunctionRuns = new Map(); /** Completed single-function texts by address, least recently used first. */ @@ -425,6 +430,22 @@ export class KunaSessionClient implements AnalysisClient { } const commands = await this.#commands.close(); if (!commands.ok) failures.push(commands.error); + try { + await this.#fileCloseResources.close(); + } catch (cause: unknown) { + const observation = ArtifactReaderFailure.cleanupObservation( + cause, + "Kuna provider file descriptor", + ); + failures.push( + new ProviderCleanupError( + KUNA_PROVIDER_ID, + observation.resources, + { reason: observation.reason }, + { cause, operation: "close_binary" }, + ), + ); + } if (failures.length > 0) return err( new ProviderCleanupError( @@ -437,6 +458,13 @@ export class KunaSessionClient implements AnalysisClient { ) .join("; "), }, + { + cause: new AggregateError( + failures, + "Kuna session cleanup remains incomplete", + ), + operation: "close_binary", + }, ), ); this.#releasePin?.(); @@ -1271,44 +1299,24 @@ export class KunaSessionClient implements AnalysisClient { async #ensureInstallation(): Promise< Result > { - this.#installation ??= inspectKunaInstallation(this.#environment).then( - (inspected): Result => { - if (!inspected.ok) { - const hashFailure = - inspected.error.code === "hash_cleanup_failed" - ? inspected.error.hashCleanupFailure - : undefined; - if (hashFailure !== undefined) { - this.#commands.retainHashCleanup(hashFailure); - const primary = this.#session.signal.aborted - ? new AnalysisCancelledError("health", { - cause: hashFailure.cause, - }) - : new ProviderAdapterError(KUNA_PROVIDER_ID, "health", { - cause: hashFailure.cause, - userMessage: inspected.error.reason, - }); - const cleanup = new ProviderCleanupError( - KUNA_PROVIDER_ID, - [hashFailure.path], - { - reason: - hashFailure.cleanupCause instanceof Error - ? hashFailure.cleanupCause.message - : String(hashFailure.cleanupCause), - ...(hashFailure instanceof StableFileHashCleanupFailure && - hashFailure.completedHash !== undefined - ? { completed_hash: hashFailure.completedHash } - : {}), - }, - { operation: "health", cause: hashFailure.cleanupCause }, - ); - return err( - analysisErrorWithCleanupFailure(primary, cleanup, "health"), - ); - } + this.#installation ??= (async () => { + const parentFileCloseResources: FileCloseResourceOwner = { + retain: (owner, resource) => + this.#fileCloseResources.retain({ + kind: "lease", + resource, + lease: { release: () => owner.close() }, + }), + run: (operation: () => Promise) => + this.#fileCloseResources.run(operation), + }; + const fileCloseScope = createKunaFileCloseScope(parentFileCloseResources); + try { + const inspected = await inspectKunaInstallation(this.#environment, { + fileCloseResources: fileCloseScope.owner, + }); + if (!inspected.ok) return err(kunaUnavailable("health", inspected.error.reason)); - } const committed = this.#profile?.parameters["kuna_sha256"]; if (committed !== undefined && committed !== inspected.value.sha256) return err( @@ -1322,8 +1330,17 @@ export class KunaSessionClient implements AnalysisClient { }), ); return ok(inspected.value); - }, - ); + } catch (cause: unknown) { + if (!(cause instanceof RegularFileCleanupFailure)) throw cause; + const error = kunaFileCleanupError( + cause, + "health", + this.#session.signal, + ); + fileCloseScope.bindFailure(cause, error); + return err(error); + } + })(); return this.#installation; } diff --git a/src/kuna/KunaTargetCommands.ts b/src/kuna/KunaTargetCommands.ts index dd6b9d8076..b99d21b5ac 100644 --- a/src/kuna/KunaTargetCommands.ts +++ b/src/kuna/KunaTargetCommands.ts @@ -10,8 +10,6 @@ import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { err, ok, type Result } from "../domain/result.js"; -import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; -import { OwnedRegularFileCleanupFailure } from "../filesystem/RegularFile.js"; import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; import { prepareProcessOwnershipInspection } from "../process/ProcessOwnershipObservation.js"; import type { @@ -26,6 +24,10 @@ import { import type { KunaInstallation } from "./KunaInstallation.js"; import { KUNA_PROVIDER_ID } from "./KunaRelease.js"; import type { KunaRunRelease } from "./KunaRunSlots.js"; +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; interface Workspace { readonly root: PrivateRuntimeRoot; @@ -50,14 +52,13 @@ export interface KunaTargetCommandRequest { /** Own the digest-checked target copy and every command that can still use it. */ export class KunaTargetCommands { - readonly #active = new Set>(); + readonly #resources = new ArtifactResourceScope(); readonly #retained: { readonly error: ProviderCleanupError; readonly owner: ProviderProcessSupervisor | undefined; readonly releaseRun: KunaRunRelease | undefined; }[] = []; readonly #roots: PrivateRuntimeRoot[] = []; - readonly #fileHandles = new Map(); #workspace: Promise> | undefined; #closing = false; @@ -78,32 +79,17 @@ export class KunaTargetCommands { err(new AnalysisCancelledError(request.operation)), ); } - return this.#own(this.#run(request)); - } - - /** Retain a hash owner whose readiness error has been projected by a session. */ - retainHashCleanup(failure: OwnedRegularFileCleanupFailure): void { - this.#fileHandles.set(failure.owner, failure.path); - } - - #own(work: Promise): Promise { - this.#active.add(work); - const release = () => this.#active.delete(work); - void work.then(release, release); - return work; + return this.#run(request); } /** Establish the immutable input before waiting for a foreground run slot. */ prepare(): Promise> { if (this.#closing || this.lifetime.signal.aborted) return Promise.resolve(err(new AnalysisCancelledError("kuna:workspace"))); - return this.#own( - (async () => { - const pending = await this.#retryFileHandles(); - if (pending !== undefined) return err(pending); - const workspace = await this.#ensureWorkspace(); - return workspace.ok ? ok(null) : workspace; - })(), + return this.#admit("kuna:workspace", () => + this.#ensureWorkspace().then((workspace) => + workspace.ok ? ok(null) : workspace, + ), ); } @@ -113,34 +99,47 @@ export class KunaTargetCommands { const { releaseRun, ...options } = request; let retained = false; try { - const pending = await this.#retryFileHandles(); - if (pending !== undefined) return err(pending); - const workspace = await this.#ensureWorkspace(); - if (!workspace.ok) return workspace; - const outcome = await runKunaCommand({ - ...options, - arguments: request.arguments(workspace.value.input), - workspace: workspace.value.root.path, - reportedTimeoutMs: request.timeoutMs, - launcher: this.lifetime.launcher, - }); - if (outcome.kind === "retained") { - this.#retained.push({ - error: outcome.error, - owner: outcome.owner, - releaseRun, + return await this.#admit(request.operation, async () => { + const workspace = await this.#ensureWorkspace(); + if (!workspace.ok) return workspace; + let failedHashClose: + | { + readonly owner: OwnedFileHandle; + readonly lease: AnalysisCleanupLease; + } + | undefined; + const outcome = await runKunaCommand({ + ...options, + arguments: request.arguments(workspace.value.input), + workspace: workspace.value.root.path, + reportedTimeoutMs: request.timeoutMs, + launcher: this.lifetime.launcher, + retainFailedFileClose: (owner, resource) => { + const lease = new AnalysisCleanupLease(() => owner.close()); + this.#resources.retain({ kind: "lease", resource, lease }); + failedHashClose = { owner, lease }; + }, }); - retained = true; + if (outcome.kind === "retained") { + this.#retained.push({ + error: outcome.error, + owner: outcome.owner, + releaseRun, + }); + retained = true; + return err(outcome.error); + } + const fileFailure = + outcome.kind === "failed" ? outcome.fileCleanupFailure : undefined; + if (outcome.kind === "ok") + return ok({ ...outcome.snapshot, input: workspace.value.input }); + if ( + fileFailure !== undefined && + fileFailure.owner === failedHashClose?.owner + ) + failedHashClose.lease.bindCleanupError(outcome.error); return err(outcome.error); - } - if (outcome.kind === "failed" && outcome.fileCleanup !== undefined) - this.#fileHandles.set( - outcome.fileCleanup.owner, - outcome.fileCleanup.path, - ); - return outcome.kind === "ok" - ? ok({ ...outcome.snapshot, input: workspace.value.input }) - : err(outcome.error); + }); } finally { if (!retained) releaseRun?.(); } @@ -149,7 +148,23 @@ export class KunaTargetCommands { /** Called after the session signal aborts; failures keep their retry owners. */ async close(): Promise> { this.#closing = true; - while (this.#active.size > 0) await Promise.allSettled(this.#active); + const failures: ProviderCleanupError[] = []; + try { + await this.#resources.close(); + } catch (cause: unknown) { + const observation = ArtifactReaderFailure.cleanupObservation( + cause, + "Kuna command file descriptor", + ); + failures.push( + new ProviderCleanupError( + KUNA_PROVIDER_ID, + observation.resources, + { reason: observation.reason }, + { cause }, + ), + ); + } for (const retained of [...this.#retained]) { const stopped = await retained.owner?.stop(); if (stopped !== undefined && stopped.status !== "incomplete") { @@ -157,11 +172,9 @@ export class KunaTargetCommands { this.#retained.splice(this.#retained.indexOf(retained), 1); } } - const failures = this.#retained.map(({ error }) => error); - const fileFailure = await this.#retryFileHandles(); - if (fileFailure !== undefined) failures.push(fileFailure); + failures.push(...this.#retained.map(({ error }) => error)); const workspace = await this.#workspace; - if (this.#retained.length === 0 && fileFailure === undefined) { + if (this.#retained.length === 0) { if (workspace?.ok === true) { this.#roots.push(workspace.value.root); this.#workspace = undefined; @@ -196,34 +209,36 @@ export class KunaTargetCommands { ) .join("; "), }, + { + cause: new AggregateError( + failures, + "Kuna command cleanup remains incomplete", + ), + }, ), ); } - async #retryFileHandles(): Promise { - const failures: ProviderCleanupError[] = []; - for (const [owner, path] of this.#fileHandles) { - try { - await owner.close(); - this.#fileHandles.delete(owner); - } catch (cause: unknown) { - failures.push( + async #admit( + operation: string, + work: () => Promise>, + ): Promise> { + try { + return await this.#resources.run(work); + } catch (cause: unknown) { + if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined) + return err( new ProviderCleanupError( KUNA_PROVIDER_ID, - [path], - { reason: cause instanceof Error ? cause.message : String(cause) }, - { cause }, + cause.cleanup.resources, + { reason: cause.cleanup.reason }, + { cause, operation }, ), ); - } + if (this.#closing && cause instanceof ArtifactReaderFailure) + return err(new AnalysisCancelledError(operation, { cause })); + throw cause; } - return failures.length === 0 - ? undefined - : new ProviderCleanupError( - KUNA_PROVIDER_ID, - failures.flatMap((failure) => failure.cleanupResources), - { reason: failures.map((failure) => failure.message).join("; ") }, - ); } async #ensureWorkspace(): Promise> { diff --git a/src/main/processShutdown.ts b/src/main/processShutdown.ts index 1daa102688..74c51f4d6a 100644 --- a/src/main/processShutdown.ts +++ b/src/main/processShutdown.ts @@ -6,7 +6,7 @@ interface ShutdownSignalSource { } interface ShutdownInputSource { - once(event: "end" | "close", listener: () => void): unknown; + on(event: "end" | "close", listener: () => void): unknown; off(event: "end" | "close", listener: () => void): unknown; } @@ -27,8 +27,8 @@ export const registerProcessShutdown = ( ): (() => void) => { events.signals.on("SIGINT", handler); events.signals.on("SIGTERM", handler); - events.stdin.once("end", handler); - events.stdin.once("close", handler); + events.stdin.on("end", handler); + events.stdin.on("close", handler); return () => { events.signals.off("SIGINT", handler); events.signals.off("SIGTERM", handler); diff --git a/src/main/transport.ts b/src/main/transport.ts index 4846bedd9d..7e5b7e07b6 100644 --- a/src/main/transport.ts +++ b/src/main/transport.ts @@ -22,6 +22,10 @@ interface ServerContext { readonly loadOptionalProviders: () => Promise; } +type ClosableOptionalProvider = { + readonly close?: () => Promise; +}; + export const startMcpTransport = async ( dependencies: RuntimeDependencies, session: BinarySession, @@ -76,8 +80,32 @@ export const startMcpTransport = async ( }; // Optional providers are shared by every server instance the SDK creates, // including a discarded discovery probe, so only process shutdown closes them. + const optionalProvidersToClose = new Set( + [ + optionalProviders.browserObservation, + optionalProviders.browserScenarioCapture, + optionalProviders.electronObservation, + ].filter((provider) => provider !== undefined), + ); + let optionalProviderClose: Promise | undefined; const closeOptionalProviders = async (): Promise => { - await optionalProviders.browserScenarioCapture?.close?.(); + optionalProviderClose ??= (async () => { + const results = await Promise.allSettled( + [...optionalProvidersToClose].map(async (provider) => { + await provider.close?.(); + optionalProvidersToClose.delete(provider); + }), + ); + const failures = results.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (failures.length > 0) + throw new AggregateError(failures, "Optional provider cleanup failed"); + })().catch((cause: unknown) => { + optionalProviderClose = undefined; + throw cause; + }); + await optionalProviderClose; }; let handle: StdioServerHandle; try { @@ -95,6 +123,7 @@ export const startMcpTransport = async ( environment: serverContext.environment, delivery: serverContext.delivery, ...optionalProviders, + optionalObservationProviderOwner: "caller", androidAnalysis: android, }, ); diff --git a/src/pcap/PcapProvider.ts b/src/pcap/PcapProvider.ts index 5f1e4e3225..2e9abed31a 100644 --- a/src/pcap/PcapProvider.ts +++ b/src/pcap/PcapProvider.ts @@ -127,6 +127,15 @@ export interface PcapProviderOptions { readonly launcher?: PcapCommandLauncher; } +interface RetainedPcapCommand { + readonly snapshot: PcapSnapshot | undefined; + readonly resources: readonly string[]; + readonly reason: string; + readonly capturedOutput: PcapCommandCleanup["capturedOutput"]; + ownerReleased: boolean; + failure: ProviderCleanupError | undefined; +} + /** Inspect saved captures through bounded, owned tool processes and one stable input copy. */ export class PcapProvider { readonly #resources = new ArtifactAnalysisResources( @@ -142,14 +151,10 @@ export class PcapProvider { private readonly environment: Readonly>; private readonly launcher: PcapCommandLauncher | undefined; private readonly snapshots = new Set(); + private readonly snapshotFailures = new Map(); private readonly commands = new Map< ProviderProcessSupervisor, - { - snapshot: PcapSnapshot | undefined; - resources: readonly string[]; - reason: string; - capturedOutput: PcapCommandCleanup["capturedOutput"]; - } + RetainedPcapCommand >(); constructor(options: PcapProviderOptions = {}) { @@ -167,6 +172,7 @@ export class PcapProvider { inputPath?: string, ) { let cleanup: PcapCommandCleanup | undefined; + let retainedCommand: RetainedPcapCommand | undefined; const output = await runPcapCommand({ operation, command, @@ -186,17 +192,20 @@ export class PcapProvider { : { nativeBound: this.nativeBound }), ...(this.launcher === undefined ? {} : { launcher: this.launcher }), retain: (owner, resources, reason, capturedOutput) => { - this.commands.set(owner, { + retainedCommand = { snapshot, resources, reason, capturedOutput, - }); + ownerReleased: false, + failure: undefined, + }; + this.commands.set(owner, retainedCommand); cleanup = { reason, resources, capturedOutput }; }, }); - if (inputPath === undefined && cleanup !== undefined) - throw new ProviderCleanupError( + if (inputPath === undefined && cleanup !== undefined) { + const failure = new ProviderCleanupError( "pcap", cleanup.resources, { @@ -210,6 +219,9 @@ export class PcapProvider { }, { operation }, ); + if (retainedCommand !== undefined) retainedCommand.failure = failure; + throw failure; + } return output; } @@ -318,7 +330,7 @@ export class PcapProvider { try { if ( [...this.commands.values()].some( - (command) => command.snapshot === snapshot, + (command) => command.snapshot === snapshot && !command.ownerReleased, ) ) throw new Error( @@ -326,6 +338,8 @@ export class PcapProvider { ); await snapshot.close(); this.snapshots.delete(snapshot); + this.snapshotFailures.get(snapshot)?.resolveCleanup(); + this.snapshotFailures.delete(snapshot); return undefined; } catch (cause) { return new ProviderCleanupError( @@ -344,20 +358,34 @@ export class PcapProvider { } catch (cause) { failures.push(cause); } - for (const owner of this.commands.keys()) { + for (const [owner, command] of this.commands) { + if (command.ownerReleased) continue; try { const result = await owner.stop(); if (result.status === "incomplete") failures.push(result.reason); else if (result.status === "not-owned") failures.push("Command ownership remains unresolved"); - else this.commands.delete(owner); + else { + command.ownerReleased = true; + if (command.snapshot === undefined) { + command.failure?.resolveCleanup(); + this.commands.delete(owner); + } + } } catch (cause) { failures.push(cause); } } for (const snapshot of this.snapshots) { const failure = await this.discard(snapshot); - if (failure !== undefined) failures.push(failure); + if (failure !== undefined) { + failures.push(failure); + } else { + for (const [owner, command] of this.commands) { + if (command.snapshot !== snapshot) continue; + this.commands.delete(owner); + } + } } if (failures.length > 0) throw new ProviderCleanupError( @@ -500,32 +528,36 @@ export class PcapProvider { const capturedOutput = result.ok ? undefined : result.error.capturedOutput; - result = err( - new ProviderCleanupError( - "pcap", - [...new Set([snapshot.path, ...pendingCommand.resources])], - { - reason: pendingCommand.reason, - previous_error: result.ok - ? null - : projectAnalysisError(result.error), - ...(capturedOutput === undefined && - pendingCommand.capturedOutput.stdout === "" && - pendingCommand.capturedOutput.stderr === "" - ? {} - : { - captured_output: { - ...(capturedOutput ?? pendingCommand.capturedOutput), - }, - }), - }, - { - operation: request.operation, - ...(partialObservation === undefined ? {} : { partialObservation }), - }, - ), + const cleanupFailure = new ProviderCleanupError( + "pcap", + [...new Set([snapshot.path, ...pendingCommand.resources])], + { + reason: pendingCommand.reason, + previous_error: result.ok ? null : projectAnalysisError(result.error), + ...(capturedOutput === undefined && + pendingCommand.capturedOutput.stdout === "" && + pendingCommand.capturedOutput.stderr === "" + ? {} + : { + captured_output: { + ...(capturedOutput ?? pendingCommand.capturedOutput), + }, + }), + }, + { + operation: request.operation, + ...(partialObservation === undefined ? {} : { partialObservation }), + }, ); + result = err(cleanupFailure); } + if ( + snapshot !== undefined && + this.snapshots.has(snapshot) && + !result.ok && + result.error.cleanupIncomplete + ) + this.snapshotFailures.set(snapshot, result.error); return result; } diff --git a/src/process/ArtifactAnalysisResources.test.ts b/src/process/ArtifactAnalysisResources.test.ts index 84ce0e6d7d..58aff744d2 100644 --- a/src/process/ArtifactAnalysisResources.test.ts +++ b/src/process/ArtifactAnalysisResources.test.ts @@ -2,9 +2,13 @@ import { EventEmitter } from "node:events"; import { PassThrough } from "node:stream"; import { expect, it } from "vitest"; import { readStableArtifact } from "../artifacts/readStableArtifact.js"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; +import { createEvidence } from "../domain/evidence.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; -import { err } from "../domain/result.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { err, ok } from "../domain/result.js"; import { ArtifactAnalysisResources } from "./ArtifactAnalysisResources.js"; import { readLinuxResourceLimits } from "./LinuxResourceLimits.js"; import { ProviderProcessSupervisor } from "./ProviderProcess.js"; @@ -83,3 +87,248 @@ it("projects the specific reason and member evidence for retained cleanup", asyn }, }); }); + +it("preserves the primary error and resolves cleanup status after retry", async () => { + let closes = 0; + const resources = new ArtifactAnalysisResources( + "fixture-provider", + "fixture-operation", + readStableArtifact, + readLinuxResourceLimits, + { message: "busy", details: {} }, + ); + resources.retain({ + path: "runtime-root", + close: async () => { + closes++; + if (closes === 1) throw new Error("temporary close failure"); + }, + }); + const primary = new AnalysisOutputError( + "fixture-operation", + "producer returned malformed output", + { + partialObservation: createEvidence( + undefined, + { + id: "fixture", + name: "Fixture", + version: null, + }, + { + operation: "inspect_signal_decoders", + parameters: {}, + result: { decoders: [] }, + confidence: "observed", + }, + ), + }, + ); + + const result = await resources.finish(err(primary), () => { + throw new Error("unexpected success observation"); + }); + + if (result.ok) throw new Error("expected cleanup failure"); + expect(result.error.cause).toBeInstanceOf(AggregateError); + expect((result.error.cause as AggregateError).errors).toEqual([ + primary, + expect.any(ProviderCleanupError), + ]); + expect(result.error.partialObservation).toEqual(primary.partialObservation); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.diagnostics).toMatchObject({ + previous_error: { code: "unreadable_output" }, + }); + + await resources.release(); + + expect(result.error.cleanupIncomplete).toBe(false); + expect(closes).toBe(2); + await resources.close(); +}); + +it("resolves each returned cleanup error with its own retained root", async () => { + let closesA = 0; + let closesB = 0; + const resources = new ArtifactAnalysisResources( + "fixture-provider", + "fixture-operation", + readStableArtifact, + readLinuxResourceLimits, + { message: "busy", details: {} }, + ); + resources.retain({ + path: "runtime-root-a", + close: async () => { + closesA++; + if (closesA === 1) throw new Error("root A close failure"); + }, + }); + const first = await resources.finish( + err(new AnalysisOutputError("fixture-operation", "primary failure")), + () => { + throw new Error("unexpected success observation"); + }, + ); + if (first.ok) throw new Error("expected root A cleanup failure"); + + resources.retain({ + path: "runtime-root-b", + close: async () => { + closesB++; + if (closesB === 1) throw new Error("root B close failure"); + }, + }); + const partial = createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: null }, + { + operation: "fixture-operation", + parameters: {}, + result: { state: "partial" }, + confidence: "observed", + }, + ); + const second = await resources.finish(ok("completed"), () => partial); + + if (second.ok) throw new Error("expected root B cleanup failure"); + expect(second.error.partialObservation).toEqual(partial); + expect(first.error.cleanupIncomplete).toBe(false); + expect(second.error.cleanupIncomplete).toBe(true); + + await resources.release(); + expect(second.error.cleanupIncomplete).toBe(false); + await resources.close(); +}); + +it("resolves an artifact cleanup error when its scope owner later closes", async () => { + let closes = 0; + const readArtifact: typeof readStableArtifact = async ( + _path, + _maximumBytes, + _signal, + _fileSystem, + scope, + ) => { + if (scope === undefined) throw new Error("expected scoped artifact read"); + if (closes === 0) { + const owner = { + kind: "lease", + resource: "fixture descriptor", + lease: { + release: async () => { + closes++; + if (closes === 1) throw new Error("descriptor remains open"); + }, + }, + } as const; + expect(await scope.release(owner)).toMatchObject({ kind: "failed" }); + throw new ArtifactReaderFailure("io", "Descriptor close failed", { + cleanup: { + reason: "descriptor remains open", + resources: ["fixture descriptor"], + }, + }); + } + return { bytes: Buffer.alloc(0), sha256: "0".repeat(64) }; + }; + const resources = new ArtifactAnalysisResources( + "fixture-provider", + "fixture-operation", + readArtifact, + readLinuxResourceLimits, + { message: "busy", details: {} }, + ); + const first = await resources.run(async () => + ok(await resources.read("fixture", 1)), + ); + + if (first.ok) throw new Error("expected descriptor cleanup failure"); + expect(first.error.cleanupIncomplete).toBe(true); + + const retried = await resources.run(async () => + ok(await resources.read("fixture", 1)), + ); + + expect(retried.ok).toBe(true); + expect(first.error.cleanupIncomplete).toBe(false); + expect(closes).toBe(2); + await resources.close(); +}); + +it("keeps a runtime root until its descriptor cleanup owner is released", async () => { + let descriptorCloses = 0; + let rootCloses = 0; + const readArtifact: typeof readStableArtifact = async ( + _path, + _maximumBytes, + _signal, + _fileSystem, + scope, + ) => { + if (scope === undefined) throw new Error("expected scoped artifact read"); + if (descriptorCloses === 0) { + const owner = { + kind: "lease", + resource: "fixture descriptor", + lease: { + release: async () => { + descriptorCloses++; + if (descriptorCloses === 1) + throw new Error("descriptor remains open"); + }, + }, + } as const; + expect(await scope.release(owner)).toMatchObject({ kind: "failed" }); + throw new ArtifactReaderFailure("io", "Descriptor close failed", { + cleanup: { + reason: "descriptor remains open", + resources: ["fixture descriptor"], + }, + }); + } + return { bytes: Buffer.alloc(0), sha256: "0".repeat(64) }; + }; + const resources = new ArtifactAnalysisResources( + "fixture-provider", + "fixture-operation", + readArtifact, + readLinuxResourceLimits, + { message: "busy", details: {} }, + ); + const first = await resources.run(async () => + ok(await resources.read("fixture", 1)), + ); + if (first.ok) throw new Error("expected descriptor cleanup failure"); + resources.retain({ + path: "runtime-root", + close: async () => { + rootCloses++; + }, + }); + + const outer = await resources.finish(first, () => { + throw new Error("unexpected success observation"); + }); + + if (outer.ok) throw new Error("expected combined cleanup failure"); + expect(rootCloses).toBe(0); + expect(outer.error.cleanupResources).toEqual([ + "fixture descriptor", + "runtime-root", + ]); + expect(outer.error.cause).toBeInstanceOf(AggregateError); + expect((outer.error.cause as AggregateError).errors[0]).toBe(first.error); + expect(first.error.cleanupIncomplete).toBe(true); + + const retried = await resources.run(async () => + ok(await resources.read("fixture", 1)), + ); + expect(retried.ok).toBe(true); + expect(first.error.cleanupIncomplete).toBe(false); + await resources.release(); + expect(rootCloses).toBe(1); + expect(outer.error.cleanupIncomplete).toBe(false); + await resources.close(); +}); diff --git a/src/process/ArtifactAnalysisResources.ts b/src/process/ArtifactAnalysisResources.ts index ff7b630488..8afad44e96 100644 --- a/src/process/ArtifactAnalysisResources.ts +++ b/src/process/ArtifactAnalysisResources.ts @@ -44,6 +44,14 @@ export class ArtifactAnalysisResources { readonly operation: string; } >(); + readonly #cleanupFailures = new Map< + AnalysisError, + { + readonly roots: Set; + readonly primaryFailure: AnalysisError | undefined; + readonly scopeOwner: boolean; + } + >(); #tail: Promise = Promise.resolve(); #busy = false; #closed = false; @@ -86,6 +94,7 @@ export class ArtifactAnalysisResources { // Failed descriptor cleanup still belongs to the preceding request until // this scope has drained it and admits acquisition for the new request. this.#artifacts.operation = operation; + this.#resolveCleanupFailures(); return work(); }), ) @@ -175,8 +184,19 @@ export class ArtifactAnalysisResources { } this.#pending.set(root, { ...retained, owner: undefined }); } + if (this.#artifacts.scope.hasPending) + throw new ProviderCleanupError( + this.providerId, + [root.path], + { + reason: + "Artifact descriptors remain open; the private runtime root is retained until descriptor cleanup succeeds.", + }, + { operation: retained.operation }, + ); await root.close(); this.#pending.delete(root); + this.#resolveCleanupFailures(root); } catch (cause: unknown) { const diagnostics = cause instanceof ProviderCleanupError ? cause.diagnostics : undefined; @@ -202,7 +222,7 @@ export class ArtifactAnalysisResources { capturedOutput?: AnalysisCapturedOutput, operation = this.operation, ): ProviderCleanupError { - return new ProviderCleanupError( + const failure = new ProviderCleanupError( this.providerId, observation.resources, { @@ -213,6 +233,8 @@ export class ArtifactAnalysisResources { }, { operation, cause }, ); + this.#trackCleanupFailure(failure, [], failure, true); + return failure; } commandFailure( @@ -283,27 +305,44 @@ export class ArtifactAnalysisResources { : result.error.partialObservation; const cleanupDiagnostics = cause instanceof ProviderCleanupError ? cause.diagnostics : undefined; - return err( - new ProviderCleanupError( - this.providerId, - this.#pendingResources, - { - ...cleanupDiagnostics, - reason: - cause instanceof ProviderCleanupError - ? String(cleanupDiagnostics?.reason ?? cause.message) - : String(cause), - previous_error: result.ok - ? null - : projectAnalysisError(result.error), - }, - { - operation, - cause, - ...(partialObservation === undefined ? {} : { partialObservation }), - }, - ), + const cleanupFailure = new ProviderCleanupError( + this.providerId, + [ + ...new Set([ + ...(!result.ok && result.error.cleanupIncomplete + ? result.error.cleanupResources + : []), + ...this.#pendingResources, + ]), + ], + { + ...cleanupDiagnostics, + reason: + cause instanceof ProviderCleanupError + ? String(cleanupDiagnostics?.reason ?? cause.message) + : String(cause), + previous_error: result.ok ? null : projectAnalysisError(result.error), + }, + { + operation, + cause: result.ok + ? cause + : new AggregateError( + [result.error, cause], + "Analysis failed and artifact cleanup remains incomplete.", + { cause: result.error }, + ), + ...(partialObservation === undefined ? {} : { partialObservation }), + }, ); + const primaryFailure = result.ok ? undefined : result.error; + this.#trackCleanupFailure( + cleanupFailure, + [...this.#pending.keys()], + primaryFailure?.cleanupIncomplete === true ? primaryFailure : undefined, + false, + ); + return err(cleanupFailure); } } @@ -318,6 +357,7 @@ export class ArtifactAnalysisResources { let descriptorFailure: AnalysisError | undefined; try { await this.#artifacts.scope.close(); + this.#resolveCleanupFailures(); } catch (cause: unknown) { descriptorFailure = this.#artifactFailure(cause); } @@ -335,6 +375,32 @@ export class ArtifactAnalysisResources { if (descriptorFailure !== undefined) throw descriptorFailure; } + #resolveCleanupFailures(releasedRoot?: ArtifactAnalysisRuntime): void { + for (const [failure, binding] of this.#cleanupFailures) { + if (releasedRoot !== undefined) binding.roots.delete(releasedRoot); + if (binding.roots.size > 0) continue; + if (binding.scopeOwner && this.#artifacts.scope.hasPending) continue; + if (binding.scopeOwner) binding.primaryFailure?.resolveCleanup(); + if (binding.primaryFailure?.cleanupIncomplete === true) continue; + failure.resolveCleanup(); + this.#cleanupFailures.delete(failure); + } + } + + #trackCleanupFailure( + failure: AnalysisError, + roots: readonly ArtifactAnalysisRuntime[], + primaryFailure: AnalysisError | undefined, + scopeOwner: boolean, + ): void { + this.#cleanupFailures.set(failure, { + roots: new Set(roots), + primaryFailure, + scopeOwner, + }); + this.#resolveCleanupFailures(); + } + #artifactFailure( cause: unknown, operation = this.#artifacts.operation, diff --git a/src/server/contractToolHandler.ts b/src/server/contractToolHandler.ts index 6ee0e5834b..592139a3c7 100644 --- a/src/server/contractToolHandler.ts +++ b/src/server/contractToolHandler.ts @@ -1,4 +1,7 @@ -import type { ServerContext } from "@modelcontextprotocol/server"; +import type { + CallToolResult, + ServerContext, +} from "@modelcontextprotocol/server"; import type { Logger } from "pino"; import type { ExecutionOptions } from "../application/AnalysisProvider.js"; @@ -11,7 +14,6 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import { logToolExecution } from "./toolLogging.js"; import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; import { runAdmittedToolOperation } from "./admittedToolOperation.js"; -import { deliverObservationEvidence } from "./observationEvidenceDelivery.js"; /** Application service executing one named contract operation. */ export interface ContractToolService { @@ -22,6 +24,35 @@ export interface ContractToolService { ): Promise>; } +/** Record and deliver one Evidence result using the tool's exact contract. */ +export const deliverEvidenceResult = ( + server: EvidenceMcpServer, + result: Result, + contract: ToolContract, + recordEvidence?: EvidenceWriter["recordEvidence"], +): CallToolResult => { + if (!result.ok) { + const partial = result.error.partialObservation; + if ( + partial !== undefined && + "evidence_id" in partial && + typeof partial.evidence_id === "string" + ) { + const recorded = recordEvidence?.(partial); + if (recorded !== undefined) + return server.delivery.toRecordedPartialErrorToolResult( + result.error, + partial.evidence_id, + recorded, + contract.returnsTargetContent, + ); + } + return server.delivery.toCallToolResult(result, contract); + } + const recorded = recordEvidence?.(result.value); + return server.delivery.toEvidenceToolResult(result.value, contract, recorded); +}; + /** * Bind one service-backed handler to its exact named contract through the * shared log/execute/record/deliver workflow. @@ -62,12 +93,7 @@ export const createEvidenceToolHandler = const result = await logToolExecution(logger, name, () => operation(input, { signal: context.mcpReq.signal }), ); - return deliverObservationEvidence( - server, - contract, - result, - recordEvidence, - ); + return deliverEvidenceResult(server, result, contract, recordEvidence); }; return runAdmittedToolOperation( server, diff --git a/src/server/createServer.ts b/src/server/createServer.ts index dcd893e588..0a5775ee10 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -176,6 +176,8 @@ export interface CreateServerOptions { readonly browserObservation?: BrowserObservationPort; readonly browserScenarioCapture?: BrowserScenarioCapturePort; readonly electronObservation?: ElectronObservationPort; + /** Selects who closes the shared optional observation providers. */ + readonly optionalObservationProviderOwner?: "server" | "caller"; readonly electronActiveObservation?: ElectronActiveObservationPort; readonly javascriptRuntimeObservation?: JavaScriptRuntimeObservationPort; readonly fridaInstrumentation?: FridaInstrumentationPort; @@ -388,9 +390,13 @@ export const createServer = ( peResources.close(), webNetworkCapture.close(), javascriptRecovery.close?.() ?? Promise.resolve(), - options.browserObservation?.close?.() ?? Promise.resolve(), - options.browserScenarioCapture?.close?.() ?? Promise.resolve(), - options.electronObservation?.close?.() ?? Promise.resolve(), + ...(options.optionalObservationProviderOwner === "caller" + ? [] + : [ + options.browserObservation?.close?.() ?? Promise.resolve(), + options.browserScenarioCapture?.close?.() ?? Promise.resolve(), + options.electronObservation?.close?.() ?? Promise.resolve(), + ]), webRuntime.provider.close?.() ?? Promise.resolve(), artifactResources.close(), reverseEngineering.close(), diff --git a/src/server/observationEvidenceDelivery.ts b/src/server/observationEvidenceDelivery.ts deleted file mode 100644 index 6338f6ce6e..0000000000 --- a/src/server/observationEvidenceDelivery.ts +++ /dev/null @@ -1,35 +0,0 @@ -import type { ToolContract } from "../contracts/toolContractTypes.js"; -import type { AnalysisError } from "../domain/analysisErrorBase.js"; -import type { Evidence } from "../domain/evidence.js"; -import type { Result } from "../domain/result.js"; -import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; -import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; - -/** Record complete or completed-partial Evidence before projecting MCP output. */ -export const deliverObservationEvidence = ( - server: EvidenceMcpServer, - contract: ToolContract, - result: Result, - recordEvidence: EvidenceWriter["recordEvidence"] | undefined, -) => { - if (!result.ok) { - const partial = result.error.partialObservation; - if ( - partial !== undefined && - "evidence_id" in partial && - typeof partial.evidence_id === "string" - ) { - const recorded = recordEvidence?.(partial); - if (recorded !== undefined) - return server.delivery.toRecordedPartialErrorToolResult( - result.error, - partial.evidence_id, - recorded, - contract.returnsTargetContent, - ); - } - return server.delivery.toCallToolResult(result, contract); - } - const recorded = recordEvidence?.(result.value); - return server.delivery.toEvidenceToolResult(result.value, contract, recorded); -}; diff --git a/src/server/registerBrowserScenarioTool.ts b/src/server/registerBrowserScenarioTool.ts index b7cf1576cc..8ecebd5340 100644 --- a/src/server/registerBrowserScenarioTool.ts +++ b/src/server/registerBrowserScenarioTool.ts @@ -15,7 +15,7 @@ import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; import { runAdmittedToolOperation } from "./admittedToolOperation.js"; -import { deliverObservationEvidence } from "./observationEvidenceDelivery.js"; +import { deliverEvidenceResult } from "./contractToolHandler.js"; interface BrowserScenarioToolRegistration { readonly logger: Logger; @@ -55,10 +55,10 @@ export const registerBrowserScenarioTool = ( signal: context.mcpReq.signal, }), ); - return deliverObservationEvidence( + return deliverEvidenceResult( server, - contract, result, + contract, options.recordEvidence, ); }, diff --git a/src/server/registerBrowserTools.ts b/src/server/registerBrowserTools.ts index 32c095f478..b744585838 100644 --- a/src/server/registerBrowserTools.ts +++ b/src/server/registerBrowserTools.ts @@ -31,7 +31,7 @@ import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; import { runAdmittedToolOperation } from "./admittedToolOperation.js"; -import { deliverObservationEvidence } from "./observationEvidenceDelivery.js"; +import { deliverEvidenceResult } from "./contractToolHandler.js"; interface BrowserToolRegistration { readonly logger: Logger; @@ -202,10 +202,10 @@ const runBrowserTool = async ( progress: mcpProgressReporter(context), }), ); - return deliverObservationEvidence( + return deliverEvidenceResult( options.server, - contract, result, + contract, options.recordEvidence, ); }, diff --git a/src/server/registerElectronTools.ts b/src/server/registerElectronTools.ts index f857702995..80f4c41ee3 100644 --- a/src/server/registerElectronTools.ts +++ b/src/server/registerElectronTools.ts @@ -34,7 +34,7 @@ import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; import { runAdmittedToolOperation } from "./admittedToolOperation.js"; import { createEvidenceToolHandler } from "./contractToolHandler.js"; -import { deliverObservationEvidence } from "./observationEvidenceDelivery.js"; +import { deliverEvidenceResult } from "./contractToolHandler.js"; interface ElectronToolRegistration { readonly artifactResources: ArtifactResourceScope; @@ -205,10 +205,10 @@ const runElectronTool = async ( progress: mcpProgressReporter(context), }), ); - return deliverObservationEvidence( + return deliverEvidenceResult( options.server, - contract, result, + contract, options.recordEvidence, ); }, diff --git a/src/server/registerJavaScriptRuntimeObservationTools.ts b/src/server/registerJavaScriptRuntimeObservationTools.ts index 1d75647925..c7621648a5 100644 --- a/src/server/registerJavaScriptRuntimeObservationTools.ts +++ b/src/server/registerJavaScriptRuntimeObservationTools.ts @@ -23,7 +23,7 @@ import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; import { runAdmittedToolOperation } from "./admittedToolOperation.js"; -import { deliverObservationEvidence } from "./observationEvidenceDelivery.js"; +import { deliverEvidenceResult } from "./contractToolHandler.js"; interface RuntimeToolRegistration { readonly logger: Logger; @@ -94,10 +94,10 @@ const runRuntimeTool = async ( const result = await logToolExecution(options.logger, contract.name, () => execute(input, context.mcpReq.signal), ); - return deliverObservationEvidence( + return deliverEvidenceResult( options.server, - contract, result, + contract, options.recordEvidence, ); }, diff --git a/src/server/shutdownRegistration.test.ts b/src/server/shutdownRegistration.test.ts new file mode 100644 index 0000000000..70da2d2dcc --- /dev/null +++ b/src/server/shutdownRegistration.test.ts @@ -0,0 +1,178 @@ +import { EventEmitter } from "node:events"; +import { expect, it } from "vitest"; +import pino from "pino"; + +import { createShutdown } from "../main/shutdown.js"; +import { registerProcessShutdown } from "../main/processShutdown.js"; +import type { RuntimeDependencies } from "../main/types.js"; +import type { StdioServerHandle } from "@modelcontextprotocol/server/stdio"; +import { createManagedBinarySession } from "../composition/binary.js"; + +const shutdownEvents = ["SIGINT", "SIGTERM", "end", "close"] as const; + +const flush = async (): Promise => { + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); +}; + +it.each(shutdownEvents)( + "retries failed cleanup after %s and unregisters every shutdown event on success", + async (event) => { + const signals = new EventEmitter(); + const input = new EventEmitter(); + const output: string[] = []; + const exitCodes: number[] = []; + let unregisterCount = 0; + let transportCloseCalls = 0; + let androidCloseCalls = 0; + let optionalCloseCalls = 0; + let sessionCloseCalls = 0; + const session = createManagedBinarySession(); + const originalSessionClose = session.close.bind(session); + session.close = async (...args) => { + sessionCloseCalls += 1; + return originalSessionClose(...args); + }; + const dependencies: RuntimeDependencies = { + env: {}, + serve: () => { + throw new Error("unused fixture transport"); + }, + writeStderr: (text) => output.push(text), + setExitCode: (code) => exitCodes.push(code), + registerShutdown: (handler) => { + const unregister = registerProcessShutdown(handler, { + signals, + stdin: input, + }); + return () => { + unregisterCount += 1; + unregister(); + }; + }, + }; + const handle: StdioServerHandle = { + close: async () => { + transportCloseCalls += 1; + if (transportCloseCalls === 1) + throw new Error("first transport close failed"); + }, + }; + const lifecycle = createShutdown({ + handle, + closeAndroid: async () => { + androidCloseCalls += 1; + }, + closeOptionalProviders: async () => { + optionalCloseCalls += 1; + }, + session, + dependencies, + serverLogger: pino({ enabled: false }), + }); + + if (event === "SIGINT" || event === "SIGTERM") signals.emit(event); + else input.emit(event); + await flush(); + + expect([ + transportCloseCalls, + androidCloseCalls, + optionalCloseCalls, + ]).toEqual([1, 1, 1]); + expect(sessionCloseCalls).toBe(1); + expect(output).toHaveLength(1); + + if (event === "SIGINT" || event === "SIGTERM") signals.emit(event); + else input.emit(event); + await lifecycle.shutdown(); + + expect([ + transportCloseCalls, + androidCloseCalls, + optionalCloseCalls, + ]).toEqual([2, 1, 1]); + expect(sessionCloseCalls).toBe(1); + expect(unregisterCount).toBe(1); + expect(signals.listenerCount("SIGINT")).toBe(0); + expect(signals.listenerCount("SIGTERM")).toBe(0); + expect(input.listenerCount("end")).toBe(0); + expect(input.listenerCount("close")).toBe(0); + + signals.emit("SIGINT"); + input.emit("end"); + await flush(); + expect([ + transportCloseCalls, + androidCloseCalls, + optionalCloseCalls, + ]).toEqual([2, 1, 1]); + }, +); + +it("joins concurrent shutdown events and retries only failed owners", async () => { + const signals = new EventEmitter(); + const input = new EventEmitter(); + const output: string[] = []; + let transportCloseCalls = 0; + let androidCloseCalls = 0; + let sessionCloseCalls = 0; + let releaseFirstClose: ((error: Error) => void) | undefined; + const session = createManagedBinarySession(); + const originalSessionClose = session.close.bind(session); + session.close = async (...args) => { + sessionCloseCalls += 1; + return originalSessionClose(...args); + }; + const dependencies: RuntimeDependencies = { + env: {}, + serve: () => { + throw new Error("unused fixture transport"); + }, + writeStderr: (text) => output.push(text), + setExitCode: () => undefined, + registerShutdown: (handler) => + registerProcessShutdown(handler, { signals, stdin: input }), + }; + const handle: StdioServerHandle = { + close: async () => { + transportCloseCalls += 1; + if (transportCloseCalls === 1) + await new Promise((_resolve, reject) => { + releaseFirstClose = reject; + }); + }, + }; + createShutdown({ + handle, + closeAndroid: async () => { + androidCloseCalls += 1; + }, + session, + dependencies, + serverLogger: pino({ enabled: false }), + }); + + signals.emit("SIGINT"); + signals.emit("SIGTERM"); + await flush(); + expect([transportCloseCalls, androidCloseCalls, sessionCloseCalls]).toEqual([ + 1, 1, 1, + ]); + + if (releaseFirstClose === undefined) + throw new Error("First close attempt did not start"); + releaseFirstClose(new Error("first transport close failed")); + await flush(); + input.emit("end"); + await flush(); + + expect([transportCloseCalls, androidCloseCalls, sessionCloseCalls]).toEqual([ + 2, 1, 1, + ]); + expect(output.length).toBeGreaterThan(0); + expect(signals.listenerCount("SIGINT")).toBe(0); + expect(signals.listenerCount("SIGTERM")).toBe(0); + expect(input.listenerCount("end")).toBe(0); + expect(input.listenerCount("close")).toBe(0); +}); diff --git a/src/wasm/WabtArtifactProvider.test.ts b/src/wasm/WabtArtifactProvider.test.ts index 25fe730f77..dc448dcb2d 100644 --- a/src/wasm/WabtArtifactProvider.test.ts +++ b/src/wasm/WabtArtifactProvider.test.ts @@ -43,6 +43,7 @@ vi.mock("node:fs/promises", async (importOriginal) => { const outcomeModes = [ "success", "invalid", + "invalid-close", "malformed", "timeout", "cancelled", @@ -57,6 +58,7 @@ type OutcomeMode = (typeof outcomeModes)[number]; const failureTags: Readonly, string>> = { invalid: "AnalysisInputError", + "invalid-close": "AnalysisInputError", timeout: "AnalysisTimeoutError", cancelled: "AnalysisCancelledError", "cancelled-close": "AnalysisCancelledError", @@ -178,7 +180,7 @@ const assertModeOutcome = ( ok: false, error: { _tag: failureTags[mode] }, }); - if (mode === "invalid") + if (mode === "invalid" || mode === "invalid-close") expect(result).toMatchObject({ error: { capturedOutput: { stderr: "module.wasm: invalid bytes\n" } }, }); @@ -413,7 +415,8 @@ const runOwnedArtifactOutcome = async (mode: OutcomeMode): Promise => { ? "unexpected decoder output\n" : '(module (@custom "test" "payload"))\n' : ""; - const invalid = !version && mode === "invalid"; + const invalid = + !version && (mode === "invalid" || mode === "invalid-close"); const stderr = invalid ? "module.wasm: invalid bytes\n" : !version && @@ -454,7 +457,8 @@ const runOwnedArtifactOutcome = async (mode: OutcomeMode): Promise => { if (mode === "cleanup" && ++cleanupAttempts <= 2) throw new Error("Source-owned cleanup uncertainty"); await root.close(); - if (mode === "cancelled-close") controller.abort(); + if (mode === "cancelled-close" || mode === "invalid-close") + controller.abort(); }, }; }, diff --git a/tests/acceptance/browser/browserScenarioCancellationCli.test.ts b/tests/acceptance/browser/browserScenarioCancellationCli.test.ts index 7983ccda01..4802ff11cf 100644 --- a/tests/acceptance/browser/browserScenarioCancellationCli.test.ts +++ b/tests/acceptance/browser/browserScenarioCancellationCli.test.ts @@ -1,8 +1,8 @@ import { resolve } from "node:path"; import { expect } from "vitest"; +import { z } from "zod"; import { browserScenarioCaptureSchema } from "../../../src/domain/browserScenarioCapture.js"; -import { parseEvidence } from "../../../src/domain/evidence.js"; import { processTest } from "../../support/process/processFixture.js"; // Actual CLI subprocess and OS signals with a controlled browser session. @@ -66,10 +66,21 @@ processTest expect(stderr).toContain("REA_CONTROLLED_CLOSE_CALLBACK"); expect(stderr).toContain("REA_CONTROLLED_PROFILE_CALLBACK"); const document: unknown = JSON.parse(stdout); - const evidence = parseEvidence(document); - expect(evidence).toEqual(document); + const cancellation = z + .object({ + error: z.literal("Browser scenario capture failed"), + code: z.literal("cancelled"), + category: z.literal("cancelled"), + details: z.object({ + partial_observation: z.object({ + kind: z.literal("browser-scenario-observation"), + capture: z.unknown(), + }), + }), + }) + .parse(document); const capture = browserScenarioCaptureSchema.parse( - evidence.normalized_result, + cancellation.details.partial_observation.capture, ); expect(capture.browser.cleanup).toBe("terminated-owned-process"); expect(capture.steps.map(({ status }) => status)).toEqual([ diff --git a/tests/boundary/browser/cdpCleanupCancellation.test.ts b/tests/boundary/browser/cdpCleanupCancellation.test.ts index 5001a73e08..10602a67b3 100644 --- a/tests/boundary/browser/cdpCleanupCancellation.test.ts +++ b/tests/boundary/browser/cdpCleanupCancellation.test.ts @@ -217,8 +217,12 @@ it("classifies cancellation that arrives during final Electron detach", async () _tag: "AnalysisCancelledError", operation: "inspect_electron_page", partialObservation: { - target: { target_id: "electron-page", file_path: page }, - dom: { total_nodes: expect.any(Number) }, + kind: "electron-page-observation", + cleanup: "transport-closed", + inspection: { + target: { target_id: "electron-page", file_path: page }, + dom: { total_nodes: expect.any(Number) }, + }, }, }, }); diff --git a/tests/boundary/browser/cdpElectronProvider.test.ts b/tests/boundary/browser/cdpElectronProvider.test.ts index 71a8e49fb0..e071f4e919 100644 --- a/tests/boundary/browser/cdpElectronProvider.test.ts +++ b/tests/boundary/browser/cdpElectronProvider.test.ts @@ -8,6 +8,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { CdpElectronProvider } from "../../../src/browser/CdpElectronProvider.js"; +import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; import { inspectElectronPageInputSchema, listElectronTargetsInputSchema, @@ -30,6 +31,44 @@ afterEach(async () => { }); describe("CdpElectronProvider target access", () => { + it("retains a completed page observation when cancellation arrives during final detach", async () => { + const root = await electronFixture(); + const controller = new AbortController(); + const browser = await startFakeCdpBrowser({ + electronFileUrl: pathToFileURL(join(root, "index.html")).href, + commandEvents: ({ method }) => { + if (method === "Target.detachFromTarget") controller.abort(); + return undefined; + }, + }); + browsers.push(browser); + + const result = await new CdpElectronProvider().inspectPage( + inspectElectronPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + target_id: "electron-page", + observation_ms: 0, + }), + { signal: controller.signal }, + ); + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected late cancellation"); + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.partialObservation).toMatchObject({ + kind: "electron-page-observation", + cleanup: "transport-closed", + inspection: { target: { file_path: join(root, "index.html") } }, + }); + expect(browser.commands.map(({ method }) => method)).toContain( + "Target.detachFromTarget", + ); + const closeDeadline = Date.now() + 1_000; + while (browser.activeConnectionCount() !== 0 && Date.now() < closeDeadline) + await new Promise((resolve) => setTimeout(resolve, 5)); + expect(browser.activeConnectionCount()).toBe(0); + }); + it("returns every in-scope resource beyond the former collection ceiling", async () => { const root = await electronFixture(); const index = join(root, "index.html"); diff --git a/tests/boundary/browser/cdpProviderCleanupOwnership.test.ts b/tests/boundary/browser/cdpProviderCleanupOwnership.test.ts index 7fd85335f0..9a3582bf5b 100644 --- a/tests/boundary/browser/cdpProviderCleanupOwnership.test.ts +++ b/tests/boundary/browser/cdpProviderCleanupOwnership.test.ts @@ -105,8 +105,12 @@ it.each([ cleanupIncomplete: true, cleanupResources: expect.arrayContaining(["electron-page"]), partialObservation: { - target: { target_id: "electron-page", file_path: page }, - dom: { total_nodes: expect.any(Number) }, + kind: "electron-page-observation", + cleanup: "incomplete", + inspection: { + target: { target_id: "electron-page", file_path: page }, + dom: { total_nodes: expect.any(Number) }, + }, }, }); } diff --git a/tests/boundary/captures/captureDescriptorCleanup.test.ts b/tests/boundary/captures/captureDescriptorCleanup.test.ts index 852d06e223..9366ab8a67 100644 --- a/tests/boundary/captures/captureDescriptorCleanup.test.ts +++ b/tests/boundary/captures/captureDescriptorCleanup.test.ts @@ -3,7 +3,7 @@ import { expectRetainedDescriptor, } from "../../fixtures/descriptorCloseFailure.js"; import { expect, it, onTestFinished, vi } from "vitest"; -import { writeFile } from "node:fs/promises"; +import { lstat, writeFile } from "node:fs/promises"; import type { FileHandle } from "node:fs/promises"; import { join } from "node:path"; import { TsharkCaptureProvider } from "../../../src/captures/TsharkCaptureProvider.js"; @@ -49,6 +49,7 @@ it { mode: 0o700 }, ); let launches = 0; + let runtimeRoot: PrivateRuntimeRoot | undefined; const provider = new TsharkCaptureProvider( { ...process.env, @@ -64,7 +65,10 @@ it expectedCommand: process.execPath, }); }, - () => PrivateRuntimeRoot.create({ parent: directory }), + async () => { + runtimeRoot = await PrivateRuntimeRoot.create({ parent: directory }); + return runtimeRoot; + }, ); const discovery = member.startsWith("field-"); const resource = discovery ? member.slice(6) : member; @@ -88,7 +92,18 @@ it const first = discovery ? await service.inspectFields({ query: "usb" }) : await service.inspect({ path, fields: ["frame.number"] }); - expectRetainedDescriptor(first, cleanup, operation); + expectRetainedDescriptor( + first, + cleanup, + operation, + runtimeRoot === undefined ? [] : [runtimeRoot.path], + ); + if (first.ok) throw new Error("Expected retained descriptor cleanup"); + const originatingCause = first.error.cause; + if (resource === "capture") { + expect(runtimeRoot).toBeDefined(); + await expect(lstat(runtimeRoot!.path)).resolves.toBeDefined(); + } const launchCount = launches; expect( await service.inspect({ path, fields: ["frame.number"] }), @@ -124,6 +139,12 @@ it }); cleanup.denied = false; await provider.close(); + expect(first.error.cleanupIncomplete).toBe(false); + expect(first.error.cause).toBe(originatingCause); + if (runtimeRoot !== undefined) + await expect(lstat(runtimeRoot.path)).rejects.toMatchObject({ + code: "ENOENT", + }); expect(cleanup.handles.every((handle) => handle.fd === -1)).toBe(true); }, ); diff --git a/tests/boundary/cli/adbCli.test.ts b/tests/boundary/cli/adbCli.test.ts index cda871ae34..a1f3dcbd25 100644 --- a/tests/boundary/cli/adbCli.test.ts +++ b/tests/boundary/cli/adbCli.test.ts @@ -1,4 +1,5 @@ import { chmod, lstat, readFile, unlink } from "node:fs/promises"; +import { dirname } from "node:path"; import { expect, it, onTestFinished } from "vitest"; import { z } from "zod"; @@ -111,6 +112,9 @@ if [ "$1" = version ]; then exit 0 fi if [ "$1" = -s ] && [ "$3" = install ]; then + if [ "$4" = -r ]; then selected="$5"; else selected="$4"; fi + cmp "$selected" "$root/app.apk" || exit 1 + printf '%s\\n' "$selected" >> "$root/snapshots" if [ -f "$root/installed" ]; then if [ "$4" != -r ]; then printf 'Failure [INSTALL_FAILED_ALREADY_EXISTS]\\n' @@ -184,14 +188,26 @@ exit 1 expect(await readFile(workspace.path("events"), "utf8")).toBe( "fresh-install\nreplacement\nfresh-install\n", ); + const snapshots = (await readFile(workspace.path("snapshots"), "utf8")) + .trim() + .split("\n"); + expect(snapshots).toHaveLength(3); + expect(new Set(snapshots).size).toBe(3); + for (const snapshot of snapshots) { + expect(snapshot).not.toBe(apk); + await expect(lstat(dirname(snapshot))).rejects.toMatchObject({ + code: "ENOENT", + }); + } + expect(await readFile(apk, "utf8")).toBe("apk"); const calls = await readFile(workspace.path("calls"), "utf8"); expect(calls.split("\n").filter((line) => line !== "")).toEqual([ "version", - `-s emulator-5554 install -r ${apk}`, + `-s emulator-5554 install -r ${snapshots[0]}`, "version", - `-s emulator-5554 install -r ${apk}`, + `-s emulator-5554 install -r ${snapshots[1]}`, "version", - `-s emulator-5554 install ${apk}`, + `-s emulator-5554 install ${snapshots[2]}`, ]); }, ); diff --git a/tests/boundary/cli/finishAnalysisCommand.test.ts b/tests/boundary/cli/finishAnalysisCommand.test.ts index 00373bd84d..627a3a8f03 100644 --- a/tests/boundary/cli/finishAnalysisCommand.test.ts +++ b/tests/boundary/cli/finishAnalysisCommand.test.ts @@ -2,9 +2,31 @@ import { expect, it } from "vitest"; import { finishAnalysisCommand } from "../../../src/cli/finishAnalysisCommand.js"; import { AnalysisOutputError } from "../../../src/domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { ProviderCleanupError } from "../../../src/domain/providerCleanupError.js"; import { err, ok } from "../../../src/domain/result.js"; import { scriptCaptureEvidenceFixture } from "../../fixtures/webScriptCapture.js"; +it("preserves producer-owned cleanup failure when service close does not resolve it", async () => { + const evidence = scriptCaptureEvidenceFixture(); + const failure = new ProviderCleanupError( + "fixture-provider", + ["producer-owned-resource"], + { reason: "producer cleanup was not confirmed" }, + { operation: "inspect_web_page", partialObservation: evidence }, + ); + const expected = projectAnalysisError(failure); + const result = await finishAnalysisCommand( + async () => err(failure), + { close: async () => {} }, + "fixture-provider", + "inspect_web_page", + ); + + expect(result).toEqual(expected); + expect(failure.cleanupIncomplete).toBe(true); +}); + it("returns completed partial Evidence when cancellation arrives during close", async () => { const evidence = scriptCaptureEvidenceFixture(); const controller = new AbortController(); diff --git a/tests/boundary/cli/wasmLateCancellationCli.test.ts b/tests/boundary/cli/wasmLateCancellationCli.test.ts new file mode 100644 index 0000000000..ba47e1e650 --- /dev/null +++ b/tests/boundary/cli/wasmLateCancellationCli.test.ts @@ -0,0 +1,50 @@ +import { EventEmitter } from "node:events"; +import { expect, it } from "vitest"; +import { WasmArtifactService } from "../../../src/application/wasm/WasmArtifactService.js"; +import { withCommandCancellation } from "../../../src/cli/commandCancellation.js"; +import { finishAnalysisCommand } from "../../../src/cli/finishAnalysisCommand.js"; +import { ok } from "../../../src/domain/result.js"; +import { + wasmArtifactExecution, + wasmArtifactFixture, +} from "../../fixtures/wasm/artifact.js"; + +it("delivers cancellation with completed WASM Evidence through the CLI boundary", async () => { + const value = wasmArtifactFixture(); + const events = new EventEmitter(); + const service = new WasmArtifactService({ + close: () => Promise.resolve(), + inspect: async (_input, options) => { + await new Promise((resolve) => + options?.signal?.addEventListener("abort", () => resolve(), { + once: true, + }), + ); + return ok(wasmArtifactExecution(value)); + }, + }); + const command = withCommandCancellation( + (signal) => + finishAnalysisCommand( + () => service.inspect({ path: value.artifact.path }, { signal }), + service, + "wabt-artifact", + "inspect_wasm_artifact", + ), + { + on: events.on.bind(events), + off: events.off.bind(events), + }, + ); + + events.emit("SIGINT"); + await expect(command).resolves.toMatchObject({ + code: "cancelled", + details: { + partial_observation: { + normalized_result: value, + confidence: "observed", + }, + }, + }); +}); diff --git a/tests/boundary/filesystem/adbInputSnapshot.test.ts b/tests/boundary/filesystem/adbInputSnapshot.test.ts new file mode 100644 index 0000000000..36bb8e82ae --- /dev/null +++ b/tests/boundary/filesystem/adbInputSnapshot.test.ts @@ -0,0 +1,71 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; +import { OwnedFileHandle } from "../../../src/filesystem/OwnedFileHandle.js"; +import { + openRegularFile, + RegularFileAdmissionFailure, +} from "../../../src/filesystem/RegularFile.js"; +import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; +import { + AdbInputSnapshotFailure, + snapshotAdbInput, +} from "../../../src/adb/AdbInputSnapshot.js"; + +it("retains and retries a source handle that fails close during admission", async () => { + const parent = await mkdtemp(join(tmpdir(), "rea-adb-snapshot-test-")); + const root = await PrivateRuntimeRoot.create({ parent }); + const resources = new ArtifactResourceScope(); + const sourcePath = join(parent, "selected.apk"); + await writeFile(sourcePath, "selected bytes"); + let closeAttempts = 0; + try { + await expect( + snapshotAdbInput( + sourcePath, + root, + resources, + "install_adb_package", + undefined, + { + openSource: async (path, options) => { + const handle = await openRegularFile(path, options); + const close = handle.close.bind(handle); + const owner = new OwnedFileHandle(handle); + handle.close = async () => { + closeAttempts += 1; + if (closeAttempts === 1) + throw new Error("controlled close failure"); + await close(); + }; + throw new RegularFileAdmissionFailure( + path, + new Error("controlled admission failure"), + new Error("first close was not confirmed"), + owner, + ); + }, + }, + ), + ).rejects.toMatchObject>({ + primaryCause: expect.objectContaining({ + message: "controlled admission failure", + }), + cleanupResources: [expect.stringContaining("source handle")], + }); + + expect(resources.hasPending).toBe(true); + expect(closeAttempts).toBe(1); + await resources.retryCleanup(); + expect(resources.hasPending).toBe(false); + expect(closeAttempts).toBe(2); + } finally { + await resources.close(); + await root.close(); + await rm(parent, { recursive: true, force: true }); + } +}); diff --git a/tests/boundary/filesystem/dependencyInstall.test.ts b/tests/boundary/filesystem/dependencyInstall.test.ts index 42ad7e564c..1eef778637 100644 --- a/tests/boundary/filesystem/dependencyInstall.test.ts +++ b/tests/boundary/filesystem/dependencyInstall.test.ts @@ -198,6 +198,7 @@ const createBuildCacheFixture = async (): Promise => { "scripts/check-dependency-install.mjs", "scripts/run-exclusive.mjs", "scripts/run-exclusive-command.mjs", + "scripts/lib/windows-owned-command.mjs", "scripts/generate-skill-metadata.mjs", "scripts/clean-build-output.mjs", "scripts/generate-package-metadata.mjs", diff --git a/tests/boundary/filesystem/stableFileHashOwnership.test.ts b/tests/boundary/filesystem/stableFileHashOwnership.test.ts index 792096b301..c58f3aeefd 100644 --- a/tests/boundary/filesystem/stableFileHashOwnership.test.ts +++ b/tests/boundary/filesystem/stableFileHashOwnership.test.ts @@ -1,177 +1,296 @@ -import { - mkdtemp, - open, - rm, - writeFile, - type FileHandle, -} from "node:fs/promises"; -import { tmpdir } from "node:os"; +import { createHash } from "node:crypto"; +import { open as openFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { afterEach, expect, it, vi } from "vitest"; +import { expect, it, vi } from "vitest"; -import { openRegularFile } from "../../../src/filesystem/RegularFile.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { OwnedFileHandle } from "../../../src/filesystem/OwnedFileHandle.js"; +import * as regularFile from "../../../src/filesystem/RegularFile.js"; +import { RegularFileAdmissionFailure } from "../../../src/filesystem/RegularFile.js"; import { hashStableFile } from "../../../src/filesystem/StableFileHash.js"; +import { LinuxHopperPublisher } from "../../../src/application/LinuxHopperPublisher.js"; +import { RegularFileCleanupFailure } from "../../../src/filesystem/OwnedRegularFile.js"; +import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; +import type { VerifiedLinuxHopperPackage } from "../../../src/application/LinuxHopperPublisher.js"; -vi.mock("../../../src/filesystem/RegularFile.js", async (importOriginal) => ({ - ...(await importOriginal< - typeof import("../../../src/filesystem/RegularFile.js") - >()), - openRegularFile: vi.fn(), -})); - -interface TestOwner { - readonly handle: FileHandle; - close(): Promise; -} - -interface OwnedHashFailure extends Error { - readonly cause?: unknown; - readonly cleanupCause?: unknown; - readonly owner: TestOwner; -} - -const roots: string[] = []; -const rawHandles: FileHandle[] = []; - -afterEach(async () => { - vi.mocked(openRegularFile).mockReset(); - for (const handle of rawHandles.splice(0)) { - if (handle.fd >= 0) await handle.close(); - } - for (const root of roots.splice(0)) - await rm(root, { recursive: true, force: true }); -}); - -const makeFile = async (options: { - readonly closeFailure?: Error; - readonly invalidateFdOnCloseFailure?: boolean; - readonly readFailure?: Error; - readonly abortAfterRead?: AbortController; - readonly abortReason?: unknown; -}) => { - const root = await mkdtemp(join(tmpdir(), "rea-stable-hash-")); - roots.push(root); +it("adopts and retries the descriptor when hashing succeeded but close failed", async () => { + const root = await createTestTempDirectory("rea-stable-hash-close-"); const path = join(root, "input.bin"); - await writeFile(path, "stable hash input"); - const raw = await open(path, "r"); - rawHandles.push(raw); - - let failedClose = false; - let invalidated = false; - const close = vi.fn(async () => { - if (options.closeFailure !== undefined && !failedClose) { - failedClose = true; - invalidated = options.invalidateFdOnCloseFailure === true; - throw options.closeFailure; + await writeFile(path, "owned bytes"); + const closeCause = new Error("injected descriptor close failure"); + const originalClose = OwnedFileHandle.prototype.close; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementationOnce(async () => { + throw closeCause; + }); + const retained: OwnedFileHandle[] = []; + try { + let failure: unknown; + try { + await hashStableFile(path, { + retainFailedClose: (owner) => retained.push(owner), + }); + } catch (cause: unknown) { + failure = cause; } - await raw.close(); - }); - const handle = { - get fd() { - return invalidated ? -1 : raw.fd; - }, - stat: () => raw.stat(), - read: async ( - buffer: Buffer, - offset: number, - length: number, - position: number | null, - ) => { - if (options.readFailure !== undefined) throw options.readFailure; - const result = await raw.read(buffer, offset, length, position); - options.abortAfterRead?.abort(options.abortReason); - return result; - }, - close, - } as unknown as FileHandle; + expect(failure).toBeInstanceOf(RegularFileCleanupFailure); + expect(retained).toHaveLength(1); + if (!(failure instanceof RegularFileCleanupFailure)) + throw new Error("Expected a typed hash cleanup failure"); + expect(failure.outcome).toMatchObject({ + kind: "completed", + value: { bytes: 11 }, + }); + expect(failure.cleanupCause).toBe(closeCause); + expect(failure.retention).toEqual({ kind: "callback_returned" }); + expect(failure.owner).toBe(retained[0]); + expect(retained[0]!.handle.fd).toBeGreaterThanOrEqual(0); + close.mockRestore(); + await retained[0]!.close(); + expect(retained[0]!.handle.fd).toBe(-1); + } finally { + close.mockRestore(); + if (retained[0] !== undefined && retained[0]!.handle.fd >= 0) + await originalClose.call(retained[0]); + } +}); - vi.mocked(openRegularFile).mockResolvedValueOnce(handle); - return { path, handle, close }; -}; +it("preserves a read cancellation cause when close also fails", async () => { + const root = await createTestTempDirectory("rea-stable-hash-primary-"); + const path = join(root, "input.bin"); + await writeFile(path, "owned bytes"); + const primary = new Error("injected cancellation after read"); + const closeCause = new Error("injected close failure after cancellation"); + const controller = new AbortController(); + const nativeOpen = regularFile.openRegularFile; + const open = vi + .spyOn(regularFile, "openRegularFile") + .mockImplementation(async (selectedPath, options) => { + const handle = await nativeOpen(selectedPath, options); + vi.spyOn(handle, "stat").mockImplementationOnce(async () => { + controller.abort(primary); + throw primary; + }); + return handle; + }); + const originalClose = OwnedFileHandle.prototype.close; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementationOnce(async () => { + throw closeCause; + }); + const retained: OwnedFileHandle[] = []; + try { + let failure: unknown; + try { + await hashStableFile(path, { + signal: controller.signal, + retainFailedClose: (owner) => retained.push(owner), + }); + } catch (cause: unknown) { + failure = cause; + } + expect(failure).toBeInstanceOf(RegularFileCleanupFailure); + if (!(failure instanceof RegularFileCleanupFailure)) + throw new Error("Expected a typed hash cleanup failure"); + expect(failure.outcome).toEqual({ kind: "failed", cause: primary }); + expect(failure.cleanupCause).toBe(closeCause); + expect(failure.cause).toBeInstanceOf(AggregateError); + expect((failure.cause as AggregateError).errors).toEqual([ + primary, + closeCause, + ]); + expect(retained).toEqual([failure.owner]); + close.mockRestore(); + await failure.owner.close(); + expect(failure.owner.handle.fd).toBe(-1); + } finally { + close.mockRestore(); + open.mockRestore(); + if (retained[0] !== undefined && retained[0]!.handle.fd >= 0) + await originalClose.call(retained[0]); + } +}); -const rejectHash = async ( - path: string, - signal?: AbortSignal, -): Promise => { - const result = await hashStableFile(path, signal).then( - () => undefined, - (cause: unknown) => cause, +it("closes a normally hashed file without adopting an owner", async () => { + const root = await createTestTempDirectory("rea-stable-hash-control-"); + const path = join(root, "input.bin"); + await writeFile(path, "owned bytes"); + const retainFailedClose = vi.fn(); + await expect( + hashStableFile(path, { retainFailedClose }), + ).resolves.toMatchObject({ bytes: 11 }); + expect(retainFailedClose).not.toHaveBeenCalled(); +}); + +it("adopts a descriptor from an admission failure and preserves both causes", async () => { + const root = await createTestTempDirectory("rea-stable-hash-admission-"); + const path = join(root, "input.bin"); + await writeFile(path, "owned bytes"); + const owner = new OwnedFileHandle(await openFile(path, "r")); + const primary = new Error("injected file admission failure"); + const cleanupCause = new Error("injected admission descriptor close failure"); + const open = vi + .spyOn(regularFile, "openRegularFile") + .mockRejectedValueOnce( + new RegularFileAdmissionFailure(path, primary, cleanupCause, owner), + ); + const retained: OwnedFileHandle[] = []; + try { + let failure: unknown; + try { + await hashStableFile(path, { + retainFailedClose: (retainedOwner) => retained.push(retainedOwner), + }); + } catch (cause: unknown) { + failure = cause; + } + expect(failure).toBeInstanceOf(RegularFileCleanupFailure); + if (!(failure instanceof RegularFileCleanupFailure)) + throw new Error("Expected a typed admission cleanup failure"); + expect(failure.outcome).toEqual({ kind: "failed", cause: primary }); + expect(failure.cleanupCause).toBe(cleanupCause); + expect(failure.retention).toEqual({ kind: "callback_returned" }); + expect((failure.cause as AggregateError).errors).toEqual([ + primary, + cleanupCause, + ]); + expect(retained).toEqual([owner]); + } finally { + open.mockRestore(); + await owner.close(); + } +}); + +it("distinguishes a thrown undefined primary failure from a clean hash", async () => { + const root = await createTestTempDirectory("rea-stable-hash-undefined-"); + const path = join(root, "input.bin"); + await writeFile(path, "owned bytes"); + const openFileAtBoundary = regularFile.openRegularFile; + const open = vi + .spyOn(regularFile, "openRegularFile") + .mockImplementation(async (selectedPath, options) => { + const handle = await openFileAtBoundary(selectedPath, options); + const originalStat = handle.stat.bind(handle); + let calls = 0; + vi.spyOn(handle, "stat").mockImplementation(async () => { + calls += 1; + if (calls === 2) throw undefined; + return originalStat(); + }); + return handle; + }); + const closeCause = new Error("injected close failure after undefined"); + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockRejectedValueOnce(closeCause); + const retained: OwnedFileHandle[] = []; + try { + let failure: unknown; + try { + await hashStableFile(path, { + retainFailedClose: (owner) => retained.push(owner), + }); + } catch (cause: unknown) { + failure = cause; + } + expect(failure).toBeInstanceOf(RegularFileCleanupFailure); + if (!(failure instanceof RegularFileCleanupFailure)) + throw new Error("Expected a typed hash cleanup failure"); + expect(failure.outcome).toEqual({ kind: "failed", cause: undefined }); + expect(failure.cleanupCause).toBe(closeCause); + expect((failure.cause as AggregateError).errors).toEqual([ + undefined, + closeCause, + ]); + expect(retained).toEqual([failure.owner]); + } finally { + close.mockRestore(); + open.mockRestore(); + if (retained[0] !== undefined) await retained[0].close(); + } +}); + +it("retries a publisher-owned launcher hash close before returning its digest", async () => { + const root = await createTestTempDirectory("rea-publisher-hash-owner-"); + const path = join(root, "launcher"); + await writeFile(path, "fixture launcher bytes"); + const publisher = new LinuxHopperPublisher({}, async () => undefined); + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockRejectedValueOnce(new Error("fixture launcher close failure")); + try { + await expect(publisher.hashLauncher(path)).resolves.toMatch( + /^[a-f0-9]{64}$/u, + ); + expect(close).toHaveBeenCalledTimes(2); + close.mockRestore(); + await publisher.close(); + } finally { + close.mockRestore(); + await publisher.close(); + } +}); + +it("retries pending cleanup before acquiring a publisher temporary root", async () => { + const root = await createTestTempDirectory("rea-publisher-admission-"); + const path = join(root, "launcher"); + const launcherBytes = "fixture launcher bytes"; + await writeFile(path, launcherBytes); + const publisher = new LinuxHopperPublisher( + { ...process.env, XDG_CACHE_HOME: join(root, "cache") }, + async () => undefined, ); - expect(result).toBeInstanceOf(Error); - expect(result).toHaveProperty("owner"); - expect(result).toHaveProperty("cleanupCause"); - return result as OwnedHashFailure; -}; + const originalClose = OwnedFileHandle.prototype.close; + const closeOwners: OwnedFileHandle[] = []; + let allowClose = false; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + closeOwners.push(this); + if (!allowClose) throw new Error("fixture close remains unavailable"); + return originalClose.call(this); + }); + const createRoot = vi.spyOn(PrivateRuntimeRoot, "create"); + try { + await expect(publisher.hashLauncher(path)).rejects.toMatchObject({ + reason: "cleanup_failed", + }); + const retainedOwner = closeOwners[0]; + if (retainedOwner === undefined) + throw new Error("Expected a retained launcher descriptor"); + const attemptsBeforeAdmission = closeOwners.length; + await expect( + publisher.retainVerifiedArchive({} as VerifiedLinuxHopperPackage, path), + ).resolves.toMatchObject({ + ok: false, + error: { + reason: "cleanup_failed", + cause: { cleanup: { resources: [path] } }, + }, + }); + expect(createRoot).not.toHaveBeenCalled(); + expect(closeOwners.length).toBeGreaterThan(attemptsBeforeAdmission); + expect(closeOwners.every((owner) => owner === retainedOwner)).toBe(true); -it("hashes and closes normally when close succeeds", async () => { - const { path, close } = await makeFile({}); - - const result = await hashStableFile(path); - - expect(result.bytes).toBe(Buffer.byteLength("stable hash input")); - expect(result.sha256).toMatch(/^[a-f0-9]{64}$/u); - expect(close).toHaveBeenCalledTimes(1); -}); - -it("retains a completed hash's live file owner after close fails and retries it", async () => { - const closeFailure = new Error("injected close failure"); - const { path, handle, close } = await makeFile({ closeFailure }); - - const failure = await rejectHash(path); - - expect(failure.cleanupCause).toBe(closeFailure); - expect(failure.owner.handle).toBe(handle); - expect(failure).toHaveProperty( - "completedHash.bytes", - Buffer.byteLength("stable hash input"), - ); - await expect(failure.owner.close()).resolves.toBeUndefined(); - expect(close).toHaveBeenCalledTimes(2); -}); - -it("preserves a read failure as primary when closing the owned file also fails", async () => { - const readFailure = new Error("injected read failure"); - const closeFailure = new Error("injected close failure"); - const { path, handle } = await makeFile({ closeFailure, readFailure }); - - const failure = await rejectHash(path); - - expect(failure.cause).toBe(readFailure); - expect(failure.cleanupCause).toBe(closeFailure); - expect(failure.owner.handle).toBe(handle); -}); - -it("preserves cancellation as primary when closing the owned file also fails", async () => { - const cancellation = new AbortController(); - const cancelCause = new Error("injected cancellation"); - const closeFailure = new Error("injected close failure"); - const { path, handle } = await makeFile({ - closeFailure, - abortAfterRead: cancellation, - abortReason: cancelCause, - }); - - const failure = await rejectHash(path, cancellation.signal); - - expect(failure.cause).toBe(cancelCause); - expect(failure.cleanupCause).toBe(closeFailure); - expect(failure.owner.handle).toBe(handle); -}); - -it("keeps an invalidated descriptor's close failure sticky without retrying raw close", async () => { - const closeFailure = new Error( - "injected close failure after fd invalidation", - ); - const { path, handle, close } = await makeFile({ - closeFailure, - invalidateFdOnCloseFailure: true, - }); - - const failure = await rejectHash(path); - - expect(failure.cleanupCause).toBe(closeFailure); - expect(failure.owner.handle).toBe(handle); - await expect(failure.owner.close()).rejects.toBe(closeFailure); - expect(close).toHaveBeenCalledTimes(1); + const attemptsBeforeRecovery = closeOwners.length; + allowClose = true; + await expect(publisher.hashLauncher(path)).resolves.toBe( + createHash("sha256").update(launcherBytes).digest("hex"), + ); + const recoveryOwners = closeOwners.slice(attemptsBeforeRecovery); + expect(recoveryOwners[0]).toBe(retainedOwner); + expect(recoveryOwners.some((owner) => owner !== retainedOwner)).toBe(true); + expect(closeOwners.every((owner) => owner.handle.fd === -1)).toBe(true); + expect(createRoot).not.toHaveBeenCalled(); + const attemptsAfterRecovery = closeOwners.length; + await publisher.close(); + expect(closeOwners).toHaveLength(attemptsAfterRecovery); + } finally { + close.mockRestore(); + createRoot.mockRestore(); + await publisher.close(); + } }); diff --git a/tests/boundary/mcp/androidLateCompletionMcp.test.ts b/tests/boundary/mcp/androidLateCompletionMcp.test.ts new file mode 100644 index 0000000000..36eaf8fece --- /dev/null +++ b/tests/boundary/mcp/androidLateCompletionMcp.test.ts @@ -0,0 +1,111 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import type { CallToolResult } from "@modelcontextprotocol/server"; +import { expect, it, onTestFinished, vi } from "vitest"; +import { z } from "zod"; + +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { silentLogger } from "../../../src/logger.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { ToolResultDelivery } from "../../../src/server/toolResult.js"; +import { createJadxProtocolFixture } from "../../fixtures/android/jadx.js"; +import { + createDeferred, + createTestBinarySession, +} from "../../fixtures/binarySession.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; + +it("retains completed Android Evidence through the MCP cancellation delivery path", async () => { + const fixture = await createJadxProtocolFixture(); + const controller = new AbortController(); + let cancelOnCompletion = false; + const session = createTestBinarySession(() => { + throw new Error("Unexpected native admission"); + }); + const server = createServer( + { kind: "session", session }, + { + logger: silentLogger, + androidAnalysis: { + inspectAvailability: fixture.provider.inspectAvailability.bind( + fixture.provider, + ), + close: fixture.provider.close.bind(fixture.provider), + execute: async (target, request, options) => { + const result = await fixture.provider.execute( + target, + request, + options, + ); + if (result.ok && cancelOnCompletion) { + const signal = options?.signal; + if (signal === undefined) + throw new Error("MCP did not forward cancellation"); + controller.abort(); + if (!signal.aborted) + await new Promise((resolve) => + signal.addEventListener("abort", () => resolve(), { + once: true, + }), + ); + } + return result; + }, + }, + }, + ); + const client = new Client({ name: "android-late-completion", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + onTestFinished(async () => { + controller.abort(); + vi.restoreAllMocks(); + await client.close(); + await server.close(); + await session.close(); + }); + await server.connect(serverTransport); + await client.connect(clientTransport); + const request = { + name: "inspect_android_package", + arguments: { path: fixture.apk }, + }; + const controlReply = await client.callTool(request); + expect(controlReply.isError).not.toBe(true); + const control = parseEvidence(controlReply.structuredContent); + const delivered = createDeferred(); + const deliver = server.delivery.toRecordedPartialErrorToolResult.bind( + server.delivery, + ); + vi.spyOn( + ToolResultDelivery.prototype, + "toRecordedPartialErrorToolResult", + ).mockImplementation((...args) => { + const result = deliver(...args); + delivered.resolve(result); + return result; + }); + cancelOnCompletion = true; + await expect( + client.callTool(request, { signal: controller.signal }), + ).rejects.toThrow(); + // The SDK suppresses a cancelled reply; observe the server's canonical + // recording/delivery boundary rather than treating client rejection as proof. + const { error } = parseMcpToolError(await delivered.promise); + expect(error.code).toBe("cancelled"); + const reference = z + .object({ + partial_observation: z.object({ + kind: z.literal("retained-evidence"), + evidence_id: z.string(), + }), + }) + .parse(error.details).partial_observation; + expect(session.evidenceById(reference.evidence_id)).toMatchObject({ + operation: control.operation, + normalized_result: control.normalized_result, + subject: control.subject, + parameters: control.parameters, + confidence: control.confidence, + }); + await expect(client.ping()).resolves.toEqual({}); +}); diff --git a/tests/boundary/mcp/browserScenarioMcp.test.ts b/tests/boundary/mcp/browserScenarioMcp.test.ts index 81efed408a..e0b5bb1e3a 100644 --- a/tests/boundary/mcp/browserScenarioMcp.test.ts +++ b/tests/boundary/mcp/browserScenarioMcp.test.ts @@ -1,6 +1,7 @@ import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { afterEach, describe, expect, test } from "vitest"; import { Ajv2020 } from "ajv/dist/2020.js"; +import { z } from "zod"; import type { BrowserScenarioCapturePort } from "../../../src/application/BrowserScenarioCapturePort.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; @@ -16,9 +17,11 @@ import { type BrowserScenarioCapture, } from "../../../src/domain/browserScenarioCapture.js"; import type { AnalysisError } from "../../../src/domain/analysisErrorBase.js"; -import { ok, type Result } from "../../../src/domain/result.js"; +import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; +import { err, ok, type Result } from "../../../src/domain/result.js"; import { createServer } from "../../../src/server/createServer.js"; import { observed } from "../../fixtures/analysisExecution.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; const providerIdentity: ProviderIdentity = { id: "test-browser-scenario", @@ -294,3 +297,65 @@ describe("browser scenario MCP tool", () => { expect(provider.scenarios).toHaveLength(3); }); }); + +test("records a completed late-close cancellation as server Evidence", async () => { + const provider: BrowserScenarioCapturePort = { + identity: () => providerIdentity, + captureScenario: async (request) => { + const capture = captureFor(request); + return err( + new AnalysisCancelledError("capture_browser_scenario", { + partialObservation: { kind: "browser-scenario-observation", capture }, + }), + ); + }, + }; + const session = createTestBinarySession(() => ({ + execute: () => Promise.resolve(observed(null)), + close: () => Promise.resolve(ok(null)), + })); + const server = createServer( + { kind: "session", session }, + { + browserScenarioCapture: provider, + availabilityPolicy: () => ({ + processCaptureEnabled: false, + investigationInputRoots: 0, + }), + }, + ); + const client = new Client({ name: "scenario-late-close", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + try { + const response = await client.callTool({ + name: "capture_browser_scenario", + arguments: minimalScenario(), + }); + const { error } = parseMcpToolError(response); + const reference = z + .object({ + partial_observation: z.object({ + kind: z.literal("retained-evidence"), + evidence_id: z.string(), + }), + }) + .parse(error.details).partial_observation; + const retained = session.evidenceById(reference.evidence_id); + if (retained === undefined) throw new Error("Evidence was not retained"); + expect(retained.operation).toBe("capture_browser_scenario"); + expect(retained.normalized_result).toMatchObject({ + browser: { cleanup: "terminated-owned-process" }, + }); + expect( + (retained.normalized_result as BrowserScenarioCapture).steps.every( + ({ status }) => status === "completed", + ), + ).toBe(true); + await expect(client.ping()).resolves.toEqual({}); + } finally { + await Promise.all([client.close(), server.close(), session.close()]); + } +}); diff --git a/tests/boundary/mcp/electronMcp.test.ts b/tests/boundary/mcp/electronMcp.test.ts index 92ec605e05..b97b7b8d55 100644 --- a/tests/boundary/mcp/electronMcp.test.ts +++ b/tests/boundary/mcp/electronMcp.test.ts @@ -13,6 +13,7 @@ import { createStrippedAsarAddon } from "../../fixtures/strippedAsarAddon.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import type { ElectronActiveObservationPort } from "../../../src/application/javascript/ElectronActiveObservationPort.js"; +import type { ElectronObservationPort } from "../../../src/application/javascript/ElectronObservationPort.js"; import { analyzeJavaScriptApplication } from "../../support/javascriptApplicationScope.js"; import { CdpElectronProvider } from "../../../src/browser/CdpElectronProvider.js"; import { createServer } from "../../../src/server/createServer.js"; @@ -26,6 +27,10 @@ import { createElectronActiveObservationFixtureResult } from "../../../src/domai import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { AsarArtifactReader } from "../../../src/artifacts/AsarArtifactReader.js"; import { toolAvailabilityInventory } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; +import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; +import { classifyBrowserCompleteness } from "../../../src/domain/browserCompleteness.js"; +import type { ElectronPageInspection } from "../../../src/domain/javascript/electronObservation.js"; +import { err } from "../../../src/domain/result.js"; const browsers: FakeCdpBrowser[] = []; const resources: Array<{ close(): Promise }> = []; @@ -93,6 +98,118 @@ afterEach(async () => { ); }); +it("records completed Electron facts after final detach as server Evidence", async () => { + const inspection: ElectronPageInspection = { + browser: { + product: "Chromium", + protocol_version: "1.3", + revision: "123", + user_agent: "fixture", + js_version: "V8", + }, + target: { + target_id: "target-1", + type: "page", + title: "fixture", + file_path: "/tmp/index.html", + attached: true, + }, + capture_window: { + started_at: "2026-01-01T00:00:00.000Z", + ended_at: "2026-01-01T00:00:00.001Z", + observation_ms: 1, + }, + completeness: classifyBrowserCompleteness({ + policyFilteredSections: new Set(), + attachLimitedSections: new Set(), + truncatedSections: new Set(), + unavailableSections: new Set(), + excluded: [], + droppedEvents: { + scripts: 0, + network_requests: 0, + console_events: 0, + websocket_connections: 0, + websocket_frames: 0, + webmcp_tools: 0, + timeline_events: 0, + }, + }), + frames: [], + dom: { total_nodes: 0, nodes: [] }, + scripts: { total: 0, items: [] }, + resources: [], + workers: [], + limitations: [], + }; + const provider: ElectronObservationPort = { + identity: () => ({ + id: "test-electron-observation", + name: "Test Electron observation provider", + version: "1", + }), + listTargets: async () => { + throw new Error("unexpected target listing"); + }, + inspectPage: async () => + err( + new AnalysisCancelledError("inspect_electron_page", { + partialObservation: { + kind: "electron-page-observation", + inspection, + cleanup: "transport-closed", + }, + }), + ), + }; + const session = createTestBinarySession(() => ({ + execute: () => Promise.resolve(observed(null)), + close: () => Promise.resolve(resultOk(null)), + })); + const server = createServer( + { kind: "session", session }, + { + electronObservation: provider, + }, + ); + const client = new Client({ name: "electron-final-detach", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + resources.push(client, server, session); + await server.connect(serverTransport); + await client.connect(clientTransport); + + const response = await client.callTool({ + name: "inspect_electron_page", + arguments: { + cdp_endpoint: "http://127.0.0.1:9222", + target_id: "target-1", + observation_ms: 0, + include_script_sources: false, + }, + }); + const { error } = parseMcpToolError(response); + expect(error.details).toMatchObject({ + partial_observation: { kind: "retained-evidence" }, + }); + const reference = z + .object({ + partial_observation: z.object({ + kind: z.literal("retained-evidence"), + evidence_id: z.string(), + }), + }) + .parse(error.details).partial_observation; + expect(session.evidenceById(reference.evidence_id)).toMatchObject({ + operation: "inspect_electron_page", + normalized_result: { target: { target_id: "target-1" } }, + limitations: expect.arrayContaining([ + expect.stringContaining("closed its CDP transport"), + ]), + }); + await expect(client.ping()).resolves.toEqual({}); +}); + it("lets MCP JavaScript analysis continue only with recorded untrusted ASAR bytes", async () => { const { archive, addon } = await createStrippedAsarAddon(); const session = createTestBinarySession(() => ({ diff --git a/tests/boundary/mcp/lateAbortApplicationEvidenceMcp.test.ts b/tests/boundary/mcp/lateAbortApplicationEvidenceMcp.test.ts new file mode 100644 index 0000000000..1932446510 --- /dev/null +++ b/tests/boundary/mcp/lateAbortApplicationEvidenceMcp.test.ts @@ -0,0 +1,227 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { expect, it, onTestFinished, vi } from "vitest"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { GoBinaryService } from "../../../src/application/go/GoBinaryService.js"; +import type { GoBinary } from "../../../src/domain/go/goBinary.js"; +import { silentLogger } from "../../../src/logger.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { createDeferred } from "../../fixtures/binarySession.js"; +import { ok } from "../../../src/domain/result.js"; +import type { JavaScriptRuntimeObservationPort } from "../../../src/application/javascript/JavaScriptRuntimeObservationPort.js"; +import type { BrowserObservationPort } from "../../../src/application/BrowserObservationPort.js"; +import type { BrowserTargetList } from "../../../src/domain/browserObservationSchemas.js"; +import type { JavaScriptRuntimeTargetList } from "../../../src/domain/javascript/javascriptRuntimeObservation.js"; + +it("retains late-cancelled Go artifact Evidence through MCP and keeps the server usable", async () => { + const gate = createDeferred(); + const started = createDeferred(); + const controller = new AbortController(); + const report: GoBinary = { + artifact: { path: "/artifacts/example", sha256: "a".repeat(64), bytes: 10 }, + format: "elf", + architecture: "amd64", + bits: 64, + byte_order: "little", + build_info: null, + limitations: [], + }; + const session = createTestBinarySession(() => { + throw new Error("unexpected binary admission"); + }); + const service = new GoBinaryService({ + identity: { id: "fixture", name: "fixture", version: "1" }, + inspect: async (_input, options) => { + started.resolve(); + await gate.promise; + if (!options?.signal) throw new Error("MCP did not forward its signal"); + return ok(report); + }, + }); + const server = createServer( + { kind: "session", session }, + { logger: silentLogger, goBinary: service }, + ); + const client = new Client({ name: "late-abort-artifact", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + onTestFinished(async () => { + gate.resolve(); + controller.abort(); + await client.close(); + await server.close(); + await session.close(); + }); + await server.connect(serverTransport); + await client.connect(clientTransport); + const call = client.callTool( + { name: "inspect_go_binary", arguments: { path: report.artifact.path } }, + { signal: controller.signal }, + ); + await started.promise; + controller.abort(); + gate.resolve(); + await expect(call).rejects.toThrow(); + await vi.waitFor(() => { + expect(session.exportEvidenceBundle().records).toContainEqual( + expect.objectContaining({ + operation: "inspect_go_binary", + normalized_result: report, + }), + ); + }); + await expect(client.ping()).resolves.toEqual({}); +}); + +it("retains late-cancelled Inspector target Evidence through MCP and keeps the server usable", async () => { + const gate = createDeferred(); + const started = createDeferred(); + const controller = new AbortController(); + const targetList: JavaScriptRuntimeTargetList = { + runtime: { product: "Node.js", protocol_version: "1.0", v8_version: null }, + targets: [], + excluded: { unsupported_location: 0, unconnectable: 0 }, + limitations: [], + }; + const provider: JavaScriptRuntimeObservationPort = { + identity: () => ({ + id: "fixture-inspector", + name: "fixture", + version: "1", + }), + listTargets: async (_input, options) => { + started.resolve(); + await gate.promise; + if (!options?.signal) throw new Error("MCP did not forward its signal"); + return ok(targetList); + }, + observe: async () => { + throw new Error("Unexpected observe call"); + }, + }; + const session = createTestBinarySession(() => { + throw new Error("unexpected binary admission"); + }); + const server = createServer( + { kind: "session", session }, + { logger: silentLogger, javascriptRuntimeObservation: provider }, + ); + const client = new Client({ name: "late-abort-inspector", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + onTestFinished(async () => { + gate.resolve(); + controller.abort(); + await client.close(); + await server.close(); + await session.close(); + }); + await server.connect(serverTransport); + await client.connect(clientTransport); + const call = client.callTool( + { + name: "list_javascript_runtime_targets", + arguments: { inspector_endpoint: "http://127.0.0.1:9222" }, + }, + { signal: controller.signal }, + ); + await started.promise; + controller.abort(); + gate.resolve(); + await expect(call).rejects.toThrow(); + await vi.waitFor(() => { + expect(session.exportEvidenceBundle().records).toContainEqual( + expect.objectContaining({ + operation: "list_javascript_runtime_targets", + normalized_result: targetList, + }), + ); + }); + await expect(client.ping()).resolves.toEqual({}); +}); + +it("records late-cancelled Browser target Evidence and keeps the server usable", async () => { + const gate = createDeferred(); + const started = createDeferred(); + const controller = new AbortController(); + const targetList: BrowserTargetList = { + browser: { + product: "Chromium", + protocol_version: "1.3", + revision: "fixture", + user_agent: "fixture", + js_version: "fixture", + }, + targets: [], + excluded: { disallowed_origin: 0, unsupported_url: 0, non_page: 0 }, + limitations: [], + }; + const browser: BrowserObservationPort = { + identity: () => ({ id: "fixture-browser", name: "fixture", version: "1" }), + listTargets: async (_input, options) => { + started.resolve(); + await gate.promise; + if (!options?.signal) throw new Error("MCP did not forward its signal"); + return ok(targetList); + }, + inspectPage: async () => { + throw new Error("Unexpected inspect call"); + }, + analyzeBundle: async () => { + throw new Error("Unexpected bundle call"); + }, + observeSession: async () => { + throw new Error("Unexpected session call"); + }, + discoverWebMcpTools: async () => { + throw new Error("Unexpected discovery call"); + }, + compareCaptures: async () => { + throw new Error("Unexpected comparison call"); + }, + captureScreenshot: async () => { + throw new Error("Unexpected screenshot call"); + }, + compareScreenshots: async () => { + throw new Error("Unexpected diff call"); + }, + }; + const session = createTestBinarySession(() => { + throw new Error("unexpected binary admission"); + }); + const server = createServer( + { kind: "session", session }, + { logger: silentLogger, browserObservation: browser }, + ); + const client = new Client({ name: "late-abort-browser", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + onTestFinished(async () => { + gate.resolve(); + controller.abort(); + await client.close(); + await server.close(); + await session.close(); + }); + await server.connect(serverTransport); + await client.connect(clientTransport); + const call = client.callTool( + { + name: "list_browser_targets", + arguments: { cdp_endpoint: "http://127.0.0.1:9222", allowed_origins: [] }, + }, + { signal: controller.signal }, + ); + await started.promise; + controller.abort(); + gate.resolve(); + await expect(call).rejects.toThrow(); + await vi.waitFor(() => { + expect(session.exportEvidenceBundle().records).toContainEqual( + expect.objectContaining({ + operation: "list_browser_targets", + normalized_result: targetList, + }), + ); + }); + await expect(client.ping()).resolves.toEqual({}); +}); diff --git a/tests/boundary/mcp/sessionMcp.test.ts b/tests/boundary/mcp/sessionMcp.test.ts index 9a173a159f..6bb80eee48 100644 --- a/tests/boundary/mcp/sessionMcp.test.ts +++ b/tests/boundary/mcp/sessionMcp.test.ts @@ -478,7 +478,7 @@ describe("target-free MCP workflow", () => { ).isError, ).not.toBe(true); await mcp.callTool({ name: "close_binary", arguments: {} }); - }, 10_000); + }); }); describe("MCP snapshot open ordering", () => { diff --git a/tests/boundary/mcp/wasmArtifactMcp.test.ts b/tests/boundary/mcp/wasmArtifactMcp.test.ts index 91c63fc8e3..1be804f61c 100644 --- a/tests/boundary/mcp/wasmArtifactMcp.test.ts +++ b/tests/boundary/mcp/wasmArtifactMcp.test.ts @@ -215,6 +215,73 @@ it("advertises exact valid schemas and records inline WASM interface evidence wi expect(close).toHaveBeenCalled(); }); +it("records complete WASM Evidence when an MCP request is cancelled during provider work", async () => { + const value = wasmArtifactFixture(); + let releaseProvider: (() => void) | undefined; + let providerStarted: (() => void) | undefined; + const started = new Promise((resolve) => { + providerStarted = resolve; + }); + const providerGate = new Promise((resolve) => { + releaseProvider = resolve; + }); + const service = new WasmArtifactService({ + close: () => Promise.resolve(), + inspect: async (_input, options) => { + providerStarted?.(); + await providerGate; + expect(options?.signal?.aborted).toBe(true); + return ok(wasmArtifactExecution(value)); + }, + }); + const inspect = vi.spyOn(service, "inspect"); + const session = createTestBinarySession(() => { + throw new Error("deep provider must not start"); + }); + const harness = await createApplicationMcpHarness({ + session, + serverOptions: { wasmArtifact: service }, + }); + onTestFinished(harness.close); + const controller = new AbortController(); + const cancellation = harness.client.callTool( + { + name: "inspect_wasm_artifact", + arguments: { path: value.artifact.path }, + }, + { signal: controller.signal }, + ); + await started; + controller.abort(new Error("cancel after provider work starts")); + releaseProvider?.(); + await expect(cancellation).rejects.toThrow(); + expect(inspect).toHaveBeenCalledOnce(); + const result = await inspect.mock.results[0]?.value; + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + partialObservation: { + operation: "inspect_wasm_artifact", + normalized_result: value, + confidence: "observed", + }, + }, + }); + await harness.client.ping(); + await vi.waitFor(() => { + expect(session.exportEvidenceBundle().records).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + operation: "inspect_wasm_artifact", + normalized_result: value, + confidence: "observed", + }), + ]), + ); + }); +}); + it("attributes rejected WASM paths to their selected fields before provider work", async () => { const value = wasmArtifactFixture(); value.glue = [ diff --git a/tests/boundary/pcap/pcapProvider.test.ts b/tests/boundary/pcap/pcapProvider.test.ts index 7797542664..9c51b96e53 100644 --- a/tests/boundary/pcap/pcapProvider.test.ts +++ b/tests/boundary/pcap/pcapProvider.test.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import { watch } from "node:fs"; import { chmod, readFile, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { expect, onTestFinished, test, vi } from "vitest"; import { PcapProvider } from "../../../src/pcap/PcapProvider.js"; @@ -184,11 +184,17 @@ it("inspects a capture with endpoints, DNS, TLS SNI, and HTTP", async () => { expect(result.coverage).toBe("complete"); }); -it.each(["capinfos", "tshark"] as const)( - "retains parsed output and cleanup ownership when %s cleanup is unresolved", - async (stage) => { +it.each([ + { stage: "capinfos", denySnapshotClose: false }, + { stage: "tshark", denySnapshotClose: false }, + { stage: "capinfos", denySnapshotClose: true }, + { stage: "tshark", denySnapshotClose: true }, +] as const)( + "retires $stage cleanup with its snapshot (snapshot close denied: $denySnapshotClose)", + async ({ stage, denySnapshotClose }) => { let mayClean = false; let producerLaunches = 0; + let cleanupAttempts = 0; const launcher: PcapCommandLauncher = async (spawn) => { const selectedStage = spawn.arguments.includes("-M") ? "capinfos" @@ -204,6 +210,7 @@ it.each(["capinfos", "tshark"] as const)( return { ...launched, cleanup: async () => { + cleanupAttempts++; const cleaned = await cleanupOwned(); if (!mayClean) throw new Error("injected retained cleanup failure"); return cleaned; @@ -246,18 +253,50 @@ it.each(["capinfos", "tshark"] as const)( }); expect(producerLaunches).toBe(1); expect(await readFile(path)).toEqual(Buffer.from("pcap-fixture-bytes")); - expect( - result.error.cleanupResources.some((resource) => - resource.endsWith("/capture.pcap"), - ), - ).toBe(true); + const snapshotPath = result.error.cleanupResources.find((resource) => + resource.endsWith("/capture.pcap"), + ); + if (snapshotPath === undefined) + throw new Error("Missing retained snapshot"); expect( result.error.cleanupResources.some((resource) => resource.startsWith("process-group:"), ), ).toBe(true); + const originatingCause = result.error.cause; + const originatingObservation = result.error.partialObservation; + const attemptsBeforeClose = cleanupAttempts; mayClean = true; + if (denySnapshotClose) { + const close = PrivateRuntimeRoot.prototype.close; + const denied = vi + .spyOn(PrivateRuntimeRoot.prototype, "close") + .mockImplementation(function (this: PrivateRuntimeRoot) { + if (this.path === dirname(snapshotPath)) + return Promise.reject(new Error("injected snapshot close failure")); + return close.call(this); + }); + try { + await expect(provider.close()).rejects.toMatchObject({ + cleanupIncomplete: true, + }); + expect(result.error.cleanupIncomplete).toBe(true); + expect(cleanupAttempts).toBe(attemptsBeforeClose + 1); + await expect(readFile(snapshotPath)).resolves.toEqual( + Buffer.from("pcap-fixture-bytes"), + ); + } finally { + denied.mockRestore(); + } + } await provider.close(); + expect(result.error.cleanupIncomplete).toBe(false); + expect(result.error.cause).toBe(originatingCause); + expect(result.error.partialObservation).toBe(originatingObservation); + expect(cleanupAttempts).toBe(attemptsBeforeClose + 1); + await expect(readFile(snapshotPath)).rejects.toMatchObject({ + code: "ENOENT", + }); }, ); @@ -616,6 +655,7 @@ it.each([false, true])( spy.mockRestore(); } await provider.close(); + if (!result.ok) expect(result.error.cleanupIncomplete).toBe(false); await expect( readFile(join(retained!.path, "capture.pcap")), ).rejects.toMatchObject({ code: "ENOENT" }); diff --git a/tests/boundary/pcap/pcapProviderCrossPlatform.test.ts b/tests/boundary/pcap/pcapProviderCrossPlatform.test.ts index c19e682fe9..6b16cdf61a 100644 --- a/tests/boundary/pcap/pcapProviderCrossPlatform.test.ts +++ b/tests/boundary/pcap/pcapProviderCrossPlatform.test.ts @@ -13,8 +13,11 @@ import { expect, it, onTestFinished } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { PcapProvider } from "../../../src/pcap/PcapProvider.js"; +import type { PcapCommandLauncher } from "../../../src/pcap/PcapCommand.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import type { PcapRequest } from "../../../src/domain/pcap/pcapAnalysis.js"; +import { cleanupOwnedProcessGroup } from "../../../src/process/ProcessOwnership.js"; +import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; const isWindows = process.platform === "win32"; const hasNativeBound = process.platform === "linux" || isWindows; @@ -59,6 +62,52 @@ it("inspects both tools through cross-platform stubs", async () => { }); }); +it("resolves a retained tool-probe error after its process owner closes", async () => { + let mayClean = false; + let launches = 0; + const launcher: PcapCommandLauncher = async (spawn) => { + const launched = await spawnOwnedProviderProcess(spawn); + launches++; + if (launches !== 1) return launched; + const cleanup = + launched.cleanup ?? (() => cleanupOwnedProcessGroup(launched.ownership)); + return { + ...launched, + cleanup: async () => { + const result = await cleanup(); + if (!mayClean) throw new Error("injected tool-probe cleanup failure"); + return result; + }, + }; + }; + const provider = new PcapProvider({ + environment: { + REA_TSHARK_COMMAND: tsharkStub, + REA_CAPINFOS_COMMAND: capinfosStub, + }, + launcher, + }); + onTestFinished(async () => { + mayClean = true; + await provider.close(); + }); + + const result = await provider.execute({ + operation: "inspect_pcap_client", + input: {}, + }); + if (result.ok) throw new Error("Expected retained tool-probe cleanup"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.diagnostics).toMatchObject({ + reason: "injected tool-probe cleanup failure", + }); + + mayClean = true; + await provider.close(); + expect(result.error.cleanupIncomplete).toBe(false); + expect(launches).toBe(1); +}); + it.runIf(hasNativeBound)( "inspects a capture with metadata, endpoints, DNS, and protocol facts", async () => { diff --git a/tests/boundary/process/adbVerifierCleanup.test.ts b/tests/boundary/process/adbVerifierCleanup.test.ts index b22e442516..42e53b85b5 100644 --- a/tests/boundary/process/adbVerifierCleanup.test.ts +++ b/tests/boundary/process/adbVerifierCleanup.test.ts @@ -1,206 +1,132 @@ -import { spawn } from "node:child_process"; -import { copyFile, mkdir, readFile, writeFile } from "node:fs/promises"; -import { join, resolve } from "node:path"; - -import { expect, it } from "vitest"; - -import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; - -const providerModule = ` -import { writeFile } from "node:fs/promises"; - -export function createAdbDeviceAnalysisProvider() { - return { - async close() { - await writeFile(process.env.REA_ADB_TEST_CLOSE_MARKER, "closed"); - if (process.env.REA_ADB_TEST_CLOSE_FAILURE === "1") - throw new Error("fake provider close failed"); - }, - }; -} -`; - -const serviceModule = ` -import { createHash } from "node:crypto"; -import { mkdir, writeFile } from "node:fs/promises"; +import { spawnSync } from "node:child_process"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { join } from "node:path"; +import { afterEach, expect, it } from "vitest"; -const transferBytes = Buffer.from("rea adb transfer verification\\n"); -const transferDigest = createHash("sha256").update(transferBytes).digest("hex"); -const packageBytes = Buffer.from("fake package artifact"); -const packageDigest = createHash("sha256").update(packageBytes).digest("hex"); -const packages = [{ - package_name: "com.example", - base_apk_device_path: "/data/app/example/base.apk", -}]; +const roots: string[] = []; +afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); +}); -export class AdbDeviceAnalysisService { - async execute(operation, input) { - if (process.env.REA_ADB_TEST_THROW === operation) - throw new Error("fake operation threw"); - - if (operation === "pull_adb_package") { - await mkdir(input.output_directory, { recursive: true }); - const local_path = join(input.output_directory, "base.apk"); - await writeFile(local_path, packageBytes); - return { - ok: true, - value: { - normalized_result: { - artifacts: [{ - local_path, - file_name: "base.apk", - bytes: packageBytes.length, - sha256: packageDigest, - role: "base", - }], - coverage: "complete", - }, - }, - }; +const runVerifier = async ( + settings: { closeFails?: boolean; thrown?: "undefined" | "null" } = {}, +) => { + const root = await mkdtemp(join(tmpdir(), "rea-adb-verifier-test-")); + roots.push(root); + await mkdir(join(root, "scripts", "lib"), { recursive: true }); + await mkdir(join(root, "dist", "application", "adb"), { recursive: true }); + await mkdir(join(root, "dist", "composition"), { recursive: true }); + await writeFile( + join(root, "scripts", "verify-adb.mjs"), + await readFile(new URL("../../../scripts/verify-adb.mjs", import.meta.url)), + ); + await writeFile( + join(root, "scripts", "lib", "verifier-cleanup.mjs"), + await readFile( + new URL("../../../scripts/lib/verifier-cleanup.mjs", import.meta.url), + ), + ); + await writeFile( + join(root, "dist", "composition", "adb.js"), + `import { writeFile } from "node:fs/promises"; +export const createAdbDeviceAnalysisProvider = () => ({ + async close() { + await writeFile(process.env.REA_ADB_CLOSE_MARKER, "closed"); + if (process.env.REA_ADB_CLOSE_FAIL === "1") { + const error = new Error("provider cleanup refused"); + error.cleanupResources = ["fixture/private-root"]; + error.diagnostics = { reason: "retry remains possible" }; + throw error; } - - const results = { - inspect_adb_client: { - client: { version: "test", binary_path: "/fake/adb" }, - }, - list_adb_devices: { - devices: [{ serial: "test-device", state: "device", kind: "emulator" }], - }, - inspect_adb_device: { - properties: [{ name: "ro.build.version.sdk", value: "35" }], - emulator_observed: true, - }, - list_adb_processes: { processes: [{ name: "system" }], columns: "PID NAME" }, + }, +});\n`, + ); + await writeFile( + join(root, "dist", "application", "adb", "AdbDeviceAnalysisService.js"), + `export class AdbDeviceAnalysisService { + async execute(operation, input) { + if (process.env.REA_ADB_THROW === "undefined") throw undefined; + if (process.env.REA_ADB_THROW === "null") throw null; + const result = { + inspect_adb_client: { client: { version: "1", binary_path: "/tmp/adb" } }, + list_adb_devices: { devices: [{ serial: "test", state: "device", kind: "emulator", kind_basis: "stub" }] }, + inspect_adb_device: { properties: [{ name: "ro.build.version.sdk", value: "35" }], emulator_observed: true }, + list_adb_processes: { processes: [{ name: "probe" }], columns: "pid name" }, inspect_adb_window: { focused_window: null }, - inspect_adb_display: { - physical_size: { width: 1, height: 1 }, - physical_density: 1, - }, + inspect_adb_display: { physical_size: { width: 2, height: 2 }, physical_density: 1 }, read_adb_setting: { value: "1" }, read_adb_logcat: { lines: [], coverage: "complete" }, - list_adb_features: { features: ["test-feature"], coverage: "complete" }, - list_adb_services: { services: ["test-service"], coverage: "complete" }, + list_adb_features: { features: ["feature"], coverage: "complete" }, + list_adb_services: { services: ["service"], coverage: "complete" }, list_adb_directory: { entries: [], coverage: "complete" }, - list_adb_packages: { packages, coverage: "complete" }, - push_adb_file: { - bytes: transferBytes.length, - sha256: transferDigest, - device_digest_source: "device_sha256sum", - device_sha256: transferDigest, - }, - pull_adb_file: { bytes: transferBytes.length, sha256: transferDigest }, - capture_adb_screen: { width: 1, height: 1, bytes: 1 }, - }; - const normalized_result = results[operation]; - if (normalized_result === undefined) - throw new Error("unexpected fake operation: " + operation); - return { ok: true, value: { normalized_result } }; + list_adb_packages: { packages: input.scope === "all" ? [{ package_name: "com.test" }] : [], coverage: "complete" }, + push_adb_file: { bytes: 3, sha256: "${"a".repeat(64)}", device_sha256: "${"a".repeat(64)}", device_digest_source: "device_sha256sum" }, + pull_adb_file: { bytes: 3, sha256: "${"a".repeat(64)}" }, + capture_adb_screen: { width: 2, height: 2, bytes: 3 }, + }[operation]; + return { ok: true, value: { normalized_result: result } }; } -} -`; - -async function createVerifierFixture() { - const root = await createTestTempDirectory("rea-adb-verifier-cleanup-"); - const scriptDirectory = join(root, "scripts"); - const scriptLibDirectory = join(scriptDirectory, "lib"); - const applicationDirectory = join(root, "dist/application/adb"); - const compositionDirectory = join(root, "dist/composition"); - await Promise.all([ - mkdir(scriptDirectory, { recursive: true }), - mkdir(scriptLibDirectory, { recursive: true }), - mkdir(applicationDirectory, { recursive: true }), - mkdir(compositionDirectory, { recursive: true }), - ]); - const scriptPath = join(scriptDirectory, "verify-adb.mjs"); - const closeMarker = join(root, "provider-closed"); - await Promise.all([ - copyFile(resolve("scripts/verify-adb.mjs"), scriptPath), - copyFile( - resolve("scripts/lib/verifier-cleanup.mjs"), - join(scriptLibDirectory, "verifier-cleanup.mjs"), - ), - writeFile(join(root, "package.json"), '{"type":"module"}\n'), - writeFile( - join(applicationDirectory, "AdbDeviceAnalysisService.js"), - serviceModule, - ), - writeFile(join(compositionDirectory, "adb.js"), providerModule), - ]); - return { root, scriptPath, closeMarker }; -} - -function runVerifier( - root: string, - scriptPath: string, - closeMarker: string, - extraEnvironment: Record = {}, -) { - return new Promise<{ code: number | null; stdout: string; stderr: string }>( - (resolveResult, reject) => { - const child = spawn(process.execPath, [scriptPath], { - cwd: root, - env: { - ...process.env, - REA_ADB_TEST_CLOSE_MARKER: closeMarker, - ...extraEnvironment, - }, - stdio: ["ignore", "pipe", "pipe"], - windowsHide: true, - }); - let stdout = ""; - let stderr = ""; - child.stdout.on("data", (chunk) => (stdout += chunk.toString())); - child.stderr.on("data", (chunk) => (stderr += chunk.toString())); - child.once("error", reject); - child.once("close", (code) => resolveResult({ code, stdout, stderr })); - }, +}\n`, ); -} - -it("fails verification when provider cleanup fails after successful checks", async () => { - const fixture = await createVerifierFixture(); - const result = await runVerifier( - fixture.root, - fixture.scriptPath, - fixture.closeMarker, - { REA_ADB_TEST_CLOSE_FAILURE: "1" }, - ); - - expect(result.code).not.toBe(0); - expect(await readFile(fixture.closeMarker, "utf8")).toBe("closed"); - expect(result.stdout + result.stderr).toContain("fake provider close failed"); - expect(result.stdout).not.toContain("all adb device checks passed"); -}); - -it("runs cleanup and retains operation and cleanup failures", async () => { - const fixture = await createVerifierFixture(); - const result = await runVerifier( - fixture.root, - fixture.scriptPath, - fixture.closeMarker, + const marker = join(root, "provider-closed"); + const run = spawnSync( + process.execPath, + [join(root, "scripts", "verify-adb.mjs")], { - REA_ADB_TEST_THROW: "inspect_adb_client", - REA_ADB_TEST_CLOSE_FAILURE: "1", + encoding: "utf8", + timeout: 10_000, + env: { + ...process.env, + REA_ADB_CLOSE_MARKER: marker, + REA_ADB_CLOSE_FAIL: settings.closeFails ? "1" : "0", + ...(settings.thrown === undefined + ? {} + : { REA_ADB_THROW: settings.thrown }), + }, }, ); + const closed = await readFile(marker, "utf8").catch(() => null); + return { ...run, closed }; +}; - expect(result.code).not.toBe(0); - expect(await readFile(fixture.closeMarker, "utf8")).toBe("closed"); - expect(result.stdout + result.stderr).toContain("fake operation threw"); - expect(result.stdout + result.stderr).toContain("fake provider close failed"); - expect(result.stdout).not.toContain("all adb device checks passed"); -}); +it("runs the production verifier successfully and closes its provider", async () => { + const result = await runVerifier(); -it("reports success after successful verification and provider cleanup", async () => { - const fixture = await createVerifierFixture(); - const result = await runVerifier( - fixture.root, - fixture.scriptPath, - fixture.closeMarker, - ); - - expect(result.code).toBe(0); - expect(await readFile(fixture.closeMarker, "utf8")).toBe("closed"); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); expect(result.stdout).toContain("all adb device checks passed"); + expect(result.closed).toBe("closed"); +}); + +it("makes provider cleanup failure fail the production verifier with resource details", async () => { + const result = await runVerifier({ closeFails: true }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("provider cleanup refused"); + expect(result.stderr).toContain("fixture/private-root"); + expect(result.stderr).toContain("retry remains possible"); + expect(result.closed).toBe("closed"); +}); + +it.each(["undefined", "null"] as const)( + "closes after the verifier throws %s and exits unsuccessfully", + async (thrown) => { + const result = await runVerifier({ thrown }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain(`verifier aborted — ${thrown}`); + expect(result.closed).toBe("closed"); + }, +); + +it("reports both an unexpected verifier throw and provider cleanup failure", async () => { + const result = await runVerifier({ closeFails: true, thrown: "null" }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("verifier aborted — null"); + expect(result.stderr).toContain("provider cleanup refused"); + expect(result.stderr).toContain("fixture/private-root"); + expect(result.closed).toBe("closed"); }); diff --git a/tests/boundary/process/browserScenarioProbeLifecycle.test.ts b/tests/boundary/process/browserScenarioProbeLifecycle.test.ts index 66dc09d6f0..bc07742389 100644 --- a/tests/boundary/process/browserScenarioProbeLifecycle.test.ts +++ b/tests/boundary/process/browserScenarioProbeLifecycle.test.ts @@ -2,6 +2,8 @@ import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; import { expect, it, vi } from "vitest"; import type { BrowserScenarioCapturePort } from "../../../src/application/BrowserScenarioCapturePort.js"; +import { createBrowserObservationProvider } from "../../../src/composition/browserObservation.js"; +import { createElectronObservationProvider } from "../../../src/composition/electronObservation.js"; import { silentLogger } from "../../../src/logger.js"; import { startMcpTransport } from "../../../src/main/transport.js"; import type { RuntimeDependencies } from "../../../src/main/types.js"; @@ -12,7 +14,18 @@ it("keeps the shared browser scenario provider open when the SDK discards its di const session = createTestBinarySession(() => { throw new Error("Unexpected deep provider"); }); - const close = vi.fn(async () => undefined); + const scenarioClose = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error("Scenario cleanup failed")) + .mockResolvedValue(undefined); + const browserObservation = createBrowserObservationProvider(); + const browserClose = vi + .spyOn(browserObservation, "close") + .mockResolvedValue(undefined); + const electronObservation = createElectronObservationProvider(); + const electronClose = vi + .spyOn(electronObservation, "close") + .mockResolvedValue(undefined); const browserScenarioCapture: BrowserScenarioCapturePort = { identity: () => ({ id: "fake-browser", @@ -22,7 +35,7 @@ it("keeps the shared browser scenario provider open when the SDK discards its di captureScenario: async () => { throw new Error("Unexpected scenario capture"); }, - close, + close: scenarioClose, }; const factories: Parameters[0][] = []; const runtime: RuntimeDependencies = { @@ -40,7 +53,11 @@ it("keeps the shared browser scenario provider open when the SDK discards its di serverLogger: silentLogger, environment: runtime.env, delivery: new ToolResultDelivery(STDIO_DEFAULT_MAX_BUFFER_SIZE), - loadOptionalProviders: async () => ({ browserScenarioCapture }), + loadOptionalProviders: async () => ({ + browserObservation, + browserScenarioCapture, + electronObservation, + }), }); if (!transport.ok) throw new Error("Transport failed"); const factory = factories[0]; @@ -50,10 +67,21 @@ it("keeps the shared browser scenario provider open when the SDK discards its di await probe.close(); const replacement = await factory({ era: "legacy" }); await replacement.close(); - expect(close).not.toHaveBeenCalled(); + expect(browserClose).not.toHaveBeenCalled(); + expect(scenarioClose).not.toHaveBeenCalled(); + expect(electronClose).not.toHaveBeenCalled(); + await expect(transport.closeOptionalProviders()).rejects.toMatchObject({ + errors: [expect.objectContaining({ message: "Scenario cleanup failed" })], + }); + expect(browserClose).toHaveBeenCalledTimes(1); + expect(scenarioClose).toHaveBeenCalledTimes(1); + expect(electronClose).toHaveBeenCalledTimes(1); await transport.closeOptionalProviders(); - expect(close).toHaveBeenCalledTimes(1); + await transport.closeOptionalProviders(); + expect(browserClose).toHaveBeenCalledTimes(1); + expect(scenarioClose).toHaveBeenCalledTimes(2); + expect(electronClose).toHaveBeenCalledTimes(1); } finally { await transport.closeAndroid(); await session.close(); @@ -64,7 +92,15 @@ it("closes optional providers when MCP server startup throws", async () => { const session = createTestBinarySession(() => { throw new Error("Unexpected deep provider"); }); - const close = vi.fn(async () => undefined); + const scenarioClose = vi.fn(async () => undefined); + const browserObservation = createBrowserObservationProvider(); + const browserClose = vi + .spyOn(browserObservation, "close") + .mockResolvedValue(undefined); + const electronObservation = createElectronObservationProvider(); + const electronClose = vi + .spyOn(electronObservation, "close") + .mockResolvedValue(undefined); const browserScenarioCapture: BrowserScenarioCapturePort = { identity: () => ({ id: "fake-browser", @@ -74,7 +110,7 @@ it("closes optional providers when MCP server startup throws", async () => { captureScenario: async () => { throw new Error("Unexpected scenario capture"); }, - close, + close: scenarioClose, }; const runtime: RuntimeDependencies = { env: {}, @@ -91,11 +127,17 @@ it("closes optional providers when MCP server startup throws", async () => { serverLogger: silentLogger, environment: runtime.env, delivery: new ToolResultDelivery(STDIO_DEFAULT_MAX_BUFFER_SIZE), - loadOptionalProviders: async () => ({ browserScenarioCapture }), + loadOptionalProviders: async () => ({ + browserObservation, + browserScenarioCapture, + electronObservation, + }), }); expect(transport).toEqual({ ok: false }); - expect(close).toHaveBeenCalledTimes(1); + expect(browserClose).toHaveBeenCalledTimes(1); + expect(scenarioClose).toHaveBeenCalledTimes(1); + expect(electronClose).toHaveBeenCalledTimes(1); } finally { await session.close(); } diff --git a/tests/boundary/process/jadxHashCleanup.test.ts b/tests/boundary/process/jadxHashCleanup.test.ts index 2f4f062bb0..a78ecffb6d 100644 --- a/tests/boundary/process/jadxHashCleanup.test.ts +++ b/tests/boundary/process/jadxHashCleanup.test.ts @@ -1,6 +1,7 @@ import { expect, it, vi } from "vitest"; import { OwnedFileHandle } from "../../../src/filesystem/OwnedFileHandle.js"; +import { RegularFileCleanupFailure } from "../../../src/filesystem/OwnedRegularFile.js"; import { AnalysisError } from "../../../src/domain/analysisErrorBase.js"; import { createJadxProtocolFixture, @@ -27,9 +28,11 @@ it("resolves descriptor uncertainty after an immediate owned hash close retry", if (result.ok) throw new Error("Expected the original close failure"); expect(result.error.cleanupIncomplete).toBe(false); expect(result.error.cause).toBeInstanceOf(AggregateError); - expect((result.error.cause as AggregateError).errors).toEqual( - expect.arrayContaining([expect.objectContaining({ cause })]), - ); + const primary = (result.error.cause as AggregateError).errors[0]; + expect(primary).toBeInstanceOf(AnalysisError); + expect(primary.cause).toBeInstanceOf(RegularFileCleanupFailure); + expect(primary.cause.cleanupCause).toBe(cause); + expect(primary.cause.outcome.kind).toBe("completed"); expect(owners).toHaveLength(2); expect(owners[0]).toBe(owners[1]); expect(owners[0]?.handle.fd).toBe(-1); @@ -61,7 +64,9 @@ it("keeps hash cleanup live until the exact retained descriptor closes", async ( }); if (result.ok) throw new Error("Expected retained descriptor cleanup"); expect(result.error.cleanupIncomplete).toBe(true); - const primary = result.error.cause; + expect(result.error.cause).toBeInstanceOf(AggregateError); + const retirement = (result.error.cause as AggregateError).errors; + const primary = retirement[0]; if (!(primary instanceof AnalysisError)) throw new Error("Missing original typed hash failure"); const originalCause = primary.cause; @@ -72,6 +77,7 @@ it("keeps hash cleanup live until the exact retained descriptor closes", async ( allowClose = true; await provider.close(); expect(primary.cleanupIncomplete).toBe(false); + expect(result.error.cleanupIncomplete).toBe(false); expect(primary.cause).toBe(originalCause); expect(owners[0]?.handle.fd).toBe(-1); expect(launches).toHaveLength(0); diff --git a/tests/boundary/process/jadxIntegration.test.ts b/tests/boundary/process/jadxIntegration.test.ts index 173f5b5227..b497cc6cf2 100644 --- a/tests/boundary/process/jadxIntegration.test.ts +++ b/tests/boundary/process/jadxIntegration.test.ts @@ -3,9 +3,12 @@ import { access, readFile, truncate, writeFile } from "node:fs/promises"; import { parseBinaryTarget } from "../../../src/application/BinaryTargetResolver.js"; import { AndroidAnalysisService } from "../../../src/application/android/AndroidAnalysisService.js"; import { JadxProvider } from "../../../src/android/JadxProvider.js"; +import { JadxSession } from "../../../src/android/JadxSession.js"; import { androidResultSchemas } from "../../../src/domain/android/androidAnalysis.js"; import { snapshotAndroidEngine } from "../../../src/android/AndroidTargetSnapshot.js"; import { hashStableFile } from "../../../src/filesystem/StableFileHash.js"; +import { OwnedFileHandle } from "../../../src/filesystem/OwnedFileHandle.js"; +import { RegularFileCleanupFailure } from "../../../src/filesystem/OwnedRegularFile.js"; import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; import { createJadxProtocolFixture as setup, @@ -15,22 +18,145 @@ import { writeJadxJarInventory } from "../../fixtures/android/jadxJar.js"; const it = test.skipIf(process.platform === "win32"); +it("retains a failed input hash descriptor until provider shutdown retries it", async () => { + const { service, apk, provider, launches } = await setup(); + const closeCause = new Error("fixture source hash close unavailable"); + const owners: OwnedFileHandle[] = []; + const originalClose = OwnedFileHandle.prototype.close; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementationOnce(async function (this: OwnedFileHandle) { + owners.push(this); + throw closeCause; + }) + .mockImplementationOnce(async function (this: OwnedFileHandle) { + owners.push(this); + throw closeCause; + }); + try { + const result = await service.execute("inspect_android_package", { + path: apk, + }); + expect(result).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + expect(close).toHaveBeenCalledTimes(2); + expect(owners).toHaveLength(2); + expect(owners[0]).toBe(owners[1]); + if (result.ok) throw new Error("Expected incomplete hash cleanup"); + expect(result.error.cause).toBeInstanceOf(AggregateError); + await provider.close(); + expect(result.error.cleanupIncomplete).toBe(false); + expect(close).toHaveBeenCalledTimes(3); + expect(owners[0]!.handle.fd).toBe(-1); + expect(launches).toHaveLength(0); + } finally { + close.mockRestore(); + await provider.close(); + await verifyCleanup(launches); + } +}); + +it("clears hash cleanup uncertainty when the provider immediately retries the same handle", async () => { + const { service, apk, provider, launches } = await setup(); + const closeCause = new Error("fixture source hash close unavailable"); + const originalClose = OwnedFileHandle.prototype.close; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockRejectedValueOnce(closeCause) + .mockImplementation(async function (this: OwnedFileHandle) { + return originalClose.call(this); + }); + try { + const result = await service.execute("inspect_android_package", { + path: apk, + }); + expect(result).toMatchObject({ ok: false }); + if (result.ok) throw new Error("Expected the hash close failure"); + expect(close).toHaveBeenCalledTimes(2); + expect(result.error.cleanupIncomplete).toBe(false); + expect(result.error.cause).toBeInstanceOf(AggregateError); + expect(launches).toHaveLength(0); + } finally { + close.mockRestore(); + await provider.close(); + await verifyCleanup(launches); + } +}); + +it("drains hash descriptors even when session shutdown fails and retains its workspace", async () => { + const { service, apk, provider, launches } = await setup(); + const initial = await service.execute("inspect_android_package", { + path: apk, + }); + if (!initial.ok) throw initial.error; + + const workspace = launches[0]?.cwd; + if (workspace === undefined) + throw new Error("Expected an acquired workspace"); + const originalClose = OwnedFileHandle.prototype.close; + let fileCloseCalls = 0; + const fileClose = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + fileCloseCalls += 1; + if (fileCloseCalls === 1) + throw new Error("fixture input hash close failure"); + return originalClose.call(this); + }); + const sessionClose = vi + .spyOn(JadxSession.prototype, "close") + .mockRejectedValueOnce(new Error("fixture session cleanup failure")); + try { + const outcome = await service.execute("inspect_android_package", { + path: apk, + }); + expect(outcome).toMatchObject({ + ok: false, + error: { + cleanupIncomplete: true, + cleanupResources: [expect.any(String), workspace], + }, + }); + if (outcome.ok) throw new Error("Expected retained session cleanup"); + expect(fileCloseCalls).toBe(2); + await expect(access(workspace)).resolves.toBeUndefined(); + + sessionClose.mockRestore(); + fileClose.mockRestore(); + await provider.close(); + expect(outcome.error.cleanupIncomplete).toBe(false); + await verifyCleanup(launches); + } finally { + sessionClose.mockRestore(); + fileClose.mockRestore(); + await provider.close(); + await verifyCleanup(launches); + } +}); + it("cancels Android admission while hashing the selected APK", async () => { - const { service, apk, launches, provider } = await setup(); + const { apk, launches, provider } = await setup(); + const target = await parseBinaryTarget(apk); + if (!target.ok) throw target.error; await writeFile(apk, Buffer.from([0x50, 0x4b, 0x03, 0x04])); await truncate(apk, 1024 * 1024 * 1024); const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), 100); + const abortCause = new Error("fixture caller cancelled JADX hashing"); + const timer = setTimeout(() => controller.abort(abortCause), 100); try { - const result = await service.execute( - "inspect_android_package", - { path: apk }, + const result = await provider.execute( + target.value, + { operation: "inspect_android_package", input: { path: apk } }, { signal: controller.signal }, ); expect(result).toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError" }, }); + if (result.ok) throw new Error("Expected Android admission cancellation"); + expect(result.error.cause).toBe(abortCause); expect(launches).toHaveLength(0); } finally { clearTimeout(timer); @@ -39,9 +165,69 @@ it("cancels Android admission while hashing the selected APK", async () => { } }); +it("preserves cancellation cause when hashing succeeded but descriptor close failed", async () => { + const { apk, launches, provider } = await setup(); + const target = await parseBinaryTarget(apk); + if (!target.ok) throw target.error; + const controller = new AbortController(); + const abortCause = new Error("fixture caller cancelled after hash read"); + const closeCause = new Error("fixture hash descriptor close failed"); + const owners: OwnedFileHandle[] = []; + const originalClose = OwnedFileHandle.prototype.close; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementationOnce(async function (this: OwnedFileHandle) { + owners.push(this); + controller.abort(abortCause); + throw closeCause; + }) + .mockImplementation(async function (this: OwnedFileHandle) { + owners.push(this); + return originalClose.call(this); + }); + try { + const result = await provider.execute( + target.value, + { operation: "inspect_android_package", input: { path: apk } }, + { signal: controller.signal }, + ); + expect(result).toMatchObject({ + ok: false, + error: { cleanupIncomplete: false }, + }); + if (result.ok) throw new Error("Expected the descriptor close failure"); + expect(close).toHaveBeenCalledTimes(2); + expect(result.error.cause).toBeInstanceOf(AggregateError); + const causes = (result.error.cause as AggregateError).errors; + expect(causes[0]).toMatchObject({ _tag: "AnalysisCancelledError" }); + expect((causes[0] as Error).cause).toBe(abortCause); + const cleanupFailure = (causes[1] as Error).cause; + if (!(cleanupFailure instanceof RegularFileCleanupFailure)) + throw new Error("Expected the retained hash cleanup failure"); + expect(cleanupFailure.cleanupCause).toBe(closeCause); + expect(owners[1]).toBe(owners[0]); + expect(owners[0]?.handle.fd).toBe(-1); + expect(launches).toHaveLength(0); + } finally { + close.mockRestore(); + await provider.close(); + await verifyCleanup(launches); + } +}); + it("rejects a copied JAR that differs from its admitted session identity", async () => { const { jar } = await setup(); - const admitted = (await hashStableFile(jar)).sha256; + const hashOwners: OwnedFileHandle[] = []; + let admitted: string; + try { + admitted = ( + await hashStableFile(jar, { + retainFailedClose: (owner) => hashOwners.push(owner), + }) + ).sha256; + } finally { + await Promise.all(hashOwners.map((owner) => owner.close())); + } await writeFile(jar, "engine changed between hashing and copying"); const root = await PrivateRuntimeRoot.create({ prefix: "rea-jadx-admission-", @@ -568,7 +754,7 @@ it("retains an uncertain workspace and blocks later launches after cleanup canno expect(launches).toHaveLength(1); }); -it("retries shutdown cleanup after ownership verification recovers", async () => { +it("resolves every reported shutdown cleanup failure after ownership recovers", async () => { let cleanupFails = true; const { service, apk, launches, provider } = await setup( "normal", @@ -589,11 +775,30 @@ it("retries shutdown cleanup after ownership verification recovers", async () => if (!outcome.ok) throw outcome.error; const firstClose = provider.close(); expect(provider.close()).toBe(firstClose); - await expect(firstClose).rejects.toMatchObject({ cleanupIncomplete: true }); + const firstFailure = await firstClose.then( + () => { + throw new Error("Expected shutdown cleanup to remain incomplete"); + }, + (cause: unknown) => cause, + ); + expect(firstFailure).toMatchObject({ cleanupIncomplete: true }); + expect(firstFailure).toHaveProperty("cause", expect.any(AggregateError)); + const secondFailure = await provider.close().then( + () => { + throw new Error( + "Expected the second cleanup retry to remain incomplete", + ); + }, + (cause: unknown) => cause, + ); + expect(secondFailure).toMatchObject({ cleanupIncomplete: true }); + expect(secondFailure).toHaveProperty("cause", expect.any(AggregateError)); const workspace = launches[0]?.cwd; if (workspace === undefined) throw new Error("Expected acquired workspace"); await expect(access(workspace)).resolves.toBeUndefined(); await closeAfterRecovery(); + expect(firstFailure).toMatchObject({ cleanupIncomplete: false }); + expect(secondFailure).toMatchObject({ cleanupIncomplete: false }); await verifyCleanup(launches); expect(launches).toHaveLength(1); } finally { @@ -668,12 +873,6 @@ it("retains the execution failure while cleanup is pending and resumes after cle ok: false, error: { cleanupIncomplete: true, - cause: { - _tag: "ProviderAdapterError", - diagnostics: { - reason: "manifest decoder rejected malformed binary XML", - }, - }, diagnostics: { previous_error: { details: { @@ -685,6 +884,15 @@ it("retains the execution failure while cleanup is pending and resumes after cle }, }, }); + if (outcome.ok) throw new Error("Expected cleanup failure"); + expect(outcome.error.cause).toBeInstanceOf(AggregateError); + const causes = (outcome.error.cause as AggregateError).errors; + expect(causes[0]).toMatchObject({ + _tag: "ProviderAdapterError", + diagnostics: { + reason: "manifest decoder rejected malformed binary XML", + }, + }); expect( await service.execute("inspect_android_package", { path: apk }), ).toMatchObject({ diff --git a/tests/boundary/process/verifierCleanup.test.ts b/tests/boundary/process/verifierCleanup.test.ts index d0bb9bda8f..8a55b162ad 100644 --- a/tests/boundary/process/verifierCleanup.test.ts +++ b/tests/boundary/process/verifierCleanup.test.ts @@ -54,3 +54,40 @@ it("retains primary verification and independent cleanup failures", async () => ]); expect(attempted).toEqual(["provider", "fixture"]); }); + +it("keeps a parent fixture when any owned child cleanup remains incomplete", async () => { + const primary = new Error("verification failed"); + const attempted: string[] = []; + let parentRemoved = false; + let failure: unknown; + try { + await closeVerifierResources( + [ + async () => { + await closeVerifierResources([ + async () => { + attempted.push("client"); + }, + async () => { + attempted.push("transport"); + throw new Error("transport close failed"); + }, + async () => { + attempted.push("file-scope"); + }, + ]); + parentRemoved = true; + }, + ], + primary, + ); + } catch (cause) { + failure = cause; + } + + expect(failure).toBeInstanceOf(AggregateError); + expect((failure as AggregateError).errors[0]).toBe(primary); + expect((failure as AggregateError).errors[1]).toBeInstanceOf(AggregateError); + expect(attempted).toEqual(["client", "transport", "file-scope"]); + expect(parentRemoved).toBe(false); +}); diff --git a/tests/boundary/process/verifierFileResources.test.ts b/tests/boundary/process/verifierFileResources.test.ts new file mode 100644 index 0000000000..366774bfb2 --- /dev/null +++ b/tests/boundary/process/verifierFileResources.test.ts @@ -0,0 +1,86 @@ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it("retains and retries the exact descriptor after a verifier hash close failure", async () => { + const directory = await createTestTempDirectory("rea-verifier-file-"); + const path = join(directory, "input.bin"); + const bytes = Buffer.from("owned verifier fixture"); + await writeFile(path, bytes); + const script = ` + import { ArtifactResourceScope } from ${JSON.stringify(new URL("../../../dist/artifacts/ArtifactResourceScope.js", import.meta.url).href)}; + import { OwnedFileHandle } from ${JSON.stringify(new URL("../../../dist/filesystem/OwnedFileHandle.js", import.meta.url).href)}; + import { hashVerifierFile } from ${JSON.stringify(new URL("../../../scripts/lib/verifier-file-resources.mjs", import.meta.url).href)}; + const path = process.argv[1]; + const resources = new ArtifactResourceScope(); + const originalClose = OwnedFileHandle.prototype.close; + let attempts = 0; + OwnedFileHandle.prototype.close = async function () { + attempts += 1; + if (attempts === 1) throw new Error("fixture close failure"); + return originalClose.call(this); + }; + let failure; + try { await hashVerifierFile(path, resources); } catch (cause) { failure = cause; } + try { + if (failure?.outcome?.kind !== "completed" || failure.retention?.kind !== "callback_returned") throw new Error("Completed hash was not retained"); + if (failure.outcome.value.sha256 !== process.env.EXPECTED_SHA256) throw new Error("Completed digest was lost"); + if (!resources.hasPending) throw new Error("Failed descriptor was not retained"); + } finally { + await resources.close(); + } + if (attempts !== 2 || resources.hasPending) throw new Error("Exact descriptor did not close on retry"); + console.log(JSON.stringify({ attempts, retained: true, retried: true })); + `; + const result = spawnSync( + process.execPath, + ["--input-type=module", "-e", script, path], + { + encoding: "utf8", + env: { + ...process.env, + EXPECTED_SHA256: createHash("sha256").update(bytes).digest("hex"), + }, + }, + ); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout.trim())).toEqual({ + attempts: 2, + retained: true, + retried: true, + }); +}); + +it("hashes and closes a clean verifier input", async () => { + const directory = await createTestTempDirectory("rea-verifier-file-"); + const path = join(directory, "input.bin"); + const bytes = Buffer.from("clean verifier fixture"); + await writeFile(path, bytes); + const script = ` + import { ArtifactResourceScope } from ${JSON.stringify(new URL("../../../dist/artifacts/ArtifactResourceScope.js", import.meta.url).href)}; + import { hashVerifierFile } from ${JSON.stringify(new URL("../../../scripts/lib/verifier-file-resources.mjs", import.meta.url).href)}; + const resources = new ArtifactResourceScope(); + const identity = await hashVerifierFile(process.argv[1], resources); + await resources.close(); + console.log(JSON.stringify(identity)); + `; + const result = spawnSync( + process.execPath, + ["--input-type=module", "-e", script, path], + { + encoding: "utf8", + }, + ); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout.trim())).toEqual({ + bytes: bytes.length, + sha256: createHash("sha256") + .update(await readFile(path)) + .digest("hex"), + }); +}); diff --git a/tests/boundary/providers/adb/adbBugreportPublication.test.ts b/tests/boundary/providers/adb/adbBugreportPublication.test.ts index 8578f1dcef..cef6cc25f1 100644 --- a/tests/boundary/providers/adb/adbBugreportPublication.test.ts +++ b/tests/boundary/providers/adb/adbBugreportPublication.test.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import { copyFile, mkdtemp, + open, readFile, readdir, rm, @@ -35,6 +36,7 @@ vi.mock("node:fs/promises", async (importOriginal) => { ...actual, copyFile: vi.fn(actual.copyFile), mkdtemp: vi.fn(actual.mkdtemp), + open: vi.fn(actual.open), rm: vi.fn(actual.rm), }; }); @@ -222,10 +224,123 @@ test("retains published evidence and retries failed staging cleanup", async () = sha256: createHash("sha256").update("bugreport-bytes").digest("hex"), }, }); + expect(result.error.cleanupIncomplete).toBe(true); + const originatingError = result.error; await f.provider.close(); + expect(originatingError.cleanupIncomplete).toBe(false); expect(await readdir(f.root)).toEqual(["bugreport test.zip"]); }); +test("provider close resolves the exact bugreport digest cleanup error", async () => { + const f = await fixture(); + let allowClose = false; + let closeAttempts = 0; + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + vi.mocked(open).mockImplementation(async (path, ...args) => { + const handle = await actualFs.open(path, ...args); + if (String(path) !== join(f.root, "bugreport test.zip")) return handle; + const close = handle.close.bind(handle); + handle.close = async () => { + closeAttempts += 1; + if (!allowClose) throw new Error("controlled digest close fault"); + await close(); + }; + return handle; + }); + + const result = await f.provider.execute(f.request); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected retained digest close failure"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.partialObservation).toMatchObject({ + operation: f.request.operation, + result: { + local_path: join(f.root, "bugreport test.zip"), + bytes: 15, + sha256: createHash("sha256").update("bugreport-bytes").digest("hex"), + }, + }); + const originatingError = result.error; + expect(closeAttempts).toBe(1); + allowClose = true; + await f.provider.close(); + expect(closeAttempts).toBe(2); + expect(originatingError.cleanupIncomplete).toBe(false); +}); + +test("keeps a bugreport error unresolved until digest and staging owners recover", async () => { + const f = await fixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let allowDigestClose = false; + let digestCloseAttempts = 0; + let stagingRemovalAttempts = 0; + vi.mocked(open).mockImplementation(async (path, ...args) => { + const handle = await actualFs.open(path, ...args); + if (String(path) !== join(f.root, "bugreport test.zip")) return handle; + const close = handle.close.bind(handle); + handle.close = async () => { + digestCloseAttempts += 1; + if (!allowDigestClose) throw new Error("controlled digest close fault"); + await close(); + }; + return handle; + }); + vi.mocked(rm).mockImplementation(async (path, options) => { + if ( + basename(String(path)).startsWith(".rea-bugreport-") && + ++stagingRemovalAttempts === 1 + ) + throw new Error("controlled staging removal fault"); + return actualFs.rm(path, options); + }); + + const result = await f.provider.execute(f.request); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected digest and staging cleanup failure"); + const originatingError = result.error; + const stagingDirectory = dirname(f.reportedPath()); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cleanupResources).toContain(stagingDirectory); + expect(result.error.cleanupResources).toContain( + `${join(f.root, "bugreport test.zip")} (ADB digest handle)`, + ); + expect(result.error.partialObservation).toMatchObject({ + operation: f.request.operation, + result: { + local_path: join(f.root, "bugreport test.zip"), + bytes: 15, + sha256: createHash("sha256").update("bugreport-bytes").digest("hex"), + }, + }); + expect(digestCloseAttempts).toBe(1); + expect(stagingRemovalAttempts).toBe(1); + + const closeError = await f.provider.close().catch((cause: unknown) => cause); + expect(closeError).toMatchObject({ cleanupIncomplete: true }); + expect(digestCloseAttempts).toBe(2); + expect(stagingRemovalAttempts).toBe(2); + expect(originatingError.cleanupIncomplete).toBe(true); + await expect(readFile(stagingDirectory)).rejects.toMatchObject({ + code: "ENOENT", + }); + expect(await readFile(join(f.root, "bugreport test.zip"), "utf8")).toBe( + "bugreport-bytes", + ); + + allowDigestClose = true; + await f.provider.close(); + expect(digestCloseAttempts).toBe(3); + expect(stagingRemovalAttempts).toBe(2); + expect(originatingError.cleanupIncomplete).toBe(false); + expect(closeError).toMatchObject({ cleanupIncomplete: false }); +}); + test("refuses an archive reported outside the owned staging directory", async () => { const f = await fixture(); const existing = join(f.root, "bugreport test.zip"); diff --git a/tests/boundary/providers/adb/adbHashCleanupConsumers.test.ts b/tests/boundary/providers/adb/adbHashCleanupConsumers.test.ts index d4ac4e9e89..4dd6b9fec0 100644 --- a/tests/boundary/providers/adb/adbHashCleanupConsumers.test.ts +++ b/tests/boundary/providers/adb/adbHashCleanupConsumers.test.ts @@ -88,7 +88,11 @@ test.each([ output_directory: root, }, }), - expected: { digest_status: "unknown", device_path: "/sdcard/payload.bin" }, + expected: { + bytes: 12, + device_path: "/sdcard/payload.bin", + sha256: expect.any(String), + }, }, { label: "package pull", @@ -116,7 +120,7 @@ test.each([ operation: "collect_adb_bugreport" as const, input: { serial: "device", output_directory: root }, }), - expected: { digest_status: "unknown", serial: "device" }, + expected: { bytes: 12, serial: "device", sha256: expect.any(String) }, }, ])( "preserves the linked hash cleanup error through $label", @@ -131,7 +135,7 @@ test.each([ expect(returnedError.cleanupIncomplete).toBe(true); expect(returnedError.diagnostics).toMatchObject({ primary_error: { - details: { diagnostics: { completed_hash: { bytes: 12 } } }, + details: { partial_observation: { result: expect.any(Object) } }, }, }); expect( diff --git a/tests/boundary/providers/adb/adbInputSnapshotOwnership.test.ts b/tests/boundary/providers/adb/adbInputSnapshotOwnership.test.ts index 2dc8d312c1..69aaa4cb2a 100644 --- a/tests/boundary/providers/adb/adbInputSnapshotOwnership.test.ts +++ b/tests/boundary/providers/adb/adbInputSnapshotOwnership.test.ts @@ -1,24 +1,19 @@ import { createHash } from "node:crypto"; -import { - copyFile, - lstat, - mkdtemp, - readFile, - rm, - writeFile, -} from "node:fs/promises"; +import { lstat, open, readFile, rm, writeFile } from "node:fs/promises"; import { basename, dirname, join } from "node:path"; import { afterEach, expect, onTestFinished, test, vi } from "vitest"; import { AdbProvider } from "../../../../src/adb/AdbProvider.js"; -import { AnalysisError } from "../../../../src/domain/analysisErrorBase.js"; -import { OwnedFileHandle } from "../../../../src/filesystem/OwnedFileHandle.js"; import { execFileBytes, execFileOutput, } from "../../../../src/process/ExecFileOutput.js"; import { parseEvidence } from "../../../../src/domain/evidence.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { PrivateRuntimeRoot } from "../../../../src/process/PrivateRuntimeRoot.js"; +import { AdbInputSnapshotFailure } from "../../../../src/adb/AdbInputSnapshot.js"; +import { AnalysisOutputError } from "../../../../src/domain/analysisErrorCore.js"; +import { OwnedFileHandle } from "../../../../src/filesystem/OwnedFileHandle.js"; vi.mock( "../../../../src/process/ExecFileOutput.js", @@ -34,8 +29,7 @@ vi.mock("node:fs/promises", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, - copyFile: vi.fn(actual.copyFile), - mkdtemp: vi.fn(actual.mkdtemp), + open: vi.fn(actual.open), rm: vi.fn(actual.rm), }; }); @@ -119,7 +113,6 @@ test("retains failed snapshot cleanup ownership and retries it on close", async const result = await f.provider.execute(f.request); expect(result.ok).toBe(false); if (result.ok) throw new Error("expected snapshot cleanup failure"); - expect(result.error.cleanupIncomplete).toBe(true); expect(result.error.cleanupResources).toContain(dirname(f.snapshotPath())); const evidence = parseEvidence(result.error.partialObservation); expect(evidence).toMatchObject({ @@ -137,100 +130,460 @@ test("retains failed snapshot cleanup ownership and retries it on close", async await readFile(f.source), ); - vi.mocked(rm).mockImplementationOnce(async (path, options) => { - if (basename(String(path)).startsWith("rea-adb-input-")) - throw new Error("snapshot cleanup retry denied"); - return actualFs.rm(path, options); - }); - await expect(f.provider.close()).rejects.toMatchObject({ - cleanupIncomplete: true, - }); - expect(result.error.cleanupIncomplete).toBe(true); - await f.provider.close(); - expect(result.error.cleanupIncomplete).toBe(false); await expect(lstat(dirname(f.snapshotPath()))).rejects.toMatchObject({ code: "ENOENT", }); }); -test("resolves a hash cleanup error only after the exact retained descriptor closes", async () => { +test("closes retained snapshot handles before removing their runtime root", async () => { const f = await fixture(); const actualFs = await vi.importActual( "node:fs/promises", ); - let stagedPath: string | undefined; - vi.mocked(copyFile).mockImplementationOnce( - async (source, destination, mode) => { - stagedPath = String(destination); - await actualFs.copyFile(source, destination, mode); - }, - ); - const closeFailure = new Error("digest descriptor close denied"); - const owners: OwnedFileHandle[] = []; - let denialsRemaining = 2; - const originalClose = OwnedFileHandle.prototype.close; - const close = vi - .spyOn(OwnedFileHandle.prototype, "close") - .mockImplementation(async function (this: OwnedFileHandle) { - owners.push(this); - if (denialsRemaining > 0) { - denialsRemaining -= 1; - throw closeFailure; - } - return originalClose.call(this); - }); - try { - const result = await f.provider.execute(f.request); - expect(result.ok).toBe(false); - if (result.ok) throw new Error("expected digest cleanup failure"); - const reported = result.error; - const originalCause = reported.cause; - expect(reported.cleanupIncomplete).toBe(true); - expect(reported.cause).toBeInstanceOf(AggregateError); - const hashFailure = (reported.cause as AggregateError).errors[0]; - if (!(hashFailure instanceof AnalysisError)) - throw new Error("Missing original typed hash failure"); - expect(hashFailure).toMatchObject({ - diagnostics: { - primary_error: { - details: { - diagnostics: { - completed_hash: { bytes: 17, sha256: f.digest }, - }, - }, - }, + let runtimePath = ""; + let snapshotHandleOpen = false; + const closeAttempts = new Map(); + let rootClosedWithHandleOpen = false; + const runtime = await PrivateRuntimeRoot.create({ prefix: "rea-adb-order-" }); + runtimePath = runtime.path; + const provider = new AdbProvider({ + environment: { REA_ADB_PATH: join(f.root, "adb") }, + createRuntimeRoot: async () => ({ + path: runtime.path, + close: async () => { + if (snapshotHandleOpen) { + rootClosedWithHandleOpen = true; + throw new Error("runtime root removed while snapshot handle is open"); + } + await runtime.close(); + }, + }), + }); + const originalOpen = actualFs.open; + vi.mocked(open).mockImplementation(async (...args) => { + const handle = await originalOpen(...args); + const path = String(args[0]); + const isSource = path === f.source; + const isSnapshot = path.startsWith(`${runtimePath}/`); + if (!isSource && !isSnapshot) return handle; + if (isSnapshot) snapshotHandleOpen = true; + return new Proxy(handle, { + get(target, property) { + if (property === "close") + return async () => { + const attempts = (closeAttempts.get(path) ?? 0) + 1; + closeAttempts.set(path, attempts); + if (attempts === 1) + throw new Error("controlled handle close fault"); + await target.close(); + if (isSnapshot) snapshotHandleOpen = false; + }; + const value = Reflect.get(target, property, target) as unknown; + return typeof value === "function" ? value.bind(target) : value; }, }); - expect(owners).toHaveLength(1); - if (stagedPath === undefined) throw new Error("Missing staged input path"); - - await expect(f.provider.close()).rejects.toMatchObject({ - cleanupIncomplete: true, - cleanupResources: [stagedPath, dirname(stagedPath)], - }); - expect(reported.cleanupIncomplete).toBe(true); - expect(reported.cause).toBe(originalCause); - await expect(readFile(stagedPath)).resolves.toEqual( - await readFile(f.source), - ); - - await f.provider.close(); - expect(owners).toHaveLength(3); - expect(owners[0]).toBe(owners[1]); - expect(owners[1]).toBe(owners[2]); - expect(reported.cleanupIncomplete).toBe(false); - expect(hashFailure.cleanupIncomplete).toBe(false); - expect(reported.cause).toBe(originalCause); - await expect(lstat(dirname(stagedPath))).rejects.toMatchObject({ - code: "ENOENT", - }); - } finally { - denialsRemaining = 0; - close.mockRestore(); - await f.provider.close(); + }); + const result = await provider.execute(f.request); + expect(result.ok).toBe(false); + expect(closeAttempts.get(f.source)).toBe(2); + expect(closeAttempts.get(join(runtimePath, "payload.bin"))).toBe(2); + expect(rootClosedWithHandleOpen).toBe(false); + if (!result.ok) { + // Both close owners failed in the snapshot boundary and recovered in the + // immediate runtime-root drain. Their old cleanup flags must not survive. + expect(result.error.cleanupIncomplete).toBe(false); + expect(result.error.cleanupResources).toEqual([]); + expect(result.error.cause).toBeDefined(); } + await provider.close(); +}); + +test("retries the same retained snapshot owner before removing its root", async () => { + const f = await fixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + const runtime = await PrivateRuntimeRoot.create({ + prefix: "rea-adb-persistent-order-", + }); + const snapshotPath = join(runtime.path, basename(f.source)); + const rawHandles = new Map< + string, + Awaited> + >(); + let rootCloseAttempts = 0; + let sourceCloseDenials = 3; + let retainedSourceOwner: OwnedFileHandle | undefined; + const sourceOwnerAttempts: OwnedFileHandle[] = []; + const destinationOwnerAttempts: OwnedFileHandle[] = []; + const originalOwnerClose = OwnedFileHandle.prototype.close; + const ownerClose = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + if (this.handle === rawHandles.get(f.source)) { + retainedSourceOwner ??= this; + sourceOwnerAttempts.push(this); + if (sourceCloseDenials > 0) { + sourceCloseDenials -= 1; + throw new Error("controlled persistent snapshot close fault"); + } + } else if (this.handle === rawHandles.get(snapshotPath)) { + destinationOwnerAttempts.push(this); + } + return originalOwnerClose.call(this); + }); + vi.mocked(open).mockImplementation(async (...args) => { + const path = String(args[0]); + const handle = await actualFs.open(...args); + if (path === f.source || path === snapshotPath) + rawHandles.set(path, handle); + return handle; + }); + + const provider = new AdbProvider({ + environment: { REA_ADB_PATH: join(f.root, "adb") }, + createRuntimeRoot: async () => ({ + path: runtime.path, + close: async () => { + rootCloseAttempts += 1; + if (rawHandles.get(f.source)?.fd !== -1) + throw new Error("runtime root removal attempted with source open"); + await runtime.close(); + }, + }), + }); + try { + const result = await provider.execute(f.request); + expect(result.ok).toBe(false); + if (result.ok) + throw new Error("expected persistent snapshot cleanup failure"); + const originatingError = result.error; + const originatingCause = originatingError.cause; + expect(originatingError.cleanupIncomplete).toBe(true); + expect(sourceOwnerAttempts).toHaveLength(2); + expect(sourceOwnerAttempts[0]).toBe(sourceOwnerAttempts[1]); + expect(sourceOwnerAttempts[0]).toBe(retainedSourceOwner); + expect(destinationOwnerAttempts).toHaveLength(1); + expect(rawHandles.get(f.source)?.fd).toBeGreaterThanOrEqual(0); + await expect(lstat(runtime.path)).resolves.toMatchObject({ + isDirectory: expect.any(Function), + }); + + await expect(provider.close()).rejects.toMatchObject({ + cleanupIncomplete: true, + cleanupResources: expect.arrayContaining([ + `${f.source} (ADB snapshot source handle)`, + ]), + }); + expect(sourceOwnerAttempts).toHaveLength(3); + expect( + sourceOwnerAttempts.every((owner) => owner === retainedSourceOwner), + ).toBe(true); + expect(destinationOwnerAttempts).toHaveLength(1); + expect(rootCloseAttempts).toBe(0); + expect(originatingError.cleanupIncomplete).toBe(true); + expect(originatingError.cause).toBe(originatingCause); + expect(rawHandles.get(f.source)?.fd).toBeGreaterThanOrEqual(0); + await expect(lstat(runtime.path)).resolves.toBeDefined(); + + await provider.close(); + expect(sourceOwnerAttempts).toHaveLength(4); + expect( + sourceOwnerAttempts.every((owner) => owner === retainedSourceOwner), + ).toBe(true); + expect(destinationOwnerAttempts).toHaveLength(1); + expect(rootCloseAttempts).toBe(1); + expect(originatingError.cleanupIncomplete).toBe(false); + expect(originatingError.cause).toBe(originatingCause); + await expect(lstat(runtime.path)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + sourceCloseDenials = 0; + ownerClose.mockRestore(); + await provider.close(); + } +}); + +test.each([ + { label: "read failure", cancel: false, tag: "AnalysisOutputError" }, + { label: "cancellation", cancel: true, tag: "AnalysisCancelledError" }, +])( + "preserves the $label chain when immediate snapshot cleanup recovers", + async ({ cancel, tag }) => { + const f = await fixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + const controller = new AbortController(); + let sourceCloseAttempts = 0; + vi.mocked(open).mockImplementation(async (...args) => { + const handle = await actualFs.open(...args); + if (String(args[0]) !== f.source) return handle; + const close = handle.close.bind(handle); + handle.read = (async () => { + if (cancel) { + controller.abort(); + } + throw new Error("controlled snapshot read failure"); + }) as typeof handle.read; + handle.close = async () => { + sourceCloseAttempts += 1; + if (sourceCloseAttempts === 1) + throw new Error("controlled snapshot handle close failure"); + await close(); + }; + return handle; + }); + + const result = await f.provider.execute(f.request, { + signal: controller.signal, + }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected snapshot operation failure"); + expect(result.error._tag).toBe(tag); + expect(result.error.cleanupIncomplete).toBe(false); + expect(result.error.cleanupResources).toEqual([]); + expect(sourceCloseAttempts).toBe(2); + const cause = result.error.cause; + expect(cause).toBeInstanceOf(AggregateError); + const secondary = (cause as AggregateError).errors.find( + (entry) => entry instanceof AnalysisOutputError, + ); + expect(secondary?.cause).toBeInstanceOf(AdbInputSnapshotFailure); + expect( + (secondary?.cause as AdbInputSnapshotFailure).cleanupCause, + ).toBeDefined(); + }, +); + +test.each([ + { cancelDuringClose: false, failRead: false }, + { cancelDuringClose: true, failRead: false }, + { cancelDuringClose: false, failRead: true }, +])( + "pull preserves the digest outcome when descriptor cleanup fails (%#)", + async ({ cancelDuringClose, failRead }) => { + const f = await fixture(); + const controller = new AbortController(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + const outputDirectory = join(f.root, "pull-output"); + const localPath = join(outputDirectory, "remote.bin"); + const pulledBytes = Buffer.from("device bytes"); + let closeAttempts = 0; + vi.mocked(execFileBytes).mockImplementation(async (_binary, args) => { + if (args[2] === "pull") { + await actualFs.writeFile(args[4]!, pulledBytes); + } + return { stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) }; + }); + vi.mocked(open).mockImplementation(async (path, ...args) => { + const handle = await actualFs.open(path, ...args); + if (String(path) === localPath) { + if (failRead) + handle.read = (async () => { + throw new Error("controlled digest read failure"); + }) as typeof handle.read; + const close = handle.close.bind(handle); + handle.close = async () => { + closeAttempts += 1; + if (closeAttempts === 1) { + if (cancelDuringClose) controller.abort(); + throw new Error("controlled descriptor close"); + } + await close(); + }; + } + return handle; + }); + + const result = await f.provider.execute( + { + operation: "pull_adb_file", + input: { + serial: "device", + device_path: "/data/local/tmp/remote.bin", + output_directory: outputDirectory, + }, + }, + cancelDuringClose ? { signal: controller.signal } : undefined, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected retained digest cleanup failure"); + expect(result.error._tag).toBe( + cancelDuringClose ? "AnalysisCancelledError" : "ProviderAdapterError", + ); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cleanupResources).toContain( + `${localPath} (ADB digest handle)`, + ); + expect(result.error.partialObservation).toMatchObject({ + provider_id: "adb", + operation: "pull_adb_file", + result: { + local_path: localPath, + device_path: "/data/local/tmp/remote.bin", + ...(failRead + ? { digest_status: "unknown" } + : { + bytes: pulledBytes.byteLength, + sha256: createHash("sha256").update(pulledBytes).digest("hex"), + }), + }, + }); + await expect(readFile(localPath)).resolves.toEqual(pulledBytes); + const originatingError = result.error; + await f.provider.close(); + expect(closeAttempts).toBe(2); + expect(originatingError.cleanupIncomplete).toBe(false); + }, +); + +test("package pull keeps earlier and completed artifacts when the digest handle cannot close", async () => { + const f = await fixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + const outputDirectory = join(f.root, "package-output"); + const packageDirectory = join(outputDirectory, "com.example.app"); + const splitPath = join(packageDirectory, "split.apk"); + const baseBytes = Buffer.from("base apk"); + const splitBytes = Buffer.from("split apk"); + let closeAttempts = 0; + const version = `Android Debug Bridge version 1.0.41\nVersion 34.0.5-stub\nInstalled as ${f.root}/adb\n`; + vi.mocked(execFileOutput).mockImplementation(async (_binary, args) => { + if (args[0] === "version") + return { + stdout: version, + stderr: "", + stdoutBytes: Buffer.byteLength(version), + stderrBytes: 0, + }; + return { stdout: "", stderr: "", stdoutBytes: 0, stderrBytes: 0 }; + }); + vi.mocked(execFileBytes).mockImplementation(async (_binary, args) => { + if (args.includes("path")) + return { + stdout: Buffer.from( + "package:/data/app/com.example.app/base.apk\npackage:/data/app/com.example.app/split.apk\n", + ), + stderr: Buffer.alloc(0), + }; + if (args[2] === "pull") + await actualFs.writeFile( + args[4]!, + args[3]!.endsWith("base.apk") ? baseBytes : splitBytes, + ); + return { stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) }; + }); + vi.mocked(open).mockImplementation(async (path, ...args) => { + const handle = await actualFs.open(path, ...args); + if (String(path) === splitPath) { + const close = handle.close.bind(handle); + handle.close = async () => { + closeAttempts += 1; + if (closeAttempts === 1) throw new Error("controlled descriptor close"); + await close(); + }; + } + return handle; + }); + + const result = await f.provider.execute({ + operation: "pull_adb_package", + input: { + serial: "device", + package: "com.example.app", + output_directory: outputDirectory, + }, + }); + expect(result.ok).toBe(false); + if (result.ok) + throw new Error("expected retained package digest cleanup failure"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.partialObservation).toMatchObject({ + provider_id: "adb", + operation: "pull_adb_package", + result: { + coverage: "partial", + artifacts: [ + { + local_path: join(packageDirectory, "base.apk"), + bytes: baseBytes.byteLength, + sha256: createHash("sha256").update(baseBytes).digest("hex"), + }, + { + local_path: splitPath, + bytes: splitBytes.byteLength, + sha256: createHash("sha256").update(splitBytes).digest("hex"), + }, + ], + failures: [ + { + stage: "digest", + device_path: "/data/app/com.example.app/split.apk", + }, + ], + }, + }); + await f.provider.close(); + expect(closeAttempts).toBe(2); +}); + +test("resolves package-directory cleanup on the originating pull error", async () => { + const f = await fixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + const outputDirectory = join(f.root, "package-cleanup-output"); + const packageDirectory = join(outputDirectory, "com.example.app"); + let removeAttempts = 0; + vi.mocked(execFileBytes).mockImplementation(async (_binary, args) => { + if (args.includes("path")) + return { + stdout: Buffer.from("package:/data/app/com.example.app/base.apk\n"), + stderr: Buffer.alloc(0), + }; + if (args[2] === "pull") throw new Error("controlled ADB pull failure"); + return { stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) }; + }); + vi.mocked(rm).mockImplementation(async (path, options) => { + if (String(path) === packageDirectory && ++removeAttempts === 1) + throw new Error("controlled package directory removal failure"); + return actualFs.rm(path, options); + }); + + const result = await f.provider.execute({ + operation: "pull_adb_package", + input: { + serial: "device", + package: "com.example.app", + output_directory: outputDirectory, + }, + }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected pull and cleanup failure"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cleanupResources).toContain(packageDirectory); + expect(result.error.cause).toBeDefined(); + const originatingError = result.error; + expect(removeAttempts).toBe(1); + + await f.provider.close(); + expect(removeAttempts).toBe(2); + expect(originatingError.cleanupIncomplete).toBe(false); + await f.provider.close(); + expect(removeAttempts).toBe(2); + await expect(lstat(packageDirectory)).rejects.toMatchObject({ + code: "ENOENT", + }); }); test.each([false, true])( @@ -238,13 +591,14 @@ test.each([false, true])( async (failCleanup) => { const f = await fixture(); const controller = new AbortController(); + const abortReason = new Error("caller stopped after snapshot cleanup"); const actualFs = await vi.importActual( "node:fs/promises", ); vi.mocked(rm).mockImplementationOnce(async (path, options) => { if (basename(String(path)).startsWith("rea-adb-input-")) { - controller.abort(); + controller.abort(abortReason); if (failCleanup) throw new Error("snapshot cleanup denied"); } return actualFs.rm(path, options); @@ -256,7 +610,15 @@ test.each([false, true])( expect(result.ok).toBe(false); if (result.ok) throw new Error("expected cancellation"); expect(result.error._tag).toBe("AnalysisCancelledError"); - const originalCause = result.error.cause; + if (failCleanup) { + expect(result.error.cause).toBeInstanceOf(AggregateError); + const causes = (result.error.cause as AggregateError).errors as [ + { readonly cause: unknown }, + { readonly cleanupIncomplete: boolean }, + ]; + expect(causes[0].cause).toBe(abortReason); + expect(causes[1].cleanupIncomplete).toBe(true); + } else expect(result.error.cause).toBe(abortReason); const evidence = parseEvidence(result.error.partialObservation); expect(evidence).toMatchObject({ provider: { id: "adb" }, @@ -270,16 +632,13 @@ test.each([false, true])( device_path: "/data/local/tmp/payload.bin", }, }); - if (failCleanup) { - expect(result.error.cleanupIncomplete).toBe(true); + if (failCleanup) expect(result.error.cleanupResources).toContain( dirname(f.snapshotPath()), ); - } else expect(result.error.cleanupIncomplete).toBe(false); + else expect(result.error.cleanupIncomplete).toBe(false); expect(await readFile(f.source)).toEqual(Buffer.from("immutable payload")); await f.provider.close(); - expect(result.error.cleanupIncomplete).toBe(false); - expect(result.error.cause).toBe(originalCause); }, ); @@ -290,9 +649,19 @@ test("cancels after the input snapshot copy and before contacting the device", a await vi.importActual( "node:fs/promises", ); - vi.mocked(copyFile).mockImplementationOnce(async (...args) => { - await actualFs.copyFile(...args); - controller.abort(); + vi.mocked(open).mockImplementation(async (path, ...args) => { + const handle = await actualFs.open(path, ...args); + if ( + String(path).endsWith("payload.bin") && + basename(dirname(String(path))).startsWith("rea-adb-input-") + ) { + const sync = handle.sync.bind(handle); + handle.sync = async () => { + await sync(); + controller.abort(); + }; + } + return handle; }); const result = await f.provider.execute(f.request, { @@ -308,10 +677,21 @@ test("cancels after the input snapshot copy and before contacting the device", a test("reports input snapshot capacity failures before contacting the device", async () => { const f = await fixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); const capacityFailure = Object.assign(new Error("no space left"), { code: "ENOSPC", }); - vi.mocked(copyFile).mockRejectedValueOnce(capacityFailure); + vi.mocked(open).mockImplementation(async (path, ...args) => { + if ( + String(path).endsWith("payload.bin") && + basename(dirname(String(path))).startsWith("rea-adb-input-") + ) + throw capacityFailure; + return actualFs.open(path, ...args); + }); const result = await f.provider.execute(f.request); expect(result.ok).toBe(false); diff --git a/tests/boundary/providers/adb/adbScreenPublicationCancellation.test.ts b/tests/boundary/providers/adb/adbScreenPublicationCancellation.test.ts index 72aa620cda..492dde7be2 100644 --- a/tests/boundary/providers/adb/adbScreenPublicationCancellation.test.ts +++ b/tests/boundary/providers/adb/adbScreenPublicationCancellation.test.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; -import { readFile, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { link, lstat, open, readFile, rm } from "node:fs/promises"; +import { basename, dirname, join } from "node:path"; import { afterEach, expect, onTestFinished, test, vi } from "vitest"; import { AdbProvider } from "../../../../src/adb/AdbProvider.js"; @@ -10,6 +10,7 @@ import { execFileOutput, } from "../../../../src/process/ExecFileOutput.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { PrivateRuntimeRoot } from "../../../../src/process/PrivateRuntimeRoot.js"; vi.mock( "../../../../src/process/ExecFileOutput.js", @@ -23,7 +24,12 @@ vi.mock( ); vi.mock("node:fs/promises", async (importOriginal) => { const actual = await importOriginal(); - return { ...actual, writeFile: vi.fn(actual.writeFile) }; + return { + ...actual, + link: vi.fn(actual.link), + open: vi.fn(actual.open), + rm: vi.fn(actual.rm), + }; }); afterEach(() => vi.restoreAllMocks()); @@ -53,8 +59,9 @@ const fixture = async () => { test("does not publish a screen capture cancelled before the commit point", async () => { const f = await fixture(); const controller = new AbortController(); + const abortReason = new Error("caller stopped after screencap"); vi.mocked(execFileBytes).mockImplementationOnce(async () => { - controller.abort(); + controller.abort(abortReason); return { stdout: f.png, stderr: Buffer.alloc(0) }; }); const output = join(f.root, "captures"); @@ -68,6 +75,7 @@ test("does not publish a screen capture cancelled before the commit point", asyn expect(result.ok).toBe(false); if (result.ok) throw new Error("expected cancellation"); expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(result.error.cause).toBe(abortReason); await expect(readFile(join(output, "screen.png"))).rejects.toMatchObject({ code: "ENOENT", }); @@ -77,6 +85,7 @@ test("does not publish a screen capture cancelled before the commit point", asyn test("retains a committed screen capture receipt when cancellation arrives after commit", async () => { const f = await fixture(); const controller = new AbortController(); + const abortReason = new Error("caller stopped after screen commit"); vi.mocked(execFileBytes).mockResolvedValueOnce({ stdout: f.png, stderr: Buffer.alloc(0), @@ -85,9 +94,9 @@ test("retains a committed screen capture receipt when cancellation arrives after await vi.importActual( "node:fs/promises", ); - vi.mocked(writeFile).mockImplementationOnce(async (...args) => { - await actualFs.writeFile(...args); - controller.abort(); + vi.mocked(link).mockImplementationOnce(async (...args) => { + await actualFs.link(...args); + controller.abort(abortReason); }); const output = join(f.root, "captures"); const result = await f.provider.execute( @@ -100,6 +109,7 @@ test("retains a committed screen capture receipt when cancellation arrives after expect(result.ok).toBe(false); if (result.ok) throw new Error("expected cancellation"); expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(result.error.cause).toBe(abortReason); const evidence = parseEvidence(result.error.partialObservation); expect(evidence).toMatchObject({ provider: { id: "adb" }, @@ -117,3 +127,610 @@ test("retains a committed screen capture receipt when cancellation arrives after expect(await readFile(join(output, "screen.png"))).toEqual(f.png); await f.provider.close(); }); + +test.each([1, 2])( + "resolves a staged screen error after %i close faults", + async (closeFaults) => { + const f = await fixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + const output = join(f.root, "captures"); + const destination = join(output, "screen.png"); + let stagedPath: string | undefined; + let stagedRoot: string | undefined; + let closeAttempts = 0; + let rawHandle: Awaited> | undefined; + const closeHandleIdentities: Awaited>[] = + []; + vi.mocked(execFileBytes).mockResolvedValueOnce({ + stdout: f.png, + stderr: Buffer.alloc(0), + }); + vi.mocked(open).mockImplementation(async (...args) => { + const path = String(args[0]); + const handle = await actualFs.open(...args); + if (basename(path) === "screen.png" && path.includes(".rea-screen-")) { + stagedPath = path; + stagedRoot = dirname(path); + rawHandle = handle; + const close = handle.close.bind(handle); + handle.close = async () => { + closeAttempts += 1; + closeHandleIdentities.push(handle); + if (closeAttempts <= closeFaults) + throw new Error("controlled staged screen close fault"); + await close(); + }; + } + return handle; + }); + + const result = await f.provider.execute({ + operation: "capture_adb_screen", + input: { serial: "device", output_directory: output }, + }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected staged screen cleanup failure"); + const originatingError = result.error; + const originatingCause = originatingError.cause; + expect(originatingError.cleanupIncomplete).toBe(closeFaults === 2); + expect(originatingError.partialObservation).toBeUndefined(); + expect(closeAttempts).toBe(2); + expect(stagedPath).toBeDefined(); + expect(stagedRoot).toBeDefined(); + if (closeFaults === 2) expect(rawHandle?.fd).toBeGreaterThanOrEqual(0); + else expect(rawHandle?.fd).toBe(-1); + expect(closeHandleIdentities).toHaveLength(2); + expect(closeHandleIdentities.every((handle) => handle === rawHandle)).toBe( + true, + ); + if (closeFaults === 2) + await expect(lstat(stagedRoot!)).resolves.toBeDefined(); + else + await expect(lstat(stagedRoot!)).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(readFile(destination)).rejects.toMatchObject({ + code: "ENOENT", + }); + + const linkedCleanupError = ( + closeFaults === 2 + ? (originatingCause as AggregateError).errors[0] + : originatingError + ) as { + readonly cause: unknown; + readonly cleanupIncomplete: boolean; + }; + const originalCloseCause = linkedCleanupError.cause; + expect(linkedCleanupError.cleanupIncomplete).toBe(closeFaults === 2); + await f.provider.close(); + expect(closeAttempts).toBe(closeFaults + 1); + expect(closeHandleIdentities.every((handle) => handle === rawHandle)).toBe( + true, + ); + expect(linkedCleanupError.cleanupIncomplete).toBe(false); + expect(linkedCleanupError.cause).toBe(originalCloseCause); + expect(originatingError.cleanupIncomplete).toBe(false); + expect(originatingError.cause).toBe(originatingCause); + await expect(lstat(stagedRoot!)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(readFile(destination)).rejects.toMatchObject({ + code: "ENOENT", + }); + }, +); + +test("retains the committed receipt and cleanup observation when cancellation races with root cleanup", async () => { + const f = await fixture(); + const runtime = await PrivateRuntimeRoot.create({ + prefix: "rea-adb-receipt-", + }); + const controller = new AbortController(); + const abortReason = new Error("caller stopped after screen commit"); + let closeAttempts = 0; + const provider = new AdbProvider({ + environment: { REA_ADB_PATH: join(f.root, "adb") }, + createRuntimeRoot: async () => ({ + path: runtime.path, + close: async () => { + closeAttempts += 1; + if (closeAttempts === 1) throw new Error("controlled cleanup fault"); + await runtime.close(); + }, + }), + }); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + vi.mocked(execFileBytes).mockResolvedValueOnce({ + stdout: f.png, + stderr: Buffer.alloc(0), + }); + vi.mocked(link).mockImplementationOnce(async (...args) => { + await actualFs.link(...args); + controller.abort(abortReason); + }); + const output = join(f.root, "captures"); + const destination = join(output, "screen.png"); + const result = await provider.execute( + { + operation: "capture_adb_screen", + input: { serial: "device", output_directory: output }, + }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected cancellation"); + expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cause).toBeInstanceOf(AggregateError); + const causes = (result.error.cause as AggregateError).errors as [ + { readonly cause: unknown }, + { readonly cleanupIncomplete: boolean }, + ]; + expect(causes[0].cause).toBe(abortReason); + expect(causes[1].cleanupIncomplete).toBe(true); + expect(result.error.cleanupResources).toContain(runtime.path); + expect(parseEvidence(result.error.partialObservation)).toMatchObject({ + operation: "capture_adb_screen", + confidence: "observed", + normalized_result: { local_path: destination, bytes: f.png.byteLength }, + }); + expect(await readFile(destination)).toEqual(f.png); + await provider.close(); + expect(closeAttempts).toBe(2); + expect(result.error.cleanupIncomplete).toBe(false); +}); + +test("publishes screen captures when the output filesystem does not support hard links", async () => { + const f = await fixture(); + vi.mocked(execFileBytes).mockResolvedValueOnce({ + stdout: f.png, + stderr: Buffer.alloc(0), + }); + vi.mocked(link).mockRejectedValueOnce( + Object.assign(new Error("hard links are unsupported"), { code: "EPERM" }), + ); + const output = join(f.root, "captures"); + const result = await f.provider.execute({ + operation: "capture_adb_screen", + input: { serial: "device", output_directory: output }, + }); + expect(result.ok).toBe(true); + expect(await readFile(join(output, "screen.png"))).toEqual(f.png); + await f.provider.close(); +}); + +test("does not replace a destination created after the initial existence check", async () => { + const f = await fixture(); + vi.mocked(execFileBytes).mockResolvedValueOnce({ + stdout: f.png, + stderr: Buffer.alloc(0), + }); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + const output = join(f.root, "captures"); + const destination = join(output, "screen.png"); + vi.mocked(link).mockImplementationOnce(async (...args) => { + await actualFs.writeFile(destination, "caller-owned"); + throw Object.assign(new Error("hard links are unsupported"), { + code: "EPERM", + }); + }); + const result = await f.provider.execute({ + operation: "capture_adb_screen", + input: { serial: "device", output_directory: output }, + }); + expect(result.ok).toBe(false); + expect(await readFile(destination, "utf8")).toBe("caller-owned"); + await f.provider.close(); +}); + +test("cancellation after a failed hard link but before exclusive creation does not publish", async () => { + const f = await fixture(); + const controller = new AbortController(); + const abortReason = new Error("caller stopped before exclusive creation"); + vi.mocked(execFileBytes).mockResolvedValueOnce({ + stdout: f.png, + stderr: Buffer.alloc(0), + }); + vi.mocked(link).mockImplementationOnce(async () => { + controller.abort(abortReason); + throw Object.assign(new Error("hard links are unsupported"), { + code: "EPERM", + }); + }); + const output = join(f.root, "captures"); + const result = await f.provider.execute( + { + operation: "capture_adb_screen", + input: { serial: "device", output_directory: output }, + }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected cancellation"); + expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(result.error.cause).toBe(abortReason); + await expect(readFile(join(output, "screen.png"))).rejects.toMatchObject({ + code: "ENOENT", + }); + await f.provider.close(); +}); + +test("retains a receipt when cancellation arrives after exclusive output creation", async () => { + const f = await fixture(); + const controller = new AbortController(); + const abortReason = new Error("caller stopped after exclusive creation"); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + const output = join(f.root, "captures"); + const destination = join(output, "screen.png"); + vi.mocked(execFileBytes).mockResolvedValueOnce({ + stdout: f.png, + stderr: Buffer.alloc(0), + }); + vi.mocked(link).mockRejectedValueOnce( + Object.assign(new Error("hard links are unsupported"), { code: "EPERM" }), + ); + vi.mocked(open).mockImplementation(async (...args) => { + const handle = await actualFs.open(...args); + if (String(args[0]) === destination) controller.abort(abortReason); + return handle; + }); + const result = await f.provider.execute( + { + operation: "capture_adb_screen", + input: { serial: "device", output_directory: output }, + }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected cancellation"); + expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(result.error.cause).toBe(abortReason); + expect(parseEvidence(result.error.partialObservation)).toMatchObject({ + operation: "capture_adb_screen", + confidence: "observed", + normalized_result: { local_path: destination, bytes: f.png.byteLength }, + }); + expect(await readFile(destination)).toEqual(f.png); + await f.provider.close(); +}); + +test("closes and retains ownership when exclusive output stat fails", async () => { + const f = await fixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + const output = join(f.root, "captures"); + const destination = join(output, "screen.png"); + let handleOpen = false; + let statAttempts = 0; + let removeAttempts = 0; + let closeAttempts = 0; + vi.mocked(execFileBytes).mockResolvedValueOnce({ + stdout: f.png, + stderr: Buffer.alloc(0), + }); + vi.mocked(link).mockRejectedValueOnce( + Object.assign(new Error("hard links are unsupported"), { code: "EPERM" }), + ); + vi.mocked(open).mockImplementation(async (...args) => { + const handle = await actualFs.open(...args); + if (String(args[0]) !== destination) return handle; + handleOpen = true; + return new Proxy(handle, { + get(target, property) { + if (property === "stat") + return async () => { + statAttempts += 1; + if (statAttempts <= 2) + throw new Error("controlled descriptor stat fault"); + return target.stat(); + }; + if (property === "close") + return async () => { + closeAttempts += 1; + await target.close(); + handleOpen = false; + }; + const value = Reflect.get(target, property, target) as unknown; + return typeof value === "function" ? value.bind(target) : value; + }, + }); + }); + const result = await f.provider.execute({ + operation: "capture_adb_screen", + input: { serial: "device", output_directory: output }, + }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected stat failure"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cleanupResources).toContain(destination); + expect(closeAttempts).toBe(0); + expect(handleOpen).toBe(true); + expect(await readFile(destination)).toEqual(Buffer.alloc(0)); + vi.mocked(rm).mockImplementation(async (path, options) => { + if (String(path) === destination && ++removeAttempts === 1) + throw new Error("controlled rollback removal fault"); + return actualFs.rm(path, options); + }); + await expect(f.provider.close()).rejects.toBeDefined(); + expect(statAttempts).toBe(3); + expect(closeAttempts).toBe(1); + expect(handleOpen).toBe(false); + await f.provider.close(); + expect(statAttempts).toBe(3); + expect(closeAttempts).toBe(1); + expect(result.error.cleanupIncomplete).toBe(false); + await expect(readFile(destination)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +test("waits for descriptor close before rolling back an incomplete exclusive output", async () => { + const f = await fixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + const output = join(f.root, "captures"); + const destination = join(output, "screen.png"); + let handleOpen = false; + let closeAttempts = 0; + const removeWhileOpen: string[] = []; + vi.mocked(execFileBytes).mockResolvedValueOnce({ + stdout: f.png, + stderr: Buffer.alloc(0), + }); + vi.mocked(link).mockRejectedValueOnce( + Object.assign(new Error("hard links are unsupported"), { code: "EPERM" }), + ); + vi.mocked(open).mockImplementation(async (...args) => { + const handle = await actualFs.open(...args); + if (String(args[0]) !== destination) return handle; + handleOpen = true; + return new Proxy(handle, { + get(target, property) { + if (property === "writeFile") + return async () => { + throw new Error("controlled screen write fault"); + }; + if (property === "close") + return async () => { + closeAttempts += 1; + if (closeAttempts === 1) + throw new Error("controlled descriptor close fault"); + await target.close(); + handleOpen = false; + }; + const value = Reflect.get(target, property, target) as unknown; + return typeof value === "function" ? value.bind(target) : value; + }, + }); + }); + vi.mocked(rm).mockImplementation(async (path, options) => { + if (String(path) === destination && handleOpen) + removeWhileOpen.push(String(path)); + return actualFs.rm(path, options); + }); + const result = await f.provider.execute({ + operation: "capture_adb_screen", + input: { serial: "device", output_directory: output }, + }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected write failure"); + expect(result.error.cleanupIncomplete).toBe(false); + expect(result.error.cause).toBeInstanceOf(AggregateError); + expect(result.error.cleanupResources).toContain(destination); + expect(closeAttempts).toBe(2); + expect(removeWhileOpen).toEqual([]); + await expect(readFile(destination)).rejects.toMatchObject({ code: "ENOENT" }); + await f.provider.close(); +}); + +const exclusiveScreenCloseFixture = async ( + fails: (attempt: number) => boolean, +) => { + const f = await fixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + const output = join(f.root, "captures"); + const destination = join(output, "screen.png"); + let closeAttempts = 0; + vi.mocked(execFileBytes).mockResolvedValueOnce({ + stdout: f.png, + stderr: Buffer.alloc(0), + }); + vi.mocked(link).mockRejectedValueOnce( + Object.assign(new Error("hard links are unsupported"), { code: "EPERM" }), + ); + vi.mocked(open).mockImplementation(async (...args) => { + const handle = await actualFs.open(...args); + if (String(args[0]) !== destination) return handle; + return new Proxy(handle, { + get(target, property) { + if (property === "close") + return async () => { + if (fails(++closeAttempts)) + throw new Error("controlled descriptor close fault"); + await target.close(); + }; + const value = Reflect.get(target, property, target) as unknown; + return typeof value === "function" ? value.bind(target) : value; + }, + }); + }); + return { ...f, output, destination, closeAttempts: () => closeAttempts }; +}; + +test.each([false, true])( + "preserves the completed receipt after handle recovery with root failure %s", + async (rootFails) => { + const f = await exclusiveScreenCloseFixture((attempt) => attempt === 1); + let runtimePath: string | undefined; + if (rootFails) + vi.spyOn(PrivateRuntimeRoot.prototype, "close").mockImplementationOnce( + async function (this: PrivateRuntimeRoot) { + runtimePath = this.path; + throw new Error("controlled workspace removal fault"); + }, + ); + const result = await f.provider.execute({ + operation: "capture_adb_screen", + input: { serial: "device", output_directory: f.output }, + }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected close failure"); + expect(parseEvidence(result.error.partialObservation)).toMatchObject({ + operation: "capture_adb_screen", + confidence: "observed", + normalized_result: { local_path: f.destination, bytes: f.png.byteLength }, + }); + expect(result.error.cleanupIncomplete).toBe(rootFails); + if (rootFails) expect(result.error.cleanupResources).toEqual([runtimePath]); + expect(await readFile(f.destination)).toEqual(f.png); + expect(f.closeAttempts()).toBe(2); + await f.provider.close(); + expect(result.error.cleanupIncomplete).toBe(false); + if (runtimePath !== undefined) + await expect( + readFile(join(runtimePath, "screen.png")), + ).rejects.toMatchObject({ code: "ENOENT" }); + }, +); + +test("keeps a sticky publication close failure until provider close retries its owner", async () => { + let allowClose = false; + const f = await exclusiveScreenCloseFixture(() => !allowClose); + const result = await f.provider.execute({ + operation: "capture_adb_screen", + input: { serial: "device", output_directory: f.output }, + }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected close failure"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cleanupResources).toContain( + `${f.destination} (ADB publication handle)`, + ); + expect(parseEvidence(result.error.partialObservation)).toMatchObject({ + operation: "capture_adb_screen", + confidence: "observed", + normalized_result: { local_path: f.destination, bytes: f.png.byteLength }, + }); + expect(await readFile(f.destination)).toEqual(f.png); + expect(f.closeAttempts()).toBe(2); + const linkedFailure = (result.error.cause as AggregateError).errors[0] as { + readonly cleanupIncomplete: boolean; + }; + expect(linkedFailure.cleanupIncomplete).toBe(true); + allowClose = true; + await f.provider.close(); + expect(f.closeAttempts()).toBe(3); + expect(linkedFailure.cleanupIncomplete).toBe(false); + expect(result.error.cleanupIncomplete).toBe(false); + expect(await readFile(f.destination)).toEqual(f.png); +}); + +test("resolves an admission cleanup error when the retained runtime-root lease closes", async () => { + let allowClose = false; + const f = await exclusiveScreenCloseFixture(() => !allowClose); + const first = await f.provider.execute({ + operation: "capture_adb_screen", + input: { serial: "device", output_directory: f.output }, + }); + expect(first.ok).toBe(false); + if (first.ok) throw new Error("expected publication close failure"); + expect(first.error.cleanupIncomplete).toBe(true); + expect(f.closeAttempts()).toBe(2); + + const admission = await f.provider.execute({ + operation: "capture_adb_screen", + input: { serial: "device", output_directory: f.output }, + }); + expect(admission.ok).toBe(false); + if (admission.ok) throw new Error("expected retained cleanup refusal"); + const admissionError = admission.error; + expect(admissionError.cleanupIncomplete).toBe(true); + expect(f.closeAttempts()).toBe(3); + + allowClose = true; + await f.provider.close(); + expect(f.closeAttempts()).toBe(4); + expect(admissionError.cleanupIncomplete).toBe(false); + expect(admissionError.cause).toBeInstanceOf(AggregateError); + expect(await readFile(f.destination)).toEqual(f.png); +}); + +test("resolves the provider close aggregate after its failed root owner recovers", async () => { + let allowClose = false; + const f = await exclusiveScreenCloseFixture(() => !allowClose); + const result = await f.provider.execute({ + operation: "capture_adb_screen", + input: { serial: "device", output_directory: f.output }, + }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected publication close failure"); + expect(result.error.cleanupIncomplete).toBe(true); + + const closeError = await f.provider.close().catch((cause: unknown) => cause); + expect(closeError).toMatchObject({ cleanupIncomplete: true }); + expect(f.closeAttempts()).toBe(3); + + allowClose = true; + await f.provider.close(); + expect(f.closeAttempts()).toBe(4); + expect(closeError).toMatchObject({ cleanupIncomplete: false }); + expect(result.error.cleanupIncomplete).toBe(false); + expect(await readFile(f.destination)).toEqual(f.png); +}); + +test("retains incomplete cleanup on a cancelled capture while keeping cancellation typed", async () => { + const f = await fixture(); + const runtime = await PrivateRuntimeRoot.create({ + prefix: "rea-adb-cancel-", + }); + const controller = new AbortController(); + let closeAttempts = 0; + const provider = new AdbProvider({ + environment: { REA_ADB_PATH: join(f.root, "adb") }, + createRuntimeRoot: async () => { + controller.abort(); + return { + path: runtime.path, + close: async () => { + closeAttempts += 1; + if (closeAttempts === 1) throw new Error("controlled cleanup fault"); + await runtime.close(); + }, + }; + }, + }); + vi.mocked(execFileBytes).mockResolvedValueOnce({ + stdout: f.png, + stderr: Buffer.alloc(0), + }); + const result = await provider.execute( + { + operation: "capture_adb_screen", + input: { serial: "device", output_directory: join(f.root, "cancelled") }, + }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected cancellation"); + expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cleanupResources).toContain(runtime.path); + await provider.close(); + expect(closeAttempts).toBe(2); +}); diff --git a/tests/boundary/providers/android/androidLateCompletion.test.ts b/tests/boundary/providers/android/androidLateCompletion.test.ts new file mode 100644 index 0000000000..a4f22bb99b --- /dev/null +++ b/tests/boundary/providers/android/androidLateCompletion.test.ts @@ -0,0 +1,74 @@ +import { expect, it } from "vitest"; + +import { AndroidAnalysisService } from "../../../../src/application/android/AndroidAnalysisService.js"; +import { AnalysisCancelledError } from "../../../../src/domain/analysisErrorCore.js"; +import { parseEvidence } from "../../../../src/domain/evidence.js"; +import type { AnalysisExecution } from "../../../../src/application/AnalysisProvider.js"; +import { createJadxProtocolFixture } from "../../../fixtures/android/jadx.js"; + +it.each([ + "inspect_android_package", + "search_android_classes", + "inspect_android_class", + "inspect_android_method", + "trace_android_references", +] as const)( + "retains %s Evidence when cancellation follows provider completion", + async (operation) => { + const fixture = await createJadxProtocolFixture(); + const input = { + path: fixture.apk, + ...(operation === "search_android_classes" ? { query: "Target" } : {}), + ...(operation === "inspect_android_class" || + operation === "inspect_android_method" || + operation === "trace_android_references" + ? { class_name: "fixture.Target" } + : {}), + ...(operation === "inspect_android_method" + ? { method_name: "onCreate" } + : {}), + }; + const control = await fixture.service.execute(operation, input); + if (!control.ok) throw control.error; + const controller = new AbortController(); + const observations: AnalysisExecution[] = []; + const service = new AndroidAnalysisService({ + inspectAvailability: fixture.provider.inspectAvailability.bind( + fixture.provider, + ), + close: fixture.provider.close.bind(fixture.provider), + execute: async (...args) => { + const completed = await fixture.provider.execute(...args); + if (completed.ok) observations.push(completed.value); + controller.abort(); + return completed; + }, + }); + const result = await service.execute(operation, input, { + signal: controller.signal, + }); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + operation, + }, + }); + if (result.ok) throw new Error("Expected cancelled provider completion"); + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + const partial = parseEvidence(result.error.partialObservation); + expect(partial).toMatchObject({ + operation, + subject: control.value.subject, + parameters: control.value.parameters, + provider: control.value.provider, + confidence: control.value.confidence, + normalized_result: control.value.normalized_result, + }); + expect(observations).toHaveLength(1); + expect(partial.raw_result).toEqual(observations[0]?.rawResult); + expect(control.value.confidence).toBe( + operation === "inspect_android_method" ? "derived" : "observed", + ); + }, +); diff --git a/tests/boundary/providers/ghidra/ghidraClient.test.ts b/tests/boundary/providers/ghidra/ghidraClient.test.ts index a210add19e..9348f001d8 100644 --- a/tests/boundary/providers/ghidra/ghidraClient.test.ts +++ b/tests/boundary/providers/ghidra/ghidraClient.test.ts @@ -743,6 +743,63 @@ it("cleans up when the final waiter cancels during lineage observation", async ( expect(launcher.processes).toHaveLength(2); }); +it("cancels a retry waiter without taking ownership of startup cleanup", async () => { + let markLaunchPending: (() => void) | undefined; + const launchPending = new Promise((resolve) => { + markLaunchPending = resolve; + }); + let releaseLaunch: (() => void) | undefined; + const launchGate = new Promise((resolve) => { + releaseLaunch = resolve; + }); + let markStartupAborted: (() => void) | undefined; + const startupAborted = new Promise((resolve) => { + markStartupAborted = resolve; + }); + const launcher = new FixtureLauncher(); + const client = clientFor({ + async launch(session, options) { + options?.signal?.addEventListener("abort", () => markStartupAborted?.(), { + once: true, + }); + markLaunchPending?.(); + await launchGate; + return launcher.launch(session); + }, + }); + const firstController = new AbortController(); + const first = client.start(firstController.signal); + let retry: ReturnType | undefined; + try { + await launchPending; + firstController.abort(); + // The retry starts after the shared owner has observed the final departure. + await startupAborted; + + const retryController = new AbortController(); + retry = client.start(retryController.signal); + retryController.abort(); + await expect(retry).resolves.toMatchObject({ + ok: false, + error: { kind: "cancelled" }, + }); + } finally { + releaseLaunch?.(); + await Promise.allSettled([first, retry]); + } + await expect(first).resolves.toMatchObject({ + ok: false, + error: { kind: "cancelled" }, + }); + expect(launcher.processes).toHaveLength(1); + const child = launcher.processes[0]; + if (child === undefined) throw new Error("fixture child was not launched"); + await expect(waitForChildExit(child, 5_000)).resolves.toBe(true); + await expect(access(launcher.runtimeRoots[0] ?? "")).rejects.toMatchObject({ + code: "ENOENT", + }); +}); + it("applies one startup deadline and removes the private runtime", async () => { vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "Date"] }); const launcher = new FixtureLauncher("silent"); diff --git a/tests/boundary/providers/kuna/kunaCommandFailures.test.ts b/tests/boundary/providers/kuna/kunaCommandFailures.test.ts index 426f18eb8a..dce2a60196 100644 --- a/tests/boundary/providers/kuna/kunaCommandFailures.test.ts +++ b/tests/boundary/providers/kuna/kunaCommandFailures.test.ts @@ -1,7 +1,7 @@ import { expect, it as test } from "vitest"; import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjection.js"; -import { inspectKunaInstallation } from "../../../../src/kuna/KunaInstallation.js"; +import { inspectKunaInstallationOwned as inspectKunaInstallation } from "./kunaFileCloseOwner.js"; import { KunaTargetCommands } from "../../../../src/kuna/KunaTargetCommands.js"; import { fakeKunaCalls } from "./fakeKuna.js"; import { session } from "./kunaSessionFixture.js"; diff --git a/tests/boundary/providers/kuna/kunaCommandLifetime.test.ts b/tests/boundary/providers/kuna/kunaCommandLifetime.test.ts index a480b6ac6c..88f0b67619 100644 --- a/tests/boundary/providers/kuna/kunaCommandLifetime.test.ts +++ b/tests/boundary/providers/kuna/kunaCommandLifetime.test.ts @@ -6,10 +6,21 @@ import { session } from "./kunaSessionFixture.js"; const SCRIPT = String.raw` import assert from "node:assert/strict"; +import { ArtifactResourceScope } from "./dist/artifacts/ArtifactResourceScope.js"; import { inspectKunaInstallation } from "./dist/kuna/KunaInstallation.js"; import { KunaTargetCommands } from "./dist/kuna/KunaTargetCommands.js"; const fixture = JSON.parse(process.env.REA_TEST_KUNA_FIXTURE); -const installation = await inspectKunaInstallation({ REA_KUNA_COMMAND: fixture.kuna, REA_KUNA_CACHE_DIR: fixture.cacheRoot }); +const fileCloseScope = new ArtifactResourceScope(); +const installation = await inspectKunaInstallation( + { REA_KUNA_COMMAND: fixture.kuna, REA_KUNA_CACHE_DIR: fixture.cacheRoot }, + { + fileCloseResources: { + retain: (handle, resource) => + fileCloseScope.retain({ kind: "file-handle", handle, resource }), + run: (operation) => fileCloseScope.run(operation), + }, + }, +); assert.equal(installation.ok, true); const controller = new AbortController(); const commands = new KunaTargetCommands(fixture.target, { signal: controller.signal }); @@ -32,6 +43,7 @@ try { } finally { controller.abort(); assert.equal((await commands.close()).ok, true); + await fileCloseScope.close(); } `; diff --git a/tests/boundary/providers/kuna/kunaDoctor.test.ts b/tests/boundary/providers/kuna/kunaDoctor.test.ts index ea648e6beb..0e63a81f14 100644 --- a/tests/boundary/providers/kuna/kunaDoctor.test.ts +++ b/tests/boundary/providers/kuna/kunaDoctor.test.ts @@ -3,7 +3,7 @@ import { join } from "node:path"; import { expect, it as test } from "vitest"; -import { inspectKunaProvider } from "../../../../src/kuna/KunaDoctor.js"; +import { inspectKunaProviderOwned as inspectKunaProvider } from "./kunaFileCloseOwner.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; import { writeFakeKuna } from "./fakeKuna.js"; diff --git a/tests/boundary/providers/kuna/kunaFileCloseOwner.ts b/tests/boundary/providers/kuna/kunaFileCloseOwner.ts new file mode 100644 index 0000000000..b047e3a9d3 --- /dev/null +++ b/tests/boundary/providers/kuna/kunaFileCloseOwner.ts @@ -0,0 +1,41 @@ +import { ArtifactResourceScope } from "../../../../src/artifacts/ArtifactResourceScope.js"; +import type { FileCloseResourceOwner } from "../../../../src/application/AnalysisProvider.js"; +import type { KunaInstallationOptions } from "../../../../src/kuna/KunaInstallation.js"; +import { inspectKunaInstallation } from "../../../../src/kuna/KunaInstallation.js"; +import { inspectKunaProvider } from "../../../../src/kuna/KunaDoctor.js"; + +export const withKunaFileCloseOwner = async ( + operation: (owner: FileCloseResourceOwner) => Promise, +): Promise => { + const scope = new ArtifactResourceScope(); + const owner: FileCloseResourceOwner = { + retain: (handle, resource) => + scope.retain({ + kind: "lease", + resource, + lease: { release: () => handle.close() }, + }), + run: (work) => scope.run(work), + }; + try { + return await operation(owner); + } finally { + await scope.close(); + } +}; + +export const inspectKunaInstallationOwned = ( + environment: Readonly, + options: Omit = {}, +) => + withKunaFileCloseOwner((fileCloseResources) => + inspectKunaInstallation(environment, { ...options, fileCloseResources }), + ); + +export const inspectKunaProviderOwned = ( + environment: Readonly, + platform?: NodeJS.Platform, +) => + withKunaFileCloseOwner((fileCloseResources) => + inspectKunaProvider(environment, platform, { fileCloseResources }), + ); diff --git a/tests/boundary/providers/kuna/kunaHashOwnerPropagation.test.ts b/tests/boundary/providers/kuna/kunaHashOwnerPropagation.test.ts index dcf9f97a3a..c033e276f3 100644 --- a/tests/boundary/providers/kuna/kunaHashOwnerPropagation.test.ts +++ b/tests/boundary/providers/kuna/kunaHashOwnerPropagation.test.ts @@ -1,10 +1,19 @@ +import { chmod, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + import { afterEach, expect, it, vi } from "vitest"; +import { ArtifactResourceScope } from "../../../../src/artifacts/ArtifactResourceScope.js"; +import type { FileCloseResourceOwner } from "../../../../src/application/AnalysisProvider.js"; + const injected = vi.hoisted(() => ({ failNextClose: false, holdCloseFailures: false, releaseCloseFailures: false, closeCalls: 0, + failCloseReadStage: undefined as "fingerprint" | "shebang" | undefined, + failReadStage: undefined as "fingerprint" | "shebang" | undefined, + abortOnReadFailure: undefined as (() => void) | undefined, handles: [] as import("node:fs/promises").FileHandle[], })); @@ -13,6 +22,9 @@ afterEach(async () => { injected.holdCloseFailures = false; injected.releaseCloseFailures = false; injected.closeCalls = 0; + injected.failCloseReadStage = undefined; + injected.failReadStage = undefined; + injected.abortOnReadFailure = undefined; for (const handle of injected.handles.splice(0)) if (handle.fd >= 0) await handle.close(); }); @@ -28,24 +40,55 @@ vi.mock("../../../../src/filesystem/RegularFile.js", async (importOriginal) => { async (...args: Parameters) => { const handle = await original.openRegularFile(...args); injected.handles.push(handle); - if (!injected.failNextClose && !injected.holdCloseFailures) + const failNextClose = injected.failNextClose; + const failCloseReadStage = injected.failCloseReadStage; + const failReadStage = injected.failReadStage; + if ( + !failNextClose && + !injected.holdCloseFailures && + !failCloseReadStage && + !failReadStage + ) return handle; injected.failNextClose = false; + let readStageMatched = false; + let failedClose = false; return { get fd() { return handle.fd; }, stat: (...statArgs: Parameters) => handle.stat(...statArgs), - read: (...readArgs: Parameters) => - handle.read(...readArgs), + read: ( + buffer: Buffer, + offset: number, + length: number, + position: number, + ) => { + if ( + (failReadStage === "fingerprint" && length > 2) || + (failReadStage === "shebang" && length === 2) + ) { + readStageMatched = true; + injected.abortOnReadFailure?.(); + throw new Error("injected read fault"); + } + if (failCloseReadStage === "fingerprint" && length > 2) + readStageMatched = true; + if (failCloseReadStage === "shebang" && length === 2) + readStageMatched = true; + return handle.read(buffer, offset, length, position); + }, close: async () => { injected.closeCalls += 1; if ( - injected.closeCalls === 1 || + (failNextClose && !failedClose) || + (readStageMatched && !failedClose) || (injected.holdCloseFailures && !injected.releaseCloseFailures) - ) + ) { + failedClose = true; throw new Error("injected close fault"); + } await handle.close(); }, } as unknown as typeof handle; @@ -57,16 +100,181 @@ vi.mock("../../../../src/filesystem/RegularFile.js", async (importOriginal) => { import { AnalysisProviderRegistry } from "../../../../src/application/binary/AnalysisProviderRegistry.js"; import { composeBinarySession } from "../../../../src/application/binary/BinarySessionComposition.js"; import { KunaTargetCommands } from "../../../../src/kuna/KunaTargetCommands.js"; +import { KunaProjectCache } from "../../../../src/kuna/KunaProjectCache.js"; import { inspectKunaProvider } from "../../../../src/kuna/KunaDoctor.js"; -import { inspectKunaInstallation } from "../../../../src/kuna/KunaInstallation.js"; +import { inspectKunaInstallationOwned } from "./kunaFileCloseOwner.js"; import { fakeKunaCalls } from "./fakeKuna.js"; -import { session } from "./kunaSessionFixture.js"; +import { run, session } from "./kunaSessionFixture.js"; + +it + .skipIf(process.platform === "win32") + .each([{ stage: "fingerprint" }, { stage: "shebang" }] as const)( + "resolves the originating installation cleanup error after a $stage close failure is retried at retirement", + async ({ stage }) => { + const fixture = await session("rea-kuna-installation-owner-resolution-"); + const client = fixture.open(); + injected.closeCalls = 0; + injected.failCloseReadStage = stage; + try { + const health = await client.execute("health", {}); + expect(health.ok).toBe(false); + if (health.ok) + throw new Error("installation inspection unexpectedly passed"); + const originatingError = health.error; + const originalCause = originatingError.cause; + expect(originatingError).toMatchObject({ + cleanupIncomplete: true, + cleanupResources: [fixture.kuna], + }); + expect(originalCause).toMatchObject({ + name: "RegularFileCleanupFailure", + }); + const failedCloseCall = stage === "fingerprint" ? 1 : 2; + expect(injected.closeCalls).toBe(failedCloseCall); + + expect(await client.close()).toEqual({ ok: true, value: null }); + expect(originatingError.cleanupIncomplete).toBe(false); + expect(originatingError.cause).toBe(originalCause); + expect(injected.closeCalls).toBe(failedCloseCall + 1); + } finally { + injected.failCloseReadStage = undefined; + const closed = await client.close(); + if (!closed.ok) throw closed.error; + } + }, +); + +it.skipIf(process.platform === "win32")( + "keeps cancellation primary when an executable hash read and close both fail", + async () => { + const fixture = await session("rea-kuna-cancelled-read-close-"); + const installed = await inspectKunaInstallationOwned({ + REA_KUNA_COMMAND: fixture.kuna, + REA_KUNA_CACHE_DIR: fixture.cacheRoot, + }); + if (!installed.ok) throw new Error(installed.error.reason); + const controller = new AbortController(); + const commands = new KunaTargetCommands(fixture.target, { + signal: controller.signal, + }); + injected.failReadStage = "fingerprint"; + injected.abortOnReadFailure = () => controller.abort(); + let originatingError: + | import("../../../../src/domain/analysisErrorBase.js").AnalysisError + | undefined; + try { + const result = await commands.run({ + installation: installed.value, + operation: "list_procedures", + arguments: (input) => ["functions", input, "--json"], + timeoutMs: 10_000, + outputBytes: 1024 * 1024, + signal: controller.signal, + }); + if (result.ok) throw new Error("injected cancelled read passed"); + originatingError = result.error; + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + cleanupIncomplete: true, + cleanupResources: [installed.value.executable], + }); + } finally { + injected.failReadStage = undefined; + injected.abortOnReadFailure = undefined; + const closed = await commands.close(); + if (!closed.ok) throw closed.error; + expect(originatingError?.cleanupIncomplete).toBe(false); + } + }, +); + +it.skipIf(process.platform === "win32")( + "resolves an export's typed read failure after the cache retries its exact descriptor", + async () => { + const fixture = await session("rea-kuna-export-read-close-"); + const installed = await inspectKunaInstallationOwned({ + REA_KUNA_COMMAND: fixture.kuna, + REA_KUNA_CACHE_DIR: fixture.cacheRoot, + }); + if (!installed.ok) throw new Error(installed.error.reason); + const cache = new KunaProjectCache(fixture.cacheRoot, { + maxBytes: 64 * 1024 ** 2, + }); + injected.failReadStage = "fingerprint"; + try { + const result = await cache.build({ + installation: installed.value, + targetPath: fixture.target.path, + targetSha256: fixture.target.sha256, + }); + if (result.ok) throw new Error("injected export hash read passed"); + const originatingError = result.error; + expect(originatingError).toMatchObject({ + cleanupIncomplete: true, + cleanupResources: [fixture.kuna], + }); + expect(originatingError.userMessage).toContain("injected read fault"); + expect(await cache.close()).toEqual({ ok: true, value: null }); + expect(originatingError.cleanupIncomplete).toBe(false); + } finally { + injected.failReadStage = undefined; + const closed = await cache.close(); + if (!closed.ok) throw closed.error; + } + }, +); + +it.skipIf(process.platform === "win32")( + "preserves a hash read failure when closing its descriptor also fails", + async () => { + const fixture = await session("rea-kuna-read-and-close-failure-"); + const installed = await inspectKunaInstallationOwned({ + REA_KUNA_COMMAND: fixture.kuna, + REA_KUNA_CACHE_DIR: fixture.cacheRoot, + }); + if (!installed.ok) throw new Error(installed.error.reason); + const controller = new AbortController(); + const commands = new KunaTargetCommands(fixture.target, { + signal: controller.signal, + }); + injected.failReadStage = "fingerprint"; + try { + const result = await commands.run({ + installation: installed.value, + operation: "list_procedures", + arguments: (input) => ["functions", input, "--json"], + timeoutMs: 10_000, + outputBytes: 1024 * 1024, + signal: controller.signal, + }); + if (result.ok) throw new Error("injected hash read failure passed"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.userMessage).toContain("injected read fault"); + expect(result.error.cause).toBeInstanceOf(AggregateError); + const errors = (result.error.cause as AggregateError).errors; + expect(errors[0]).toMatchObject({ + cause: { message: "injected read fault" }, + }); + expect(errors[1]).toMatchObject({ + cleanupResources: [installed.value.executable], + }); + expect(result.error.cleanupResources).toEqual([ + installed.value.executable, + ]); + } finally { + injected.failReadStage = undefined; + controller.abort(); + const closed = await commands.close(); + if (!closed.ok) throw closed.error; + } + }, +); it.skipIf(process.platform === "win32")( "retains and retries the exact Kuna executable hash owner without running Kuna", async () => { const fixture = await session("rea-kuna-hash-owner-"); - const installed = await inspectKunaInstallation({ + const installed = await inspectKunaInstallationOwned({ REA_KUNA_COMMAND: fixture.kuna, REA_KUNA_CACHE_DIR: fixture.cacheRoot, }); @@ -88,11 +296,15 @@ it.skipIf(process.platform === "win32")( }); expect(result.ok).toBe(false); if (result.ok) throw new Error("Kuna command unexpectedly ran"); + const originatingError = result.error; + const originalCause = originatingError.cause; expect(result.error).toMatchObject({ cleanupIncomplete: true, cleanupResources: [installed.value.executable], }); expect(await commands.close()).toEqual({ ok: true, value: null }); + expect(originatingError.cleanupIncomplete).toBe(false); + expect(originatingError.cause).toBe(originalCause); expect(injected.closeCalls).toBe(2); expect( (await fakeKunaCalls(fixture.kuna)).map(({ argv }) => argv), @@ -105,6 +317,165 @@ it.skipIf(process.platform === "win32")( }, ); +it.skipIf(process.platform === "win32")( + "blocks command and prepare admission while one retained executable hash owner keeps failing", + async () => { + const fixture = await session("rea-kuna-hash-owner-admission-"); + const installed = await inspectKunaInstallationOwned({ + REA_KUNA_COMMAND: fixture.kuna, + REA_KUNA_CACHE_DIR: fixture.cacheRoot, + }); + if (!installed.ok) throw new Error(installed.error.reason); + const controller = new AbortController(); + const commands = new KunaTargetCommands(fixture.target, { + signal: controller.signal, + }); + const originalHandleCount = injected.handles.length; + let releasedAdmissions = 0; + injected.closeCalls = 0; + injected.holdCloseFailures = true; + injected.releaseCloseFailures = false; + try { + const request = () => + commands.run({ + installation: installed.value, + operation: "list_procedures", + arguments: (input) => ["functions", input, "--json"], + timeoutMs: 10_000, + outputBytes: 1024 * 1024, + signal: controller.signal, + releaseRun: () => { + releasedAdmissions += 1; + }, + }); + + const first = await request(); + expect(first.ok).toBe(false); + if (first.ok) + throw new Error("hash descriptor close unexpectedly passed"); + const blocked = await request(); + expect(blocked.ok).toBe(false); + if (blocked.ok) throw new Error("Kuna ran with retained hash cleanup"); + expect(blocked.error).toMatchObject({ cleanupIncomplete: true }); + const prepared = await commands.prepare(); + expect(prepared.ok).toBe(false); + if (prepared.ok) throw new Error("prepare passed retained hash cleanup"); + expect(prepared.error).toMatchObject({ cleanupIncomplete: true }); + + expect(injected.closeCalls).toBe(3); + expect(injected.handles.length - originalHandleCount).toBe(1); + expect(releasedAdmissions).toBe(2); + expect( + (await fakeKunaCalls(fixture.kuna)).map(({ argv }) => argv[0]), + ).not.toContain("functions"); + } finally { + injected.releaseCloseFailures = true; + controller.abort(); + const closed = await commands.close(); + if (!closed.ok) throw closed.error; + } + }, +); + +it.skipIf(process.platform === "win32")( + "resolves the originating hash cleanup error when its exact owner closes on retry", + async () => { + const fixture = await session("rea-kuna-hash-owner-resolution-"); + const installed = await inspectKunaInstallationOwned({ + REA_KUNA_COMMAND: fixture.kuna, + REA_KUNA_CACHE_DIR: fixture.cacheRoot, + }); + if (!installed.ok) throw new Error(installed.error.reason); + const controller = new AbortController(); + const commands = new KunaTargetCommands(fixture.target, { + signal: controller.signal, + }); + injected.closeCalls = 0; + injected.holdCloseFailures = true; + injected.releaseCloseFailures = false; + try { + const request = () => + commands.run({ + installation: installed.value, + operation: "list_procedures", + arguments: (input) => ["functions", input, "--json"], + timeoutMs: 10_000, + outputBytes: 1024 * 1024, + signal: controller.signal, + }); + + const first = await request(); + expect(first.ok).toBe(false); + if (first.ok) + throw new Error("hash descriptor close unexpectedly passed"); + const originatingError = first.error; + const originalCause = originatingError.cause; + expect(originatingError).toMatchObject({ cleanupIncomplete: true }); + expect(originalCause).toMatchObject({ + name: "RegularFileCleanupFailure", + }); + + const blocked = await request(); + expect(blocked.ok).toBe(false); + if (blocked.ok) throw new Error("Kuna ran with retained hash cleanup"); + expect(blocked.error).toMatchObject({ cleanupIncomplete: true }); + expect(injected.closeCalls).toBe(2); + + injected.releaseCloseFailures = true; + const recovered = await request(); + expect(recovered.ok).toBe(true); + expect(originatingError.cleanupIncomplete).toBe(false); + expect(originatingError.cause).toBe(originalCause); + expect(injected.closeCalls).toBe(4); + expect( + (await fakeKunaCalls(fixture.kuna)).filter( + ({ argv }) => argv[0] === "functions", + ), + ).toHaveLength(1); + } finally { + injected.releaseCloseFailures = true; + controller.abort(); + const closed = await commands.close(); + if (!closed.ok) throw closed.error; + } + }, +); + +it + .skipIf(process.platform === "win32") + .each([{ stage: "fingerprint" }, { stage: "shebang" }] as const)( + "resolves the Kuna profile cleanup error after its $stage descriptor closes", + async ({ stage }) => { + const fixture = await session("rea-kuna-profile-owner-resolution-"); + const binary = composeBinarySession( + new AnalysisProviderRegistry([fixture.provider]), + ); + injected.closeCalls = 0; + injected.failCloseReadStage = stage; + try { + const resolved = await binary.previewTarget(fixture.target); + expect(resolved.ok).toBe(false); + if (resolved.ok) + throw new Error("Kuna profile inspection unexpectedly passed"); + const originatingError = resolved.error; + expect(originatingError).toMatchObject({ + cleanupIncomplete: true, + cleanupResources: [fixture.kuna], + }); + expect(binary.status()).toMatchObject({ cleanup_pending: true }); + + expect(await binary.close()).toEqual({ ok: true, value: null }); + expect(originatingError.cleanupIncomplete).toBe(false); + expect(injected.closeCalls).toBe(stage === "fingerprint" ? 2 : 3); + expect(binary.status()).toMatchObject({ cleanup_pending: false }); + } finally { + injected.failCloseReadStage = undefined; + const closed = await binary.close(); + if (!closed.ok) throw closed.error; + } + }, +); + it.skipIf(process.platform === "win32")( "drains a Kuna candidate's retained hash owner when the binary session closes without an active client", async () => { @@ -116,34 +487,28 @@ it.skipIf(process.platform === "win32")( injected.holdCloseFailures = true; injected.releaseCloseFailures = false; - const resolved = await fixture.provider.resolveAnalysisProfile( - fixture.target, - ); + const resolved = await binary.previewTarget(fixture.target); expect(resolved.ok).toBe(false); if (resolved.ok) throw new Error("Kuna readiness unexpectedly succeeded"); expect(resolved.error).toMatchObject({ cleanupIncomplete: true, cleanupResources: [fixture.kuna], }); - expect(injected.closeCalls).toBe(2); + expect(injected.closeCalls).toBe(1); expect(binary.status()).toMatchObject({ open: false, cleanup_pending: true, }); - expect(await binary.close()).toMatchObject({ ok: false, - error: { - cleanupIncomplete: true, - cleanupResources: [fixture.kuna], - }, + error: { cleanupIncomplete: true, cleanupResources: [fixture.kuna] }, }); - expect(injected.closeCalls).toBe(3); + expect(injected.closeCalls).toBe(2); expect(binary.status()).toMatchObject({ cleanup_pending: true }); injected.releaseCloseFailures = true; expect(await binary.close()).toEqual({ ok: true, value: null }); - expect(injected.closeCalls).toBe(4); + expect(injected.closeCalls).toBe(3); expect(binary.status()).toMatchObject({ cleanup_pending: false }); expect((await fakeKunaCalls(fixture.kuna)).map(({ argv }) => argv)).toEqual( [["--version"], ["--version"]], @@ -152,33 +517,83 @@ it.skipIf(process.platform === "win32")( ); it.skipIf(process.platform === "win32")( - "returns the exact Kuna doctor owner when no lifecycle sink was supplied", + "retains a failed Kuna doctor hash in the supplied lifecycle owner", async () => { const fixture = await session("rea-kuna-doctor-hash-owner-"); injected.closeCalls = 0; injected.failNextClose = true; - let failure: unknown; - try { - await inspectKunaProvider({ - REA_KUNA_COMMAND: fixture.kuna, - REA_KUNA_CACHE_DIR: fixture.cacheRoot, - }); - } catch (cause: unknown) { - failure = cause; - } - expect(failure).toMatchObject({ - name: "KunaDoctorHashCleanupFailure", - readinessFailure: { code: "hash_cleanup_failed" }, - cleanupCause: expect.any(Error), - owner: { handle: { fd: expect.any(Number) } }, - }); - const owned = failure as { - readonly owner: { close(): Promise }; + const resources = new ArtifactResourceScope(); + const fileCloseResources: FileCloseResourceOwner = { + retain: (owner, resource) => + resources.retain({ + kind: "lease", + resource, + lease: { release: () => owner.close() }, + }), + run: (operation) => resources.run(operation), }; - await owned.owner.close(); - expect(injected.closeCalls).toBe(2); + try { + const inspection = await inspectKunaProvider( + { + REA_KUNA_COMMAND: fixture.kuna, + REA_KUNA_CACHE_DIR: fixture.cacheRoot, + }, + process.platform, + { fileCloseResources }, + ); + expect(inspection.checks[0]).toMatchObject({ + ok: false, + code: "cleanup_incomplete", + }); + expect(injected.closeCalls).toBe(1); + await resources.close(); + expect(injected.closeCalls).toBe(2); + } finally { + await resources.close(); + } expect((await fakeKunaCalls(fixture.kuna)).map(({ argv }) => argv)).toEqual( [["--version"], ["--version"]], ); }, ); + +it.skipIf(process.platform === "win32")( + "drains shared cache cleanup outside a retired client's target key", + async () => { + const fixture = await session("rea-kuna-session-cache-cleanup-"); + const binary = composeBinarySession( + new AnalysisProviderRegistry([fixture.provider]), + ); + const retired = join(fixture.cacheRoot, "trash", "retired"); + await mkdir(retired, { recursive: true }); + await writeFile(join(retired, "evidence"), "retained export"); + await chmod(retired, 0o500); + const client = fixture.open(); + try { + await run(client, "procedure_assembly", { procedure: "main" }); + // Target-key cleanup cannot release maintenance retained for another key. + expect(await client.close()).toEqual({ ok: true, value: null }); + expect(await binary.close()).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true, cleanupResources: [retired] }, + }); + expect(binary.status()).toMatchObject({ + open: false, + cleanup_pending: true, + }); + expect(await readFile(join(retired, "evidence"), "utf8")).toBe( + "retained export", + ); + await chmod(retired, 0o700); + expect(await binary.close()).toEqual({ ok: true, value: null }); + expect(binary.status()).toMatchObject({ cleanup_pending: false }); + await expect(readFile(join(retired, "evidence"))).rejects.toMatchObject({ + code: "ENOENT", + }); + } finally { + await chmod(retired, 0o700).catch(() => {}); + await client.close(); + await binary.close(); + } + }, +); diff --git a/tests/boundary/providers/kuna/kunaInstallation.test.ts b/tests/boundary/providers/kuna/kunaInstallation.test.ts index dfd7b3e291..6e5401dd5f 100644 --- a/tests/boundary/providers/kuna/kunaInstallation.test.ts +++ b/tests/boundary/providers/kuna/kunaInstallation.test.ts @@ -5,7 +5,7 @@ import { join } from "node:path"; import { expect, it as test } from "vitest"; import { readKunaConfiguration } from "../../../../src/kuna/KunaConfiguration.js"; -import { inspectKunaInstallation } from "../../../../src/kuna/KunaInstallation.js"; +import { inspectKunaInstallationOwned as inspectKunaInstallation } from "./kunaFileCloseOwner.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; import { fakeKunaCalls, writeFakeKuna } from "./fakeKuna.js"; diff --git a/tests/boundary/providers/kuna/kunaInstallationLifetime.test.ts b/tests/boundary/providers/kuna/kunaInstallationLifetime.test.ts new file mode 100644 index 0000000000..d8c987e5a5 --- /dev/null +++ b/tests/boundary/providers/kuna/kunaInstallationLifetime.test.ts @@ -0,0 +1,139 @@ +import { afterEach, expect, it, vi } from "vitest"; + +const gates = vi.hoisted(() => { + let releaseHashClose!: () => void; + let releaseShebangRead!: () => void; + let hashCloseEntered!: () => void; + let shebangReadEntered!: () => void; + const hashClose = new Promise((resolve) => { + releaseHashClose = resolve; + }); + const shebangRead = new Promise((resolve) => { + releaseShebangRead = resolve; + }); + const hashCloseStarted = new Promise((resolve) => { + hashCloseEntered = resolve; + }); + const shebangReadStarted = new Promise((resolve) => { + shebangReadEntered = resolve; + }); + return { + hashClose, + hashCloseStarted, + shebangRead, + shebangReadStarted, + releaseHashClose, + releaseShebangRead, + hashCloseEntered, + shebangReadEntered, + opens: 0, + }; +}); + +vi.mock("../../../../src/filesystem/RegularFile.js", async (importOriginal) => { + const original = + await importOriginal< + typeof import("../../../../src/filesystem/RegularFile.js") + >(); + return { + ...original, + openRegularFile: vi.fn( + async (...args: Parameters) => { + const handle = await original.openRegularFile(...args); + const index = gates.opens++; + return { + get fd() { + return handle.fd; + }, + stat: (...statArgs: Parameters) => + handle.stat(...statArgs), + read: async (...readArgs: Parameters) => { + if (index === 1) { + gates.shebangReadEntered(); + await gates.shebangRead; + } + return handle.read(...readArgs); + }, + close: async () => { + if (index === 0) { + gates.hashCloseEntered(); + await gates.hashClose; + } + await handle.close(); + }, + } as unknown as typeof handle; + }, + ), + }; +}); + +import { ArtifactResourceScope } from "../../../../src/artifacts/ArtifactResourceScope.js"; +import type { FileCloseResourceOwner } from "../../../../src/application/AnalysisProvider.js"; +import { inspectKunaInstallation } from "../../../../src/kuna/KunaInstallation.js"; +import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { writeFakeKuna } from "./fakeKuna.js"; + +afterEach(() => { + gates.releaseHashClose(); + gates.releaseShebangRead(); +}); + +it.skipIf(process.platform === "win32")( + "keeps installation inspection inside the resource owner's lifetime through shebang reading", + async () => { + const root = await createTestTempDirectory("rea-kuna-inspection-lifetime-"); + const executable = await writeFakeKuna(root); + const scope = new ArtifactResourceScope(); + const fileCloseResources: FileCloseResourceOwner = { + retain: (owner, resource) => + scope.retain({ + kind: "lease", + resource, + lease: { release: () => owner.close() }, + }), + run: (operation) => scope.run(operation), + }; + const controller = new AbortController(); + const inspection = inspectKunaInstallation( + { REA_KUNA_COMMAND: executable }, + { fileCloseResources, signal: controller.signal }, + ); + + const inspectionOutcome = inspection.then( + (value) => ({ ok: true as const, value }), + (error: unknown) => ({ ok: false as const, error }), + ); + let cleanup: Promise | undefined; + try { + await gates.hashCloseStarted; + cleanup = scope.retryCleanup(); + controller.abort(); + gates.releaseHashClose(); + const reachedShebang = await Promise.race([ + gates.shebangReadStarted.then(() => true), + inspection.then( + () => false, + () => false, + ), + ]); + expect(reachedShebang).toBe(true); + + let cleanupFinished = false; + void cleanup.then(() => { + cleanupFinished = true; + }); + await Promise.resolve(); + expect(cleanupFinished).toBe(false); + + gates.releaseShebangRead(); + await expect(inspection).rejects.toMatchObject({ name: "AbortError" }); + await expect(cleanup).resolves.toBeUndefined(); + } finally { + gates.releaseHashClose(); + gates.releaseShebangRead(); + await inspectionOutcome; + await cleanup; + await scope.close(); + } + }, +); diff --git a/tests/boundary/providers/kuna/kunaProjectCache.test.ts b/tests/boundary/providers/kuna/kunaProjectCache.test.ts index b445169e09..9fc96fdcb9 100644 --- a/tests/boundary/providers/kuna/kunaProjectCache.test.ts +++ b/tests/boundary/providers/kuna/kunaProjectCache.test.ts @@ -12,9 +12,9 @@ import { } from "node:fs/promises"; import { join } from "node:path"; -import { expect, it as test } from "vitest"; +import { expect, it as test, vi } from "vitest"; -import { inspectKunaInstallation } from "../../../../src/kuna/KunaInstallation.js"; +import { inspectKunaInstallationOwned as inspectKunaInstallation } from "./kunaFileCloseOwner.js"; import { cleanupOwnedProcessGroup } from "../../../../src/process/ProcessOwnership.js"; import { spawnOwnedProviderProcess } from "../../../../src/process/ProviderProcess.js"; import type { KunaInstallation } from "../../../../src/kuna/KunaInstallation.js"; @@ -24,6 +24,10 @@ import { } from "../../../../src/kuna/KunaProjectCache.js"; import type { KunaExportStatus } from "../../../../src/kuna/KunaProjectExport.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { OwnedFileHandle } from "../../../../src/filesystem/OwnedFileHandle.js"; +import { RegularFileCleanupFailure } from "../../../../src/filesystem/OwnedRegularFile.js"; +import * as stableFileHash from "../../../../src/filesystem/StableFileHash.js"; +import { ProviderCleanupError } from "../../../../src/domain/providerCleanupError.js"; import { fakeKunaCalls, setFakeKunaBehavior, @@ -98,6 +102,29 @@ const processGone = (pid: number): boolean => { } }; +const deferred = () => { + let resolve!: (value: T) => void; + let reject!: (cause: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +const causesOf = (cause: unknown): unknown[] => { + const pending = [cause]; + const seen = new Set(); + while (pending.length > 0) { + const current = pending.pop(); + if (!(current instanceof Error) || seen.has(current)) continue; + seen.add(current); + if (current.cause !== undefined) pending.push(current.cause); + if (current instanceof AggregateError) pending.push(...current.errors); + } + return [...seen]; +}; + it("publishes a verified export and serves later requests from the cache", async () => { const { kuna, installation, cacheRoot, cache, target } = await fixture( "rea-kuna-cache-", @@ -188,6 +215,355 @@ it("publishes a verified export and serves later requests from the cache", async expect(await callsOf(kuna, "decompile-project")).toHaveLength(1); }); +it("retains a failed warm-cache hash descriptor and retries it at cache close", async () => { + const { installation, cache, target } = await fixture( + "rea-kuna-cache-hash-owner-", + ); + const input = await target("program"); + const built = await cache.build({ installation, ...input }); + if (!built.ok) throw built.error; + const closeCause = new Error("injected warm-cache descriptor close failure"); + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockRejectedValueOnce(closeCause); + const warmCache = newCache(installation); + try { + const failure = await warmCache + .lookup(built.value.key) + .catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(ProviderCleanupError); + if (!(failure instanceof ProviderCleanupError)) throw failure; + expect(failure.cause).toBeInstanceOf(RegularFileCleanupFailure); + close.mockRestore(); + expect(await warmCache.close()).toEqual({ ok: true, value: null }); + expect(failure.cleanupIncomplete).toBe(false); + expect(await warmCache.lookup(built.value.key)).toMatchObject({ + key: built.value.key, + }); + } finally { + close.mockRestore(); + expect(await warmCache.close()).toEqual({ ok: true, value: null }); + await cache.close(); + } +}); + +it.each(["success", "failure"] as const)( + "keeps hash cleanup bound to its exact owner when a concurrent lookup retry %s", + async (retryOutcome) => { + const { installation, cache, target } = await fixture( + `rea-kuna-cache-hash-race-${retryOutcome}-`, + ); + const input = await target("program"); + const built = await cache.build({ installation, ...input }); + if (!built.ok) throw built.error; + + const warmCache = newCache(installation); + const initialCloseFailure = new Error("initial descriptor close failure"); + const retryCloseFailure = new Error("concurrent retry close failure"); + const initialClose = deferred(); + const retryGate = deferred(); + const initialCloseEntered = deferred(); + const retryEntered = deferred(); + const closeCounts = new Map(); + let retainedOwner: OwnedFileHandle | undefined; + let concurrentLookup: Promise | undefined; + const closeOwner = OwnedFileHandle.prototype.close; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(function (this: OwnedFileHandle) { + const count = (closeCounts.get(this) ?? 0) + 1; + closeCounts.set(this, count); + if (count === 1 && retainedOwner === undefined) { + retainedOwner = this; + initialCloseEntered.resolve(); + return initialClose.promise; + } + if (this === retainedOwner && count === 2) { + retryEntered.resolve(); + return retryGate.promise.then(async () => { + if (retryOutcome === "failure") throw retryCloseFailure; + await closeOwner.call(this); + }); + } + return closeOwner.call(this); + }); + + try { + const firstLookup = warmCache + .lookup(built.value.key) + .catch((cause) => cause); + await initialCloseEntered.promise; + initialClose.reject(initialCloseFailure); + // The failed descriptor is retained before the first lookup binds its + // originating error, so this lookup begins the retry in that window. + queueMicrotask(() => { + concurrentLookup = warmCache + .lookup(built.value.key) + .catch((cause: unknown) => cause); + }); + await retryEntered.promise; + + const firstFailure = await firstLookup; + expect(firstFailure).toBeInstanceOf(ProviderCleanupError); + if (!(firstFailure instanceof ProviderCleanupError)) throw firstFailure; + expect(firstFailure.cleanupIncomplete).toBe(true); + retryGate.resolve(); + if (concurrentLookup === undefined) + throw new Error("Concurrent lookup did not start"); + const retryResult = await concurrentLookup; + + if (retryOutcome === "success") { + expect(retryResult).toMatchObject({ key: built.value.key }); + // Closing the retained owner resolves the already-bound operation error. + expect(firstFailure.cleanupIncomplete).toBe(false); + } else { + expect(retryResult).toBeInstanceOf(ProviderCleanupError); + if (!(retryResult instanceof ProviderCleanupError)) throw retryResult; + const retryCause = retryResult.cause; + expect(retryCause).toBeInstanceOf(AggregateError); + if (!(retryCause instanceof AggregateError)) throw retryCause; + const retryCauses = causesOf(retryCause); + expect(retryCauses).toContain(firstFailure); + expect(retryCauses).toContain(retryCloseFailure); + + // The next confirmed close recovers both errors without admitting an + // extra read in between the failed retry and recovery. + close.mockRestore(); + expect(await warmCache.lookup(built.value.key)).toMatchObject({ + key: built.value.key, + }); + expect(firstFailure.cleanupIncomplete).toBe(false); + expect(retryResult.cleanupIncomplete).toBe(false); + } + } finally { + close.mockRestore(); + expect(await warmCache.close()).toEqual({ ok: true, value: null }); + await cache.close(); + } + }, +); + +it("resolves a hash failure delivered after its retained owner was released", async () => { + const { installation, cache, target } = await fixture( + "rea-kuna-cache-late-hash-", + ); + const built = await cache.build({ + installation, + ...(await target("program")), + }); + if (!built.ok) throw built.error; + const warmCache = newCache(installation); + const failureRetained = deferred(); + const deliverFailure = deferred(); + const hashFile = stableFileHash.hashStableFile; + const hash = vi + .spyOn(stableFileHash, "hashStableFile") + .mockImplementation(async (...args) => { + try { + return await hashFile(...args); + } catch (cause: unknown) { + if (!(cause instanceof RegularFileCleanupFailure)) throw cause; + failureRetained.resolve(); + await deliverFailure.promise; + throw cause; + } + }); + const closeCause = new Error("initial hash descriptor close failure"); + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockRejectedValueOnce(closeCause); + const firstLookup = warmCache + .lookup(built.value.key) + .catch((cause: unknown) => cause); + try { + await failureRetained.promise; + expect(warmCache.hasPendingCleanup).toBe(true); + expect(await warmCache.lookup(built.value.key)).toMatchObject({ + key: built.value.key, + }); + expect(warmCache.hasPendingCleanup).toBe(false); + deliverFailure.resolve(); + const error = await firstLookup; + expect(error).toBeInstanceOf(ProviderCleanupError); + if (!(error instanceof ProviderCleanupError)) throw error; + expect(error.cleanupIncomplete).toBe(false); + expect(error.cause).toBeInstanceOf(RegularFileCleanupFailure); + if (!(error.cause instanceof RegularFileCleanupFailure)) throw error.cause; + expect(error.cause.cleanupCause).toBe(closeCause); + expect(error.cause.owner.handle.fd).toBe(-1); + } finally { + deliverFailure.resolve(); + await firstLookup; + hash.mockRestore(); + close.mockRestore(); + expect(await warmCache.close()).toEqual({ ok: true, value: null }); + await cache.close(); + } +}); + +it("blocks repeated cache reads while a retained hash descriptor cannot close", async () => { + const { installation, cache, target } = await fixture( + "rea-kuna-cache-hash-admission-", + ); + const input = await target("program"); + const built = await cache.build({ installation, ...input }); + if (!built.ok) throw built.error; + const otherBuilt = await cache.build({ + installation, + ...(await target("other")), + }); + if (!otherBuilt.ok) throw otherBuilt.error; + const keys = [built.value.key, otherBuilt.value.key] as const; + + const warmCache = newCache(installation); + const owners = new Set(); + const persistentCloseCause = new Error("persistent descriptor close failure"); + const blocked: ProviderCleanupError[] = []; + const blockedByKey = new Map(); + let closeAggregate: ProviderCleanupError | undefined; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(function (this: OwnedFileHandle) { + owners.add(this); + return Promise.reject(persistentCloseCause); + }); + try { + for (const key of keys) + await expect(warmCache.lookup(key)).rejects.toMatchObject({ + cleanupIncomplete: true, + cause: expect.any(RegularFileCleanupFailure), + }); + expect(owners.size).toBe(2); + expect(warmCache.hasPendingCleanup).toBe(true); + + for (let attempt = 0; attempt < 3; attempt += 1) { + for (const key of keys) { + try { + await warmCache.lookup(key); + throw new Error("Lookup admitted a read with a retained descriptor"); + } catch (cause: unknown) { + expect(cause).toBeInstanceOf(ProviderCleanupError); + if (!(cause instanceof ProviderCleanupError)) throw cause; + blocked.push(cause); + const failures = blockedByKey.get(key) ?? []; + failures.push(cause); + blockedByKey.set(key, failures); + } + } + } + + // Each key retries its exact descriptor without admitting another read. + expect(owners.size).toBe(2); + expect(warmCache.hasPendingCleanup).toBe(true); + + for (const failures of blockedByKey.values()) { + expect(failures).toHaveLength(3); + const originals = new Set(); + for (const failure of failures) { + const retry = (failure.cause as AggregateError).errors[0]; + expect(retry).toBeInstanceOf(ProviderCleanupError); + if (!(retry instanceof ProviderCleanupError)) throw retry; + const causes = (retry.cause as AggregateError).errors; + expect(causes).toHaveLength(2); + originals.add(causes[0]); + expect(causes[1]).toBe(persistentCloseCause); + } + expect(originals.size).toBe(1); + } + const closed = await warmCache.close(); + expect(closed.ok).toBe(false); + if (closed.ok) throw new Error("Close lost the retained hash owner"); + expect(closed.error.cleanupIncomplete).toBe(true); + expect(new Set(closed.error.cleanupResources).size).toBe(2); + closeAggregate = closed.error; + blocked.push(closed.error); + } finally { + close.mockRestore(); + } + + expect(await warmCache.lookup(keys[0])).toMatchObject({ key: keys[0] }); + expect(warmCache.hasPendingCleanup).toBe(true); + if (closeAggregate === undefined) throw new Error("Missing close aggregate"); + expect(closeAggregate.cleanupIncomplete).toBe(true); + for (const failure of blockedByKey.get(keys[0]) ?? []) + expect(failure.cleanupIncomplete).toBe(false); + for (const failure of blockedByKey.get(keys[1]) ?? []) + expect(failure.cleanupIncomplete).toBe(true); + expect(await warmCache.lookup(keys[1])).toMatchObject({ key: keys[1] }); + expect(warmCache.hasPendingCleanup).toBe(false); + for (const [index, failure] of blocked.entries()) + expect(failure.cleanupIncomplete, `failed aggregate ${String(index)}`).toBe( + false, + ); + expect(await warmCache.close()).toEqual({ ok: true, value: null }); + await cache.close(); +}); + +it("keeps hash cleanup aggregates pending through retained staging removal", async () => { + const { installation, cacheRoot, cache, target } = await fixture( + "rea-kuna-cache-hash-staging-owner-", + ); + const persistentCloseCause = new Error( + "persistent staging hash close failure", + ); + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockRejectedValue(persistentCloseCause); + let stagingPath: string | undefined; + let retainedPath: string | undefined; + let buildFailure: ProviderCleanupError | undefined; + const aggregates: ProviderCleanupError[] = []; + try { + const built = await cache.build({ + installation, + ...(await target("program")), + }); + expect(built.ok).toBe(false); + if (built.ok) throw new Error("Expected the retained hash close to fail"); + expect(built.error).toBeInstanceOf(ProviderCleanupError); + if (!(built.error instanceof ProviderCleanupError)) throw built.error; + buildFailure = built.error; + + const [staging] = await entries(cacheRoot, "staging"); + expect(staging).toBeDefined(); + if (staging === undefined) throw new Error("Missing retained staging"); + stagingPath = join(cacheRoot, "staging", staging); + await chmod(stagingPath, 0o500); + + const firstClose = await cache.close(); + expect(firstClose.ok).toBe(false); + if (firstClose.ok) throw new Error("Close lost the retained descriptor"); + aggregates.push(firstClose.error); + + close.mockRestore(); + const stagingClose = await cache.close(); + expect(stagingClose.ok).toBe(false); + if (stagingClose.ok) throw new Error("Close lost the retained staging"); + aggregates.push(stagingClose.error); + retainedPath = stagingClose.error.cleanupResources[0]; + expect(retainedPath).toBeDefined(); + if (retainedPath === undefined) + throw new Error("Missing failed staging cleanup resource"); + expect(buildFailure.cleanupIncomplete).toBe(false); + for (const failure of aggregates) + expect(failure.cleanupIncomplete).toBe(true); + + await chmod(retainedPath, 0o700); + expect(await cache.close()).toEqual({ ok: true, value: null }); + expect(buildFailure.cleanupIncomplete).toBe(false); + expect( + aggregates.every(({ cleanupIncomplete }) => !cleanupIncomplete), + ).toBe(true); + } finally { + close.mockRestore(); + if (stagingPath !== undefined) + await chmod(stagingPath, 0o700).catch(() => undefined); + if (retainedPath !== undefined) + await chmod(retainedPath, 0o700).catch(() => undefined); + await cache.close(); + } +}); + it("shares one Kuna run between concurrent requests for one export", async () => { const { kuna, installation, cache, target } = await fixture( "rea-kuna-singleflight-", diff --git a/tests/boundary/providers/kuna/kunaProjectConcurrency.test.ts b/tests/boundary/providers/kuna/kunaProjectConcurrency.test.ts index 700ef91620..9f0c302e99 100644 --- a/tests/boundary/providers/kuna/kunaProjectConcurrency.test.ts +++ b/tests/boundary/providers/kuna/kunaProjectConcurrency.test.ts @@ -5,7 +5,7 @@ import { join } from "node:path"; import { afterEach, expect, it as test, onTestFinished, vi } from "vitest"; -import { inspectKunaInstallation } from "../../../../src/kuna/KunaInstallation.js"; +import { inspectKunaInstallationOwned as inspectKunaInstallation } from "./kunaFileCloseOwner.js"; import { KunaProjectCache } from "../../../../src/kuna/KunaProjectCache.js"; import type { KunaExportStatus } from "../../../../src/kuna/KunaProjectExport.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; @@ -130,7 +130,9 @@ it.each([true, false])( if (!closed.ok) throw closed.error; }); await expect - .poll(() => actual.readdir(listing.directory).catch(() => [])) + .poll(() => actual.readdir(listing.directory).catch(() => []), { + timeout: 10_000, + }) .toHaveLength(2); listing.names = await actual.readdir(listing.directory); releaseListing(); diff --git a/tests/boundary/providers/kuna/kunaProvider.test.ts b/tests/boundary/providers/kuna/kunaProvider.test.ts index 9c755a0e26..593b952c76 100644 --- a/tests/boundary/providers/kuna/kunaProvider.test.ts +++ b/tests/boundary/providers/kuna/kunaProvider.test.ts @@ -16,6 +16,9 @@ import type { BinaryTarget } from "../../../../src/domain/binaryTargetTypes.js"; import type { JsonValue } from "../../../../src/domain/jsonValue.js"; import type { KunaProcessLauncher } from "../../../../src/kuna/KunaCommand.js"; import type { AnalysisError } from "../../../../src/domain/analysisErrorBase.js"; +import { OwnedFileHandle } from "../../../../src/filesystem/OwnedFileHandle.js"; +import { AnalysisProviderRegistry } from "../../../../src/application/binary/AnalysisProviderRegistry.js"; +import { composeBinarySession } from "../../../../src/application/binary/BinarySessionComposition.js"; import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjection.js"; import { KunaProvider } from "../../../../src/kuna/KunaProvider.js"; import { KUNA_FUNCTION_TEXT_BYTES } from "../../../../src/kuna/KunaLimits.js"; @@ -97,6 +100,65 @@ it("discovers configuration without probing and admits only verified targets", a }); }); +it("waits for an aborted pre-session hash before retrying its late close owner", async () => { + const fixture = await session("rea-kuna-profile-hash-owner-"); + const binary = composeBinarySession( + new AnalysisProviderRegistry([fixture.provider], "kuna"), + ); + const nativeClose = OwnedFileHandle.prototype.close; + let closeCalls = 0; + let enteredClose!: () => void; + let releaseClose!: () => void; + const closeEntered = new Promise((resolve) => { + enteredClose = resolve; + }); + const closeGate = new Promise((resolve) => { + releaseClose = resolve; + }); + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + closeCalls += 1; + if (closeCalls === 1) { + enteredClose(); + await closeGate; + } + if (closeCalls <= 2) + throw new Error("injected Kuna profile close failure"); + return nativeClose.call(this); + }); + try { + const controller = new AbortController(); + const previewWork = binary.previewTarget(fixture.target, { + signal: controller.signal, + }); + await closeEntered; + controller.abort(); + const preview = await previewWork; + expect(preview).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + + const closing = binary.close(); + releaseClose(); + const firstClose = await closing; + expect(firstClose).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + expect(binary.status()).toMatchObject({ cleanup_pending: true }); + + close.mockRestore(); + expect(await binary.close()).toEqual({ ok: true, value: null }); + expect(binary.status()).toMatchObject({ cleanup_pending: false }); + } finally { + releaseClose(); + close.mockRestore(); + await binary.close(); + } +}); + it("commits the observed Kuna release and digest to the analysis profile", async () => { const { profile } = await session("rea-kuna-profile-"); diff --git a/tests/boundary/providers/kuna/kunaRunOwnership.test.ts b/tests/boundary/providers/kuna/kunaRunOwnership.test.ts index f0ca09a081..63b6503823 100644 --- a/tests/boundary/providers/kuna/kunaRunOwnership.test.ts +++ b/tests/boundary/providers/kuna/kunaRunOwnership.test.ts @@ -2,7 +2,7 @@ import { setImmediate } from "node:timers/promises"; import { expect, it as test, vi } from "vitest"; -import { inspectKunaInstallation } from "../../../../src/kuna/KunaInstallation.js"; +import { inspectKunaInstallationOwned as inspectKunaInstallation } from "./kunaFileCloseOwner.js"; import { KunaRunSlots } from "../../../../src/kuna/KunaRunSlots.js"; import { KunaTargetCommands } from "../../../../src/kuna/KunaTargetCommands.js"; import { spawnOwnedProviderProcess } from "../../../../src/process/ProviderProcess.js"; diff --git a/tests/boundary/providers/kuna/kunaSessionFixture.ts b/tests/boundary/providers/kuna/kunaSessionFixture.ts index dc53b29a0e..7fa749afa9 100644 --- a/tests/boundary/providers/kuna/kunaSessionFixture.ts +++ b/tests/boundary/providers/kuna/kunaSessionFixture.ts @@ -9,6 +9,7 @@ import { KunaProvider } from "../../../../src/kuna/KunaProvider.js"; import { cleanupOwnedProcessGroup } from "../../../../src/process/ProcessOwnership.js"; import { spawnOwnedProviderProcess } from "../../../../src/process/ProviderProcess.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { withKunaFileCloseOwner } from "./kunaFileCloseOwner.js"; import { fakeKunaCalls, writeFakeKuna, @@ -53,7 +54,9 @@ export const session = async ( ...(singleFunctionRuns === undefined ? {} : { singleFunctionRuns }), }, ); - const profile = await provider.resolveAnalysisProfile(target); + const profile = await withKunaFileCloseOwner((fileCloseResources) => + provider.resolveAnalysisProfile(target, { fileCloseResources }), + ); if (!profile.ok || profile.value.profile === null) throw new Error("profile did not resolve"); const resolved = profile.value.profile; diff --git a/tests/boundary/setup/linuxHopperPublisherVerification.test.ts b/tests/boundary/setup/linuxHopperPublisherVerification.test.ts index 527a5946a0..28d9791a9f 100644 --- a/tests/boundary/setup/linuxHopperPublisherVerification.test.ts +++ b/tests/boundary/setup/linuxHopperPublisherVerification.test.ts @@ -11,6 +11,7 @@ import { readFile, readdir, rm, + stat, symlink, writeFile, } from "node:fs/promises"; @@ -33,6 +34,7 @@ import { import * as signatures from "../../../src/process/DetachedPgpSignature.js"; import { downloadHopperPackage } from "../../../src/application/HopperDownload.js"; import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; +import { OwnedFileHandle } from "../../../src/filesystem/OwnedFileHandle.js"; import * as leases from "../../../src/process/HopperLease.js"; import * as nativeCommands from "../../../src/process/ExecFileOutput.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -43,6 +45,7 @@ const verifySignature = signatures.verifyDetachedPgpSignature; const sha256 = (bytes: Uint8Array) => createHash("sha256").update(bytes).digest("hex"); const linux = it.skipIf(process.platform !== "linux"); +const portableTar = it.skipIf(process.platform === "win32"); afterEach(() => { vi.restoreAllMocks(); @@ -702,6 +705,79 @@ it("cancels an oversized chunked metadata response before retaining it", async ( expect(cancelled).toHaveBeenCalledOnce(); }); +portableTar( + "retains the temporary root until its failed hash descriptor closes", + async () => { + const f = await fixture("arch"); + admitFixturePublisher(f.fingerprint); + const events: string[] = []; + let targetCloseAttempts = 0; + let failTargetClose = true; + const targetSize = (await stat(f.launcher)).size; + let temporaryRoot: string | undefined; + const originalFileClose = OwnedFileHandle.prototype.close; + const fileClose = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + if ((await this.handle.stat()).size === targetSize) { + targetCloseAttempts += 1; + if (failTargetClose) { + events.push(`hash-close-failed-${targetCloseAttempts}`); + throw new Error("Fixture extracted launcher close unavailable"); + } + events.push("hash-close-retried"); + } + return originalFileClose.call(this); + }); + const originalRootClose = PrivateRuntimeRoot.prototype.close; + const originalRootCreate = PrivateRuntimeRoot.create; + const rootCreate = vi + .spyOn(PrivateRuntimeRoot, "create") + .mockImplementation(async (options) => { + const root = await originalRootCreate.call(PrivateRuntimeRoot, options); + temporaryRoot = root.path; + return root; + }); + const rootClose = vi + .spyOn(PrivateRuntimeRoot.prototype, "close") + .mockImplementation(async function (this: PrivateRuntimeRoot) { + temporaryRoot = this.path; + events.push("temporary-root-close"); + return originalRootClose.call(this); + }); + try { + expect( + await f.publisher.verifyArchive( + "arch", + f.release, + f.archive, + undefined, + f.publicKey, + ), + ).toMatchObject({ ok: false, error: { reason: "cleanup_failed" } }); + expect(targetCloseAttempts).toBeGreaterThanOrEqual(2); + expect(events).not.toContain("temporary-root-close"); + const attemptsBeforeClose = targetCloseAttempts; + await expect(f.publisher.close()).rejects.toThrow(); + expect(targetCloseAttempts).toBeGreaterThan(attemptsBeforeClose); + expect(events).not.toContain("temporary-root-close"); + expect(temporaryRoot).toBeDefined(); + await expect(access(temporaryRoot!)).resolves.toBeUndefined(); + failTargetClose = false; + await f.publisher.close(); + expect(events.slice(-2)).toEqual([ + "hash-close-retried", + "temporary-root-close", + ]); + } finally { + fileClose.mockRestore(); + rootClose.mockRestore(); + rootCreate.mockRestore(); + await f.publisher.close(); + } + }, +); + linux("retains and retries failed verification cleanup", async () => { const f = await fixture(); admitFixturePublisher(f.fingerprint); @@ -726,6 +802,156 @@ linux("retains and retries failed verification cleanup", async () => { } }); +portableTar( + "reports pending publisher cleanup as a typed failure before retrying the operation", + async () => { + const f = await fixture("arch"); + admitFixturePublisher(f.fingerprint); + const closeRoot = PrivateRuntimeRoot.prototype.close; + let rootCloseAttempts = 0; + vi.spyOn(PrivateRuntimeRoot.prototype, "close").mockImplementation( + async function (this: PrivateRuntimeRoot) { + rootCloseAttempts += 1; + if (rootCloseAttempts < 3) + throw new Error("Fixture cleanup remains unavailable"); + return await closeRoot.apply(this); + }, + ); + try { + expect( + await f.publisher.verifyArchive( + "arch", + f.release, + f.archive, + undefined, + f.publicKey, + ), + ).toMatchObject({ ok: false, error: { reason: "cleanup_failed" } }); + + const closeHandle = vi.spyOn(OwnedFileHandle.prototype, "close"); + expect(await f.publisher.inspectLauncher(f.launcher)).toMatchObject({ + ok: false, + error: { reason: "cleanup_failed" }, + }); + expect(closeHandle).not.toHaveBeenCalled(); + + expect( + await f.publisher.verifyArchive( + "arch", + f.release, + f.archive, + undefined, + f.publicKey, + ), + ).toMatchObject({ ok: true }); + expect(closeHandle).toHaveBeenCalled(); + expect(rootCloseAttempts).toBe(4); + } finally { + await f.publisher.close(); + } + }, +); + +it("retries a launcher hash descriptor after a transient close failure", async () => { + const launcher = join( + await createTestTempDirectory("rea-linux-publisher-hash-"), + "Hopper", + ); + const bytes = Buffer.from("fixture launcher bytes"); + await writeFile(launcher, bytes); + const publisher = new LinuxHopperPublisher({}, async () => undefined); + const closeCause = new Error("Fixture hash descriptor close unavailable"); + const owners: OwnedFileHandle[] = []; + const originalClose = OwnedFileHandle.prototype.close; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + owners.push(this); + if (owners.length === 1) throw closeCause; + return originalClose.call(this); + }); + try { + await expect(publisher.hashLauncher(launcher)).resolves.toBe(sha256(bytes)); + expect(close).toHaveBeenCalledTimes(2); + expect(owners).toHaveLength(2); + expect(owners[1]).toBe(owners[0]); + expect(owners[0]?.handle.fd).toBe(-1); + await publisher.close(); + expect(close).toHaveBeenCalledTimes(2); + } finally { + close.mockRestore(); + await publisher.close(); + } +}); + +portableTar( + "lets overlapping verification and descriptor cleanup settle without re-entering publisher admission", + async () => { + const f = await fixture("arch"); + admitFixturePublisher(f.fingerprint); + const verified = await f.publisher.verifyArchive( + "arch", + f.release, + f.archive, + undefined, + f.publicKey, + ); + if (!verified.ok) throw verified.error; + expect( + await f.publisher.retainVerifiedArchive(verified.value, f.archive), + ).toMatchObject({ ok: true }); + + let reachedMetadataClose!: () => void; + const metadataCloseStarted = new Promise((resolve) => { + reachedMetadataClose = resolve; + }); + let releaseMetadataClose!: () => void; + const metadataCloseGate = new Promise((resolve) => { + releaseMetadataClose = resolve; + }); + const originalClose = OwnedFileHandle.prototype.close; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + const attempt = close.mock.calls.length; + if (attempt === 2) { + reachedMetadataClose(); + await metadataCloseGate; + } + if (attempt === 3 || attempt === 4 || attempt === 5) + throw new Error("Fixture descriptor close temporarily unavailable"); + return await originalClose.apply(this); + }); + try { + const inspection = f.publisher.inspectLauncher(f.launcher); + await metadataCloseStarted; + const hashFailure = await f.publisher.hashLauncher(f.launcher).then( + () => undefined, + (cause: unknown) => cause, + ); + expect(hashFailure).toMatchObject({ + reason: "cleanup_failed", + cause: expect.any(AggregateError), + }); + const cleanupOutcomes = (hashFailure as Error & { cause: AggregateError }) + .cause.errors; + expect(cleanupOutcomes).toHaveLength(2); + expect(cleanupOutcomes[0]).toMatchObject({ reason: "cleanup_failed" }); + releaseMetadataClose(); + await expect(inspection).resolves.toMatchObject({ + ok: true, + value: { source: "signed_package_cache" }, + }); + await expect(f.publisher.close()).resolves.toBeUndefined(); + expect(close).toHaveBeenCalledTimes(9); + } finally { + releaseMetadataClose(); + close.mockRestore(); + await f.publisher.close(); + } + }, +); + linux( "preserves cleanup ownership when verification is cancelled", async () => { diff --git a/tests/boundary/signals/recordingDescriptorCleanup.test.ts b/tests/boundary/signals/recordingDescriptorCleanup.test.ts index f600f0477a..618e3404fc 100644 --- a/tests/boundary/signals/recordingDescriptorCleanup.test.ts +++ b/tests/boundary/signals/recordingDescriptorCleanup.test.ts @@ -3,7 +3,7 @@ import { expectRetainedDescriptor, } from "../../fixtures/descriptorCloseFailure.js"; import { expect, it, onTestFinished, vi } from "vitest"; -import { writeFile } from "node:fs/promises"; +import { lstat, writeFile } from "node:fs/promises"; import type { FileHandle } from "node:fs/promises"; import { join } from "node:path"; import { SigrokRecordingProvider } from "../../../src/signals/SigrokRecordingProvider.js"; @@ -49,6 +49,7 @@ it { mode: 0o700 }, ); let launches = 0; + let runtimeRoot: PrivateRuntimeRoot | undefined; const provider = new SigrokRecordingProvider( { ...process.env, @@ -64,7 +65,10 @@ it expectedCommand: process.execPath, }); }, - () => PrivateRuntimeRoot.create({ parent: directory }), + async () => { + runtimeRoot = await PrivateRuntimeRoot.create({ parent: directory }); + return runtimeRoot; + }, ); const discovery = member.startsWith("field-"); const resource = discovery ? member.slice(6) : member; @@ -88,7 +92,18 @@ it const first = discovery ? await service.inspectDecoders({ query: "uart" }) : await service.decode({ path, decoders: "uart" }); - expectRetainedDescriptor(first, cleanup, operation); + expectRetainedDescriptor( + first, + cleanup, + operation, + runtimeRoot === undefined ? [] : [runtimeRoot.path], + ); + if (first.ok) throw new Error("Expected retained descriptor cleanup"); + const originatingCause = first.error.cause; + if (resource === "capture") { + expect(runtimeRoot).toBeDefined(); + await expect(lstat(runtimeRoot!.path)).resolves.toBeDefined(); + } const launchCount = launches; expect(await service.decode({ path, decoders: "uart" })).toMatchObject({ ok: false, @@ -102,6 +117,12 @@ it }); cleanup.denied = false; await provider.close(); + expect(first.error.cleanupIncomplete).toBe(false); + expect(first.error.cause).toBe(originatingCause); + if (runtimeRoot !== undefined) + await expect(lstat(runtimeRoot.path)).rejects.toMatchObject({ + code: "ENOENT", + }); expect(cleanup.handles[0]?.fd).toBe(-1); }, ); diff --git a/tests/boundary/signals/recordingProvider.test.ts b/tests/boundary/signals/recordingProvider.test.ts index 789a2010ac..f99b252dbf 100644 --- a/tests/boundary/signals/recordingProvider.test.ts +++ b/tests/boundary/signals/recordingProvider.test.ts @@ -575,6 +575,8 @@ it("retains completed discovery when cleanup fails, then retries before acceptin partialObservation: { operation: "inspect_signal_decoders" }, }, }); + if (first.ok) throw new Error("expected retained cleanup failure"); + expect(first.error.cleanupIncomplete).toBe(true); await f.service.inspectDecoders({}); await f.service.decode({ path: f.path, decoders: "uart" }); expect(f.calls()).toBe(2); @@ -585,6 +587,7 @@ it("retains completed discovery when cleanup fails, then retries before acceptin }); f.allowCleanup(); await f.service.close(); + expect(first.error.cleanupIncomplete).toBe(false); }); it("rejects ambiguous decoder selectors before launch and retains malformed help output", async () => { diff --git a/tests/conformance/providerCompletion.test.ts b/tests/conformance/providerCompletion.test.ts new file mode 100644 index 0000000000..4a20cdaa60 --- /dev/null +++ b/tests/conformance/providerCompletion.test.ts @@ -0,0 +1,308 @@ +import { expect, it } from "vitest"; +import { ok } from "../../src/domain/result.js"; +import { WebNetworkCaptureService } from "../../src/application/WebNetworkCaptureService.js"; +import { WebSourceLocationService } from "../../src/application/WebSourceLocationService.js"; +import { + webSourceLocationArgs, + webSourceLocationFixture, + webSourceMapDecoderFixture, +} from "../fixtures/webSourceLocation.js"; +import { historicalHar } from "../fixtures/historicalHar.js"; +import { BinaryLayoutService } from "../../src/application/binaryDiagnostics/BinaryLayoutService.js"; +import { + binaryLayoutFixture, + BINARY_LAYOUT_TEST_PROVIDER, +} from "../fixtures/binaryDiagnostics/layout.js"; +import { PeResourcesService } from "../../src/application/binaryDiagnostics/PeResourcesService.js"; +import { PE_RESOURCES_PROVIDER } from "../../src/application/InvestigationProviders.js"; +import { inspectPeResourcesInputSchema } from "../../src/domain/native/peResources.js"; +import { parsePeResources } from "../../src/native/pe/PeResourceParser.js"; +import { peResourceFixture } from "../../src/native/pe/PeResources.fixture.js"; +import { createHash } from "node:crypto"; +import { RecordedCrashService } from "../../src/application/binaryDiagnostics/RecordedCrashService.js"; +import { + recordedCrashFixture, + RECORDED_CRASH_TEST_PROVIDER, +} from "../fixtures/binaryDiagnostics/recordedCrash.js"; +import { EvmInterfaceService } from "../../src/application/evm/EvmInterfaceService.js"; +import { + evmInterfaceExecution, + evmInterfaceFixture, +} from "../fixtures/evm/interface.js"; +import type { EvmInterfaceExecution } from "../../src/application/evm/EvmInterfacePort.js"; + +it("retains valid web-network observations and validates malformed ones before cancellation", async () => { + const report = { + decoder: { id: "fixture", name: "fixture", version: "1" }, + container: { + reported: null, + numeric_literals: [], + redactions: [], + records_pointer: "/log/entries", + }, + total_records: 1, + records: [ + { + ordinal: 0, + location: { kind: "json-pointer" as const, pointer: "/log/entries/0" }, + reported: historicalHar().log.entries[0] ?? null, + numeric_literals: [], + binary_fields: [], + redactions: [], + limitations: [], + }, + ], + artifact: { path: "/capture.har", sha256: "a".repeat(64), bytes: 10 }, + format: "har" as const, + runtime_attribution: "unknown" as const, + limitations: ["fixture"], + }; + const controller = new AbortController(); + const valid = await new WebNetworkCaptureService({ + inspect: () => { + controller.abort(); + return Promise.resolve(ok(report)); + }, + }).inspect( + { capture_path: report.artifact.path, format: "har" }, + { signal: controller.signal }, + ); + expect(valid).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + partialObservation: { + normalized_result: expect.objectContaining({ + artifact: report.artifact, + }), + }, + }, + }); + const malformed = { + ...report, + artifact: { ...report.artifact, path: "/other.har" }, + }; + const invalidController = new AbortController(); + const invalid = await new WebNetworkCaptureService({ + inspect: () => { + invalidController.abort(); + return Promise.resolve(ok(malformed)); + }, + }).inspect( + { capture_path: report.artifact.path, format: "har" }, + { signal: invalidController.signal }, + ); + expect(invalid).toMatchObject({ + ok: false, + error: { _tag: "AnalysisOutputError" }, + }); +}); + +it("retains valid source-map traces and preserves malformed decoder output over late cancellation", async () => { + const controller = new AbortController(); + const valid = await new WebSourceLocationService( + { load: () => Promise.resolve(ok(webSourceLocationFixture())) }, + { + trace: async (input) => { + const result = await webSourceMapDecoderFixture.trace(input); + controller.abort(); + return result; + }, + }, + ).trace(webSourceLocationArgs, { signal: controller.signal }); + expect(valid).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + partialObservation: { operation: "trace_web_source_location" }, + }, + }); + const invalidController = new AbortController(); + const invalid = await new WebSourceLocationService( + { load: () => Promise.resolve(ok(webSourceLocationFixture())) }, + { + trace: async (input) => { + const result = await webSourceMapDecoderFixture.trace(input); + if (!result.ok) return result; + result.value.url_context = "https://wrong.test/"; + invalidController.abort(); + return result; + }, + }, + ).trace(webSourceLocationArgs, { signal: invalidController.signal }); + expect(invalid).toMatchObject({ + ok: false, + error: { _tag: "AnalysisOutputError" }, + }); +}); + +it("retains valid binary-layout Evidence while preserving malformed output precedence", async () => { + const controller = new AbortController(); + const layoutInspect = () => { + controller.abort(); + return Promise.resolve(ok(binaryLayoutFixture())); + }; + const valid = await new BinaryLayoutService({ + identity: BINARY_LAYOUT_TEST_PROVIDER, + inspect: layoutInspect, + inspectAndClose: layoutInspect, + }).inspect( + { path: "/artifacts/source-owned.elf" }, + { signal: controller.signal }, + ); + expect(valid).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + partialObservation: { operation: "inspect_binary_layout" }, + }, + }); + const invalidController = new AbortController(); + const malformed = binaryLayoutFixture(); + malformed.artifact.path = "/other.elf"; + const malformedLayoutInspect = () => { + invalidController.abort(); + return Promise.resolve(ok(malformed)); + }; + const invalid = await new BinaryLayoutService({ + identity: BINARY_LAYOUT_TEST_PROVIDER, + inspect: malformedLayoutInspect, + inspectAndClose: malformedLayoutInspect, + }).inspect( + { path: "/artifacts/source-owned.elf" }, + { signal: invalidController.signal }, + ); + expect(invalid).toMatchObject({ + ok: false, + error: { _tag: "AnalysisOutputError" }, + }); +}); + +const peReport = async () => { + const { bytes } = peResourceFixture(); + return await parsePeResources( + bytes, + { + path: "/fixtures/example.exe", + bytes: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + }, + inspectPeResourcesInputSchema.parse({ path: "/fixtures/example.exe" }), + ); +}; +it("retains valid PE-resource Evidence and preserves malformed output precedence", async () => { + const controller = new AbortController(); + const valid = await new PeResourcesService({ + identity: PE_RESOURCES_PROVIDER, + inspect: () => { + controller.abort(); + return peReport().then(ok); + }, + }).inspect({ path: "/fixtures/example.exe" }, { signal: controller.signal }); + expect(valid).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + partialObservation: { operation: "inspect_pe_resources" }, + }, + }); + const invalidController = new AbortController(); + const malformed = await peReport(); + malformed.artifact.path = "/other.exe"; + const invalid = await new PeResourcesService({ + identity: PE_RESOURCES_PROVIDER, + inspect: () => { + invalidController.abort(); + return Promise.resolve(ok(malformed)); + }, + }).inspect( + { path: "/fixtures/example.exe" }, + { signal: invalidController.signal }, + ); + expect(invalid).toMatchObject({ + ok: false, + error: { _tag: "AnalysisOutputError" }, + }); +}); + +it("retains valid recorded-crash Evidence and preserves malformed output precedence", async () => { + const controller = new AbortController(); + const crashInspect = () => { + controller.abort(); + return Promise.resolve(ok(recordedCrashFixture())); + }; + const valid = await new RecordedCrashService({ + identity: RECORDED_CRASH_TEST_PROVIDER, + inspect: crashInspect, + inspectAndClose: crashInspect, + }).inspect( + { path: "/artifacts/source-owned.core", include_debugger_context: false }, + { signal: controller.signal }, + ); + expect(valid).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + partialObservation: { operation: "inspect_recorded_crash" }, + }, + }); + const invalidController = new AbortController(); + const malformed = recordedCrashFixture(); + malformed.artifact.path = "/other.core"; + const malformedCrashInspect = () => { + invalidController.abort(); + return Promise.resolve(ok(malformed)); + }; + const invalid = await new RecordedCrashService({ + identity: RECORDED_CRASH_TEST_PROVIDER, + inspect: malformedCrashInspect, + inspectAndClose: malformedCrashInspect, + }).inspect( + { path: "/artifacts/source-owned.core", include_debugger_context: false }, + { signal: invalidController.signal }, + ); + expect(invalid).toMatchObject({ + ok: false, + error: { _tag: "AnalysisOutputError" }, + }); +}); + +it("retains valid EVM interface Evidence and preserves malformed output precedence", async () => { + const controller = new AbortController(); + const valid = await new EvmInterfaceService({ + inspect: () => { + controller.abort(); + return Promise.resolve(ok(evmInterfaceExecution(evmInterfaceFixture()))); + }, + }).inspect( + { path: "/artifacts/source-owned.hex", encoding: "hex" }, + { signal: controller.signal }, + ); + expect(valid).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + partialObservation: { operation: "inspect_evm_interface" }, + }, + }); + const invalidController = new AbortController(); + const malformed = evmInterfaceFixture(); + malformed.artifact.path = "/other.hex"; + const invalid = await new EvmInterfaceService({ + inspect: () => { + invalidController.abort(); + return Promise.resolve( + ok( + evmInterfaceExecution(malformed) as unknown as EvmInterfaceExecution, + ), + ); + }, + }).inspect( + { path: "/artifacts/source-owned.hex", encoding: "hex" }, + { signal: invalidController.signal }, + ); + expect(invalid).toMatchObject({ + ok: false, + error: { _tag: "AnalysisOutputError" }, + }); +}); diff --git a/tests/fixtures/descriptorCloseFailure.ts b/tests/fixtures/descriptorCloseFailure.ts index 04cf29cde5..86ccfe3cca 100644 --- a/tests/fixtures/descriptorCloseFailure.ts +++ b/tests/fixtures/descriptorCloseFailure.ts @@ -14,12 +14,13 @@ export const expectRetainedDescriptor = ( result: Result, cleanup: DescriptorCloseFailure, operation: string, + additionalResources: readonly string[] = [], ): void => { expect(result).toMatchObject({ ok: false, error: { cleanupIncomplete: true, - cleanupResources: [cleanup.path], + cleanupResources: [cleanup.path, ...additionalResources], operation, }, }); diff --git a/tests/process-global/curatedWindowsLane.test.ts b/tests/process-global/curatedWindowsLane.test.ts index 8cd9c902cc..52510e055a 100644 --- a/tests/process-global/curatedWindowsLane.test.ts +++ b/tests/process-global/curatedWindowsLane.test.ts @@ -12,7 +12,14 @@ const workflowSchema = z.object({ jobs: z.record( z.string(), z.object({ - steps: z.array(z.object({ run: z.string().optional() })).optional(), + steps: z + .array( + z.object({ + run: z.string().optional(), + with: z.object({ name: z.string().optional() }).optional(), + }), + ) + .optional(), }), ), }); @@ -75,6 +82,34 @@ describe("curated Windows test lane", () => { "tests/boundary/process/packageNpmInvocation.test.ts", ); }); + + it("keeps Windows runner cancellation coverage in the lane", async () => { + expect(await explicitPaths("windows")).toEqual( + expect.arrayContaining([ + "tests/process-global/runnerCancellation.test.ts", + "tests/process-global/windowsOwnedCommand.test.ts", + ]), + ); + }); + + it("installs and verifies native command ownership before the source build", async () => { + const workflow = workflowSchema.parse( + parse(await readFile(WORKFLOW, "utf8")), + ); + const steps = workflow.jobs["windows-curated"]?.steps ?? []; + const download = steps.findIndex( + (step) => step.with?.name === "windows-native-controls", + ); + const verify = steps.findIndex( + (step) => step.run === "npm run verify:windows-native:artifact", + ); + const build = steps.findIndex( + (step) => step.run === "npm run build:cached", + ); + expect(download).toBeGreaterThanOrEqual(0); + expect(verify).toBeGreaterThan(download); + expect(build).toBeGreaterThan(verify); + }); }); it("keeps every explicit focused CI test path executable after file moves", async () => { diff --git a/tests/process-global/runnerCancellation.test.ts b/tests/process-global/runnerCancellation.test.ts index a84d54c560..2560f67870 100644 --- a/tests/process-global/runnerCancellation.test.ts +++ b/tests/process-global/runnerCancellation.test.ts @@ -1,21 +1,29 @@ import { spawn } from "node:child_process"; -import { copyFile, mkdir, readFile, utimes, writeFile } from "node:fs/promises"; +import { + access, + copyFile, + mkdir, + readFile, + utimes, + writeFile, +} from "node:fs/promises"; import { createRequire } from "node:module"; import { join } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { setTimeout as delay } from "node:timers/promises"; -import { expect, it } from "vitest"; +import { expect, it, onTestFinished, vi } from "vitest"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; -const runnerPath = fileURLToPath( - new URL("../../scripts/run-exclusive.mjs", import.meta.url), -); const supervisorPath = fileURLToPath( new URL("../../scripts/run-exclusive-command.mjs", import.meta.url), ); +const runnerPath = fileURLToPath( + new URL("../../scripts/run-exclusive.mjs", import.meta.url), +); + it.runIf(process.platform !== "win32")( "holds ownership until an uncooperative command has finished cancellation", async () => { @@ -93,6 +101,183 @@ it.runIf(process.platform !== "win32")( }, ); +it.runIf(process.platform === "win32")( + "terminates the owned command after supervisor IPC cancellation", + async () => { + const directory = await createTestTempDirectory("rea-runner-win-cancel-"); + const gate = join(directory, "release-child"); + const started = join(directory, "child-started"); + const runner = launchSupervisor(directory, gatedCommand(gate, started)); + onTestFinished(async () => { + await writeFile(gate, "release"); + await runner.closed(); + }); + try { + await runner.ready; + await expect(readFile(started, "utf8")).resolves.toBe("started"); + runner.child.send("SIGTERM"); + await expect(runner.closed()).resolves.toMatchObject({ code: 143 }); + expect(await pathExists(gate)).toBe(false); + await expect( + launch(directory, 'process.stdout.write("released");').closed, + ).resolves.toMatchObject({ code: 0, stdout: "released" }); + } finally { + await writeFile(gate, "release"); + } + }, +); + +it.runIf(process.platform === "win32")( + "keeps the lock after leader exit and cancels a surviving grandchild by message or disconnect", + async () => { + await exerciseWindowsGrandchild("message"); + await exerciseWindowsGrandchild("disconnect"); + }, +); + +async function exerciseWindowsGrandchild( + cancellation: "message" | "disconnect", +): Promise { + const directory = await createTestTempDirectory("rea-runner-win-tree-"); + const activityPath = join(directory, "grandchild-activity.log"); + const readyPath = join(directory, "grandchild-ready"); + const leaderExitedPath = join(directory, "leader-exited"); + const childExitedPath = join(directory, "child-exited"); + const childErrorPath = join(directory, "child-error"); + const grandchildExitedPath = join(directory, "grandchild-exited"); + const grandchildErrorPath = join(directory, "grandchild-error"); + const gatePath = join(directory, "grandchild-stop"); + const grandchild = ` + const fs = require("node:fs"); + const activity = ${JSON.stringify(activityPath)}; + const gate = ${JSON.stringify(gatePath)}; + process.on("exit", (code) => fs.writeFileSync(${JSON.stringify(grandchildExitedPath)}, String(code))); + process.on("uncaughtExceptionMonitor", (error) => fs.writeFileSync(${JSON.stringify(grandchildErrorPath)}, String(error))); + fs.writeFileSync(${JSON.stringify(readyPath)}, "ready"); + const write = () => fs.appendFileSync(activity, "active\\n"); + write(); + const timer = setInterval(() => { + if (fs.existsSync(gate)) { + clearInterval(timer); + process.exit(0); + } + write(); + }, 20); + `; + const child = ` + const fs = require("node:fs"); + const grandchildProcess = require("node:child_process").spawn( + process.execPath, + ["-e", ${JSON.stringify(grandchild)}], + { detached: true, stdio: "ignore" }, + ); + grandchildProcess.once("error", (error) => fs.writeFileSync(${JSON.stringify(childErrorPath)}, String(error))); + grandchildProcess.unref(); + process.on("exit", (code) => fs.writeFileSync(${JSON.stringify(childExitedPath)}, String(code))); + `; + const leader = ` + const fs = require("node:fs"); + process.on("exit", () => fs.writeFileSync(${JSON.stringify(leaderExitedPath)}, "exited")); + const child = require("node:child_process").spawn( + process.execPath, + ["-e", ${JSON.stringify(child)}], + { detached: true, stdio: "ignore" }, + ); + child.unref(); + const waitForGrandchild = setInterval(() => { + if (!fs.existsSync(${JSON.stringify(readyPath)})) return; + clearInterval(waitForGrandchild); + process.stdout.write("ready"); + }, 10); + `; + const runner = launchSupervisor(directory, leader); + onTestFinished(async () => { + await writeFile(gatePath, "stop"); + await runner.closed(); + }); + + try { + await runner.ready; + await expect(readFile(readyPath, "utf8")).resolves.toBe("ready"); + await vi.waitFor(async () => + expect(await readFile(leaderExitedPath, "utf8")).toBe("exited"), + ); + const activityBefore = await readFile(activityPath, "utf8"); + await expect + .poll(async () => (await readFile(activityPath, "utf8")).length, { + timeout: 5_000, + }) + .toBeGreaterThan(activityBefore.length); + const activityDuring = await readFile(activityPath, "utf8"); + const contender = await launch( + directory, + 'process.stdout.write("unexpected");', + ).closed; + const evidence = JSON.stringify({ + cancellation, + supervisorAlive: runner.child.exitCode === null, + leaderExited: await pathExists(leaderExitedPath), + descendantReady: await pathExists(readyPath), + descendantWroteDuringProbe: activityDuring.length > activityBefore.length, + descendantExited: await pathExists(grandchildExitedPath), + descendantError: await pathExists(grandchildErrorPath), + childExited: await pathExists(childExitedPath), + childError: await pathExists(childErrorPath), + supervisorCode: runner.child.exitCode, + supervisorStdout: runner.stdout, + supervisorStderr: runner.stderr, + contender, + }); + expect( + contender, + `Windows process-tree ownership evidence: ${evidence}`, + ).toMatchObject({ + code: 1, + stderr: expect.stringContaining("already running"), + }); + + if (cancellation === "disconnect") runner.child.disconnect(); + else runner.child.send("SIGTERM"); + await expect(runner.closed()).resolves.toMatchObject({ code: 143 }); + const stopped = await readFile(activityPath, "utf8"); + await delay(100); + expect(await readFile(activityPath, "utf8")).toBe(stopped); + await expect( + launch(directory, 'process.stdout.write("released");').closed, + ).resolves.toMatchObject({ code: 0, stdout: "released" }); + } finally { + await writeFile(gatePath, "stop"); + } +} + +it.runIf(process.platform === "win32")( + "stops the owned command after the supervisor IPC disconnects", + async () => { + const directory = await createTestTempDirectory( + "rea-runner-win-disconnect-", + ); + const gate = join(directory, "release-child"); + const started = join(directory, "child-started"); + const runner = launchSupervisor(directory, gatedCommand(gate, started)); + onTestFinished(async () => { + await writeFile(gate, "release"); + await runner.closed(); + }); + try { + await runner.ready; + await expect(readFile(started, "utf8")).resolves.toBe("started"); + runner.child.disconnect(); + await expect(runner.closed()).resolves.toMatchObject({ code: 143 }); + expect(await pathExists(gate)).toBe(false); + await expect( + launch(directory, 'process.stdout.write("released");').closed, + ).resolves.toMatchObject({ code: 0, stdout: "released" }); + } finally { + await writeFile(gate, "release"); + } + }, +); + it("does not treat an old database timestamp as permission to replace a live owner", async () => { const directory = await createTestTempDirectory("rea-runner-live-"); const runner = launch( @@ -170,6 +355,27 @@ it.runIf(process.platform === "win32")( expect(npm).toMatchObject({ code: 0, stderr: "" }); expect(npm.stdout.trim()).toMatch(/^\d+\.\d+\.\d+/u); + const npmProject = join(directory, "npm project"); + await mkdir(npmProject, { recursive: true }); + await writeFile( + join(npmProject, "package.json"), + JSON.stringify({ + scripts: { + "print-argv": + 'node -e "process.stdout.write(JSON.stringify(process.argv.slice(1)))" --', + }, + }), + ); + const shimPayload = 'argument with spaces and "quotes" & ^ % !'; + const npmArgv = await execute(npmProject, "npm", [ + "run", + "print-argv", + "--", + shimPayload, + ]); + expect(npmArgv).toMatchObject({ code: 0, stderr: "" }); + expect(npmArgv.stdout).toContain(JSON.stringify([shimPayload])); + const turbo = await execute( directory, join(process.cwd(), "node_modules", ".bin", "turbo"), @@ -283,6 +489,99 @@ function launch(directory: string, command: string) { return { child, ready, closed }; } +function launchSupervisor(directory: string, command: string) { + const child = spawn( + process.execPath, + [ + fileURLToPath( + new URL("../../scripts/run-exclusive-command.mjs", import.meta.url), + ), + "test", + process.execPath, + "-e", + command, + ], + { cwd: directory, stdio: ["ignore", "pipe", "pipe", "ipc"] }, + ); + if (child.stdout === null || child.stderr === null) + throw new Error("runner supervisor streams were not piped"); + const stdoutStream = child.stdout; + const stderrStream = child.stderr; + let stdout = ""; + let stderr = ""; + let notifyReady: (() => void) | undefined; + const ready = withTimeout( + new Promise((resolveReady) => { + notifyReady = resolveReady; + }), + "runner supervisor child did not start", + ); + void ready.catch(() => {}); + stdoutStream.on("data", (chunk) => { + stdout += chunk.toString(); + if (stdout.includes("ready")) notifyReady?.(); + }); + stderrStream.on("data", (chunk) => (stderr += chunk.toString())); + const exited = new Promise<{ + code: number | null; + signal: NodeJS.Signals | null; + }>((resolveExit, reject) => { + child.once("error", reject); + child.once("exit", (code, signal) => { + notifyReady?.(); + resolveExit({ code, signal }); + }); + }); + const stdoutClosed = new Promise((resolveClose) => { + stdoutStream.once("close", resolveClose); + }); + const stderrClosed = new Promise((resolveClose) => { + stderrStream.once("close", resolveClose); + }); + const ipcDisconnected = new Promise((resolveDisconnect) => { + child.once("disconnect", resolveDisconnect); + }); + const finished = Promise.all([ + exited, + stdoutClosed, + stderrClosed, + ipcDisconnected, + ]).then(([{ code, signal }]) => ({ code, signal, stdout, stderr })); + return { + child, + ready, + closed: () => withTimeout(finished, "runner supervisor did not finish"), + get stdout() { + return stdout; + }, + get stderr() { + return stderr; + }, + }; +} + +function gatedCommand(gate: string, started: string): string { + return ` + const fs = require("node:fs"); + const gate = ${JSON.stringify(gate)}; + fs.writeFileSync(${JSON.stringify(started)}, "started"); + process.stdout.write("ready"); + const poll = setInterval(() => { + if (fs.existsSync(gate)) { + clearInterval(poll); + process.exit(0); + } + }, 10); + `; +} + +function pathExists(path: string): Promise { + return access(path).then( + () => true, + () => false, + ); +} + function execute( directory: string, command: string, @@ -307,140 +606,34 @@ function execute( }); } -function withTimeout(promise: Promise, message: string): Promise { +function withTimeout( + promise: Promise, + message: string, + timeoutMs = 5_000, +): Promise { let timeout: NodeJS.Timeout | undefined; return Promise.race([ promise, new Promise((_, reject) => { - timeout = setTimeout(() => reject(new Error(message)), 5_000); + timeout = setTimeout(() => reject(new Error(message)), timeoutMs); }), ]).finally(() => clearTimeout(timeout)); } -/** True while the directly owned command PID remains signalable. */ -function isAlive(pid: number): boolean { - try { - process.kill(pid, 0); - return true; - } catch { - return false; - } -} - -/** Run the supervisor over IPC to cover Windows cancellation and disconnect. */ -function launchSupervisor(directory: string) { - const pidPath = join(directory, "command.pid"); - const gated = ` - const fs = require("node:fs"); - fs.writeFileSync(${JSON.stringify(pidPath)}, String(process.pid)); - process.stdout.write("ready"); - setInterval(() => {}, 1000); - `; - const supervisor = spawn( - process.execPath, - [supervisorPath, "test", process.execPath, "-e", gated], - { cwd: directory, stdio: ["ignore", "pipe", "pipe", "ipc"] }, - ); - if (supervisor.stdout === null || supervisor.stderr === null) { - supervisor.kill("SIGKILL"); - throw new Error("supervisor output pipes were not created"); - } - const stdoutStream = supervisor.stdout; - const stderrStream = supervisor.stderr; - let stdout = ""; - let stderr = ""; - let notifyReady: (() => void) | undefined; - const ready = withTimeout( - new Promise((resolveReady) => { - notifyReady = resolveReady; - }), - "supervisor command did not start", - ); - void ready.catch(() => undefined); - stdoutStream.on("data", (chunk) => { - stdout += chunk.toString(); - if (stdout.includes("ready")) notifyReady?.(); - }); - stderrStream.on("data", (chunk) => (stderr += chunk.toString())); - const closed = withTimeout( - new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( - (resolveClosed, reject) => { - supervisor.once("error", reject); - supervisor.once("close", (code, signal) => - resolveClosed({ code, signal }), - ); - }, - ), - "supervisor did not finish", - ); - const commandPid = async () => - Number(await readFile(pidPath, "utf8").catch(() => "0")); - const stop = async () => { - const pid = await commandPid(); - if (pid > 0) stopIfPresent(pid); - if (supervisor.exitCode === null && supervisor.signalCode === null) - supervisor.kill("SIGKILL"); - await closed.catch(() => undefined); - }; - return { supervisor, ready, closed, commandPid, stop, stderr: () => stderr }; -} - -it.runIf(process.platform === "win32")( - "terminates the owned command and releases the lock after forwarded cancellation", +it.runIf(process.platform !== "win32")( + "retains lock ownership through a post-spawn child error until close", async () => { - const directory = await createTestTempDirectory("rea-runner-win-ipc-"); - const runner = launchSupervisor(directory); - try { - await runner.ready; - const pid = await runner.commandPid(); - expect(pid).toBeGreaterThan(0); - expect(isAlive(pid)).toBe(true); - runner.supervisor.send("SIGTERM"); - expect(await runner.closed).toEqual({ code: 143, signal: null }); - expect(isAlive(pid)).toBe(false); - expect( - await execute(directory, 'process.stdout.write("lock released");'), - ).toMatchObject({ code: 0, stdout: "lock released" }); - } finally { - await runner.stop(); - } - }, -); - -it.runIf(process.platform === "win32")( - "terminates the owned command and releases the lock after parent disconnect", - async () => { - const directory = await createTestTempDirectory("rea-runner-win-drop-"); - const runner = launchSupervisor(directory); - try { - await runner.ready; - const pid = await runner.commandPid(); - expect(pid).toBeGreaterThan(0); - expect(isAlive(pid)).toBe(true); - runner.supervisor.disconnect(); - expect(await runner.closed).toEqual({ code: 143, signal: null }); - expect(isAlive(pid)).toBe(false); - expect( - await execute(directory, 'process.stdout.write("lock released");'), - ).toMatchObject({ code: 0, stdout: "lock released" }); - } finally { - await runner.stop(); - } - }, -); - -it("retains lock ownership through a post-spawn child error until close", async () => { - const directory = await createTestTempDirectory("rea-runner-child-error-"); - const readyPath = join(directory, "command-ready"); - const gatePath = join(directory, "allow-command-exit"); - const closedPath = join(directory, "command-closed"); - const source = await readFile(supervisorPath, "utf8"); - const crossSpawn = pathToFileURL( - createRequire(import.meta.url).resolve("cross-spawn"), - ).href; - const spawnCall = - /child = spawn\(command\[0\], command\.slice\(1\), \{\n\s+cwd: process\.cwd\(\),\n\s+env: process\.env,\n\s+stdio: "inherit",\n\s+\}\);/u; - const postSpawnInjection = `child.once("spawn", () => { + const directory = await createTestTempDirectory("rea-runner-child-error-"); + const readyPath = join(directory, "command-ready"); + const gatePath = join(directory, "allow-command-exit"); + const closedPath = join(directory, "command-closed"); + const source = await readFile(supervisorPath, "utf8"); + const crossSpawn = pathToFileURL( + createRequire(import.meta.url).resolve("cross-spawn"), + ).href; + const spawnCall = + /child = spawn\(command\[0\], command\.slice\(1\), \{\n\s+cwd: process\.cwd\(\),\n\s+env: process\.env,\n\s+stdio: "inherit",\n\s+\}\);/u; + const postSpawnInjection = `child.once("spawn", () => { const waitForReady = setInterval(() => { if (!existsSync(${JSON.stringify(readyPath)})) return; clearInterval(waitForReady); @@ -449,19 +642,25 @@ it("retains lock ownership through a post-spawn child error until close", async child.emit("error", new Error("controlled secondary child error")); }, 5); });`; - expect(source).toMatch(spawnCall); - const injectedSource = source - .replace( - 'import spawn from "cross-spawn";', - `import spawn from ${JSON.stringify(crossSpawn)};\nimport { existsSync } from "node:fs";`, - ) - .replace(spawnCall, (call) => `${call}\n ${postSpawnInjection}`); - expect(injectedSource).toContain(postSpawnInjection); - expect(injectedSource).toContain("controlled post-spawn child error"); - const copiedSupervisor = join(directory, "run-exclusive-command.mjs"); - await writeFile(copiedSupervisor, injectedSource); + expect(source).toMatch(spawnCall); + const injectedSource = source + .replace( + 'import spawn from "cross-spawn";', + `import spawn from ${JSON.stringify(crossSpawn)};\nimport { existsSync } from "node:fs";`, + ) + .replace(spawnCall, (call) => `${call}\n ${postSpawnInjection}`); + expect(injectedSource).toContain(postSpawnInjection); + expect(injectedSource).toContain("controlled post-spawn child error"); + const copiedSupervisor = join(directory, "run-exclusive-command.mjs"); + const copiedHelperDirectory = join(directory, "lib"); + await mkdir(copiedHelperDirectory, { recursive: true }); + await copyFile( + join(process.cwd(), "scripts/lib/windows-owned-command.mjs"), + join(copiedHelperDirectory, "windows-owned-command.mjs"), + ); + await writeFile(copiedSupervisor, injectedSource); - const gatedCommand = ` + const gatedCommand = ` const fs = require("node:fs"); fs.writeFileSync(${JSON.stringify(readyPath)}, "ready"); process.stdout.write("command-ready\\n"); @@ -472,79 +671,80 @@ it("retains lock ownership through a post-spawn child error until close", async process.stdout.write("command-closed\\n"); }, 5); `; - const supervisor = spawn( - process.execPath, - [copiedSupervisor, "test", process.execPath, "-e", gatedCommand], - { cwd: directory, stdio: ["ignore", "pipe", "pipe", "ipc"] }, - ); - if (supervisor.stdout === null || supervisor.stderr === null) { - supervisor.kill("SIGKILL"); - throw new Error("supervisor output pipes were not created"); - } - const stdoutStream = supervisor.stdout; - const stderrStream = supervisor.stderr; - let stdout = ""; - let stderr = ""; - const outputWaiters = new Map void)[]>(); - stdoutStream.on("data", (chunk) => { - stdout += chunk.toString(); - for (const [marker, waiters] of outputWaiters) { - if (!stdout.includes(marker)) continue; - outputWaiters.delete(marker); - for (const notify of waiters) notify(); - } - }); - stderrStream.on("data", (chunk) => (stderr += chunk.toString())); - const waitForOutput = (marker: string) => { - if (stdout.includes(marker)) return Promise.resolve(); - return withTimeout( - new Promise((resolveOutput) => { - outputWaiters.set(marker, [ - ...(outputWaiters.get(marker) ?? []), - resolveOutput, - ]); - }), - `supervisor did not write ${marker}`, + const supervisor = spawn( + process.execPath, + [copiedSupervisor, "test", process.execPath, "-e", gatedCommand], + { cwd: directory, stdio: ["ignore", "pipe", "pipe", "ipc"] }, ); - }; - const exited = withTimeout( - new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( - (resolveExit, reject) => { - supervisor.once("error", reject); - supervisor.once("exit", (code, signal) => - resolveExit({ code, signal }), - ); - }, - ), - "supervisor did not exit", - ); - try { - await waitForOutput("controlled-post-spawn-error"); - const contender = await execute(directory, process.execPath, [ - "-e", - 'process.stdout.write("lock acquired");', - ]); - expect(contender).toMatchObject({ - code: 1, - stdout: "", - stderr: expect.stringContaining("already running"), + if (supervisor.stdout === null || supervisor.stderr === null) { + supervisor.kill("SIGKILL"); + throw new Error("supervisor output pipes were not created"); + } + const stdoutStream = supervisor.stdout; + const stderrStream = supervisor.stderr; + let stdout = ""; + let stderr = ""; + const outputWaiters = new Map void)[]>(); + stdoutStream.on("data", (chunk) => { + stdout += chunk.toString(); + for (const [marker, waiters] of outputWaiters) { + if (!stdout.includes(marker)) continue; + outputWaiters.delete(marker); + for (const notify of waiters) notify(); + } }); - expect(stdout).not.toContain("command-closed"); - - await writeFile(gatePath, "open"); - await waitForOutput("command-closed"); - expect(await exited).toEqual({ code: 1, signal: null }); - expect(stderr).toContain("controlled post-spawn child error"); - expect(stderr).not.toContain("controlled secondary child error"); - expect( - await execute(directory, process.execPath, [ + stderrStream.on("data", (chunk) => (stderr += chunk.toString())); + const waitForOutput = (marker: string) => { + if (stdout.includes(marker)) return Promise.resolve(); + return withTimeout( + new Promise((resolveOutput) => { + outputWaiters.set(marker, [ + ...(outputWaiters.get(marker) ?? []), + resolveOutput, + ]); + }), + `supervisor did not write ${marker}`, + ); + }; + const exited = withTimeout( + new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolveExit, reject) => { + supervisor.once("error", reject); + supervisor.once("exit", (code, signal) => + resolveExit({ code, signal }), + ); + }, + ), + "supervisor did not exit", + ); + try { + await waitForOutput("controlled-post-spawn-error"); + const contender = await execute(directory, process.execPath, [ "-e", 'process.stdout.write("lock acquired");', - ]), - ).toMatchObject({ code: 0, stdout: "lock acquired" }); - } finally { - await writeFile(gatePath, "open"); - await waitForOutput("command-closed").catch(() => undefined); - await exited.catch(() => undefined); - } -}); + ]); + expect(contender).toMatchObject({ + code: 1, + stdout: "", + stderr: expect.stringContaining("already running"), + }); + expect(stdout).not.toContain("command-closed"); + + await writeFile(gatePath, "open"); + await waitForOutput("command-closed"); + expect(await exited).toEqual({ code: 1, signal: null }); + expect(stderr).toContain("controlled post-spawn child error"); + expect(stderr).not.toContain("controlled secondary child error"); + expect( + await execute(directory, process.execPath, [ + "-e", + 'process.stdout.write("lock acquired");', + ]), + ).toMatchObject({ code: 0, stdout: "lock acquired" }); + } finally { + await writeFile(gatePath, "open"); + await waitForOutput("command-closed").catch(() => undefined); + await exited.catch(() => undefined); + } + }, +); diff --git a/tests/process-global/windowsOwnedCommand.test.ts b/tests/process-global/windowsOwnedCommand.test.ts new file mode 100644 index 0000000000..1e83fbc1f9 --- /dev/null +++ b/tests/process-global/windowsOwnedCommand.test.ts @@ -0,0 +1,422 @@ +import { EventEmitter, once } from "node:events"; +import { readFile, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; + +import { expect, it, vi } from "vitest"; + +import { + createWindowsOwnedCommand, + loadWindowsOwnedProcessClass, + superviseWindowsOwnedProcess, +} from "../../scripts/lib/windows-owned-command.mjs"; +import { requireWindowsNativeAuthority } from "../../src/windows/WindowsNativeLoader.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +const require = createRequire(import.meta.url); + +async function readOptional(path: string): Promise { + return readFile(path, "utf8").catch(() => undefined); +} + +it("imports the source Windows owner without dist using a fake native authority", async () => { + const calls: string[] = []; + const WindowsOwnedProcess = await loadWindowsOwnedProcessClass(); + const child = new WindowsOwnedProcess( + "ignored.exe", + [], + undefined, + {}, + false, + { + authority: { + inspection: { + abiVersion: 1, + nodeApiVersion: 8, + architecture: "x64", + filesystem: { + requestedPath: "C:\\probe", + finalPath: "C:\\probe", + filesystem: "NTFS", + volumeSerial: "0".repeat(16), + fileId: "0".repeat(32), + size: 0, + directory: true, + }, + privateDacl: true, + privateBridgeDescriptorAccess: true, + atomicJobAssignment: true, + killOnOwnerClose: true, + }, + call(operation: string) { + calls.push(operation); + if (operation === "process_spawn") + return { + pid: 42, + handle: {}, + jobAssigned: true, + killOnOwnerClose: true, + }; + if (operation === "process_poll") + return { + stdout: Buffer.alloc(0), + stderr: Buffer.alloc(0), + stdoutEnded: true, + stderrEnded: true, + exitCode: 0, + activeProcesses: 0, + }; + if (operation === "process_close") return null; + throw new Error(`Unmodeled fake native operation ${operation}`); + }, + }, + }, + ); + + await expect(once(child, "close")).resolves.toEqual([0, null]); + expect(calls).toEqual(["process_spawn", "process_poll", "process_close"]); +}); + +it("passes cross-spawn's resolved executable to the native owner", () => { + const parserPath = require.resolve("cross-spawn/lib/parse"); + require(parserPath); + const parserModule = require.cache[parserPath]; + if (parserModule === undefined) + throw new Error("cross-spawn parser was not loaded"); + const originalParser = parserModule.exports; + let received: unknown[] | undefined; + class CaptureOwnedProcess { + constructor(...arguments_: unknown[]) { + received = arguments_; + } + } + parserModule.exports = () => ({ + command: "node", + file: "C:\\runtime\\node.exe", + args: ["-e", "code"], + options: {}, + }); + try { + createWindowsOwnedCommand({ + WindowsOwnedProcess: CaptureOwnedProcess as never, + command: "node", + arguments: ["-e", "code"], + cwd: undefined, + environment: {}, + }); + } finally { + parserModule.exports = originalParser; + } + + expect(received?.[0]).toBe("C:\\runtime\\node.exe"); + expect(received?.[1]).toEqual(["-e", "code"]); +}); + +it("supervises protocol stdin errors until the owned job and stream close", async () => { + const calls: string[] = []; + let activeProcesses = 1; + const writeFailure = new Error("controlled native stdin failure"); + const WindowsOwnedProcess = await loadWindowsOwnedProcessClass(); + const child = createWindowsOwnedCommand({ + WindowsOwnedProcess, + command: "ignored.exe", + arguments: [], + cwd: undefined, + environment: {}, + authority: { + inspection: { + abiVersion: 1, + nodeApiVersion: 8, + architecture: "x64", + filesystem: { + requestedPath: "C:\\probe", + finalPath: "C:\\probe", + filesystem: "NTFS", + volumeSerial: "0".repeat(16), + fileId: "0".repeat(32), + size: 0, + directory: true, + }, + privateDacl: true, + privateBridgeDescriptorAccess: true, + atomicJobAssignment: true, + killOnOwnerClose: true, + protocolStdin: true, + }, + call(operation: string) { + calls.push(operation); + if (operation === "process_spawn") + return { + pid: 42, + handle: {}, + jobAssigned: true, + killOnOwnerClose: true, + }; + if (operation === "process_stdin_write") throw writeFailure; + if (operation === "process_stdin_close") return null; + if (operation === "process_terminate") { + activeProcesses = 0; + return true; + } + if (operation === "process_poll") + return { + stdout: Buffer.alloc(0), + stderr: Buffer.alloc(0), + stdoutEnded: true, + stderrEnded: true, + exitCode: 0, + activeProcesses, + }; + if (operation === "process_close") return null; + throw new Error(`Unmodeled fake native operation ${operation}`); + }, + }, + }); + const lifecycle = superviseWindowsOwnedProcess(child, { + retryIntervalMs: 10, + }); + if (child.stdin === null) throw new Error("fixture protocol stdin missing"); + const stdin = child.stdin; + const write = new Promise((resolve) => { + stdin.write(Buffer.from("x"), (cause?: Error | null) => + resolve(cause ?? null), + ); + }); + + await expect(write).resolves.toBe(writeFailure); + await expect(lifecycle.finished).rejects.toBe(writeFailure); + expect(calls).toContain("process_terminate"); + expect(calls).toContain("process_close"); +}); + +it.runIf(process.platform === "win32")( + "supervises native protocol stdin through Job Object close", + async () => { + const directory = await createTestTempDirectory("rea-owned-stdin-probe-"); + const WindowsOwnedProcess = await loadWindowsOwnedProcessClass(); + const child = new WindowsOwnedProcess( + process.execPath, + ["-e", "setInterval(() => {}, 1000)"], + directory, + process.env, + false, + { + protocolStdin: true, + authority: requireWindowsNativeAuthority(), + }, + ); + child.stdout.resume(); + child.stderr.resume(); + const lifecycle = superviseWindowsOwnedProcess(child); + const stdin = child.stdin; + if (stdin === null) throw new Error("fixture protocol stdin missing"); + let stdinClosed = false; + stdin.once("close", () => { + stdinClosed = true; + }); + process.stdin.pipe(stdin); + + try { + lifecycle.requestCleanup(); + let settled = false; + void lifecycle.finished.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + await vi.waitFor(() => expect(settled).toBe(true), { timeout: 15_000 }); + await expect(lifecycle.finished).resolves.toMatchObject({ + code: expect.any(Number), + }); + expect(stdinClosed).toBe(true); + } finally { + process.stdin.unpipe(stdin); + lifecycle.requestCleanup(); + } + }, +); + +it.runIf(process.platform === "win32")( + "keeps native Job Object ownership after the leader exits", + async () => { + const directory = await createTestTempDirectory("rea-owned-tree-probe-"); + const activityPath = join(directory, "descendant-activity.log"); + const readyPath = join(directory, "descendant-ready"); + const exitedPath = join(directory, "descendant-exited"); + const errorPath = join(directory, "descendant-error"); + const stopPath = join(directory, "descendant-stop"); + const grandchild = ` + const fs = require("node:fs"); + process.on("exit", (code) => fs.writeFileSync(${JSON.stringify(exitedPath)}, String(code))); + process.on("uncaughtExceptionMonitor", (error) => fs.writeFileSync(${JSON.stringify(errorPath)}, String(error))); + fs.writeFileSync(${JSON.stringify(readyPath)}, "ready"); + const timer = setInterval(() => { + if (fs.existsSync(${JSON.stringify(stopPath)})) { + clearInterval(timer); + process.exit(0); + } + fs.appendFileSync(${JSON.stringify(activityPath)}, "active\\n"); + }, 20); + `; + const leader = ` + const fs = require("node:fs"); + const child = require("node:child_process").spawn( + process.execPath, + ["-e", ${JSON.stringify(grandchild)}], + { detached: true, stdio: "ignore" }, + ); + child.unref(); + const waitForDescendant = setInterval(() => { + if (!fs.existsSync(${JSON.stringify(readyPath)})) return; + clearInterval(waitForDescendant); + process.exit(0); + }, 10); + `; + const WindowsOwnedProcess = await loadWindowsOwnedProcessClass(); + const native = requireWindowsNativeAuthority(); + const observedCounts: number[] = []; + const authority = { + inspection: native.inspection, + call(operation: string, arguments_: readonly unknown[]) { + const result = native.call(operation, arguments_); + if ( + operation === "process_poll" && + typeof result === "object" && + result !== null && + "activeProcesses" in result && + typeof result.activeProcesses === "number" + ) + observedCounts.push(result.activeProcesses); + return result; + }, + }; + const child = new WindowsOwnedProcess( + process.execPath, + ["-e", leader], + directory, + process.env, + false, + { authority }, + ); + child.stdout.resume(); + child.stderr.resume(); + const exited = once(child, "exit"); + const closed = once(child, "close"); + let hasClosed = false; + void closed.then(() => { + hasClosed = true; + }); + + try { + await exited; + await expect(readFile(readyPath, "utf8")).resolves.toBe("ready"); + const before = await readOptional(activityPath); + await expect + .poll(async () => (await readFile(activityPath, "utf8")).length, { + timeout: 5_000, + }) + .toBeGreaterThan(before?.length ?? 0); + expect( + hasClosed, + JSON.stringify({ + activeProcessCounts: observedCounts, + descendantExitCode: await readOptional(exitedPath), + descendantError: await readOptional(errorPath), + activity: await readOptional(activityPath), + }), + ).toBe(false); + await expect(child.cleanup()).resolves.toMatchObject({ cleaned: true }); + await closed; + const activityAtCleanup = await readFile(activityPath, "utf8"); + await delay(100); + expect(await readFile(activityPath, "utf8")).toBe(activityAtCleanup); + } finally { + await writeFile(stopPath, "stop"); + if (!hasClosed) { + const cleanup = await child.cleanup(); + if (cleanup.cleaned) await closed; + else child.unref(); + } + await delay(100); + } + }, +); + +it("retries thrown and false cleanup results, then preserves the first error", async () => { + vi.useFakeTimers(); + try { + class ControlledOwnedProcess extends EventEmitter { + attempts = 0; + + cleanup() { + this.attempts += 1; + if (this.attempts === 1) + throw new Error("controlled synchronous cleanup throw"); + if (this.attempts === 2) + return Promise.resolve({ + cleaned: false, + reason: "controlled poll failure", + }); + this.emit("close", 7, null); + return Promise.resolve({ cleaned: true, signaled: true }); + } + } + + const child = new ControlledOwnedProcess(); + const diagnostics: { type: string; reason?: string }[] = []; + const lifecycle = superviseWindowsOwnedProcess(child, { + retryIntervalMs: 25, + onCleanupDiagnostic: (diagnostic) => diagnostics.push(diagnostic), + }); + let completed = false; + const completion = lifecycle.finished + .catch((cause: unknown) => cause) + .then((cause) => { + completed = true; + return cause; + }); + + child.emit("error", undefined); + expect(child.attempts).toBe(1); + await vi.waitFor(() => + expect(diagnostics).toEqual([ + { + type: "cleanup-failed", + reason: "controlled synchronous cleanup throw", + }, + ]), + ); + expect(completed).toBe(false); + + child.emit("error", new Error("secondary native error")); + expect(child.attempts).toBe(2); + await vi.waitFor(() => + expect(diagnostics).toEqual([ + { + type: "cleanup-failed", + reason: "controlled synchronous cleanup throw", + }, + { type: "cleanup-failed", reason: "controlled poll failure" }, + ]), + ); + expect(completed).toBe(false); + + await vi.advanceTimersByTimeAsync(25); + await expect(completion).resolves.toBeUndefined(); + expect(child.attempts).toBe(3); + expect(diagnostics).toEqual([ + { + type: "cleanup-failed", + reason: "controlled synchronous cleanup throw", + }, + { type: "cleanup-failed", reason: "controlled poll failure" }, + { type: "cleanup-recovered" }, + ]); + } finally { + vi.useRealTimers(); + } +});